From 117aea8c053827e12261a28a32e80217024d557b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 30 Oct 2026 21:42:34 +0000 Subject: [PATCH 01/59] feat: checkpoint native orchestration backend Preserve in-progress storage, scheduling, claims, credentials and native API/CLI work. The orchestration backend remains incomplete and is not ready for delivery. Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- Cargo.lock | 2125 ++++++++++++++++- Cargo.toml | 18 + crates/sqlbuild-cli/Cargo.toml | 5 + .../sqlbuild-cli/src/argv/command_table.json | 139 +- .../src/launcher/_helpers/dispatch.rs | 19 +- .../src/native_commands/_helpers/decision.rs | 4 +- .../src/native_commands/constants.rs | 2 + .../src/native_commands/main/_run_control.rs | 135 ++ .../src/native_commands/main/mod.rs | 3 + .../main/parse_native_invocation.rs | 49 + .../main/run_control_invocation.rs | 63 + .../main/run_native_invocation.rs | 41 +- .../src/native_commands/models.rs | 12 + crates/sqlbuild-cli/tests/native_commands.rs | 6 + .../tests/native_commands/helpers.rs | 108 + .../tests/native_commands/test_control.rs | 100 + .../tests/native_commands/test_types.rs | 15 + crates/sqlbuild-orchestration/Cargo.toml | 36 + .../src/api/_helpers/authentication.rs | 38 + .../src/api/_helpers/mod.rs | 3 + .../src/api/_helpers/runs.rs | 65 + .../src/api/_helpers/snapshots.rs | 65 + .../sqlbuild-orchestration/src/api/errors.rs | 54 + .../src/api/main/mod.rs | 2 + .../src/api/main/openapi_document.rs | 19 + .../src/api/main/serve_api.rs | 39 + crates/sqlbuild-orchestration/src/api/mod.rs | 6 + .../sqlbuild-orchestration/src/api/models.rs | 24 + .../src/credentials/_helpers/audit.rs | 31 + .../src/credentials/_helpers/mod.rs | 2 + .../src/credentials/_helpers/scopes.rs | 40 + .../src/credentials/constants.rs | 4 + .../credentials/main/authenticate_token.rs | 59 + .../src/credentials/main/create_principal.rs | 55 + .../src/credentials/main/issue_token.rs | 78 + .../src/credentials/main/mod.rs | 4 + .../src/credentials/main/revoke_token.rs | 48 + .../src/credentials/mod.rs | 6 + .../src/credentials/models.rs | 63 + .../src/declarations/main/change_trigger.rs | 87 + .../declarations/main/list_trigger_events.rs | 28 + .../src/declarations/main/list_triggers.rs | 30 + .../src/declarations/main/mod.rs | 5 + .../declarations/main/publish_definitions.rs | 94 + .../declarations/main/snapshot_definitions.rs | 79 + .../src/declarations/mod.rs | 2 + .../src/declarations/models.rs | 139 ++ crates/sqlbuild-orchestration/src/lib.rs | 11 + .../src/runs/_helpers/changes.rs | 120 + .../src/runs/_helpers/claims.rs | 84 + .../src/runs/_helpers/events.rs | 63 + .../src/runs/_helpers/mod.rs | 5 + .../src/runs/_helpers/nodes.rs | 83 + .../src/runs/_helpers/projection.rs | 115 + .../src/runs/main/change_run.rs | 104 + .../src/runs/main/claim_next_run.rs | 94 + .../src/runs/main/heartbeat_run.rs | 71 + .../src/runs/main/list_run_nodes.rs | 29 + .../src/runs/main/list_runs.rs | 30 + .../src/runs/main/mod.rs | 7 + .../src/runs/main/queue_run.rs | 57 + .../src/runs/main/request_run.rs | 67 + crates/sqlbuild-orchestration/src/runs/mod.rs | 3 + .../sqlbuild-orchestration/src/runs/models.rs | 250 ++ .../src/scheduler/_helpers/calendar.rs | 31 + .../src/scheduler/_helpers/expansion.rs | 58 + .../src/scheduler/_helpers/mod.rs | 4 + .../src/scheduler/_helpers/persistence.rs | 76 + .../src/scheduler/_helpers/ticks.rs | 38 + .../src/scheduler/constants.rs | 8 + .../src/scheduler/main/acquire_lease.rs | 37 + .../src/scheduler/main/check_lease.rs | 22 + .../src/scheduler/main/evaluate_trigger.rs | 81 + .../src/scheduler/main/list_ticks.rs | 29 + .../src/scheduler/main/mod.rs | 12 + .../src/scheduler/main/next_cron_time.rs | 28 + .../src/scheduler/main/next_schedule_time.rs | 28 + .../src/scheduler/main/parse_cron.rs | 24 + .../src/scheduler/main/plan_ticks.rs | 84 + .../src/scheduler/main/previous_cron_time.rs | 30 + .../src/scheduler/main/renew_lease.rs | 30 + .../src/scheduler/main/run_scheduler.rs | 26 + .../src/scheduler/main/scheduler_cycle.rs | 75 + .../src/scheduler/mod.rs | 9 + .../src/scheduler/models.rs | 150 ++ .../src/scheduler/tests.rs | 12 + .../src/scheduler/tests/helpers.rs | 129 + .../src/scheduler/tests/test_backlog.rs | 107 + .../src/scheduler/tests/test_cron.rs | 158 ++ .../src/scheduler/tests/test_simulation.rs | 32 + .../src/scheduler/tests/test_ticks.rs | 146 ++ .../src/scheduler/tests/test_types.rs | 52 + .../src/snapshots/_helpers/content.rs | 72 + .../src/snapshots/_helpers/mod.rs | 4 + .../src/snapshots/_helpers/preparation.rs | 74 + .../src/snapshots/_helpers/publication.rs | 34 + .../src/snapshots/_helpers/reading.rs | 57 + .../src/snapshots/main/current_snapshot.rs | 53 + .../src/snapshots/main/mod.rs | 2 + .../src/snapshots/main/publish_snapshot.rs | 118 + .../src/snapshots/mod.rs | 3 + .../src/snapshots/models.rs | 79 + .../src/store/_helpers/database.rs | 154 ++ .../src/store/_helpers/migration.rs | 13 + .../src/store/_helpers/mod.rs | 5 + .../src/store/_helpers/open_sqlite.rs | 37 + .../src/store/_helpers/recall_rows.rs | 111 + .../src/store/_helpers/recall_schema.rs | 13 + .../src/store/errors.rs | 22 + .../src/store/main/append_recall.rs | 36 + .../src/store/main/begin.rs | 59 + .../src/store/main/commit.rs | 18 + .../src/store/main/create_location.rs | 30 + .../src/store/main/decode_row.rs | 12 + .../src/store/main/migrate_store.rs | 70 + .../src/store/main/mod.rs | 12 + .../src/store/main/open_store.rs | 23 + .../src/store/main/publish_recall.rs | 25 + .../src/store/main/query.rs | 35 + .../src/store/main/read_recall_events.rs | 28 + .../src/store/main/read_recall_view.rs | 46 + .../src/store/main/refresh_clock.rs | 26 + .../sqlbuild-orchestration/src/store/mod.rs | 5 + .../src/store/models.rs | 59 + .../src/store/schema.sql | 264 ++ .../sqlbuild-orchestration/src/store/types.rs | 3 + crates/sqlbuild-orchestration/tests/api.rs | 12 + .../tests/api/helpers.rs | 52 + .../tests/api/test_contract.rs | 331 +++ .../tests/api/test_types.rs | 7 + .../tests/credentials.rs | 12 + .../tests/credentials/test_contract.rs | 241 ++ crates/sqlbuild-orchestration/tests/recall.rs | 12 + .../tests/recall/helpers.rs | 137 ++ .../tests/recall/test_contract.rs | 28 + .../tests/recall/test_types.rs | 13 + .../tests/recall/test_views.rs | 192 ++ crates/sqlbuild-orchestration/tests/runs.rs | 18 + .../tests/runs/helpers.rs | 118 + .../tests/runs/test_claims.rs | 252 ++ .../tests/runs/test_coalesce.rs | 103 + .../tests/runs/test_faults.rs | 162 ++ .../tests/runs/test_lifecycle.rs | 427 ++++ .../sqlbuild-orchestration/tests/scheduler.rs | 10 + .../tests/scheduler/helpers.rs | 91 + .../tests/scheduler/test_contract.rs | 28 + .../tests/scheduler/test_types.rs | 10 + .../sqlbuild-orchestration/tests/snapshots.rs | 10 + .../tests/snapshots/helpers.rs | 294 +++ .../tests/snapshots/test_contract.rs | 48 + .../tests/snapshots/test_types.rs | 22 + .../tests/store/helpers.rs | 96 + .../tests/store/test_types.rs | 16 + crates/sqlbuild-orchestration/tests/ticks.rs | 16 + .../tests/ticks/fault_helpers.rs | 66 + .../tests/ticks/helpers.rs | 240 ++ .../tests/ticks/test_contract.rs | 32 + .../tests/ticks/test_loop.rs | 98 + .../tests/ticks/test_triggers.rs | 195 ++ .../tests/ticks/test_types.rs | 23 + crates/sqlbuild-recall/Cargo.toml | 1 + .../src/clickhouse/_helpers/literals.rs | 7 +- .../src/records/_helpers/rows.rs | 11 + .../src/schema/_helpers/mod.rs | 4 +- .../src/schema/_helpers/postgres_views.rs | 91 + .../src/schema/_helpers/relational_tables.rs | 69 + .../{sqlite_views.rs => relational_views.rs} | 43 +- .../schema/main/control_table_statements.rs | 14 + .../schema/main/control_view_statements.rs | 34 + crates/sqlbuild-recall/src/schema/main/mod.rs | 3 + .../main/postgres_partition_statement.rs | 24 + .../src/schema/main/table_statements.rs | 40 +- .../src/schema/main/view_statements.rs | 4 +- .../sqlbuild-recall/src/store/_helpers/mod.rs | 1 + .../src/store/_helpers/relational.rs | 12 + .../src/store/_helpers/rows.rs | 79 +- .../src/store/main/conflict_evidence.rs | 23 + crates/sqlbuild-recall/src/store/main/mod.rs | 3 + .../src/store/main/relational_insert.rs | 48 + .../src/store/main/relational_lookup.rs | 48 + crates/sqlbuild-recall/src/store/models.rs | 9 +- fensu.toml | 1 + 182 files changed, 12144 insertions(+), 197 deletions(-) create mode 100644 crates/sqlbuild-cli/src/native_commands/main/_run_control.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/main/parse_native_invocation.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs create mode 100644 crates/sqlbuild-cli/tests/native_commands.rs create mode 100644 crates/sqlbuild-cli/tests/native_commands/helpers.rs create mode 100644 crates/sqlbuild-cli/tests/native_commands/test_control.rs create mode 100644 crates/sqlbuild-cli/tests/native_commands/test_types.rs create mode 100644 crates/sqlbuild-orchestration/Cargo.toml create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/authentication.rs create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/runs.rs create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/snapshots.rs create mode 100644 crates/sqlbuild-orchestration/src/api/errors.rs create mode 100644 crates/sqlbuild-orchestration/src/api/main/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/api/main/openapi_document.rs create mode 100644 crates/sqlbuild-orchestration/src/api/main/serve_api.rs create mode 100644 crates/sqlbuild-orchestration/src/api/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/api/models.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/_helpers/audit.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/_helpers/scopes.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/constants.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/main/authenticate_token.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/main/create_principal.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/main/issue_token.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/main/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/main/revoke_token.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/models.rs create mode 100644 crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs create mode 100644 crates/sqlbuild-orchestration/src/declarations/main/list_trigger_events.rs create mode 100644 crates/sqlbuild-orchestration/src/declarations/main/list_triggers.rs create mode 100644 crates/sqlbuild-orchestration/src/declarations/main/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs create mode 100644 crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs create mode 100644 crates/sqlbuild-orchestration/src/declarations/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/declarations/models.rs create mode 100644 crates/sqlbuild-orchestration/src/lib.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/changes.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/claims.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/events.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/nodes.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/projection.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/change_run.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/claim_next_run.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/heartbeat_run.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/list_run_nodes.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/list_runs.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/queue_run.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/request_run.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/models.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/_helpers/calendar.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/_helpers/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/_helpers/ticks.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/constants.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/main/acquire_lease.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/main/check_lease.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/main/evaluate_trigger.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/main/list_ticks.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/main/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/main/next_cron_time.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/main/next_schedule_time.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/main/parse_cron.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/main/plan_ticks.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/main/previous_cron_time.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/main/renew_lease.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/main/scheduler_cycle.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/models.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/tests.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/tests/test_backlog.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/tests/test_cron.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/tests/test_simulation.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/tests/test_ticks.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/tests/test_types.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/_helpers/content.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/_helpers/preparation.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/_helpers/publication.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/_helpers/reading.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/main/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/main/publish_snapshot.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/models.rs create mode 100644 crates/sqlbuild-orchestration/src/store/_helpers/database.rs create mode 100644 crates/sqlbuild-orchestration/src/store/_helpers/migration.rs create mode 100644 crates/sqlbuild-orchestration/src/store/_helpers/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/store/_helpers/open_sqlite.rs create mode 100644 crates/sqlbuild-orchestration/src/store/_helpers/recall_rows.rs create mode 100644 crates/sqlbuild-orchestration/src/store/_helpers/recall_schema.rs create mode 100644 crates/sqlbuild-orchestration/src/store/errors.rs create mode 100644 crates/sqlbuild-orchestration/src/store/main/append_recall.rs create mode 100644 crates/sqlbuild-orchestration/src/store/main/begin.rs create mode 100644 crates/sqlbuild-orchestration/src/store/main/commit.rs create mode 100644 crates/sqlbuild-orchestration/src/store/main/create_location.rs create mode 100644 crates/sqlbuild-orchestration/src/store/main/decode_row.rs create mode 100644 crates/sqlbuild-orchestration/src/store/main/migrate_store.rs create mode 100644 crates/sqlbuild-orchestration/src/store/main/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/store/main/open_store.rs create mode 100644 crates/sqlbuild-orchestration/src/store/main/publish_recall.rs create mode 100644 crates/sqlbuild-orchestration/src/store/main/query.rs create mode 100644 crates/sqlbuild-orchestration/src/store/main/read_recall_events.rs create mode 100644 crates/sqlbuild-orchestration/src/store/main/read_recall_view.rs create mode 100644 crates/sqlbuild-orchestration/src/store/main/refresh_clock.rs create mode 100644 crates/sqlbuild-orchestration/src/store/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/store/models.rs create mode 100644 crates/sqlbuild-orchestration/src/store/schema.sql create mode 100644 crates/sqlbuild-orchestration/src/store/types.rs create mode 100644 crates/sqlbuild-orchestration/tests/api.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/test_contract.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/test_types.rs create mode 100644 crates/sqlbuild-orchestration/tests/credentials.rs create mode 100644 crates/sqlbuild-orchestration/tests/credentials/test_contract.rs create mode 100644 crates/sqlbuild-orchestration/tests/recall.rs create mode 100644 crates/sqlbuild-orchestration/tests/recall/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/recall/test_contract.rs create mode 100644 crates/sqlbuild-orchestration/tests/recall/test_types.rs create mode 100644 crates/sqlbuild-orchestration/tests/recall/test_views.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/test_claims.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/test_faults.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs create mode 100644 crates/sqlbuild-orchestration/tests/scheduler.rs create mode 100644 crates/sqlbuild-orchestration/tests/scheduler/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/scheduler/test_contract.rs create mode 100644 crates/sqlbuild-orchestration/tests/scheduler/test_types.rs create mode 100644 crates/sqlbuild-orchestration/tests/snapshots.rs create mode 100644 crates/sqlbuild-orchestration/tests/snapshots/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/snapshots/test_contract.rs create mode 100644 crates/sqlbuild-orchestration/tests/snapshots/test_types.rs create mode 100644 crates/sqlbuild-orchestration/tests/store/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/store/test_types.rs create mode 100644 crates/sqlbuild-orchestration/tests/ticks.rs create mode 100644 crates/sqlbuild-orchestration/tests/ticks/fault_helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/ticks/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/ticks/test_contract.rs create mode 100644 crates/sqlbuild-orchestration/tests/ticks/test_loop.rs create mode 100644 crates/sqlbuild-orchestration/tests/ticks/test_triggers.rs create mode 100644 crates/sqlbuild-orchestration/tests/ticks/test_types.rs create mode 100644 crates/sqlbuild-recall/src/schema/_helpers/postgres_views.rs create mode 100644 crates/sqlbuild-recall/src/schema/_helpers/relational_tables.rs rename crates/sqlbuild-recall/src/schema/_helpers/{sqlite_views.rs => relational_views.rs} (92%) create mode 100644 crates/sqlbuild-recall/src/schema/main/control_table_statements.rs create mode 100644 crates/sqlbuild-recall/src/schema/main/control_view_statements.rs create mode 100644 crates/sqlbuild-recall/src/schema/main/postgres_partition_statement.rs create mode 100644 crates/sqlbuild-recall/src/store/_helpers/relational.rs create mode 100644 crates/sqlbuild-recall/src/store/main/conflict_evidence.rs create mode 100644 crates/sqlbuild-recall/src/store/main/relational_insert.rs create mode 100644 crates/sqlbuild-recall/src/store/main/relational_lookup.rs diff --git a/Cargo.lock b/Cargo.lock index 5e2d5968a1..cef38f434c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aho-corasick" version = "1.1.5" @@ -11,6 +17,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + [[package]] name = "ar_archive_writer" version = "0.5.3" @@ -32,6 +53,21 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "attribute-derive" version = "0.10.5" @@ -68,6 +104,85 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "bit-set" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56d87354e4229f54a44f7bf2435906a4656dba36026ab6eaca629a2c436a691c" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5727b15fa97d4f4fee0a3b7c3d550ed0269f54329207b86388de918604e31269" +dependencies = [ + "borsh", + "serde", +] + [[package]] name = "bitflags" version = "2.13.1" @@ -106,6 +221,30 @@ dependencies = [ "objc2", ] +[[package]] +name = "borsh" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "553c5d846a6ba5150c65e3b1b8ec073bcf1abc20f9b7220de384a4443ea4e20a" +dependencies = [ + "borsh-derive", + "bytes", + "cfg_aliases", +] + +[[package]] +name = "borsh-derive" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12cdfe656708a01f89b451a7d36466e6fe6c414de0aa18fc54f864f6f9ca9f56" +dependencies = [ + "once_cell", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "bstr" version = "1.13.1" @@ -117,6 +256,24 @@ dependencies = [ "serde_core", ] +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + [[package]] name = "castaway" version = "0.2.4" @@ -148,6 +305,41 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "rand_core 0.10.1", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "chrono-tz" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" +dependencies = [ + "chrono", + "phf 0.12.1", +] + [[package]] name = "collection_literals" version = "1.0.3" @@ -184,6 +376,18 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "core_detect" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -202,6 +406,41 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + +[[package]] +name = "crc32fast" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ba106e844045ddc02c20768ac4b0b7cb45f2cc2ecbd833ac9f5dacb183c2055" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "croner" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aa42bcd3d846ebf66e15bd528d1087f75d1c6c1c66ebff626178a106353c576" +dependencies = [ + "chrono", + "derive_builder", + "strum", +] + [[package]] name = "crossbeam-deque" version = "0.8.8" @@ -221,6 +460,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03e8bd762f7479489c70ed6c768ddca99d7296857de437a68dcb2a94365b3fae" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.23" @@ -245,7 +493,42 @@ checksum = "e0b1fab2ae45819af2d0731d60f2afe17227ebb1a1538a236da84c93e9a60162" dependencies = [ "dispatch2", "nix 0.31.3", - "windows-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.119", ] [[package]] @@ -259,6 +542,37 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.119", +] + [[package]] name = "digest" version = "0.10.7" @@ -267,6 +581,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", + "subtle", ] [[package]] @@ -281,6 +596,23 @@ dependencies = [ "objc2", ] +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "either" version = "1.18.0" @@ -300,7 +632,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", ] [[package]] @@ -340,12 +693,98 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foldhash" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -397,8 +836,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", ] [[package]] @@ -408,8 +861,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", + "js-sys", "libc", - "r-efi", + "r-efi 6.0.0", + "rand_core 0.10.1", + "wasm-bindgen", ] [[package]] @@ -431,6 +887,8 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ + "allocator-api2", + "equivalent", "foldhash", ] @@ -456,13 +914,287 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] -name = "indexmap" -version = "2.14.2" +name = "hex" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" -dependencies = [ - "equivalent", +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" +dependencies = [ + "atomic-waker", + "bytes", + "futures-core", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots 1.0.11", +] + +[[package]] +name = "hyper-util" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" +dependencies = [ + "base64 0.23.1", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "httparse", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe090a42a0d3140fed8c63263ed1fecc02422cb2f4bfe55cd8de2de1b8e105dd" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25d29b57c17c6528d0dc4c2141cfa2d08200465acb537a3909c8b9b8a4a984c0" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "404a3b8c486f205c87fe7f23ab6894d84a4ad76d6e4671ae1981ba7def87c599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20f0088bfb7441f7a37fa0c9e20cb1e9cba1cac6b9ce8cd2a6dc44294f8e8cbc" + +[[package]] +name = "icu_properties" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ea83d991c56158f7cf898bf86436c1dfe7c07ee82fe88951f2695c9f37a379f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98f29af82d02b163fe50161388350e743bb3107cc8e8c2f7314cb505faddfcf" + +[[package]] +name = "icu_provider" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413099315636c588ee8f011190b329b971a09a8fae9d157e01dcae8379468514" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", "hashbrown 0.17.1", + "serde", + "serde_core", ] [[package]] @@ -480,6 +1212,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71dd52191aae121e8611f1e8dc3e324dd0dd1dee1e6dd91d10ee07a3cfb4d9d8" +[[package]] +name = "ipnet" +version = "2.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" + [[package]] name = "is-macro" version = "0.3.8" @@ -507,6 +1245,17 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "js-sys" +version = "0.3.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dc09d2154098bbf4ef54eab6b117a3c24cbbffe94046b16ee6d6d4510584d3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + [[package]] name = "junction" version = "2.1.0" @@ -514,7 +1263,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cde48f36530cbd91b65f050101e6c056d6eb743cea2a8f0f19beef69c6742ff8" dependencies = [ "scopeguard", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -532,6 +1281,18 @@ dependencies = [ "cc", ] +[[package]] +name = "libredox" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d" +dependencies = [ + "bitflags", + "libc", + "plain", + "redox_syscall 0.9.5", +] + [[package]] name = "libsqlite3-sys" version = "0.35.0" @@ -549,12 +1310,33 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "lru-slab" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" + [[package]] name = "manyhow" version = "0.11.4" @@ -578,6 +1360,22 @@ dependencies = [ "quote", ] +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.8.3" @@ -602,6 +1400,33 @@ dependencies = [ "libmimalloc-sys", ] +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + [[package]] name = "nix" version = "0.27.1" @@ -625,6 +1450,15 @@ dependencies = [ "libc", ] +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "objc2" version = "0.6.5" @@ -664,13 +1498,57 @@ dependencies = [ "indexmap", ] +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.18", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + [[package]] name = "phf" version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" dependencies = [ - "phf_shared", + "phf_shared 0.11.3", +] + +[[package]] +name = "phf" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" +dependencies = [ + "phf_shared 0.12.1", ] [[package]] @@ -680,7 +1558,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" dependencies = [ "phf_generator", - "phf_shared", + "phf_shared 0.11.3", ] [[package]] @@ -689,8 +1567,8 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ - "phf_shared", - "rand", + "phf_shared 0.11.3", + "rand 0.8.8", ] [[package]] @@ -702,12 +1580,33 @@ dependencies = [ "siphasher", ] +[[package]] +name = "phf_shared" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + [[package]] name = "pkg-config" version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + [[package]] name = "polyglot-sql" version = "0.13.9" @@ -744,6 +1643,15 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -753,6 +1661,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro-utils" version = "0.10.0" @@ -773,6 +1690,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proptest" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8530004ccb15eae51c7e40009fbe317f341f804db54dc033eec1c50be28cfa0" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags", + "chacha20", + "core_detect", + "num-traits", + "rand 0.10.3", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + [[package]] name = "psm" version = "0.1.32" @@ -845,7 +1782,69 @@ dependencies = [ ] [[package]] -name = "quote" +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quinn" +version = "0.11.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.21", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" +dependencies = [ + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand 0.10.3", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.21", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af66907df18639dcf4db56ca65490cabc4b27a97dbadd96f2926cca73298f016" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "quote" version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" @@ -875,6 +1874,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "r-efi" version = "6.0.0" @@ -889,7 +1894,18 @@ checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" dependencies = [ "libc", "rand_chacha", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", ] [[package]] @@ -899,7 +1915,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -911,6 +1927,30 @@ dependencies = [ "getrandom 0.2.17", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + +[[package]] +name = "rand_xorshift" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60aa6af80be32871323012e02e6e65f8a7cc7890931ae421d217ad8fe0df2ccf" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "rayon" version = "1.12.0" @@ -931,6 +1971,24 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_syscall" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "580c875845b41ecb1564ad1ae4703044cef774145e3d7a94dc2b3fa7c97868fd" +dependencies = [ + "bitflags", +] + [[package]] name = "regex" version = "1.13.1" @@ -960,6 +2018,58 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots 1.0.11", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "ruff_python_ast" version = "0.0.4" @@ -1065,7 +2175,42 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", ] [[package]] @@ -1074,6 +2219,18 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" +[[package]] +name = "rusty-fork" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + [[package]] name = "ryu" version = "1.0.23" @@ -1105,6 +2262,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + [[package]] name = "serde" version = "1.0.229" @@ -1149,6 +2312,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + [[package]] name = "serde_spanned" version = "1.1.1" @@ -1158,6 +2332,24 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + [[package]] name = "sha2" version = "0.10.9" @@ -1195,17 +2387,42 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + [[package]] name = "siphasher" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + [[package]] name = "smallvec" version = "1.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" +dependencies = [ + "serde", +] + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] [[package]] name = "sqlbuild-analysis" @@ -1252,7 +2469,7 @@ dependencies = [ "syn 2.0.119", "syn 3.0.3", "thiserror 2.0.21", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1275,6 +2492,8 @@ dependencies = [ "blake3", "ctrlc", "getrandom 0.4.3", + "reqwest", + "rusqlite", "rustix", "serde", "serde_json", @@ -1284,8 +2503,11 @@ dependencies = [ "sqlbuild-core", "sqlbuild-cursor", "sqlbuild-model-config", + "sqlbuild-orchestration", "sqlbuild-rules", "tempfile", + "tokio", + "uuid", ] [[package]] @@ -1343,6 +2565,33 @@ dependencies = [ "sqlbuild-core", ] +[[package]] +name = "sqlbuild-orchestration" +version = "0.186.2" +dependencies = [ + "axum", + "chrono", + "chrono-tz", + "croner", + "flate2", + "getrandom 0.4.3", + "proptest", + "reqwest", + "rusqlite", + "semver", + "serde", + "serde_json", + "sqlbuild-recall", + "sqlx-core", + "sqlx-postgres", + "subtle", + "tempfile", + "thiserror 2.0.21", + "tokio", + "utoipa", + "uuid", +] + [[package]] name = "sqlbuild-project" version = "0.186.2" @@ -1403,6 +2652,7 @@ name = "sqlbuild-recall" version = "0.186.2" dependencies = [ "blake3", + "chrono", "rusqlite", "rustix", "serde_json", @@ -1495,6 +2745,87 @@ dependencies = [ "sqlbuild-cursor", ] +[[package]] +name = "sqlx-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" +dependencies = [ + "base64 0.22.1", + "bytes", + "chrono", + "crossbeam-queue", + "either", + "event-listener", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashbrown 0.15.5", + "hashlink", + "indexmap", + "log", + "memchr", + "once_cell", + "percent-encoding", + "rustls", + "serde", + "serde_json", + "smallvec", + "thiserror 2.0.21", + "tokio", + "tokio-stream", + "tracing", + "url", + "uuid", + "webpki-roots 0.26.11", +] + +[[package]] +name = "sqlx-postgres" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags", + "byteorder", + "chrono", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand 0.8.8", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.21", + "tracing", + "uuid", + "whoami", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + [[package]] name = "stacker" version = "0.1.25" @@ -1505,7 +2836,7 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1514,6 +2845,50 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.119" @@ -1536,6 +2911,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "target-lexicon" version = "0.13.5" @@ -1552,7 +2947,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1601,6 +2996,16 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "tinystr" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abc840c238154d58f9e86b2792261edfde565a17696e1cda1a449de27979801a" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinyvec" version = "1.13.3" @@ -1608,25 +3013,73 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" [[package]] -name = "toml" -version = "0.9.12+spec-1.1.0" +name = "tokio" +version = "1.54.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" +checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3" dependencies = [ - "indexmap", - "serde_core", - "serde_spanned", - "toml_datetime 0.7.5+spec-1.1.0", - "toml_parser", - "toml_writer", - "winnow 0.7.15", + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", ] [[package]] -name = "toml_datetime" -version = "0.7.5+spec-1.1.0" +name = "tokio-macros" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "0.9.12+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime 0.7.5+spec-1.1.0", + "toml_parser", + "toml_writer", + "winnow 0.7.15", +] + +[[package]] +name = "toml_datetime" +version = "0.7.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" dependencies = [ "serde_core", ] @@ -1667,12 +3120,107 @@ version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "typenum" version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -1688,6 +3236,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + [[package]] name = "unicode-segmentation" version = "1.13.3" @@ -1706,7 +3260,7 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d1673eca9782c84de5f81b82e4109dcfb3611c8ba0d52930ec4a9478f547b2dd" dependencies = [ - "phf", + "phf 0.11.3", "unicode_names2_generator", ] @@ -1719,7 +3273,7 @@ dependencies = [ "getopts", "log", "phf_codegen", - "rand", + "rand 0.8.8", ] [[package]] @@ -1728,6 +3282,76 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af3bd9d3e571c26fcce9d033b87ea5ab061b8d2ffe9b7a5675d810d68f03edf0" +dependencies = [ + "utf_types", +] + +[[package]] +name = "utf_types" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a3b425286980a367278453f3b6d28cdcb134b574e5a35824a6ff5eebad50bb6" + +[[package]] +name = "utoipa" +version = "5.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bde15df68e80b16c7d16b9616e80770ad158988daa56a27dccd1e55558b0160" +dependencies = [ + "indexmap", + "serde", + "serde_json", + "utoipa-gen", +] + +[[package]] +name = "utoipa-gen" +version = "5.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba0b99ee52df3028635d93840c797102da61f8a7bb3cf751032455895b52ef8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "uuid", +] + +[[package]] +name = "uuid" +version = "1.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" +dependencies = [ + "getrandom 0.3.4", + "js-sys", + "serde", + "sha1_smol", + "wasm-bindgen", +] + [[package]] name = "vcpkg" version = "0.2.15" @@ -1740,6 +3364,15 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + [[package]] name = "walkdir" version = "2.5.0" @@ -1750,12 +3383,139 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0181c14ddd6c7b8b0b0132a5f1ea2f32be0d893cd3ea97b6e3b28886b8a3d1c" +dependencies = [ + "cfg-if", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e14e0327cb5368ad60ccb22d04a9827a693c95a926d8d125e9fa39e0baec501" +dependencies = [ + "js-sys", + "tokio", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e931a02a39abdbf66966fc35902b5ed58344750a70885bed4ebb3d8eaf28ed9" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ca4b54280e455bd831e33a6b8e734baf432fe63fbc35c42ac14ce67a043a53a" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.3", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b1f9c3798725016c6ee5f2f57869ec0831e8bbc0ab0310a86e787e97f41389e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7d179596ff5ec0f6de4db64d54011bde6289cc49603d3a6e4ce3f329207ab88" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.11", +] + +[[package]] +name = "webpki-roots" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5f280bbb2b287bc8e7de7e3852945a0ff50d1bbcb5ed42b6c7a568c546df171" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "whoami" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" +dependencies = [ + "libredox", + "wasite", +] + [[package]] name = "winapi" version = "0.3.9" @@ -1778,7 +3538,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1787,12 +3547,83 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -1802,6 +3633,127 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "winnow" version = "0.7.15" @@ -1817,6 +3769,41 @@ dependencies = [ "memchr", ] +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yoke" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "417ef3fcf6c300b7dbd25c85531a746931c02ad536f3cf4786ed1e88223340b3" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.8.62" @@ -1837,6 +3824,72 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f58a2799cfe9a1670c7ea8edbd52ef8edb93cb37a7f83bf31ba9e35d9dae6a40" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3743ddbf47738d929cd1ae2e199d40d0162979dfe31aef8ae646d2bf00bf97a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + [[package]] name = "zmij" version = "1.0.23" diff --git a/Cargo.toml b/Cargo.toml index 6014fffc19..ea73843c9b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ members = [ "crates/sqlbuild-config", "crates/sqlbuild-host", "crates/sqlbuild-recall", + "crates/sqlbuild-orchestration", "crates/sqlbuild-discovery", "crates/sqlbuild-scopes", "crates/sqlbuild-model-config", @@ -39,6 +40,7 @@ order = [ "sqlbuild-config", "sqlbuild-host", "sqlbuild-recall", + "sqlbuild-orchestration", "sqlbuild-model-config", "sqlbuild-discovery", "sqlbuild-render", @@ -62,6 +64,7 @@ publish = false rust-version = "1.88" [workspace.dependencies] +axum = { version = "0.8", default-features = false, features = ["http1", "json", "tokio"] } fensu-policy = "=0.22.0" # `sqb` names compute log captures with random ids, as `uuid.uuid4().hex`. getrandom = "0.4" @@ -115,10 +118,23 @@ rusqlite = { version = "0.37", features = ["bundled", "column_decltype"] } rustix = { version = "1.1.4", default-features = false, features = ["fs", "std", "termios"] } saphyr-parser = "0.1" regex = "1" +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } serde = { version = "1", features = ["derive"] } # Every crate relies on insertion-ordered JSON objects, which the extension has always built with. serde_json = { version = "1", features = ["preserve_order"] } sha2 = "0.10" +chrono = { version = "0.4", features = ["serde"] } +chrono-tz = "0.10" +croner = "=3.0.1" +flate2 = "1" +proptest = "1" +semver = "1" +# Pin SQLx's driver crates together: its umbrella also resolves an incompatible SQLite engine. +sqlx-core = { version = "=0.8.6", default-features = false, features = ["_rt-tokio", "_tls-rustls-ring-webpki", "chrono", "uuid", "json"] } +sqlx-postgres = { version = "=0.8.6", default-features = false, features = ["chrono", "uuid", "json"] } +tokio = { version = "1", features = ["macros", "rt-multi-thread", "sync", "time", "signal", "process", "net"] } +uuid = { version = "=1.18.1", features = ["v5", "v7", "serde"] } +utoipa = { version = "5", features = ["chrono", "uuid"] } sqlbuild-analysis = { path = "crates/sqlbuild-analysis" } sqlbuild-attachments = { path = "crates/sqlbuild-attachments" } sqlbuild-build-features = { path = "crates/sqlbuild-build-features" } @@ -133,6 +149,7 @@ sqlbuild-render = { path = "crates/sqlbuild-render" } sqlbuild-refactor = { path = "crates/sqlbuild-refactor" } sqlbuild-rules = { path = "crates/sqlbuild-rules" } sqlbuild-model-config = { path = "crates/sqlbuild-model-config" } +sqlbuild-orchestration = { path = "crates/sqlbuild-orchestration" } sqlbuild-project = { path = "crates/sqlbuild-project" } sqlbuild-pysource = { path = "crates/sqlbuild-pysource" } sqlbuild-recall = { path = "crates/sqlbuild-recall" } @@ -141,6 +158,7 @@ sqlbuild-sqltext = { path = "crates/sqlbuild-sqltext" } sqlbuild-state = { path = "crates/sqlbuild-state" } stacker = "0.1" syn = { version = "2", features = ["full", "visit", "extra-traits"] } +subtle = "2" tempfile = "3" toml = "0.9" toml_edit = { version = "0.25", default-features = false, features = ["parse"] } diff --git a/crates/sqlbuild-cli/Cargo.toml b/crates/sqlbuild-cli/Cargo.toml index 4a2253fb56..686080a599 100644 --- a/crates/sqlbuild-cli/Cargo.toml +++ b/crates/sqlbuild-cli/Cargo.toml @@ -26,7 +26,10 @@ sqlbuild-config.workspace = true sqlbuild-core.workspace = true sqlbuild-cursor.workspace = true sqlbuild-model-config.workspace = true +sqlbuild-orchestration.workspace = true sqlbuild-rules.workspace = true +tokio.workspace = true +uuid.workspace = true [target.'cfg(unix)'.dependencies] rustix.workspace = true @@ -35,6 +38,8 @@ rustix.workspace = true ctrlc.workspace = true [dev-dependencies] +reqwest.workspace = true +rusqlite.workspace = true tempfile.workspace = true [lints] diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index d2b8b61be1..ec58647f34 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -78,6 +78,133 @@ "nargs": "A...", "group": 0, "parsers": [ + { + "name": "server", + "parser": { + "prog": "sqb server", + "description": "Run the native control API and scheduler without importing project Python.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--database-env" + ], + "dest": "control_database_env", + "default": "SQLBUILD_CONTROL_DATABASE_URL", + "help": "environment variable holding the control database URL (default: SQLBUILD_CONTROL_DATABASE_URL; SQLite when unset)", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--host" + ], + "dest": "control_host", + "default": "127.0.0.1", + "help": "IP address to listen on (default: 127.0.0.1)", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--port" + ], + "dest": "control_port", + "default": 8000, + "type": "int", + "help": "TCP port, or 0 to choose a free port (default: 8000)", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--no-scheduler" + ], + "dest": "control_no_scheduler", + "nargs": 0, + "const": true, + "default": false, + "help": "serve the API without the embedded scheduler", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--openapi" + ], + "dest": "control_openapi", + "nargs": 0, + "const": true, + "default": false, + "help": "print the OpenAPI contract as JSON without opening a database", + "group": 1 + } + ] + } + }, + { + "name": "scheduler", + "parser": { + "prog": "sqb scheduler", + "description": "Run the lease-guarded native scheduler against the server's control database.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--database-env" + ], + "dest": "control_database_env", + "default": "SQLBUILD_CONTROL_DATABASE_URL", + "help": "environment variable holding the control database URL (default: SQLBUILD_CONTROL_DATABASE_URL; SQLite when unset)", + "group": 1 + } + ] + } + }, { "name": "compile", "parser": { @@ -7592,6 +7719,16 @@ } ], "choices_help": [ + { + "dest": "server", + "metavar": "server", + "help": "run the native control API and scheduler" + }, + { + "dest": "scheduler", + "metavar": "scheduler", + "help": "run a standalone lease-guarded scheduler" + }, { "dest": "cost", "metavar": "cost", @@ -7620,4 +7757,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs b/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs index e6f689c242..441901c2b6 100644 --- a/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs +++ b/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs @@ -10,7 +10,10 @@ use crate::launcher::_helpers::messages::{print_version, report_missing_python_c use crate::launcher::constants::{ HANDOFF_FAILURE_EXIT_CODE, LAUNCH_RECORD_OPTION, VERSION_OPTIONS, }; +use crate::native_commands::main::parse_native_invocation::parse_native_invocation; +use crate::native_commands::main::run_control_invocation::run_control_invocation; use crate::native_commands::main::run_native_invocation::run_native_invocation; +use crate::native_commands::models::NativeParse; use crate::replay::main::serve_reused_compile::serve_reused_compile; /// The launch record path and key for these arguments, when the directory is known. @@ -38,6 +41,15 @@ pub(crate) fn launch(arguments: Vec, started: (Instant, SystemTime)) - print_version(); return 0; } + let native = parse_native_invocation(&arguments); + if let Ok(NativeParse::Exit(code)) = &native { + return *code; + } + if let Ok(NativeParse::Invocation { namespace, .. }) = &native + && let Some(code) = run_control_invocation(namespace) + { + return code; + } let program: PathBuf = std::env::current_exe().unwrap_or_default(); let Some(script) = python_cli(&program) else { report_missing_python_cli(&program); @@ -50,7 +62,12 @@ pub(crate) fn launch(arguments: Vec, started: (Instant, SystemTime)) - { return exit_code(code); } - if let Ok(code) = run_native_invocation(&arguments, &script) { + if let Ok(NativeParse::Invocation { + namespace, + use_color, + }) = native + && let Ok(code) = run_native_invocation(&namespace, &script, use_color) + { return code; } if let Some((path, _)) = record { diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index 1ab59675b6..9355478a2d 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -25,14 +25,14 @@ pub(crate) fn review_json_failure(arguments: &[String], outcome: &ArgvOutcome) - .is_some_and(|failure| failure.prog == REVIEW_PROG) } -fn text(namespace: &Namespace, name: &str) -> Option { +pub(crate) fn text(namespace: &Namespace, name: &str) -> Option { match namespace.get(name) { Some(Value::Str(text) | Value::Path(text)) => Some(text.clone()), _ => None, } } -fn flag(namespace: &Namespace, name: &str) -> bool { +pub(crate) fn flag(namespace: &Namespace, name: &str) -> bool { namespace.get(name) == Some(&Value::Bool(true)) } diff --git a/crates/sqlbuild-cli/src/native_commands/constants.rs b/crates/sqlbuild-cli/src/native_commands/constants.rs index 867ce8bb82..7f993a2320 100644 --- a/crates/sqlbuild-cli/src/native_commands/constants.rs +++ b/crates/sqlbuild-cli/src/native_commands/constants.rs @@ -41,6 +41,8 @@ pub(crate) const RUNNING_BUILD_STATUS: &str = "running"; pub(crate) const CLEAN_COMMAND: &str = "clean"; pub(crate) const INIT_COMMAND: &str = "init"; pub(crate) const SKILLS_COMMAND: &str = "skills"; +pub(crate) const SERVER_COMMAND: &str = "server"; +pub(crate) const SCHEDULER_COMMAND: &str = "scheduler"; pub(crate) const RULES_COMMAND: &str = "rules"; pub(crate) const RULES_LIST_ACTION: &str = "list"; pub(crate) const RULES_SHOW_ACTION: &str = "show"; diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs new file mode 100644 index 0000000000..111fc3a1a1 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs @@ -0,0 +1,135 @@ +//! Supervise native API/scheduler roles together; a failed role stops its siblings. + +use std::net::{IpAddr, SocketAddr}; +use std::path::PathBuf; + +use sqlbuild_orchestration::api::main::serve_api::serve_api; +use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler; +use sqlbuild_orchestration::scheduler::models::SchedulerState; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use sqlbuild_orchestration::store::main::open_store::open_store; +use tokio::sync::watch; +use uuid::Uuid; + +use crate::argv::models::{Namespace, Stream, Value}; +use crate::native_commands::_helpers::decision::{flag, text}; +use crate::native_commands::_helpers::terminal::write_output; + +async fn stopped(mut shutdown: watch::Receiver) { + while !*shutdown.borrow_and_update() { + if shutdown.changed().await.is_err() { + break; + } + } +} + +async fn shutdown_signal() -> Result<(), ControlError> { + #[cfg(unix)] + { + let mut terminate = + tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?; + tokio::select! { + result = tokio::signal::ctrl_c() => result?, + _ = terminate.recv() => {}, + } + } + #[cfg(not(unix))] + tokio::signal::ctrl_c().await?; + Ok(()) +} + +pub(crate) async fn run_control( + namespace: &Namespace, + standalone: bool, +) -> Result<(), ControlError> { + let address = if standalone { + None + } else { + let host: IpAddr = text(namespace, "control_host") + .unwrap_or_else(|| "127.0.0.1".into()) + .parse() + .map_err(|_| ControlError::Invalid("--host must be an IP address".into()))?; + let port = match namespace.get("control_port") { + Some(Value::Int(value)) => value + .parse::() + .map_err(|_| ControlError::Invalid("--port must be between 0 and 65535".into()))?, + _ => 8000, + }; + Some(SocketAddr::new(host, port)) + }; + let variable = text(namespace, "control_database_env") + .unwrap_or_else(|| "SQLBUILD_CONTROL_DATABASE_URL".into()); + let url = match std::env::var(&variable) { + Ok(value) => value, + Err(std::env::VarError::NotPresent) => { + PathBuf::from(text(namespace, "project_dir").unwrap_or_else(|| ".".into())) + .join(".sqlbuild/control.sqlite3") + .to_string_lossy() + .into_owned() + } + Err(std::env::VarError::NotUnicode(_)) => { + return Err(ControlError::Invalid( + "control database variable must be UTF-8".into(), + )); + } + }; + drop(write_output(&[( + Stream::Stderr, + "Opening control store...\n", + )])); + let store = open_store(&url).await?; + migrate_store(&store).await?; + drop(write_output(&[(Stream::Stderr, "Control store ready\n")])); + let (stop, shutdown) = watch::channel(false); + let (_changes, changes) = watch::channel(()); + let mut tasks = tokio::task::JoinSet::new(); + if let Some(address) = address { + drop(write_output(&[( + Stream::Stderr, + "Starting control API...\n", + )])); + let listener = tokio::net::TcpListener::bind(address).await?; + drop(write_output(&[( + Stream::Stderr, + &format!("Control API ready: {}\n", listener.local_addr()?), + )])); + tasks.spawn(serve_api( + store.clone(), + listener, + stopped(shutdown.clone()), + )); + } + if standalone || !flag(namespace, "control_no_scheduler") { + drop(write_output(&[(Stream::Stderr, "Starting scheduler...\n")])); + let scheduler_store = store.clone(); + tasks.spawn(async move { + run_scheduler( + &scheduler_store, + SchedulerState::new(Uuid::now_v7()), + shutdown, + changes, + ) + .await + }); + drop(write_output(&[( + Stream::Stderr, + "Scheduler loop started; leadership is lease-guarded\n", + )])); + } + let mut result = tokio::select! { + result = tasks.join_next() => match result { + Some(result) => result.map_err(ControlError::from).and_then(std::convert::identity), + None => Ok(()), + }, + result = shutdown_signal() => result, + }; + stop.send_replace(true); + while let Some(joined) = tasks.join_next().await { + let joined = joined + .map_err(ControlError::from) + .and_then(std::convert::identity); + result = result.and(joined); + } + result +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/mod.rs b/crates/sqlbuild-cli/src/native_commands/main/mod.rs index 034b2c7bcd..c52a4a29a2 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/mod.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/mod.rs @@ -1,3 +1,6 @@ pub(super) mod _cost_report_options; +pub(super) mod _run_control; pub(super) mod _run_cost; +pub(crate) mod parse_native_invocation; +pub(crate) mod run_control_invocation; pub mod run_native_invocation; diff --git a/crates/sqlbuild-cli/src/native_commands/main/parse_native_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/parse_native_invocation.rs new file mode 100644 index 0000000000..707115b4f9 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/parse_native_invocation.rs @@ -0,0 +1,49 @@ +//! Parse once with the shared command table before resolving the optional Python CLI. + +use std::ffi::OsString; + +use crate::argv::main::parse_cli_arguments::parse_cli_arguments; +use crate::argv::models::{ArgvOutcome, ArgvSettings, Namespace}; +use crate::native_commands::_helpers::argv_host::{NativeArgvHost, namespace_defaults}; +use crate::native_commands::_helpers::decision::review_json_failure; +use crate::native_commands::_helpers::terminal::{ + finish_rejected_parse, stdout_supports_color, terminal_columns, +}; +use crate::native_commands::constants::NO_COLOR_OPTION; +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::NativeParse; + +pub(crate) fn parse_native_invocation(arguments: &[OsString]) -> Result { + let arguments: Vec = arguments + .iter() + .map(|argument| argument.to_str().map(str::to_owned)) + .collect::>() + .ok_or(PythonNeeded::Arguments)?; + let use_color = + !arguments.iter().any(|argument| argument == NO_COLOR_OPTION) && stdout_supports_color(); + let settings = ArgvSettings { + columns: terminal_columns()?, + use_color, + }; + let mut host = NativeArgvHost::default(); + let outcome: ArgvOutcome = + parse_cli_arguments(&arguments, namespace_defaults(), settings, &mut host); + if host.needs_python { + return Err(PythonNeeded::Arguments); + } + let namespace: Namespace = match outcome.parsed { + Err(_) if review_json_failure(&arguments, &outcome) => return Err(PythonNeeded::Arguments), + Err(code) => { + return Ok(NativeParse::Exit(finish_rejected_parse( + &outcome.writes, + code, + ))); + } + Ok(namespace) if outcome.writes.is_empty() => namespace, + Ok(_) => return Err(PythonNeeded::Arguments), + }; + Ok(NativeParse::Invocation { + namespace, + use_color, + }) +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs new file mode 100644 index 0000000000..699407935b --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs @@ -0,0 +1,63 @@ +//! Control roles are served before Python/package discovery, including their JSON contract. + +use sqlbuild_orchestration::api::main::openapi_document::openapi_document; + +use crate::argv::models::{Namespace, Stream}; +use crate::native_commands::_helpers::decision::{flag, text}; +use crate::native_commands::_helpers::terminal::{finish_lost_stdout, write_output}; +use crate::native_commands::constants::{SCHEDULER_COMMAND, SERVER_COMMAND}; +use crate::native_commands::main::_run_control::run_control; + +pub(crate) fn run_control_invocation(namespace: &Namespace) -> Option { + let command = text(namespace, "command")?; + if !matches!(command.as_str(), SERVER_COMMAND | SCHEDULER_COMMAND) { + return None; + } + if flag(namespace, "control_openapi") { + return Some(match serde_json::to_string(&openapi_document()) { + Ok(document) => { + let document = document + "\n"; + let lost = write_output(&[(Stream::Stdout, document.as_str())]); + finish_lost_stdout(lost.as_ref(), 0) + } + Err(_) => { + drop(write_output(&[( + Stream::Stderr, + "Control API contract generation failed\n", + )])); + 1 + } + }); + } + let runtime = match tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(error) => { + drop(write_output(&[( + Stream::Stderr, + &format!("Control service failed to start: {error}\n"), + )])); + return Some(1); + } + }; + Some( + match runtime.block_on(run_control(namespace, command == SCHEDULER_COMMAND)) { + Ok(()) => { + drop(write_output(&[( + Stream::Stderr, + "Control service stopped successfully\n", + )])); + 0 + } + Err(error) => { + drop(write_output(&[( + Stream::Stderr, + &format!("Control service failed: {error}\n"), + )])); + 1 + } + }, + ) +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs index 40360a7de8..c92474afef 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs @@ -1,50 +1,19 @@ //! Serve one invocation without Python when its parse and project facts allow it. -use std::ffi::OsString; use std::path::Path; -use crate::argv::main::parse_cli_arguments::parse_cli_arguments; -use crate::argv::models::{ArgvOutcome, ArgvSettings, Namespace}; -use crate::native_commands::_helpers::argv_host::{NativeArgvHost, namespace_defaults}; -use crate::native_commands::_helpers::decision::{native_invocation, review_json_failure}; +use crate::argv::models::Namespace; +use crate::native_commands::_helpers::decision::native_invocation; use crate::native_commands::_helpers::execution::run_command; -use crate::native_commands::_helpers::terminal::{ - finish_rejected_parse, stdout_supports_color, terminal_columns, -}; -use crate::native_commands::constants::NO_COLOR_OPTION; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::models::NativeInvocation; /// The exit status of an invocation `sqb` served, or why the same arguments go to Python. pub fn run_native_invocation( - arguments: &[OsString], + namespace: &Namespace, python_cli: &Path, + use_color: bool, ) -> Result { - let arguments: Vec = arguments - .iter() - .map(|argument| argument.to_str().map(str::to_owned)) - .collect::>() - .ok_or(PythonNeeded::Arguments)?; - let use_color: bool = - !arguments.iter().any(|argument| argument == NO_COLOR_OPTION) && stdout_supports_color(); - let settings: ArgvSettings = ArgvSettings { - columns: terminal_columns()?, - use_color, - }; - let mut host: NativeArgvHost = NativeArgvHost::default(); - let outcome: ArgvOutcome = - parse_cli_arguments(&arguments, namespace_defaults(), settings, &mut host); - if host.needs_python { - return Err(PythonNeeded::Arguments); - } - let namespace: Namespace = match outcome.parsed { - Err(_) if review_json_failure(&arguments, &outcome) => { - return Err(PythonNeeded::Arguments); - } - Err(code) => return Ok(finish_rejected_parse(&outcome.writes, code)), - Ok(namespace) if outcome.writes.is_empty() => namespace, - Ok(_) => return Err(PythonNeeded::Arguments), - }; - let invocation: NativeInvocation = native_invocation(&namespace)?; + let invocation: NativeInvocation = native_invocation(namespace)?; run_command(&invocation, python_cli, use_color) } diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 93db875a33..f328de4e67 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -2,6 +2,18 @@ use std::path::PathBuf; +use crate::argv::models::Namespace; + +/// Parsing can finish an invocation before package discovery or any Python handoff. +#[derive(Debug)] +pub(crate) enum NativeParse { + Exit(i32), + Invocation { + namespace: Namespace, + use_color: bool, + }, +} + /// What `sqb skills` installs. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct SkillsOptions { diff --git a/crates/sqlbuild-cli/tests/native_commands.rs b/crates/sqlbuild-cli/tests/native_commands.rs new file mode 100644 index 0000000000..23a9b5ba97 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands.rs @@ -0,0 +1,6 @@ +#[path = "native_commands/helpers.rs"] +mod helpers; +#[path = "native_commands/test_control.rs"] +mod test_control; +#[path = "native_commands/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-cli/tests/native_commands/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/helpers.rs new file mode 100644 index 0000000000..5e4ae4bfde --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/helpers.rs @@ -0,0 +1,108 @@ +use std::path::Path; +use std::process::{Command, Stdio}; +use std::time::Duration; + +use crate::test_types::TestControl; + +pub(crate) fn isolated_binary(directory: &Path) -> std::path::PathBuf { + let binary = directory.join(if cfg!(windows) { "sqb.exe" } else { "sqb" }); + std::fs::copy(env!("CARGO_BIN_EXE_sqb"), &binary) + .expect("copy standalone binary without Python CLI"); + binary +} + +pub(crate) fn start_control(arguments: &[&str]) -> TestControl { + let directory = tempfile::tempdir().expect("isolated native control project"); + let binary = isolated_binary(directory.path()); + let stderr = directory.path().join("control.log"); + let child = Command::new(binary) + .args(arguments) + .current_dir(directory.path()) + .env("PATH", "") + .env_remove("SQLBUILD_CONTROL_DATABASE_URL") + .stdout(Stdio::null()) + .stderr(std::fs::File::create(&stderr).expect("control stderr")) + .spawn() + .expect("start native control process"); + TestControl { + child, + directory, + stderr, + } +} + +pub(crate) async fn api_ready(control: &mut TestControl) -> String { + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(1)) + .build() + .expect("HTTP client"); + tokio::time::timeout(Duration::from_secs(10), async { + loop { + assert!( + control.child.try_wait().expect("child status").is_none(), + "control process exited: {}", + std::fs::read_to_string(&control.stderr).unwrap_or_default() + ); + let log = std::fs::read_to_string(&control.stderr).expect("read control log"); + if let Some(address) = log + .lines() + .find_map(|line| line.strip_prefix("Control API ready: ")) + { + let url = format!("http://{address}"); + if client + .get(format!("{url}/api/openapi.json")) + .send() + .await + .is_ok_and(|response| response.status().is_success()) + { + return url; + } + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("native API becomes ready") +} + +pub(crate) fn leader_count(control: &TestControl) -> Option { + let connection = rusqlite::Connection::open_with_flags( + control.directory.path().join(".sqlbuild/control.sqlite3"), + rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, + ) + .ok()?; + connection + .query_row("SELECT COUNT(*) FROM sqb_scheduler_lease", [], |row| { + row.get(0) + }) + .ok() +} + +#[cfg(unix)] +pub(crate) async fn terminate(control: &mut TestControl) { + let status = Command::new("kill") + .args(["-TERM", &control.child.id().to_string()]) + .status() + .expect("send SIGTERM"); + assert!(status.success()); + let status = tokio::time::timeout(Duration::from_secs(5), async { + loop { + if let Some(status) = control.child.try_wait().expect("child exit") { + return status; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("control roles stop promptly"); + assert!( + status.success(), + "shutdown failed: {}", + std::fs::read_to_string(&control.stderr).unwrap_or_default() + ); + assert!( + std::fs::read_to_string(&control.stderr) + .expect("shutdown log") + .ends_with("Control service stopped successfully\n") + ); +} diff --git a/crates/sqlbuild-cli/tests/native_commands/test_control.rs b/crates/sqlbuild-cli/tests/native_commands/test_control.rs new file mode 100644 index 0000000000..de16cd7367 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/test_control.rs @@ -0,0 +1,100 @@ +use std::process::Command; +use std::time::Duration; + +use crate::helpers::{api_ready, isolated_binary, leader_count, start_control}; + +#[test] +fn given_a_standalone_binary_when_printing_control_help_or_openapi_then_python_and_database_access_are_not_required() + { + let directory = tempfile::tempdir().expect("isolated CLI directory"); + let binary = isolated_binary(directory.path()); + for arguments in [ + ["server", "--help"], + ["scheduler", "--help"], + ["server", "--openapi"], + ] { + let output = Command::new(&binary) + .args(arguments) + .current_dir(directory.path()) + .env("PATH", "") + .env("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://not-opened") + .output() + .expect("standalone native command"); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!(output.stderr.is_empty()); + if arguments[1] == "--openapi" { + let document: serde_json::Value = + serde_json::from_slice(&output.stdout).expect("machine-readable OpenAPI"); + assert!(document["paths"]["/api/runs/{run_id}/events"].is_object()); + } else { + assert!(String::from_utf8_lossy(&output.stdout).starts_with("usage: sqb ")); + } + } + assert!(!directory.path().join(".sqlbuild").exists()); + let output = Command::new(&binary) + .args(["server", "--port", "65536"]) + .current_dir(directory.path()) + .env("PATH", "") + .output() + .expect("invalid port"); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("--port must be between 0 and 65535")); + assert!( + !directory.path().join(".sqlbuild").exists(), + "invalid input cannot initialize a database" + ); +} + +#[tokio::test] +async fn given_a_standalone_binary_when_serving_with_or_without_a_scheduler_then_migrations_auth_and_shutdown_are_native() + { + for scheduler in [false, true] { + let mut arguments = vec!["server", "--port", "0"]; + if !scheduler { + arguments.push("--no-scheduler"); + } + let mut control = start_control(&arguments); + let url = api_ready(&mut control).await; + let response = reqwest::Client::new() + .post(format!("{url}/api/runs/{}/heartbeat", uuid::Uuid::now_v7())) + .send() + .await + .expect("protected native API"); + assert_eq!(response.status(), reqwest::StatusCode::UNAUTHORIZED); + tokio::time::timeout(Duration::from_secs(5), async { + while leader_count(&control) != Some(i64::from(scheduler)) { + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("embedded scheduler enabled exactly when requested"); + #[cfg(unix)] + crate::helpers::terminate(&mut control).await; + } +} + +#[tokio::test] +async fn given_a_standalone_scheduler_when_started_without_python_then_it_migrates_and_takes_the_durable_lease() + { + let mut control = start_control(&["scheduler"]); + tokio::time::timeout(Duration::from_secs(5), async { + while leader_count(&control) != Some(1) { + assert!( + control + .child + .try_wait() + .expect("scheduler status") + .is_none() + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("standalone scheduler acquires leadership"); + #[cfg(unix)] + crate::helpers::terminate(&mut control).await; +} diff --git a/crates/sqlbuild-cli/tests/native_commands/test_types.rs b/crates/sqlbuild-cli/tests/native_commands/test_types.rs new file mode 100644 index 0000000000..35842e7f61 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/test_types.rs @@ -0,0 +1,15 @@ +use std::path::PathBuf; +use std::process::Child; + +pub(crate) struct TestControl { + pub(crate) child: Child, + pub(crate) directory: tempfile::TempDir, + pub(crate) stderr: PathBuf, +} + +impl Drop for TestControl { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} diff --git a/crates/sqlbuild-orchestration/Cargo.toml b/crates/sqlbuild-orchestration/Cargo.toml new file mode 100644 index 0000000000..7e9e614a65 --- /dev/null +++ b/crates/sqlbuild-orchestration/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "sqlbuild-orchestration" +version.workspace = true +description = "SQLBuild control-plane storage, snapshots, scheduling and run coordination." +edition.workspace = true +license.workspace = true +publish.workspace = true +rust-version.workspace = true + +[dependencies] +axum.workspace = true +chrono.workspace = true +chrono-tz.workspace = true +croner.workspace = true +flate2.workspace = true +getrandom.workspace = true +rusqlite.workspace = true +semver.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlbuild-recall.workspace = true +sqlx-core.workspace = true +sqlx-postgres.workspace = true +subtle.workspace = true +thiserror.workspace = true +tokio.workspace = true +utoipa.workspace = true +uuid.workspace = true + +[dev-dependencies] +proptest.workspace = true +reqwest.workspace = true +tempfile.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication.rs new file mode 100644 index 0000000000..2920fd6ea0 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication.rs @@ -0,0 +1,38 @@ +//! Authenticate before body buffering; scope checks remain in each owning endpoint. + +use axum::extract::{Request, State}; +use axum::http::{HeaderValue, StatusCode, header}; +use axum::middleware::Next; +use axum::response::Response; + +use crate::api::errors::ApiError; +use crate::credentials::main::authenticate_token::authenticate_token; +use crate::store::models::ControlStore; + +pub(crate) async fn authenticate( + State(store): State, + mut request: Request, + next: Next, +) -> Result { + let unauthorized = || ApiError { + status: StatusCode::UNAUTHORIZED, + message: "valid bearer credential required".into(), + }; + let bearer = request + .headers() + .get(header::AUTHORIZATION) + .ok_or_else(unauthorized)? + .to_str() + .map_err(|_| unauthorized())? + .strip_prefix("Bearer ") + .ok_or_else(unauthorized)?; + let identity = authenticate_token(&store, bearer) + .await? + .ok_or_else(unauthorized)?; + request.extensions_mut().insert(identity); + let mut response = next.run(request).await; + response + .headers_mut() + .insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); + Ok(response) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs new file mode 100644 index 0000000000..41d8c461c5 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod authentication; +pub(crate) mod runs; +pub(crate) mod snapshots; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs b/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs new file mode 100644 index 0000000000..419ec2681a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs @@ -0,0 +1,65 @@ +//! Worker reports are bound to the authenticated run and claim, never a caller-supplied actor. + +use axum::extract::{Path, State}; +use axum::http::StatusCode; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::errors::ApiError; +use crate::api::models::{HeartbeatResponse, MutationResponse}; +use crate::credentials::models::{CredentialScope, TokenIdentity}; +use crate::runs::main::change_run::change_run; +use crate::runs::main::heartbeat_run::heartbeat_run; +use crate::runs::models::{RunAuthority, RunChangeRequest}; +use crate::store::models::ControlStore; + +#[utoipa::path(post, path = "/api/runs/{run_id}/events", params(("run_id" = Uuid, Path)), + request_body = RunChangeRequest, + responses((status = 200, body = MutationResponse), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Run not authorized")), + security(("bearer" = [])))] +pub(crate) async fn report( + State(store): State, + Extension(identity): Extension, + Path(run_id): Path, + Json(request): Json, +) -> Result, ApiError> { + let claim_id = require_worker(&identity, run_id)?; + if request.run_id != run_id || request.claim_id != Some(claim_id) { + return Err(ApiError { + status: StatusCode::FORBIDDEN, + message: "report does not match the authenticated run and claim".into(), + }); + } + let applied = change_run(&store, RunAuthority::Worker { run_id, claim_id }, &request).await?; + Ok(Json(MutationResponse { applied })) +} + +#[utoipa::path(post, path = "/api/runs/{run_id}/heartbeat", params(("run_id" = Uuid, Path)), + responses((status = 200, body = HeartbeatResponse), (status = 401, description = "Invalid credential"), + (status = 403, description = "Run not authorized")), security(("bearer" = [])))] +pub(crate) async fn heartbeat( + State(store): State, + Extension(identity): Extension, + Path(run_id): Path, +) -> Result, ApiError> { + let claim_id = require_worker(&identity, run_id)?; + Ok(Json(HeartbeatResponse { + cancel_requested: heartbeat_run(&store, (run_id, claim_id)).await?, + })) +} + +fn require_worker(identity: &TokenIdentity, run_id: Uuid) -> Result { + if let CredentialScope::Run { + run_id: permitted, + claim_id, + } = identity.scope + && permitted == run_id + { + return Ok(claim_id); + } + Err(ApiError { + status: StatusCode::FORBIDDEN, + message: "credential does not authorize this run".into(), + }) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/snapshots.rs b/crates/sqlbuild-orchestration/src/api/_helpers/snapshots.rs new file mode 100644 index 0000000000..198925e8fc --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/snapshots.rs @@ -0,0 +1,65 @@ +//! A location credential publishes and reads only its own location's snapshots. + +use axum::extract::{Path, State}; +use axum::http::StatusCode; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::errors::ApiError; +use crate::credentials::models::{CredentialScope, TokenIdentity}; +use crate::snapshots::main::current_snapshot::current_snapshot; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::models::{CurrentSnapshot, SnapshotInput, SnapshotRecord}; +use crate::store::models::ControlStore; + +#[utoipa::path(post, path = "/api/locations/{location_id}/targets/{target}/snapshots", + params(("location_id" = Uuid, Path), ("target" = String, Path)), + request_body = SnapshotInput, + responses((status = 200, body = SnapshotRecord), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized"), + (status = 409, description = "Stale generation")), security(("bearer" = [])))] +pub(crate) async fn publish( + State(store): State, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, + Json(input): Json, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + if input.location_id != location_id || input.target != target { + return Err(ApiError { + status: StatusCode::BAD_REQUEST, + message: "snapshot does not match the requested location and target".into(), + }); + } + Ok(Json(publish_snapshot(&store, &input).await?)) +} + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/snapshots", + params(("location_id" = Uuid, Path), ("target" = String, Path)), + responses((status = 200, body = CurrentSnapshot), (status = 401, description = "Invalid credential"), + (status = 403, description = "Location not authorized"), (status = 404, description = "No successful snapshot")), + security(("bearer" = [])))] +pub(crate) async fn current( + State(store): State, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + let current = current_snapshot(&store, location_id, &target) + .await? + .ok_or_else(|| ApiError { + status: StatusCode::NOT_FOUND, + message: "no successful snapshot".into(), + })?; + Ok(Json(current)) +} + +fn require_location(identity: &TokenIdentity, location_id: Uuid) -> Result<(), ApiError> { + if identity.scope != (CredentialScope::Location { location_id }) { + return Err(ApiError { + status: StatusCode::FORBIDDEN, + message: "credential does not authorize this location".into(), + }); + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/api/errors.rs b/crates/sqlbuild-orchestration/src/api/errors.rs new file mode 100644 index 0000000000..011ed4974a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/errors.rs @@ -0,0 +1,54 @@ +//! HTTP errors never return database diagnostics, connection strings, or credentials. + +use axum::Json; +use axum::http::{HeaderValue, StatusCode, header}; +use axum::response::{IntoResponse, Response}; + +use crate::api::models::ApiErrorBody; +use crate::store::errors::ControlError; + +#[derive(Debug)] +pub struct ApiError { + pub(crate) status: StatusCode, + pub(crate) message: String, +} + +impl From for ApiError { + fn from(error: ControlError) -> Self { + match error { + ControlError::Invalid(message) => Self { + status: StatusCode::BAD_REQUEST, + message, + }, + ControlError::StaleGeneration | ControlError::Fenced => Self { + status: StatusCode::CONFLICT, + message: error.to_string(), + }, + _ => Self { + status: StatusCode::INTERNAL_SERVER_ERROR, + message: "control-store operation failed".into(), + }, + } + } +} + +impl IntoResponse for ApiError { + fn into_response(self) -> Response { + let mut response = ( + self.status, + Json(ApiErrorBody { + error: self.message, + }), + ) + .into_response(); + response + .headers_mut() + .insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); + if self.status == StatusCode::UNAUTHORIZED { + response + .headers_mut() + .insert(header::WWW_AUTHENTICATE, HeaderValue::from_static("Bearer")); + } + response + } +} diff --git a/crates/sqlbuild-orchestration/src/api/main/mod.rs b/crates/sqlbuild-orchestration/src/api/main/mod.rs new file mode 100644 index 0000000000..e72ebe13fc --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/main/mod.rs @@ -0,0 +1,2 @@ +pub mod openapi_document; +pub mod serve_api; diff --git a/crates/sqlbuild-orchestration/src/api/main/openapi_document.rs b/crates/sqlbuild-orchestration/src/api/main/openapi_document.rs new file mode 100644 index 0000000000..597993eda8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/main/openapi_document.rs @@ -0,0 +1,19 @@ +//! Generate the HTTP contract from the request/response types used by the actual endpoints. + +use utoipa::OpenApi; +use utoipa::openapi::security::{HttpAuthScheme, HttpBuilder, SecurityScheme}; + +use crate::api::models::ControlApi; + +#[must_use] +pub fn openapi_document() -> utoipa::openapi::OpenApi { + let mut document = ControlApi::openapi(); + document + .components + .get_or_insert_with(Default::default) + .add_security_scheme( + "bearer", + SecurityScheme::Http(HttpBuilder::new().scheme(HttpAuthScheme::Bearer).build()), + ); + document +} diff --git a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs new file mode 100644 index 0000000000..51d3b48cf4 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs @@ -0,0 +1,39 @@ +//! One native server owns migrations and all storage writes; locations and workers speak HTTP. + +use axum::extract::DefaultBodyLimit; +use axum::routing::{get, post}; +use axum::{Json, Router, middleware}; + +use crate::api::_helpers::authentication::authenticate; +use crate::api::_helpers::{runs, snapshots}; +use crate::api::main::openapi_document::openapi_document; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::models::ControlStore; + +pub async fn serve_api( + store: ControlStore, + listener: tokio::net::TcpListener, + shutdown: impl Future + Send + 'static, +) -> Result<(), ControlError> { + migrate_store(&store).await?; + let protected = Router::new() + .route( + "/api/locations/{location_id}/targets/{target}/snapshots", + get(snapshots::current).post(snapshots::publish), + ) + .route("/api/runs/{run_id}/events", post(runs::report)) + .route("/api/runs/{run_id}/heartbeat", post(runs::heartbeat)) + .layer(DefaultBodyLimit::max(64 * 1024 * 1024)) + .route_layer(middleware::from_fn_with_state(store.clone(), authenticate)); + let application = protected + .route( + "/api/openapi.json", + get(|| async { Json(openapi_document()) }), + ) + .with_state(store); + axum::serve(listener, application) + .with_graceful_shutdown(shutdown) + .await?; + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/api/mod.rs b/crates/sqlbuild-orchestration/src/api/mod.rs new file mode 100644 index 0000000000..8fa5a583a4 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/mod.rs @@ -0,0 +1,6 @@ +//! Python-free HTTP endpoints; locations and workers have no direct database access. + +pub(crate) mod _helpers; +pub mod errors; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs new file mode 100644 index 0000000000..3c3590a1ba --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -0,0 +1,24 @@ +//! Typed HTTP responses shared with the generated OpenAPI contract. + +use serde::{Deserialize, Serialize}; + +use crate::api::_helpers::{runs, snapshots}; + +#[derive(Debug, utoipa::OpenApi)] +#[openapi(paths(snapshots::publish, snapshots::current, runs::report, runs::heartbeat))] +pub(crate) struct ControlApi; + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct MutationResponse { + pub applied: bool, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct HeartbeatResponse { + pub cancel_requested: bool, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct ApiErrorBody { + pub error: String, +} diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/audit.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/audit.rs new file mode 100644 index 0000000000..7a78c89f96 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/audit.rs @@ -0,0 +1,31 @@ +//! Security changes and their audit records commit together, without bearer material. + +use serde_json::Value; +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn audit( + transaction: ControlTransaction, + context: &AuditContext, + (event_id, action): (Uuid, &str), + (scope, before, after): (Value, Value, Value), +) -> Result { + if context.actor.trim().is_empty() || context.granting_role.trim().is_empty() { + return Err(ControlError::Invalid( + "audit actor and granting role are required".into(), + )); + } + let now = transaction.now; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_security_audit (event_id, occurred_at, actor, action, granting_role, request_id, ip_address, scope_json, before_json, after_json) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)", + vec![Parameter::Id(event_id), Parameter::Time(now), Parameter::Text(context.actor.clone()), + Parameter::Text(action.into()), Parameter::Text(context.granting_role.clone()), Parameter::Id(context.request_id), + Parameter::OptionalText(context.ip_address.map(|value| value.to_string())), + Parameter::Json(scope), Parameter::Json(before), Parameter::Json(after)]).await?; + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs new file mode 100644 index 0000000000..7093af372d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod audit; +pub(crate) mod scopes; diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/scopes.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/scopes.rs new file mode 100644 index 0000000000..151288c4c4 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/scopes.rs @@ -0,0 +1,40 @@ +//! Service credentials stop being valid with their live principal, location, or run claim. + +use crate::credentials::models::CredentialScope; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +/// Bound values in principal/location/run/claim column order for the credential insert. +pub(crate) fn scope_parameters(scope: CredentialScope) -> Vec { + match scope { + CredentialScope::Principal { principal_id } => [Some(principal_id), None, None, None], + CredentialScope::Location { location_id } => [None, Some(location_id), None, None], + CredentialScope::Run { run_id, claim_id } => [None, None, Some(run_id), Some(claim_id)], + } + .into_iter() + .map(Parameter::OptionalId) + .collect() +} + +pub(crate) async fn scope_is_active( + transaction: ControlTransaction, + scope: CredentialScope, +) -> Result<(ControlTransaction, bool), ControlError> { + let (sql, parameters) = match scope { + CredentialScope::Principal { principal_id } => ( + "SELECT principal_id FROM sqb_principals WHERE principal_id=$1 AND disabled_at IS NULL", + vec![Parameter::Id(principal_id)], + ), + CredentialScope::Location { location_id } => ( + "SELECT location_id FROM sqb_locations WHERE location_id=$1 AND archived_at IS NULL", + vec![Parameter::Id(location_id)], + ), + CredentialScope::Run { run_id, claim_id } => ( + "SELECT run_id FROM sqb_run_liveness WHERE run_id=$1 AND worker_instance=$2", + vec![Parameter::Id(run_id), Parameter::Id(claim_id)], + ), + }; + let (transaction, rows) = query(transaction, sql, parameters).await?; + Ok((transaction, !rows.is_empty())) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/constants.rs b/crates/sqlbuild-orchestration/src/credentials/constants.rs new file mode 100644 index 0000000000..ce3675812c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/constants.rs @@ -0,0 +1,4 @@ +//! Credential secrets use 256 bits of entropy, represented as hexadecimal in bearer strings. + +pub(crate) const TOKEN_SECRET_BYTES: usize = 32; +pub(crate) const TOKEN_SECRET_HEX_LENGTH: usize = TOKEN_SECRET_BYTES * 2; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/authenticate_token.rs b/crates/sqlbuild-orchestration/src/credentials/main/authenticate_token.rs new file mode 100644 index 0000000000..4349c0d08b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/authenticate_token.rs @@ -0,0 +1,59 @@ +//! Every request sees revocation and current scope state, including across server replicas. + +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use subtle::ConstantTimeEq; +use uuid::Uuid; + +use crate::credentials::_helpers::scopes::scope_is_active; +use crate::credentials::constants::TOKEN_SECRET_HEX_LENGTH; +use crate::credentials::models::{StoredToken, TokenIdentity}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Invalid, revoked, expired and inactive-scope credentials are indistinguishable to callers. +pub async fn authenticate_token( + store: &ControlStore, + bearer: &str, +) -> Result, ControlError> { + let Some((id, secret)) = bearer.split_once('.') else { + return Ok(None); + }; + if secret.len() != TOKEN_SECRET_HEX_LENGTH + || !secret.bytes().all(|value| value.is_ascii_hexdigit()) + { + return Ok(None); + } + let Ok(id) = Uuid::parse_str(id) else { + return Ok(None); + }; + let transaction = begin(store, TransactionMode::Read).await?; + let now = transaction.now; + let (transaction, rows) = query( + transaction, + "SELECT token_id, scope_json, secret_hash, expires_at FROM sqb_access_tokens \ + WHERE token_id=$1 AND revoked_at IS NULL AND expires_at>$2", + vec![Parameter::Id(id), Parameter::Time(now)], + ) + .await?; + let Some(row) = rows.into_iter().next() else { + commit(transaction).await?; + return Ok(None); + }; + let token: StoredToken = serde_json::from_value(serde_json::json!(row)) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let digest = sha256_hex(bearer.as_bytes()); + if !bool::from(digest.as_bytes().ct_eq(token.secret_hash.as_bytes())) { + commit(transaction).await?; + return Ok(None); + } + let (transaction, active) = scope_is_active(transaction, token.scope_json).await?; + commit(transaction).await?; + Ok(active.then_some(TokenIdentity { + token_id: token.token_id, + scope: token.scope_json, + expires_at: token.expires_at, + })) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/create_principal.rs b/crates/sqlbuild-orchestration/src/credentials/main/create_principal.rs new file mode 100644 index 0000000000..b85b43d51e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/create_principal.rs @@ -0,0 +1,55 @@ +//! Server-owned people and service accounts; creating one never grants a role implicitly. + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::_helpers::audit::audit; +use crate::credentials::models::{AuditContext, PrincipalKind}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn create_principal( + store: &ControlStore, + context: &AuditContext, + (name, kind): (&str, PrincipalKind), +) -> Result { + if name.trim().is_empty() { + return Err(ControlError::Invalid("principal name is required".into())); + } + let transaction = begin(store, TransactionMode::Write).await?; + let id = Uuid::now_v7(); + let now = transaction.now; + let kind = match kind { + PrincipalKind::Person => "person", + PrincipalKind::ServiceAccount => "service_account", + }; + let (transaction, _) = query( + transaction, + "INSERT INTO sqb_principals (principal_id, name, kind, created_at) VALUES ($1,$2,$3,$4)", + vec![ + Parameter::Id(id), + Parameter::Text(name.into()), + Parameter::Text(kind.into()), + Parameter::Time(now), + ], + ) + .await?; + commit( + audit( + transaction, + context, + (Uuid::new_v5(&id, b"created"), "principal.created"), + ( + json!({"principal_id": id}), + json!(null), + json!({"name": name, "kind": kind}), + ), + ) + .await?, + ) + .await?; + Ok(id) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/issue_token.rs b/crates/sqlbuild-orchestration/src/credentials/main/issue_token.rs new file mode 100644 index 0000000000..1ccb56aba1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/issue_token.rs @@ -0,0 +1,78 @@ +//! Random bearer secrets are returned once; only their hashes and bounded scopes are durable. + +use std::fmt::Write; + +use chrono::{DateTime, SubsecRound, Utc}; +use serde_json::json; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use uuid::Uuid; + +use crate::credentials::_helpers::audit::audit; +use crate::credentials::_helpers::scopes::{scope_is_active, scope_parameters}; +use crate::credentials::constants::TOKEN_SECRET_BYTES; +use crate::credentials::models::{AuditContext, CredentialScope, IssuedToken, TokenIdentity}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// The caller authorizes token management before entering this storage operation. +pub async fn issue_token( + store: &ControlStore, + context: &AuditContext, + scope: CredentialScope, + expires_at: DateTime, +) -> Result { + let expires_at = expires_at.trunc_subsecs(6); + let transaction = begin(store, TransactionMode::Write).await?; + if expires_at <= transaction.now { + return Err(ControlError::Invalid( + "token expiry must be in the future".into(), + )); + } + let (transaction, active) = scope_is_active(transaction, scope).await?; + if !active { + return Err(ControlError::Invalid("token scope is not active".into())); + } + let token_id = Uuid::now_v7(); + let mut random = [0_u8; TOKEN_SECRET_BYTES]; + getrandom::fill(&mut random) + .map_err(|_| ControlError::Invalid("could not generate a secure token".into()))?; + let mut bearer = format!("{token_id}."); + for byte in random { + write!(bearer, "{byte:02x}") + .map_err(|_| ControlError::Invalid("could not format a secure token".into()))?; + } + let now = transaction.now; + let mut parameters: Vec = vec![ + Parameter::Id(token_id), + Parameter::Text(sha256_hex(bearer.as_bytes())), + ]; + parameters.extend(scope_parameters(scope)); + parameters.extend([ + Parameter::Json(json!(scope)), + Parameter::Time(now), + Parameter::Time(expires_at), + ]); + let (transaction, _) = query(transaction, + "INSERT INTO sqb_access_tokens (token_id, secret_hash, principal_id, location_id, run_id, claim_id, scope_json, created_at, expires_at) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)", + parameters).await?; + let identity = TokenIdentity { + token_id, + scope, + expires_at, + }; + commit( + audit( + transaction, + context, + (Uuid::new_v5(&token_id, b"issued"), "token.issued"), + (json!(scope), json!(null), json!(identity)), + ) + .await?, + ) + .await?; + Ok(IssuedToken { identity, bearer }) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/mod.rs b/crates/sqlbuild-orchestration/src/credentials/main/mod.rs new file mode 100644 index 0000000000..6b63069d7b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/mod.rs @@ -0,0 +1,4 @@ +pub mod authenticate_token; +pub mod create_principal; +pub mod issue_token; +pub mod revoke_token; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/revoke_token.rs b/crates/sqlbuild-orchestration/src/credentials/main/revoke_token.rs new file mode 100644 index 0000000000..6be1bb1e78 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/revoke_token.rs @@ -0,0 +1,48 @@ +//! Revocation and its security audit are one transaction, without a per-process token cache. + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::_helpers::audit::audit; +use crate::credentials::models::AuditContext; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Repeating a revocation does not create a second audit entry or rewrite its actor. +pub async fn revoke_token( + store: &ControlStore, + context: &AuditContext, + token_id: Uuid, +) -> Result { + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "UPDATE sqb_access_tokens SET revoked_at=$1 WHERE token_id=$2 AND revoked_at IS NULL RETURNING scope_json", + vec![Parameter::Time(now), Parameter::Id(token_id)]).await?; + let Some(row) = rows.into_iter().next() else { + commit(transaction).await?; + return Ok(false); + }; + let scope = row + .get("scope_json") + .cloned() + .ok_or_else(|| ControlError::Invalid("stored token scope is missing".into()))?; + commit( + audit( + transaction, + context, + (Uuid::new_v5(&token_id, b"revoked"), "token.revoked"), + ( + scope, + json!({"token_id": token_id, "revoked": false}), + json!({"token_id": token_id, "revoked": true}), + ), + ) + .await?, + ) + .await?; + Ok(true) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/mod.rs b/crates/sqlbuild-orchestration/src/credentials/mod.rs new file mode 100644 index 0000000000..a3ce0d1f39 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/mod.rs @@ -0,0 +1,6 @@ +//! Hashed, revocable credentials scoped to one principal, location, or claimed run. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-orchestration/src/credentials/models.rs b/crates/sqlbuild-orchestration/src/credentials/models.rs new file mode 100644 index 0000000000..f8f7d956f1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/models.rs @@ -0,0 +1,63 @@ +//! Credentials identify a caller; principal permissions are a separate authorization decision. + +use std::net::IpAddr; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PrincipalKind { + Person, + ServiceAccount, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum CredentialScope { + Principal { principal_id: Uuid }, + Location { location_id: Uuid }, + Run { run_id: Uuid, claim_id: Uuid }, +} + +/// Server-derived request metadata, never copied from an action's JSON body. +#[derive(Clone, Debug)] +pub struct AuditContext { + pub actor: String, + pub granting_role: String, + pub request_id: Uuid, + pub ip_address: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct TokenIdentity { + pub token_id: Uuid, + pub scope: CredentialScope, + pub expires_at: DateTime, +} + +/// The bearer is returned once. Debug output must never contain it. +#[derive(Serialize)] +pub struct IssuedToken { + pub identity: TokenIdentity, + pub bearer: String, +} + +impl std::fmt::Debug for IssuedToken { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("IssuedToken") + .field("identity", &self.identity) + .field("bearer", &"[redacted]") + .finish() + } +} + +#[derive(Debug, Deserialize)] +pub(crate) struct StoredToken { + pub(crate) token_id: Uuid, + pub(crate) scope_json: CredentialScope, + pub(crate) secret_hash: String, + pub(crate) expires_at: DateTime, +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs b/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs new file mode 100644 index 0000000000..e7d2165cf9 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs @@ -0,0 +1,87 @@ +//! Apply an operator fact and its projection in one transaction, with retry-safe request identity. + +use chrono::TimeDelta; + +use crate::declarations::models::{ + EditableTriggerState, TriggerChange, TriggerChangeRequest, TriggerStatus, +}; +use crate::scheduler::main::next_schedule_time::next_schedule_time; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Authorization belongs to the API boundary; `actor` is the authenticated principal, not client input. +pub async fn change_trigger( + store: &ControlStore, + request: &TriggerChangeRequest, +) -> Result { + if request.actor.trim().is_empty() { + return Err(ControlError::Invalid( + "trigger changes require an actor".into(), + )); + } + let body = + serde_json::to_value(request).map_err(|error| ControlError::Invalid(error.to_string()))?; + let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, previous) = query( + transaction, + "SELECT request_json FROM sqb_trigger_state_events WHERE event_id=$1", + vec![Parameter::Id(request.event_id)], + ) + .await?; + if let Some(previous) = previous.first() { + if previous.get("request_json") != Some(&body) { + return Err(ControlError::Invalid( + "trigger change event ID already has different content".into(), + )); + } + commit(transaction).await?; + return Ok(false); + } + let (transaction, rows) = query( + transaction, + "SELECT t.definition_json, t.snapshot_id, s.started_at, s.last_tick_for, s.status_override \ + FROM sqb_triggers t JOIN sqb_trigger_state s ON s.trigger_id=t.trigger_id \ + WHERE t.trigger_id=$1 AND t.removed_in_snapshot_id IS NULL", + vec![Parameter::Id(request.trigger_id)], + ) + .await?; + let mut state: EditableTriggerState = + decode_row(rows.into_iter().next().ok_or_else(|| { + ControlError::Invalid("trigger does not exist in the current snapshot".into()) + })?)?; + match request.change { + TriggerChange::Enable => state.status_override = Some(TriggerStatus::Running), + TriggerChange::Pause => state.status_override = Some(TriggerStatus::Paused), + TriggerChange::ResetDefault => state.status_override = None, + TriggerChange::SetCursor { time } => state.last_tick_for = Some(time), + TriggerChange::ResetCursor => state.last_tick_for = None, + } + let status = state + .status_override + .unwrap_or(state.definition_json.default_status); + let after = state.last_tick_for.unwrap_or( + state + .started_at + .checked_sub_signed(TimeDelta::microseconds(1)) + .ok_or_else(|| ControlError::Invalid("trigger start is out of range".into()))?, + ); + let next = if status == TriggerStatus::Running { + next_schedule_time(&state.definition_json.schedule, after)? + } else { + None + }; + let now = transaction.now; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_trigger_state_events (event_id, trigger_id, occurred_at, snapshot_id, request_json) VALUES ($1,$2,$3,$4,$5)", + vec![Parameter::Id(request.event_id), Parameter::Id(request.trigger_id), Parameter::Time(now), Parameter::Id(state.snapshot_id), Parameter::Json(body)]).await?; + let (transaction, _) = query(transaction, + "UPDATE sqb_trigger_state SET status=$1, status_override=$2, last_tick_for=$3, next_due_at=$4, updated_at=$5 WHERE trigger_id=$6", + vec![Parameter::Text(status.as_str().into()), Parameter::OptionalText(state.status_override.map(|value| value.as_str().into())), Parameter::OptionalTime(state.last_tick_for), + Parameter::OptionalTime(next), Parameter::Time(now), Parameter::Id(request.trigger_id)]).await?; + commit(transaction).await?; + Ok(true) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/list_trigger_events.rs b/crates/sqlbuild-orchestration/src/declarations/main/list_trigger_events.rs new file mode 100644 index 0000000000..10d6d3f28e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/list_trigger_events.rs @@ -0,0 +1,28 @@ +//! Read immutable operator changes in committed order. + +use uuid::Uuid; + +use crate::declarations::models::TriggerChangeRecord; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn list_trigger_events( + store: &ControlStore, + trigger_id: Uuid, + limit: u32, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query(transaction, + "SELECT request_json, occurred_at, snapshot_id FROM sqb_trigger_state_events WHERE trigger_id=$1 ORDER BY occurred_at, event_id LIMIT $2", + vec![Parameter::Id(trigger_id), Parameter::Integer(i64::from(limit.min(1000)))]).await?; + let events = rows + .into_iter() + .map(decode_row) + .collect::, _>>()?; + commit(transaction).await?; + Ok(events) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/list_triggers.rs b/crates/sqlbuild-orchestration/src/declarations/main/list_triggers.rs new file mode 100644 index 0000000000..c3ead93c9c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/list_triggers.rs @@ -0,0 +1,30 @@ +//! Current declarations and runtime state are read in one consistent transaction. + +use uuid::Uuid; + +use crate::declarations::models::TriggerRecord; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn list_triggers( + store: &ControlStore, + location_id: Uuid, + target: &str, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query(transaction, + "SELECT t.trigger_id, t.name, t.job_key, t.definition_json, s.status, s.last_tick_for, s.next_due_at \ + FROM sqb_triggers t JOIN sqb_trigger_state s ON s.trigger_id=t.trigger_id \ + WHERE t.location_id=$1 AND t.target=$2 AND t.removed_in_snapshot_id IS NULL ORDER BY t.name", + vec![Parameter::Id(location_id), Parameter::Text(target.into())]).await?; + let triggers = rows + .into_iter() + .map(decode_row) + .collect::, _>>()?; + commit(transaction).await?; + Ok(triggers) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/mod.rs b/crates/sqlbuild-orchestration/src/declarations/main/mod.rs new file mode 100644 index 0000000000..a9078f70e6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/mod.rs @@ -0,0 +1,5 @@ +pub mod change_trigger; +pub mod list_trigger_events; +pub mod list_triggers; +pub(crate) mod publish_definitions; +pub mod snapshot_definitions; diff --git a/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs b/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs new file mode 100644 index 0000000000..c65b13468f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs @@ -0,0 +1,94 @@ +//! Publish definition projections inside the current-snapshot transaction. + +use chrono::TimeDelta; +use uuid::Uuid; + +use crate::declarations::models::{Definitions, PreviousTriggerState, TriggerStatus}; +use crate::scheduler::main::next_schedule_time::next_schedule_time; +use crate::snapshots::models::SnapshotRecord; +use crate::store::errors::ControlError; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn publish_definitions( + mut transaction: ControlTransaction, + snapshot: &SnapshotRecord, + definitions: Definitions, +) -> Result { + for table in ["sqb_job_nodes", "sqb_jobs"] { + (transaction, _) = query( + transaction, + &format!("DELETE FROM {table} WHERE location_id=$1 AND target=$2"), + vec![ + Parameter::Id(snapshot.location_id), + Parameter::Text(snapshot.target.clone()), + ], + ) + .await?; + } + for (name, job) in definitions.jobs { + let definition = + serde_json::to_value(&job).map_err(|error| ControlError::Invalid(error.to_string()))?; + (transaction, _) = query(transaction, + "INSERT INTO sqb_jobs (location_id, target, job_key, snapshot_id, kind, definition_json, overlap_policy) VALUES ($1,$2,$3,$4,'job',$5,$6)", + vec![Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone()), Parameter::Text(name.clone()), + Parameter::Id(snapshot.snapshot_id), Parameter::Json(definition), Parameter::Text(job.overlap.as_str().into())]).await?; + for node in job.selection { + (transaction, _) = query(transaction, + "INSERT INTO sqb_job_nodes (location_id, target, job_key, node_key) VALUES ($1,$2,$3,$4)", + vec![Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone()), Parameter::Text(name.clone()), Parameter::Text(node)]).await?; + } + } + (transaction, _) = query(transaction, + "UPDATE sqb_triggers SET removed_in_snapshot_id=$1 WHERE location_id=$2 AND target=$3 AND removed_in_snapshot_id IS NULL", + vec![Parameter::Id(snapshot.snapshot_id), Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone())]).await?; + for (name, trigger) in definitions.triggers { + let id = Uuid::new_v5( + &snapshot.location_id, + &serde_json::to_vec(&(&snapshot.target, "schedule", &name)) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + ); + let definition = serde_json::to_value(&trigger) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + (transaction, _) = query(transaction, + "INSERT INTO sqb_triggers (trigger_id, location_id, target, kind, name, job_key, definition_json, snapshot_id) VALUES ($1,$2,$3,'schedule',$4,$5,$6,$7) \ + ON CONFLICT (trigger_id) DO UPDATE SET job_key=excluded.job_key, definition_json=excluded.definition_json, snapshot_id=excluded.snapshot_id, removed_in_snapshot_id=NULL", + vec![Parameter::Id(id), Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone()), Parameter::Text(name), + Parameter::Text(trigger.job_key.clone()), Parameter::Json(definition), Parameter::Id(snapshot.snapshot_id)]).await?; + let previous; + (transaction, previous) = query(transaction, "SELECT started_at, last_tick_for, status_override FROM sqb_trigger_state WHERE trigger_id=$1", vec![Parameter::Id(id)]).await?; + let state = previous + .into_iter() + .next() + .map(decode_row::) + .transpose()?; + let started_at = state.as_ref().map_or( + trigger.starts_at.unwrap_or(snapshot.published_at), + |state| state.started_at, + ); + let default = trigger.default_status; + let status = state + .as_ref() + .and_then(|state| state.status_override) + .unwrap_or(default); + let after = state + .as_ref() + .and_then(|state| state.last_tick_for) + .unwrap_or( + started_at + .checked_sub_signed(TimeDelta::microseconds(1)) + .ok_or_else(|| ControlError::Invalid("trigger start is out of range".into()))?, + ); + let next = if status == TriggerStatus::Running { + next_schedule_time(&trigger.schedule, after)? + } else { + None + }; + (transaction, _) = query(transaction, + "INSERT INTO sqb_trigger_state (trigger_id, status, started_at, next_due_at, updated_at) VALUES ($1,$2,$3,$4,$5) \ + ON CONFLICT (trigger_id) DO UPDATE SET status=excluded.status, next_due_at=excluded.next_due_at, updated_at=excluded.updated_at", + vec![Parameter::Id(id), Parameter::Text(status.as_str().into()), Parameter::Time(started_at), Parameter::OptionalTime(next), Parameter::Time(snapshot.published_at)]).await?; + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs b/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs new file mode 100644 index 0000000000..96987a439c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs @@ -0,0 +1,79 @@ +//! Validate resolved declarations before acquiring the snapshot publication lock. + +use std::collections::BTreeMap; + +use serde::de::DeserializeOwned; + +use crate::declarations::models::{Definitions, Schedule}; +use crate::scheduler::constants::MAX_TICKS_PER_BATCH; +use crate::scheduler::main::parse_cron::parse_cron; +use crate::scheduler::models::CatchUp; +use crate::snapshots::models::SnapshotInput; +use crate::store::errors::ControlError; + +fn document( + input: &SnapshotInput, + name: &str, +) -> Result, ControlError> { + input.documents.get(name).map_or_else( + || Ok(BTreeMap::new()), + |value| { + serde_json::from_value(value.clone()) + .map_err(|error| ControlError::Invalid(format!("{name}: {error}"))) + }, + ) +} + +/// Names, references and resolved selections are checked again at the native trust boundary. +pub fn snapshot_definitions(input: &SnapshotInput) -> Result { + let definitions = Definitions { + jobs: document(input, "jobs")?, + triggers: document(input, "triggers")?, + }; + for (name, job) in &definitions.jobs { + if name.trim().is_empty() + || job.description.trim().is_empty() + || job.selection.is_empty() + || job + .selection + .iter() + .any(|key| !input.nodes.contains_key(key)) + { + return Err(ControlError::Invalid(format!( + "job {name} needs a description and a nonempty selection of snapshot nodes" + ))); + } + } + for (name, trigger) in &definitions.triggers { + if name.trim().is_empty() + || trigger.description.trim().is_empty() + || !definitions.jobs.contains_key(&trigger.job_key) + { + return Err(ControlError::Invalid(format!( + "trigger {name} needs a description and a known job" + ))); + } + if let CatchUp::All { max_runs } = trigger.catch_up + && max_runs.get() > MAX_TICKS_PER_BATCH + { + return Err(ControlError::Invalid(format!( + "trigger {name} catch-up exceeds {MAX_TICKS_PER_BATCH}" + ))); + } + match &trigger.schedule { + Schedule::Cron { + expression, + timezone, + } => { + parse_cron(expression, timezone)?; + } + Schedule::Calendar { times, .. } if times.len() > MAX_TICKS_PER_BATCH => { + return Err(ControlError::Invalid(format!( + "calendar {name} exceeds {MAX_TICKS_PER_BATCH} upcoming times" + ))); + } + Schedule::Calendar { .. } => {} + } + } + Ok(definitions) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/mod.rs b/crates/sqlbuild-orchestration/src/declarations/mod.rs new file mode 100644 index 0000000000..5355003ff2 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/mod.rs @@ -0,0 +1,2 @@ +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-orchestration/src/declarations/models.rs b/crates/sqlbuild-orchestration/src/declarations/models.rs new file mode 100644 index 0000000000..9561c50ca1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/models.rs @@ -0,0 +1,139 @@ +//! Resolved, serializable declarations; Python objects never enter scheduler state. + +use std::collections::{BTreeMap, BTreeSet}; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +use crate::scheduler::models::{CatchUp, Overlap}; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum OnFailure { + #[default] + SkipDependents, + SkipDownstreamJobs, + StopGroup, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TriggerStatus { + #[default] + Running, + Paused, +} + +impl TriggerStatus { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Running => "running", + Self::Paused => "paused", + } + } +} + +/// Selections have already been resolved by the code location against the compiled snapshot. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct JobDefinition { + pub description: String, + pub selection: BTreeSet, + #[serde(default)] + pub settings: BTreeMap, + #[serde(default)] + pub overlap: Overlap, + #[serde(default)] + pub on_failure: OnFailure, + #[serde(default)] + pub concurrency_groups: BTreeSet, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum Schedule { + Cron { + expression: String, + timezone: String, + }, + Calendar { + times: BTreeSet>, + refresh_at: DateTime, + }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TriggerDefinition { + pub description: String, + pub job_key: String, + pub schedule: Schedule, + #[serde(default)] + pub starts_at: Option>, + #[serde(default)] + pub default_status: TriggerStatus, + #[serde(default)] + pub catch_up: CatchUp, +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Definitions { + pub jobs: BTreeMap, + pub triggers: BTreeMap, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct TriggerRecord { + pub trigger_id: uuid::Uuid, + pub name: String, + pub job_key: String, + pub definition_json: TriggerDefinition, + pub status: TriggerStatus, + pub last_tick_for: Option>, + pub next_due_at: Option>, +} + +/// Operator changes are immutable facts; clearing an override restores the published code default. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum TriggerChange { + Enable, + Pause, + ResetDefault, + SetCursor { time: DateTime }, + ResetCursor, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TriggerChangeRequest { + pub event_id: uuid::Uuid, + pub trigger_id: uuid::Uuid, + pub actor: String, + pub reason: String, + pub change: TriggerChange, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct TriggerChangeRecord { + pub request_json: TriggerChangeRequest, + pub occurred_at: DateTime, + pub snapshot_id: uuid::Uuid, +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct EditableTriggerState { + pub(crate) definition_json: TriggerDefinition, + pub(crate) snapshot_id: uuid::Uuid, + pub(crate) started_at: DateTime, + pub(crate) last_tick_for: Option>, + pub(crate) status_override: Option, +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct PreviousTriggerState { + pub(crate) started_at: DateTime, + pub(crate) last_tick_for: Option>, + pub(crate) status_override: Option, +} diff --git a/crates/sqlbuild-orchestration/src/lib.rs b/crates/sqlbuild-orchestration/src/lib.rs new file mode 100644 index 0000000000..d1bc86c3ef --- /dev/null +++ b/crates/sqlbuild-orchestration/src/lib.rs @@ -0,0 +1,11 @@ +//! The Python-free SQLBuild control plane. + +#![forbid(unsafe_code)] + +pub mod api; +pub mod credentials; +pub mod declarations; +pub mod runs; +pub mod scheduler; +pub mod snapshots; +pub mod store; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/changes.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/changes.rs new file mode 100644 index 0000000000..ba4b35be6e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/changes.rs @@ -0,0 +1,120 @@ +//! Validate lifecycle authority before publishing any fact or releasing physical write claims. + +use crate::runs::models::{ + ChangingRun, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, +}; +use crate::store::errors::ControlError; + +pub(crate) fn validate_authority( + authority: RunAuthority<'_>, + request: &RunChangeRequest, +) -> Result<(), ControlError> { + let allowed = match authority { + RunAuthority::Scheduler(_) => !matches!( + request.change, + RunChange::CancelRequested { .. } + | RunChange::Started + | RunChange::NodeStarted { .. } + | RunChange::NodeFinished { .. } + ), + RunAuthority::Worker { run_id, claim_id } => { + run_id == request.run_id + && request.claim_id == Some(claim_id) + && matches!( + request.change, + RunChange::Started + | RunChange::NodeStarted { .. } + | RunChange::NodeFinished { .. } + | RunChange::Recovered + | RunChange::Finished { + outcome: RunOutcome::Success + | RunOutcome::Warn + | RunOutcome::Partial + | RunOutcome::Failed + | RunOutcome::Cancelled, + .. + } + ) + } + RunAuthority::Operator(actor) => { + matches!(&request.change, RunChange::CancelRequested { actor: recorded } if !actor.trim().is_empty() && actor == recorded) + } + }; + if !allowed { + return Err(ControlError::Invalid( + "authority cannot publish this run change".into(), + )); + } + Ok(()) +} + +pub(crate) fn validate_change( + state: &ChangingRun, + request: &RunChangeRequest, +) -> Result<(), ControlError> { + use RunStatus::{Launched, Queued, Running, Unresponsive}; + if !matches!(state.status, Queued | Launched | Running | Unresponsive) { + return Err(ControlError::Invalid("terminal runs cannot change".into())); + } + if !matches!(request.change, RunChange::CancelRequested { .. }) + && request.claim_id != state.worker_instance + { + return Err(ControlError::Invalid( + "run change does not belong to the current claim".into(), + )); + } + let valid = match request.change { + RunChange::Launched { .. } => state.worker_instance.is_some(), + RunChange::Started => { + state.worker_instance.is_some() && matches!(state.status, Queued | Launched) + } + RunChange::NodeStarted { .. } | RunChange::NodeFinished { .. } => { + matches!(state.status, Running | Unresponsive) + } + RunChange::Unresponsive => state.status == Running, + RunChange::Recovered => state.status == Unresponsive, + RunChange::CancelRequested { .. } => state.cancel_requested_at.is_none(), + RunChange::Finished { outcome, .. } => { + state.worker_instance.is_some() + || matches!(outcome, RunOutcome::Failed | RunOutcome::Cancelled) + } + }; + if !valid { + return Err(ControlError::Invalid( + "run change is inconsistent with the current lifecycle".into(), + )); + } + if let RunChange::Finished { + outcome: RunOutcome::Success | RunOutcome::Warn | RunOutcome::Partial, + .. + } = request.change + { + let count = |status: &str| { + state + .node_counts_json + .get(status) + .copied() + .unwrap_or_default() + }; + let good = count("success") + count("warn"); + let bad = count("failed") + count("skipped") + count("cancelled"); + let expected = if bad > 0 && good > 0 { + RunOutcome::Partial + } else if bad > 0 { + RunOutcome::Failed + } else if count("warn") > 0 { + RunOutcome::Warn + } else { + RunOutcome::Success + }; + if count("pending") > 0 + || count("running") > 0 + || !matches!(request.change, RunChange::Finished { outcome, .. } if outcome == expected) + { + return Err(ControlError::Invalid( + "run outcome does not match its completed node facts".into(), + )); + } + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/claims.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/claims.rs new file mode 100644 index 0000000000..2b65b19559 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/claims.rs @@ -0,0 +1,84 @@ +//! A rejected candidate rolls back its entire savepoint, including any partial write set. + +use std::collections::{BTreeMap, BTreeSet}; +use std::num::NonZeroU32; + +use crate::runs::models::RunRequest; +use crate::scheduler::models::Overlap; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn reserve_claims( + mut transaction: ControlTransaction, + request: &RunRequest, + limits: &BTreeMap, +) -> Result<(ControlTransaction, bool), ControlError> { + if request.overlap != Overlap::Allow + && let Some(job_key) = &request.job_key + { + let rows; + (transaction, rows) = query( + transaction, + "SELECT r.run_id FROM sqb_runs r JOIN sqb_run_liveness l ON l.run_id=r.run_id \ + WHERE r.location_id=$1 AND r.target=$2 AND r.job_key=$3 LIMIT 1", + vec![ + Parameter::Id(request.location_id), + Parameter::Text(request.target.clone()), + Parameter::Text(job_key.clone()), + ], + ) + .await?; + if !rows.is_empty() { + return Ok((transaction, false)); + } + } + let now = transaction.now; + for node in &request.write_set { + let inserted; + (transaction, inserted) = query(transaction, + "INSERT INTO sqb_write_claims (target, node_key, run_id, claimed_at) VALUES ($1,$2,$3,$4) \ + ON CONFLICT (target, node_key) DO NOTHING RETURNING node_key", + vec![Parameter::Text(request.target.clone()), Parameter::Text(node.clone()), Parameter::Id(request.run_id), Parameter::Time(now)]).await?; + if inserted.is_empty() { + return Ok((transaction, false)); + } + } + for group in &request.concurrency_groups { + let limit = limits + .get(group) + .ok_or_else(|| { + ControlError::Invalid(format!("concurrency group {group} has no configured limit")) + })? + .get(); + let slots; + (transaction, slots) = query( + transaction, + "SELECT slot FROM sqb_concurrency_slots WHERE group_key=$1 ORDER BY slot", + vec![Parameter::Text(group.clone())], + ) + .await?; + let occupied: BTreeSet = slots + .iter() + .filter_map(|row| row.get("slot").and_then(serde_json::Value::as_i64)) + .collect(); + // Lowering a limit also counts occupied slots above the new maximum. + if occupied.len() >= limit as usize { + return Ok((transaction, false)); + } + let slot = (1..=i64::from(limit)) + .find(|slot| !occupied.contains(slot)) + .ok_or_else(|| { + ControlError::Invalid("concurrency slot accounting is inconsistent".into()) + })?; + let inserted; + (transaction, inserted) = query(transaction, + "INSERT INTO sqb_concurrency_slots (group_key, slot, run_id, claimed_at) VALUES ($1,$2,$3,$4) \ + ON CONFLICT (group_key, slot) DO NOTHING RETURNING slot", + vec![Parameter::Text(group.clone()), Parameter::Integer(slot), Parameter::Id(request.run_id), Parameter::Time(now)]).await?; + if inserted.is_empty() { + return Ok((transaction, false)); + } + } + Ok((transaction, true)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/events.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/events.rs new file mode 100644 index 0000000000..7ef23c279b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/events.rs @@ -0,0 +1,63 @@ +//! All orchestration transitions share Recall's invocation sequence and transaction. + +use chrono::SecondsFormat; +use serde_json::{Map, Value, json}; +use sqlbuild_recall::records::main::event_object_row::event_object_row; +use sqlbuild_recall::records::models::RowEnvelope; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::append_recall::append_recall; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn append_run_event( + transaction: ControlTransaction, + (run_id, event_id): (Uuid, Uuid), + kind: &str, + payload: Value, +) -> Result<(ControlTransaction, i64), ControlError> { + let invocation_id = format!("scheduler/{run_id}"); + let timestamp = transaction.now.to_rfc3339_opts(SecondsFormat::Micros, true); + let (transaction, rows) = query(transaction, + "SELECT invocation_sequence AS last_sequence, invocation_started_at AS started_at FROM recall_events \ + WHERE invocation_id=$1 ORDER BY invocation_sequence DESC LIMIT 1", + vec![Parameter::Text(invocation_id.clone())]).await?; + let previous = rows.first(); + let sequence = previous + .and_then(|row| row.get("last_sequence")) + .and_then(Value::as_i64) + .map_or(Some(0), |value| value.checked_add(1)) + .ok_or_else(|| ControlError::Invalid("run event sequence is exhausted".into()))?; + let envelope = RowEnvelope { + invocation_id, + invocation_started_at: previous + .and_then(|row| row.get("started_at")) + .and_then(Value::as_str) + .unwrap_or(×tamp) + .into(), + producer_version: env!("CARGO_PKG_VERSION").into(), + }; + let event = Map::from_iter([ + ("event_id".into(), json!(event_id)), + ("event_type".into(), json!(kind)), + ("schema_version".into(), json!(1)), + ("occurred_at".into(), json!(timestamp)), + ("producer".into(), json!("scheduler")), + ("invocation_sequence".into(), json!(sequence)), + ("run_id".into(), json!(run_id)), + ("payload".into(), payload), + ]); + let row = event_object_row(&envelope, &event) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let (transaction, outcome) = append_recall(transaction, &row).await?; + if outcome.conflict.is_some() { + return Err(ControlError::Invalid( + "run event conflicts with a durable fact".into(), + )); + } + let cursor = outcome + .storage_id + .ok_or_else(|| ControlError::Invalid("run event has no storage cursor".into()))?; + Ok((transaction, cursor)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs new file mode 100644 index 0000000000..5994d61750 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod changes; +pub(crate) mod claims; +pub(crate) mod events; +pub(crate) mod nodes; +pub(crate) mod projection; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/nodes.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/nodes.rs new file mode 100644 index 0000000000..cb8d430a89 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/nodes.rs @@ -0,0 +1,83 @@ +//! Updating one node adjusts bounded status counters instead of rescanning every node per event. + +use std::collections::BTreeMap; + +use serde_json::json; + +use crate::runs::models::{RunChange, RunChangeRequest}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn project_node( + transaction: ControlTransaction, + request: &RunChangeRequest, + counts: &BTreeMap, +) -> Result { + let (node, status, rows, error, skipped) = match &request.change { + RunChange::NodeStarted { node_key } => (node_key, "running", None, None, None), + RunChange::NodeFinished { + node_key, + outcome, + rows_affected, + error_code, + skipped_because_of, + } => ( + node_key, + outcome.as_str(), + *rows_affected, + error_code.clone(), + skipped_because_of.clone(), + ), + _ => { + return Err(ControlError::Invalid( + "node projection requires a node event".into(), + )); + } + }; + let now = transaction.now; + let (transaction, nodes) = query( + transaction, + "SELECT status FROM sqb_run_nodes WHERE run_id=$1 AND node_key=$2 AND attempt=1", + vec![Parameter::Id(request.run_id), Parameter::Text(node.clone())], + ) + .await?; + let previous = nodes + .first() + .and_then(|row| row.get("status")) + .and_then(serde_json::Value::as_str) + .ok_or_else(|| ControlError::Invalid("node is outside the frozen run selection".into()))?; + if previous != "pending" && (previous != "running" || status == "running") { + return Err(ControlError::Invalid( + "node change is inconsistent with its current outcome".into(), + )); + } + let mut counts = counts.clone(); + let old = counts + .get_mut(previous) + .ok_or_else(|| ControlError::Invalid("node counter is missing".into()))?; + *old -= 1; + if *old < 0 { + return Err(ControlError::Invalid("node counter is negative".into())); + } + if *old == 0 { + counts.remove(previous); + } + *counts.entry(status.into()).or_default() += 1; + let (transaction, _) = query(transaction, + "UPDATE sqb_run_nodes SET status=$1, started_at=CASE WHEN $1='running' THEN $2 ELSE started_at END, \ + ended_at=$3, rows_affected=$4, error_code=$5, skipped_because_of=$6 WHERE run_id=$7 AND node_key=$8 AND attempt=1", + vec![Parameter::Text(status.into()), Parameter::Time(now), Parameter::OptionalTime((status != "running").then_some(now)), + rows.map_or(Parameter::Null(sqlbuild_recall::schema::models::ColumnType::Integer), Parameter::Integer), + Parameter::OptionalText(error), Parameter::OptionalText(skipped), Parameter::Id(request.run_id), Parameter::Text(node.clone())]).await?; + let (transaction, _) = query( + transaction, + "UPDATE sqb_runs SET node_counts_json=$1 WHERE run_id=$2", + vec![ + Parameter::Json(json!(counts)), + Parameter::Id(request.run_id), + ], + ) + .await?; + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/projection.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/projection.rs new file mode 100644 index 0000000000..9d0629b574 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/projection.rs @@ -0,0 +1,115 @@ +//! Lifecycle projections and claim cleanup are part of the fact's transaction, never a later task. + +use std::collections::BTreeMap; + +use serde_json::json; + +use crate::runs::_helpers::nodes::project_node; +use crate::runs::models::{ChangingRun, RunChange, RunChangeRequest, RunOutcome}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn project_change( + mut transaction: ControlTransaction, + request: &RunChangeRequest, + state: &ChangingRun, +) -> Result { + let now = transaction.now; + match &request.change { + RunChange::NodeStarted { .. } | RunChange::NodeFinished { .. } => { + return project_node(transaction, request, &state.node_counts_json).await; + } + RunChange::Launched { container_id } => { + (transaction, _) = query(transaction, + "UPDATE sqb_runs SET status=CASE WHEN status='queued' THEN 'launched' ELSE status END, launched_at=COALESCE(launched_at,$1) WHERE run_id=$2", + vec![Parameter::Time(now), Parameter::Id(request.run_id)]).await?; + (transaction, _) = query( + transaction, + "UPDATE sqb_run_liveness SET container_id=$1 WHERE run_id=$2", + vec![ + Parameter::OptionalText(container_id.clone()), + Parameter::Id(request.run_id), + ], + ) + .await?; + } + RunChange::Started => { + (transaction, _) = query(transaction, + "UPDATE sqb_runs SET status='running', launched_at=COALESCE(launched_at,$1), started_at=$1 WHERE run_id=$2", + vec![Parameter::Time(now), Parameter::Id(request.run_id)]).await?; + } + RunChange::Unresponsive | RunChange::Recovered => { + let status = if request.change == RunChange::Unresponsive { + "unresponsive" + } else { + "running" + }; + (transaction, _) = query( + transaction, + "UPDATE sqb_runs SET status=$1 WHERE run_id=$2", + vec![ + Parameter::Text(status.into()), + Parameter::Id(request.run_id), + ], + ) + .await?; + } + RunChange::CancelRequested { .. } => { + (transaction, _) = query( + transaction, + "UPDATE sqb_runs SET cancel_requested_at=$1 WHERE run_id=$2", + vec![Parameter::Time(now), Parameter::Id(request.run_id)], + ) + .await?; + } + RunChange::Finished { + outcome, reason, .. + } => { + let cancelled = *outcome == RunOutcome::Cancelled; + (transaction, _) = query(transaction, + "UPDATE sqb_run_nodes SET status=CASE WHEN status='running' THEN $1 ELSE $2 END, ended_at=$3 \ + WHERE run_id=$4 AND status IN ('pending','running')", + vec![Parameter::Text(if cancelled { "cancelled" } else { "failed" }.into()), + Parameter::Text(if cancelled { "cancelled" } else { "skipped" }.into()), Parameter::Time(now), Parameter::Id(request.run_id)]).await?; + let counts; + (transaction, counts) = query(transaction, + "SELECT status, COUNT(*) AS count FROM sqb_run_nodes WHERE run_id=$1 GROUP BY status", + vec![Parameter::Id(request.run_id)]).await?; + let counts: BTreeMap = counts + .into_iter() + .filter_map(|row| { + Some(( + row.get("status")?.as_str()?.into(), + row.get("count")?.as_i64()?, + )) + }) + .collect(); + (transaction, _) = query( + transaction, + "UPDATE sqb_runs SET status=$1, status_reason=$2, ended_at=$3, node_counts_json=$5 WHERE run_id=$4", + vec![ + Parameter::Text(outcome.as_str().into()), + Parameter::Text(reason.clone()), + Parameter::Time(now), + Parameter::Id(request.run_id), + Parameter::Json(json!(counts)), + ], + ) + .await?; + for table in [ + "sqb_write_claims", + "sqb_concurrency_slots", + "sqb_run_liveness", + ] { + (transaction, _) = query( + transaction, + &format!("DELETE FROM {table} WHERE run_id=$1"), + vec![Parameter::Id(request.run_id)], + ) + .await?; + } + } + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/change_run.rs b/crates/sqlbuild-orchestration/src/runs/main/change_run.rs new file mode 100644 index 0000000000..f3884cb69f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/change_run.rs @@ -0,0 +1,104 @@ +//! Retry-safe facts, lifecycle projection and terminal claim release form one transaction. + +use serde_json::json; + +use crate::runs::_helpers::changes::{validate_authority, validate_change}; +use crate::runs::_helpers::events::append_run_event; +use crate::runs::_helpers::projection::project_change; +use crate::runs::models::{ChangingRun, RunAuthority, RunChange, RunChangeRequest, RunOutcome}; +use crate::scheduler::main::check_lease::check_lease; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Authority comes from authenticated API state, not client-supplied actor or scope fields. +pub async fn change_run( + store: &ControlStore, + authority: RunAuthority<'_>, + request: &RunChangeRequest, +) -> Result { + validate_authority(authority, request)?; + let mut transaction = begin(store, TransactionMode::Write).await?; + if let RunAuthority::Scheduler(lease) = authority { + transaction = check_lease(transaction, lease).await?; + } + let body = json!({"change": request}); + let (transaction, previous) = query( + transaction, + "SELECT invocation_id, payload FROM recall_events WHERE event_id=$1 LIMIT 1", + vec![Parameter::Text(request.event_id.to_string())], + ) + .await?; + if let Some(previous) = previous.first() { + if previous.get("payload") != Some(&body) + || previous + .get("invocation_id") + .and_then(serde_json::Value::as_str) + != Some(format!("scheduler/{}", request.run_id).as_str()) + { + return Err(ControlError::Invalid( + "run change event ID already has different content".into(), + )); + } + commit(transaction).await?; + return Ok(false); + } + let (transaction, rows) = query( + transaction, + "SELECT r.status, r.cancel_requested_at, r.node_counts_json, l.worker_instance FROM sqb_runs r \ + LEFT JOIN sqb_run_liveness l ON l.run_id=r.run_id WHERE r.run_id=$1", + vec![Parameter::Id(request.run_id)], + ) + .await?; + let state: ChangingRun = decode_row( + rows.into_iter() + .next() + .ok_or_else(|| ControlError::Invalid("run does not exist".into()))?, + )?; + validate_change(&state, request)?; + let (transaction, mut cursor) = append_run_event( + transaction, + (request.run_id, request.event_id), + request.change.event_type(), + body, + ) + .await?; + let mut transaction = project_change(transaction, request, &state).await?; + // No process can exist before a durable claim. Cancel queued work without leaving it inert forever. + if matches!(request.change, RunChange::CancelRequested { .. }) + && state.worker_instance.is_none() + { + let terminal = RunChangeRequest { + event_id: uuid::Uuid::new_v5(&request.event_id, b"cancelled-before-launch"), + run_id: request.run_id, + claim_id: None, + change: RunChange::Finished { + outcome: RunOutcome::Cancelled, + reason: "cancelled before launch".into(), + exit_code: None, + }, + }; + (transaction, cursor) = append_run_event( + transaction, + (terminal.run_id, terminal.event_id), + terminal.change.event_type(), + json!({"change": terminal}), + ) + .await?; + transaction = project_change(transaction, &terminal, &state).await?; + } + let (mut transaction, _) = query( + transaction, + "UPDATE sqb_runs SET last_event_storage_id=$1 WHERE run_id=$2", + vec![Parameter::Integer(cursor), Parameter::Id(request.run_id)], + ) + .await?; + if let RunAuthority::Scheduler(lease) = authority { + transaction = check_lease(transaction, lease).await?; + } + commit(transaction).await?; + Ok(true) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/claim_next_run.rs b/crates/sqlbuild-orchestration/src/runs/main/claim_next_run.rs new file mode 100644 index 0000000000..3a86ee6d75 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/claim_next_run.rs @@ -0,0 +1,94 @@ +//! Claim priority-ordered work without allowing a blocked page to starve later candidates. + +use std::collections::BTreeMap; +use std::num::NonZeroU32; + +use serde_json::json; +use sqlbuild_recall::schema::models::ColumnType; +use uuid::Uuid; + +use crate::runs::_helpers::claims::reserve_claims; +use crate::runs::_helpers::events::append_run_event; +use crate::runs::models::{QueueCandidate, RunClaim}; +use crate::scheduler::main::check_lease::check_lease; +use crate::scheduler::models::SchedulerLease; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{Connection, ControlStore, Parameter, TransactionMode}; + +/// Limits come from one installation configuration, never from competing per-job declarations. +pub async fn claim_next_run( + store: &ControlStore, + lease: &SchedulerLease, + limits: &BTreeMap, +) -> Result, ControlError> { + let mut transaction = check_lease(begin(store, TransactionMode::Write).await?, lease).await?; + let locking = if matches!(transaction.connection, Some(Connection::Postgres(_))) { + " FOR UPDATE OF r SKIP LOCKED" + } else { + "" + }; + let mut after: Option = None; + loop { + let rows; + (transaction, rows) = query(transaction, + &format!("SELECT r.request_json, r.queued_at FROM sqb_runs r WHERE r.status='queued' AND r.cancel_requested_at IS NULL \ + AND NOT EXISTS (SELECT 1 FROM sqb_run_liveness l WHERE l.run_id=r.run_id) \ + AND ($1 IS NULL OR r.priority<$1 OR (r.priority=$1 AND (r.queued_at>$2 OR (r.queued_at=$2 AND r.run_id>$3)))) \ + ORDER BY r.priority DESC, r.queued_at, r.run_id LIMIT 100{locking}"), + vec![after.as_ref().map_or(Parameter::Null(ColumnType::Integer), |value| Parameter::Integer(value.request_json.priority)), + Parameter::OptionalTime(after.as_ref().map(|value| value.queued_at)), + Parameter::OptionalId(after.as_ref().map(|value| value.request_json.run_id))]).await?; + if rows.is_empty() { + commit(check_lease(transaction, lease).await?).await?; + return Ok(None); + } + for row in rows { + let candidate: QueueCandidate = decode_row(row)?; + (transaction, _) = query(transaction, "SAVEPOINT candidate_claim", vec![]).await?; + let reserved; + (transaction, reserved) = + reserve_claims(transaction, &candidate.request_json, limits).await?; + if !reserved { + (transaction, _) = + query(transaction, "ROLLBACK TO SAVEPOINT candidate_claim", vec![]).await?; + (transaction, _) = + query(transaction, "RELEASE SAVEPOINT candidate_claim", vec![]).await?; + after = Some(candidate); + continue; + } + let claim = RunClaim { + claim_id: Uuid::new_v5(&candidate.request_json.run_id, b"run_claimed"), + request: candidate.request_json, + }; + let now = transaction.now; + (transaction, _) = query(transaction, + "INSERT INTO sqb_run_liveness (run_id, worker_instance, claimed_at, last_heartbeat_at) VALUES ($1,$2,$3,$3)", + vec![Parameter::Id(claim.request.run_id), Parameter::Id(claim.claim_id), Parameter::Time(now)]).await?; + let cursor; + (transaction, cursor) = append_run_event( + transaction, + (claim.request.run_id, claim.claim_id), + "run_claimed", + json!({"claim": claim}), + ) + .await?; + (transaction, _) = query( + transaction, + "UPDATE sqb_runs SET last_event_storage_id=$1 WHERE run_id=$2", + vec![ + Parameter::Integer(cursor), + Parameter::Id(claim.request.run_id), + ], + ) + .await?; + (transaction, _) = + query(transaction, "RELEASE SAVEPOINT candidate_claim", vec![]).await?; + commit(check_lease(transaction, lease).await?).await?; + return Ok(Some(claim)); + } + } +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/heartbeat_run.rs b/crates/sqlbuild-orchestration/src/runs/main/heartbeat_run.rs new file mode 100644 index 0000000000..a5399c6cdf --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/heartbeat_run.rs @@ -0,0 +1,71 @@ +//! Heartbeats update only liveness; recovery is a durable transition in that same transaction. + +use serde_json::json; +use uuid::Uuid; + +use crate::runs::_helpers::events::append_run_event; +use crate::runs::models::{RunChange, RunChangeRequest}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// The worker identity is taken from its authenticated run token. Return whether cancellation is pending. +pub async fn heartbeat_run( + store: &ControlStore, + (run_id, claim_id): (Uuid, Uuid), +) -> Result { + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "UPDATE sqb_run_liveness SET last_heartbeat_at=$1 WHERE run_id=$2 AND worker_instance=$3 RETURNING run_id", + vec![Parameter::Time(now), Parameter::Id(run_id), Parameter::Id(claim_id)]).await?; + if rows.is_empty() { + return Err(ControlError::Invalid( + "heartbeat does not belong to an active run claim".into(), + )); + } + let (mut transaction, rows) = query( + transaction, + "SELECT status, last_event_storage_id, cancel_requested_at FROM sqb_runs WHERE run_id=$1", + vec![Parameter::Id(run_id)], + ) + .await?; + let row = rows + .first() + .ok_or_else(|| ControlError::Invalid("claimed run does not exist".into()))?; + let cancelled = row + .get("cancel_requested_at") + .is_some_and(|value| !value.is_null()); + if row.get("status").and_then(serde_json::Value::as_str) == Some("unresponsive") { + let previous = row + .get("last_event_storage_id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| { + ControlError::Invalid("run recovery is missing its preceding event".into()) + })?; + let request = RunChangeRequest { + event_id: Uuid::new_v5(&claim_id, format!("recovery/{previous}").as_bytes()), + run_id, + claim_id: Some(claim_id), + change: RunChange::Recovered, + }; + let cursor; + (transaction, cursor) = append_run_event( + transaction, + (run_id, request.event_id), + request.change.event_type(), + json!({"change": request}), + ) + .await?; + (transaction, _) = query( + transaction, + "UPDATE sqb_runs SET status='running', last_event_storage_id=$1 WHERE run_id=$2", + vec![Parameter::Integer(cursor), Parameter::Id(run_id)], + ) + .await?; + } + commit(transaction).await?; + Ok(cancelled) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/list_run_nodes.rs b/crates/sqlbuild-orchestration/src/runs/main/list_run_nodes.rs new file mode 100644 index 0000000000..f0312d6f5c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/list_run_nodes.rs @@ -0,0 +1,29 @@ +//! Node outcomes are scoped to one frozen run, independently of current code snapshots. + +use uuid::Uuid; + +use crate::runs::models::RunNodeRecord; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// The API authorizes visibility of the run before reading its nodes. +pub async fn list_run_nodes( + store: &ControlStore, + run_id: Uuid, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query(transaction, + "SELECT node_key, attempt, status, started_at, ended_at, rows_affected, error_code, skipped_because_of \ + FROM sqb_run_nodes WHERE run_id=$1 ORDER BY node_key, attempt", + vec![Parameter::Id(run_id)]).await?; + let nodes = rows + .into_iter() + .map(decode_row) + .collect::, _>>()?; + commit(transaction).await?; + Ok(nodes) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/list_runs.rs b/crates/sqlbuild-orchestration/src/runs/main/list_runs.rs new file mode 100644 index 0000000000..516302af18 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/list_runs.rs @@ -0,0 +1,30 @@ +//! Installation API reads never mix locations or targets. + +use uuid::Uuid; + +use crate::runs::models::RunRecord; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Return recent projected runs; per-run events remain in Recall. +pub async fn list_runs( + store: &ControlStore, + (location_id, target): (Uuid, &str), + limit: u32, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query(transaction, + "SELECT run_id, location_id, target, snapshot_id, job_key, status, queued_at, launched_at, started_at, ended_at, selection_json, node_counts_json, last_event_storage_id \ + FROM sqb_runs WHERE location_id=$1 AND target=$2 ORDER BY queued_at DESC, run_id DESC LIMIT $3", + vec![Parameter::Id(location_id), Parameter::Text(target.into()), Parameter::Integer(i64::from(limit.min(1000)))]).await?; + let runs = rows + .into_iter() + .map(decode_row) + .collect::, _>>()?; + commit(transaction).await?; + Ok(runs) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/mod.rs b/crates/sqlbuild-orchestration/src/runs/main/mod.rs new file mode 100644 index 0000000000..e3e4c9a4d4 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/mod.rs @@ -0,0 +1,7 @@ +pub mod change_run; +pub mod claim_next_run; +pub mod heartbeat_run; +pub mod list_run_nodes; +pub mod list_runs; +pub(crate) mod queue_run; +pub mod request_run; diff --git a/crates/sqlbuild-orchestration/src/runs/main/queue_run.rs b/crates/sqlbuild-orchestration/src/runs/main/queue_run.rs new file mode 100644 index 0000000000..08a7101e43 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/queue_run.rs @@ -0,0 +1,57 @@ +//! Publish request facts, queued facts and initial node projections in the caller's transaction. + +use serde_json::json; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use uuid::Uuid; + +use crate::runs::_helpers::events::append_run_event; +use crate::runs::models::RunRequest; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn queue_run( + mut transaction: ControlTransaction, + request: &RunRequest, +) -> Result { + let now = transaction.now; + let mut cursor = None; + for kind in ["run_requested", "run_queued"] { + let storage_id; + (transaction, storage_id) = append_run_event( + transaction, + ( + request.run_id, + Uuid::new_v5(&request.run_id, kind.as_bytes()), + ), + kind, + json!({"request": request}), + ) + .await?; + cursor = Some(storage_id); + } + let cursor = + cursor.ok_or_else(|| ControlError::Invalid("queued event has no storage cursor".into()))?; + let selection = serde_json::to_value(&request.selection) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let digest = sha256_hex( + &serde_json::to_vec(&request.write_set) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + ); + (transaction, _) = query(transaction, + "INSERT INTO sqb_runs (run_id, location_id, target, snapshot_id, image_id, kind, origin, job_key, parent_run_id, root_run_id, trigger_id, tick_id, rerun_of_run_id, \ + requested_by, priority, selection_json, write_set_digest, status, queued_at, node_counts_json, last_event_storage_id, request_json) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,'queued',$18,$19,$20,$21)", + vec![Parameter::Id(request.run_id), Parameter::Id(request.location_id), Parameter::Text(request.target.clone()), Parameter::Id(request.snapshot_id), + Parameter::OptionalText(request.image_id.clone()), Parameter::Text(request.kind.as_str().into()), Parameter::Text(request.origin.as_str().into()), + Parameter::OptionalText(request.job_key.clone()), Parameter::OptionalId(request.parent_run_id), Parameter::Id(request.root_run_id), + Parameter::OptionalId(request.trigger_id), Parameter::OptionalId(request.tick_id), Parameter::OptionalId(request.rerun_of_run_id), + Parameter::Text(request.requested_by.clone()), Parameter::Integer(request.priority), Parameter::Json(selection), Parameter::Text(digest), + Parameter::Time(now), Parameter::Json(json!({"pending": request.selection.len()})), Parameter::Integer(cursor), Parameter::Json(json!(request))]).await?; + for node in &request.selection { + (transaction, _) = query(transaction, + "INSERT INTO sqb_run_nodes (run_id, node_key, attempt, status, writes) VALUES ($1,$2,1,'pending',$3)", + vec![Parameter::Id(request.run_id), Parameter::Text(node.clone()), Parameter::Boolean(request.write_set.contains(node))]).await?; + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/request_run.rs b/crates/sqlbuild-orchestration/src/runs/main/request_run.rs new file mode 100644 index 0000000000..eb55625e5d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/request_run.rs @@ -0,0 +1,67 @@ +//! Queue a frozen, already authorized request without recreating work on transport retries. + +use std::collections::BTreeSet; + +use serde_json::json; + +use crate::runs::main::queue_run::queue_run; +use crate::runs::models::RunRequest; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// The API must compute and authorize this selection; callers cannot supply their own write scope. +pub async fn request_run(store: &ControlStore, request: &RunRequest) -> Result { + if request.requested_by.trim().is_empty() + || request.selection.is_empty() + || !request.write_set.is_subset(&request.selection) + { + return Err(ControlError::Invalid( + "run needs an actor, selection and a contained write set".into(), + )); + } + let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, previous) = query( + transaction, + "SELECT request_json FROM sqb_runs WHERE run_id=$1", + vec![Parameter::Id(request.run_id)], + ) + .await?; + if let Some(previous) = previous.first() { + if previous.get("request_json") != Some(&json!(request)) { + return Err(ControlError::Invalid( + "run ID already has different request content".into(), + )); + } + commit(transaction).await?; + return Ok(false); + } + let (transaction, nodes) = query( + transaction, + "SELECT node_key FROM sqb_snapshot_nodes WHERE snapshot_id=$1 AND EXISTS \ + (SELECT 1 FROM sqb_snapshots WHERE snapshot_id=$1 AND location_id=$2 AND target=$3)", + vec![ + Parameter::Id(request.snapshot_id), + Parameter::Id(request.location_id), + Parameter::Text(request.target.clone()), + ], + ) + .await?; + let keys: BTreeSet<&str> = nodes + .iter() + .filter_map(|row| row.get("node_key").and_then(serde_json::Value::as_str)) + .collect(); + if request + .selection + .iter() + .any(|key| !keys.contains(key.as_str())) + { + return Err(ControlError::Invalid( + "run selection is not in the pinned location and target snapshot".into(), + )); + } + commit(queue_run(transaction, request).await?).await?; + Ok(true) +} diff --git a/crates/sqlbuild-orchestration/src/runs/mod.rs b/crates/sqlbuild-orchestration/src/runs/mod.rs new file mode 100644 index 0000000000..5359000d16 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod _helpers; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs new file mode 100644 index 0000000000..270381a7f4 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -0,0 +1,250 @@ +//! Frozen requests and rebuildable run projections; lifecycle authority remains in Recall. + +use std::collections::{BTreeMap, BTreeSet}; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RunKind { + Job, + Group, + Adhoc, +} + +impl RunKind { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Job => "job", + Self::Group => "group", + Self::Adhoc => "adhoc", + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RunOrigin { + Manual, + Schedule, + Sensor, + Rerun, + Backfill, +} + +impl RunOrigin { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Manual => "manual", + Self::Schedule => "schedule", + Self::Sensor => "sensor", + Self::Rerun => "rerun", + Self::Backfill => "backfill", + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RunStatus { + Queued, + Launched, + Running, + Unresponsive, + Success, + Warn, + Partial, + Failed, + Cancelled, + PresumedFailed, +} + +/// The authorized selection and pinned execution inputs are immutable once requested. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct RunRequest { + pub run_id: Uuid, + pub location_id: Uuid, + pub target: String, + pub snapshot_id: Uuid, + pub image_id: Option, + pub kind: RunKind, + pub origin: RunOrigin, + pub job_key: Option, + pub parent_run_id: Option, + pub root_run_id: Uuid, + pub trigger_id: Option, + pub tick_id: Option, + pub rerun_of_run_id: Option, + pub requested_by: String, + pub priority: i64, + pub selection: BTreeSet, + pub write_set: BTreeSet, + pub settings: BTreeMap, + pub concurrency_groups: BTreeSet, + pub overlap: crate::scheduler::models::Overlap, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct RunRecord { + pub run_id: Uuid, + pub location_id: Uuid, + pub target: String, + pub snapshot_id: Uuid, + pub job_key: Option, + pub status: RunStatus, + pub queued_at: DateTime, + pub launched_at: Option>, + pub started_at: Option>, + pub ended_at: Option>, + pub selection_json: BTreeSet, + pub node_counts_json: BTreeMap, + pub last_event_storage_id: i64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum NodeOutcome { + Success, + Warn, + Failed, + Skipped, + Cancelled, +} + +impl NodeOutcome { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Success => "success", + Self::Warn => "warn", + Self::Failed => "failed", + Self::Skipped => "skipped", + Self::Cancelled => "cancelled", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RunNodeRecord { + pub node_key: String, + pub attempt: i64, + pub status: String, + pub started_at: Option>, + pub ended_at: Option>, + pub rows_affected: Option, + pub error_code: Option, + pub skipped_because_of: Option, +} + +/// The database owns the claim before a launcher sees the pinned request. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct RunClaim { + pub claim_id: Uuid, + pub request: RunRequest, +} + +/// Terminal outcomes cannot be confused with a request to change an active status. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum RunOutcome { + Success, + Warn, + Partial, + Failed, + Cancelled, + PresumedFailed, +} + +impl RunOutcome { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Success => "success", + Self::Warn => "warn", + Self::Partial => "partial", + Self::Failed => "failed", + Self::Cancelled => "cancelled", + Self::PresumedFailed => "presumed_failed", + } + } +} + +/// This is a server-owned instruction, not an arbitrary status supplied by a worker. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum RunChange { + Launched { + container_id: Option, + }, + Started, + NodeStarted { + node_key: String, + }, + NodeFinished { + node_key: String, + outcome: NodeOutcome, + rows_affected: Option, + error_code: Option, + skipped_because_of: Option, + }, + Unresponsive, + Recovered, + CancelRequested { + actor: String, + }, + Finished { + outcome: RunOutcome, + reason: String, + exit_code: Option, + }, +} + +impl RunChange { + pub(crate) const fn event_type(&self) -> &'static str { + match self { + Self::Launched { .. } => "run_launched", + Self::Started => "run_started", + Self::NodeStarted { .. } => "run_node_started", + Self::NodeFinished { .. } => "run_node_finished", + Self::Unresponsive => "run_unresponsive", + Self::Recovered => "run_recovered", + Self::CancelRequested { .. } => "run_cancel_requested", + Self::Finished { .. } => "run_terminated", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct RunChangeRequest { + pub event_id: Uuid, + pub run_id: Uuid, + pub claim_id: Option, + pub change: RunChange, +} + +/// Constructed after authentication; this authority is never deserialized from a request body. +#[derive(Clone, Copy, Debug)] +pub enum RunAuthority<'a> { + Scheduler(&'a crate::scheduler::models::SchedulerLease), + Worker { run_id: Uuid, claim_id: Uuid }, + Operator(&'a str), +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct QueueCandidate { + pub(crate) request_json: RunRequest, + pub(crate) queued_at: DateTime, +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct ChangingRun { + pub(crate) status: RunStatus, + pub(crate) worker_instance: Option, + pub(crate) cancel_requested_at: Option>, + pub(crate) node_counts_json: BTreeMap, +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/calendar.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/calendar.rs new file mode 100644 index 0000000000..7a27408a66 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/calendar.rs @@ -0,0 +1,31 @@ +//! Map local cron times without using the cron dependency's different fold policy. + +use chrono::{DateTime, LocalResult, NaiveDateTime, TimeZone, Utc}; +use chrono_tz::{GapInfo, Tz}; + +use crate::store::errors::ControlError; + +/// During the first fold, smaller wall times in the second fold may still be in the future. +pub(crate) fn search_start(timezone: Tz, after: DateTime) -> NaiveDateTime { + let local = after.with_timezone(&timezone).naive_local(); + match timezone.from_local_datetime(&local) { + LocalResult::Ambiguous(earlier, later) if after < later.to_utc() => { + local - (later.to_utc() - earlier.to_utc()) + } + _ => local, + } +} + +pub(crate) fn resolve_wall_time( + timezone: Tz, + wall_time: NaiveDateTime, +) -> Result, ControlError> { + match timezone.from_local_datetime(&wall_time) { + LocalResult::Single(time) => Ok(time.to_utc()), + LocalResult::Ambiguous(_, later) => Ok(later.to_utc()), + LocalResult::None => GapInfo::new(&wall_time, &timezone) + .and_then(|gap| gap.end) + .map(|time| time.to_utc()) + .ok_or_else(|| ControlError::Invalid("cron time has no representable gap end".into())), + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs new file mode 100644 index 0000000000..a3af3bf96e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs @@ -0,0 +1,58 @@ +//! Bound cron backlog work without losing omitted times or requesting an obsolete latest run. + +use std::ops::Bound::{Excluded, Unbounded}; + +use chrono::{DateTime, Utc}; + +use crate::declarations::models::{Schedule, TriggerDefinition}; +use crate::scheduler::constants::MAX_TICKS_PER_BATCH; +use crate::scheduler::main::next_cron_time::next_cron_time; +use crate::scheduler::main::parse_cron::parse_cron; +use crate::scheduler::main::previous_cron_time::previous_cron_time; +use crate::scheduler::models::CatchUp; +use crate::store::errors::ControlError; + +pub(crate) fn expand_schedule( + trigger: &TriggerDefinition, + from: DateTime, + now: DateTime, +) -> Result<(Vec>, Option>), ControlError> { + let Schedule::Cron { + expression, + timezone, + } = &trigger.schedule + else { + let Schedule::Calendar { times, .. } = &trigger.schedule else { + unreachable!() + }; + return Ok(( + times.range((Excluded(from), Unbounded)).copied().collect(), + None, + )); + }; + let cron = parse_cron(expression, timezone)?; + let latest = trigger.catch_up == CatchUp::Latest; + let mut due = Vec::new(); + let mut cursor = from; + for _ in 0..(MAX_TICKS_PER_BATCH - usize::from(latest)) { + cursor = next_cron_time(&cron, cursor)?; + due.push(cursor); + if cursor > now { + return Ok((due, None)); + } + } + let continuation = next_cron_time(&cron, cursor)?; + if latest && continuation <= now { + let latest = previous_cron_time(&cron, now)?; + due.push(latest); + return Ok(( + due, + Some(if latest == continuation { + next_cron_time(&cron, latest)? + } else { + continuation + }), + )); + } + Ok((due, Some(continuation))) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/mod.rs new file mode 100644 index 0000000000..85bb8cc89b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod calendar; +pub(crate) mod expansion; +pub(crate) mod persistence; +pub(crate) mod ticks; diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs new file mode 100644 index 0000000000..e7c466d77b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs @@ -0,0 +1,76 @@ +//! Deterministic tick insertion gates all run facts; retries cannot queue a second run. + +use serde_json::json; +use std::collections::BTreeSet; + +use crate::runs::main::queue_run::queue_run; +use crate::runs::models::{RunKind, RunOrigin, RunRequest}; +use crate::scheduler::models::{DueTrigger, SchedulerLease, TickDecision, TickPlan}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn persist_ticks( + transaction: ControlTransaction, + (trigger, lease): (&DueTrigger, &SchedulerLease), + plan: &TickPlan, +) -> Result { + let now = transaction.now; + let (mut transaction, rows) = query(transaction, + "SELECT n.node_key FROM sqb_snapshot_nodes n JOIN sqb_job_nodes j ON j.node_key=n.node_key \ + WHERE n.snapshot_id=$1 AND j.location_id=$2 AND j.target=$3 AND j.job_key=$4 AND n.node_kind IN ('model','seed','function')", + vec![Parameter::Id(trigger.snapshot_id), Parameter::Id(trigger.location_id), Parameter::Text(trigger.target.clone()), Parameter::Text(trigger.job_key.clone())]).await?; + let write_set: BTreeSet = rows + .into_iter() + .filter_map(|row| { + row.get("node_key") + .and_then(serde_json::Value::as_str) + .map(str::to_owned) + }) + .collect(); + for action in &plan.actions { + let (outcome, run, covered) = match action.decision { + TickDecision::Requested { run_id } => ("requested", Some(run_id), None), + TickDecision::Covered { covered_by_run_id } => { + ("covered", None, Some(covered_by_run_id)) + } + }; + let inserted; + (transaction, inserted) = query(transaction, + "INSERT INTO sqb_ticks (tick_id, trigger_id, scheduled_for, evaluated_at, epoch, outcome, run_id, covered_by_run_id, details_json) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) ON CONFLICT (trigger_id, scheduled_for) DO NOTHING RETURNING tick_id", + vec![Parameter::Id(action.tick_id), Parameter::Id(trigger.trigger_id), Parameter::Time(action.scheduled_for), Parameter::Time(now), Parameter::Integer(lease.epoch), + Parameter::Text(outcome.into()), Parameter::OptionalId(run), Parameter::OptionalId(covered), Parameter::Json(json!({}))]).await?; + if let Some(run_id) = run + && !inserted.is_empty() + { + let request = RunRequest { + run_id, + location_id: trigger.location_id, + target: trigger.target.clone(), + snapshot_id: trigger.snapshot_id, + image_id: trigger.image_id.clone(), + kind: RunKind::Job, + origin: RunOrigin::Schedule, + job_key: Some(trigger.job_key.clone()), + parent_run_id: None, + root_run_id: run_id, + trigger_id: Some(trigger.trigger_id), + tick_id: Some(action.tick_id), + rerun_of_run_id: None, + requested_by: "system:scheduler".into(), + priority: 0, + selection: trigger.job_json.selection.clone(), + write_set: write_set.clone(), + settings: trigger.job_json.settings.clone(), + concurrency_groups: trigger.job_json.concurrency_groups.clone(), + overlap: trigger.job_json.overlap, + }; + transaction = queue_run(transaction, &request).await?; + } + } + (transaction, _) = query(transaction, + "UPDATE sqb_trigger_state SET last_tick_for=$1, next_due_at=$2, updated_at=$3 WHERE trigger_id=$4", + vec![Parameter::OptionalTime(plan.last_tick_for), Parameter::OptionalTime(plan.next_due_at), Parameter::Time(now), Parameter::Id(trigger.trigger_id)]).await?; + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/ticks.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/ticks.rs new file mode 100644 index 0000000000..4cb2683e54 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/ticks.rs @@ -0,0 +1,38 @@ +//! Stable IDs and dispositions, shared by catch-up and ordinary schedule evaluation. + +use chrono::{DateTime, SecondsFormat, Utc}; +use uuid::Uuid; + +use crate::scheduler::models::{TickAction, TickDecision}; + +pub(crate) fn tick_id(trigger_id: Uuid, scheduled_for: DateTime) -> Uuid { + Uuid::new_v5( + &trigger_id, + scheduled_for + .to_rfc3339_opts(SecondsFormat::Nanos, true) + .as_bytes(), + ) +} + +pub(crate) fn requested_run(tick_id: Uuid) -> Uuid { + Uuid::new_v5(&tick_id, b"run") +} + +pub(crate) fn action( + trigger_id: Uuid, + scheduled_for: DateTime, + covering_run: Option, +) -> TickAction { + let tick_id = tick_id(trigger_id, scheduled_for); + let decision = match covering_run { + Some(covered_by_run_id) => TickDecision::Covered { covered_by_run_id }, + None => TickDecision::Requested { + run_id: requested_run(tick_id), + }, + }; + TickAction { + tick_id, + scheduled_for, + decision, + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/constants.rs b/crates/sqlbuild-orchestration/src/scheduler/constants.rs new file mode 100644 index 0000000000..b6d1c0ff0f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/constants.rs @@ -0,0 +1,8 @@ +//! Leadership timing is shared by embedded and standalone schedulers. + +/// Renew well before another process may take over a stopped scheduler. +pub const RENEW_INTERVAL_SECONDS: u64 = 5; +/// A stopped holder cannot delay failover for more than this database-clock interval. +pub const LEASE_TTL_SECONDS: i64 = 30; +/// Bound each decision batch, including covered ticks, before opening a write transaction. +pub const MAX_TICKS_PER_BATCH: usize = 10_000; diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/acquire_lease.rs b/crates/sqlbuild-orchestration/src/scheduler/main/acquire_lease.rs new file mode 100644 index 0000000000..1c1bf2ec6a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/acquire_lease.rs @@ -0,0 +1,37 @@ +//! Elect one scheduler under the control store's serialized writer lock. + +use chrono::TimeDelta; +use uuid::Uuid; + +use crate::scheduler::constants::LEASE_TTL_SECONDS; +use crate::scheduler::models::SchedulerLease; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// An existing live lease is never stolen, including by a duplicate caller with the same holder. +pub async fn acquire_lease( + store: &ControlStore, + holder: Uuid, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let expires_at = now + TimeDelta::seconds(LEASE_TTL_SECONDS); + let (transaction, rows) = query(transaction, + "INSERT INTO sqb_scheduler_lease (name, holder, epoch, expires_at) VALUES ('scheduler',$1,1,$2) \ + ON CONFLICT (name) DO UPDATE SET holder=excluded.holder, epoch=sqb_scheduler_lease.epoch+1, expires_at=excluded.expires_at \ + WHERE sqb_scheduler_lease.expires_at<=$3 RETURNING epoch", + vec![Parameter::Id(holder), Parameter::Time(expires_at), Parameter::Time(now)]).await?; + let lease = rows + .first() + .and_then(|row| row["epoch"].as_i64()) + .map(|epoch| SchedulerLease { + holder, + epoch, + expires_at, + }); + commit(transaction).await?; + Ok(lease) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/check_lease.rs b/crates/sqlbuild-orchestration/src/scheduler/main/check_lease.rs new file mode 100644 index 0000000000..4c9d7a9247 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/check_lease.rs @@ -0,0 +1,22 @@ +//! Shared fence validation using a fresh database-clock reading in the writer transaction. + +use crate::scheduler::models::SchedulerLease; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::main::refresh_clock::refresh_clock; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn check_lease( + transaction: ControlTransaction, + lease: &SchedulerLease, +) -> Result { + let transaction = refresh_clock(transaction).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "SELECT epoch FROM sqb_scheduler_lease WHERE name='scheduler' AND holder=$1 AND epoch=$2 AND expires_at>$3", + vec![Parameter::Id(lease.holder), Parameter::Integer(lease.epoch), Parameter::Time(now)]).await?; + if rows.is_empty() { + return Err(ControlError::Fenced); + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/evaluate_trigger.rs b/crates/sqlbuild-orchestration/src/scheduler/main/evaluate_trigger.rs new file mode 100644 index 0000000000..0c079450a9 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/evaluate_trigger.rs @@ -0,0 +1,81 @@ +//! Fence, decide and publish using one serialized transaction and its database clock. + +use chrono::TimeDelta; +use uuid::Uuid; + +use crate::declarations::models::TriggerStatus; +use crate::scheduler::_helpers::expansion::expand_schedule; +use crate::scheduler::_helpers::persistence::persist_ticks; +use crate::scheduler::main::check_lease::check_lease; +use crate::scheduler::main::plan_ticks::plan_ticks; +use crate::scheduler::models::{DueTrigger, SchedulerLease, TickPlan, TickState}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// A removed or paused trigger is inert; a lost fence rolls back every write, including Recall. +pub async fn evaluate_trigger( + store: &ControlStore, + lease: &SchedulerLease, + trigger_id: Uuid, +) -> Result, ControlError> { + let transaction = check_lease(begin(store, TransactionMode::Write).await?, lease).await?; + let (transaction, rows) = query(transaction, + "SELECT t.trigger_id, t.location_id, t.target, t.job_key, t.snapshot_id, s.image_id, t.definition_json, j.definition_json AS job_json, \ + state.started_at, state.last_tick_for, state.status FROM sqb_triggers t \ + JOIN sqb_trigger_state state ON state.trigger_id=t.trigger_id JOIN sqb_snapshots s ON s.snapshot_id=t.snapshot_id \ + JOIN sqb_jobs j ON j.location_id=t.location_id AND j.target=t.target AND j.job_key=t.job_key \ + WHERE t.trigger_id=$1 AND t.kind='schedule' AND t.removed_in_snapshot_id IS NULL", + vec![Parameter::Id(trigger_id)]).await?; + let Some(trigger) = rows + .into_iter() + .next() + .map(decode_row::) + .transpose()? + else { + return Ok(None); + }; + if trigger.status != TriggerStatus::Running { + return Ok(None); + } + let now = transaction.now; + let from = trigger.last_tick_for.unwrap_or( + trigger + .started_at + .checked_sub_signed(TimeDelta::microseconds(1)) + .ok_or_else(|| ControlError::Invalid("trigger start is out of range".into()))?, + ); + let (due_times, continuation) = expand_schedule(&trigger.definition_json, from, now)?; + let (transaction, rows) = query(transaction, + "SELECT r.run_id FROM sqb_runs r WHERE r.location_id=$1 AND r.target=$2 AND r.job_key=$3 \ + AND r.status='queued' AND r.cancel_requested_at IS NULL \ + AND NOT EXISTS (SELECT 1 FROM sqb_run_liveness l WHERE l.run_id=r.run_id) ORDER BY r.queued_at, r.run_id LIMIT 1", + vec![Parameter::Id(trigger.location_id), Parameter::Text(trigger.target.clone()), Parameter::Text(trigger.job_key.clone())]).await?; + let waiting_run = rows + .first() + .and_then(|row| row.get("run_id")) + .and_then(serde_json::Value::as_str) + .map(Uuid::parse_str) + .transpose() + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let plan = plan_ticks( + &TickState { + trigger_id, + enabled: true, + last_tick_for: trigger.last_tick_for, + due_times, + continuation, + catch_up: trigger.definition_json.catch_up, + overlap: trigger.job_json.overlap, + waiting_run, + }, + now, + )?; + let transaction = persist_ticks(transaction, (&trigger, lease), &plan).await?; + let transaction = check_lease(transaction, lease).await?; + commit(transaction).await?; + Ok(Some(plan)) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/list_ticks.rs b/crates/sqlbuild-orchestration/src/scheduler/main/list_ticks.rs new file mode 100644 index 0000000000..0ffa73f861 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/list_ticks.rs @@ -0,0 +1,29 @@ +//! Tick history contains immutable evaluations, including covered schedule times. + +use uuid::Uuid; + +use crate::scheduler::models::TickRecord; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn list_ticks( + store: &ControlStore, + trigger_id: Uuid, + limit: u32, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query(transaction, + "SELECT tick_id, trigger_id, scheduled_for, evaluated_at, epoch, outcome, run_id, covered_by_run_id, reason \ + FROM sqb_ticks WHERE trigger_id=$1 ORDER BY scheduled_for DESC LIMIT $2", + vec![Parameter::Id(trigger_id), Parameter::Integer(i64::from(limit.min(1000)))]).await?; + let ticks = rows + .into_iter() + .map(decode_row) + .collect::, _>>()?; + commit(transaction).await?; + Ok(ticks) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/mod.rs b/crates/sqlbuild-orchestration/src/scheduler/main/mod.rs new file mode 100644 index 0000000000..0e3a7afb49 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/mod.rs @@ -0,0 +1,12 @@ +pub mod acquire_lease; +pub(crate) mod check_lease; +pub mod evaluate_trigger; +pub mod list_ticks; +pub mod next_cron_time; +pub(crate) mod next_schedule_time; +pub mod parse_cron; +pub mod plan_ticks; +pub mod previous_cron_time; +pub mod renew_lease; +pub mod run_scheduler; +pub mod scheduler_cycle; diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/next_cron_time.rs b/crates/sqlbuild-orchestration/src/scheduler/main/next_cron_time.rs new file mode 100644 index 0000000000..8964653860 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/next_cron_time.rs @@ -0,0 +1,28 @@ +//! Calculate the first distinct UTC tick after a persisted cursor. + +use chrono::{DateTime, Utc}; + +use crate::scheduler::_helpers::calendar::{resolve_wall_time, search_start}; +use crate::scheduler::models::CronSchedule; +use crate::store::errors::ControlError; + +/// Ambiguities select the later fold; missing times move to the end of the actual IANA gap. +pub fn next_cron_time( + schedule: &CronSchedule, + after: DateTime, +) -> Result, ControlError> { + let mut cursor = search_start(schedule.timezone, after).and_utc(); + loop { + let candidate = schedule + .expression + .find_next_occurrence(&cursor, false) + .map_err(|error| { + ControlError::Invalid(format!("cron has no next occurrence: {error}")) + })?; + let time = resolve_wall_time(schedule.timezone, candidate.naive_utc())?; + if time > after { + return Ok(time); + } + cursor = candidate; + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/next_schedule_time.rs b/crates/sqlbuild-orchestration/src/scheduler/main/next_schedule_time.rs new file mode 100644 index 0000000000..c4675ccbef --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/next_schedule_time.rs @@ -0,0 +1,28 @@ +//! Find the next due time for a resolved cron or calendar declaration. + +use std::ops::Bound::{Excluded, Unbounded}; + +use chrono::{DateTime, Utc}; + +use crate::declarations::models::Schedule; +use crate::scheduler::main::next_cron_time::next_cron_time; +use crate::scheduler::main::parse_cron::parse_cron; +use crate::store::errors::ControlError; + +pub(crate) fn next_schedule_time( + schedule: &Schedule, + after: DateTime, +) -> Result>, ControlError> { + match schedule { + Schedule::Cron { + expression, + timezone, + } => Ok(Some(next_cron_time( + &parse_cron(expression, timezone)?, + after, + )?)), + Schedule::Calendar { times, .. } => { + Ok(times.range((Excluded(after), Unbounded)).next().copied()) + } + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/parse_cron.rs b/crates/sqlbuild-orchestration/src/scheduler/main/parse_cron.rs new file mode 100644 index 0000000000..50dbeee4b6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/parse_cron.rs @@ -0,0 +1,24 @@ +//! Validate POSIX minute-resolution schedules and IANA time zones before publication. + +use croner::parser::{CronParser, Seconds, Year}; + +use crate::scheduler::models::CronSchedule; +use crate::store::errors::ControlError; + +/// Day-of-month and day-of-week use POSIX OR, not Quartz AND semantics. +pub fn parse_cron(expression: &str, timezone: &str) -> Result { + let expression = CronParser::builder() + .seconds(Seconds::Disallowed) + .year(Year::Disallowed) + .dom_and_dow(false) + .build() + .parse(expression) + .map_err(|error| ControlError::Invalid(format!("invalid cron expression: {error}")))?; + let timezone = timezone + .parse() + .map_err(|error| ControlError::Invalid(format!("invalid IANA time zone: {error}")))?; + Ok(CronSchedule { + expression, + timezone, + }) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/plan_ticks.rs b/crates/sqlbuild-orchestration/src/scheduler/main/plan_ticks.rs new file mode 100644 index 0000000000..6ddee7f820 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/plan_ticks.rs @@ -0,0 +1,84 @@ +//! Pure scheduling decisions: crashes and clock jumps cannot change a tick's identity. + +use std::collections::BTreeSet; + +use chrono::{DateTime, Utc}; + +use crate::scheduler::_helpers::ticks::{action, requested_run, tick_id}; +use crate::scheduler::constants::MAX_TICKS_PER_BATCH; +use crate::scheduler::models::{CatchUp, Overlap, TickAction, TickPlan, TickState}; +use crate::store::errors::ControlError; + +/// Return facts to append and the cursor to commit atomically with them. +pub fn plan_ticks(state: &TickState, now: DateTime) -> Result { + if !state.enabled { + return Ok(TickPlan { + actions: vec![], + last_tick_for: state.last_tick_for, + next_due_at: None, + }); + } + if state.due_times.len() > MAX_TICKS_PER_BATCH { + return Err(ControlError::Invalid( + "schedule expansion exceeds the bounded tick batch".into(), + )); + } + let pending: BTreeSet> = state + .due_times + .iter() + .copied() + .filter(|time| state.last_tick_for.is_none_or(|last| *time > last)) + .collect(); + let due: Vec> = pending + .iter() + .copied() + .take_while(|time| *time <= now) + .collect(); + let covering_run = (state.overlap == Overlap::Coalesce) + .then_some(state.waiting_run) + .flatten(); + let actions: Vec = match (covering_run, state.catch_up) { + (Some(run_id), _) => due + .iter() + .map(|time| action(state.trigger_id, *time, Some(run_id))) + .collect(), + (None, CatchUp::All { max_runs }) => due + .iter() + .take(max_runs.get()) + .map(|time| action(state.trigger_id, *time, None)) + .collect(), + (None, CatchUp::Latest) => { + let latest = due.last().copied(); + let run_id = latest.map(|time| requested_run(tick_id(state.trigger_id, time))); + due.iter() + .map(|time| { + action( + state.trigger_id, + *time, + (Some(*time) != latest).then_some(run_id).flatten(), + ) + }) + .collect() + } + }; + let last_tick_for = actions + .iter() + .rev() + .find(|action| { + state + .continuation + .is_none_or(|next| action.scheduled_for < next) + }) + .map(|action| action.scheduled_for) + .or(state.last_tick_for); + let next_due_at = pending + .into_iter() + .chain(state.continuation) + .filter(|time| last_tick_for.is_none_or(|last| *time > last)) + .min(); + Ok(TickPlan { + actions, + last_tick_for, + next_due_at, + }) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/previous_cron_time.rs b/crates/sqlbuild-orchestration/src/scheduler/main/previous_cron_time.rs new file mode 100644 index 0000000000..de74a44c70 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/previous_cron_time.rs @@ -0,0 +1,30 @@ +//! Locate the latest due tick without expanding an unbounded catch-up backlog. + +use chrono::{DateTime, Utc}; + +use crate::scheduler::_helpers::calendar::resolve_wall_time; +use crate::scheduler::models::CronSchedule; +use crate::store::errors::ControlError; + +/// Fold and gap mapping is identical to forward expansion, including the later ambiguous instant. +pub fn previous_cron_time( + schedule: &CronSchedule, + at: DateTime, +) -> Result, ControlError> { + let mut cursor = at.with_timezone(&schedule.timezone).naive_local().and_utc(); + let mut inclusive = true; + loop { + let candidate = schedule + .expression + .find_previous_occurrence(&cursor, inclusive) + .map_err(|error| { + ControlError::Invalid(format!("cron has no previous occurrence: {error}")) + })?; + let time = resolve_wall_time(schedule.timezone, candidate.naive_utc())?; + if time <= at { + return Ok(time); + } + cursor = candidate; + inclusive = false; + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/renew_lease.rs b/crates/sqlbuild-orchestration/src/scheduler/main/renew_lease.rs new file mode 100644 index 0000000000..5e9811a5a0 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/renew_lease.rs @@ -0,0 +1,30 @@ +//! A paused or superseded scheduler cannot renew an expired fence. + +use chrono::TimeDelta; + +use crate::scheduler::constants::LEASE_TTL_SECONDS; +use crate::scheduler::main::check_lease::check_lease; +use crate::scheduler::models::SchedulerLease; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Renewal preserves the epoch; only takeover allocates a new epoch. +pub async fn renew_lease( + store: &ControlStore, + lease: &SchedulerLease, +) -> Result { + let transaction = begin(store, TransactionMode::Write).await?; + let transaction = check_lease(transaction, lease).await?; + let expires_at = transaction.now + TimeDelta::seconds(LEASE_TTL_SECONDS); + let (transaction, _) = query(transaction, + "UPDATE sqb_scheduler_lease SET expires_at=$1 WHERE name='scheduler' AND holder=$2 AND epoch=$3", + vec![Parameter::Time(expires_at), Parameter::Id(lease.holder), Parameter::Integer(lease.epoch)]).await?; + commit(transaction).await?; + Ok(SchedulerLease { + expires_at, + ..lease.clone() + }) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs b/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs new file mode 100644 index 0000000000..4bf8917b6f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs @@ -0,0 +1,26 @@ +//! Embedded and standalone roles share the same interruptible, durable scheduling loop. + +use tokio::sync::watch; + +use crate::scheduler::main::scheduler_cycle::scheduler_cycle; +use crate::scheduler::models::SchedulerState; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; + +/// Notifications shorten the wait; durable database state, not receipt of a notification, drives work. +pub async fn run_scheduler( + store: &ControlStore, + mut state: SchedulerState, + mut shutdown: watch::Receiver, + mut changes: watch::Receiver<()>, +) -> Result<(), ControlError> { + while !*shutdown.borrow() { + let cycle = scheduler_cycle(store, &mut state).await?; + tokio::select! { + _ = tokio::time::sleep(cycle.wait_for) => {}, + result = changes.changed() => { if result.is_err() { break; } }, + result = shutdown.changed() => { if result.is_err() { break; } }, + } + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/scheduler_cycle.rs b/crates/sqlbuild-orchestration/src/scheduler/main/scheduler_cycle.rs new file mode 100644 index 0000000000..9e6edf9a32 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/scheduler_cycle.rs @@ -0,0 +1,75 @@ +//! A cycle renews leadership and drains a bounded set of persisted due triggers. + +use std::time::Duration; + +use chrono::{DateTime, Utc}; +use uuid::Uuid; + +use crate::scheduler::constants::RENEW_INTERVAL_SECONDS; +use crate::scheduler::main::acquire_lease::acquire_lease; +use crate::scheduler::main::evaluate_trigger::evaluate_trigger; +use crate::scheduler::main::renew_lease::renew_lease; +use crate::scheduler::models::{SchedulerCycle, SchedulerState}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn scheduler_cycle( + store: &ControlStore, + state: &mut SchedulerState, +) -> Result { + let renewal = match state.lease.as_ref() { + Some(lease) => renew_lease(store, lease).await.map(Some), + None => acquire_lease(store, state.holder).await, + }; + state.lease = match renewal { + Ok(lease) => lease, + Err(ControlError::Fenced) => None, + Err(error) => return Err(error), + }; + let mut cycle = SchedulerCycle { + leader: state.lease.is_some(), + evaluated: 0, + ticks: 0, + wait_for: Duration::from_secs(RENEW_INTERVAL_SECONDS), + }; + let Some(mut lease) = state.lease.clone() else { + return Ok(cycle); + }; + let transaction = begin(store, TransactionMode::Read).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "SELECT s.trigger_id FROM sqb_trigger_state s JOIN sqb_triggers t ON t.trigger_id=s.trigger_id \ + JOIN sqb_locations l ON l.location_id=t.location_id \ + WHERE s.status='running' AND s.next_due_at<=$1 AND t.removed_in_snapshot_id IS NULL AND l.archived_at IS NULL \ + ORDER BY s.next_due_at, s.trigger_id LIMIT 100", vec![Parameter::Time(now)]).await?; + commit(transaction).await?; + for row in rows { + let trigger_id: Uuid = serde_json::from_value(row["trigger_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if let Some(plan) = evaluate_trigger(store, &lease, trigger_id).await? { + cycle.evaluated += 1; + cycle.ticks += plan.actions.len(); + } + lease = renew_lease(store, &lease).await?; + } + state.lease = Some(lease); + let transaction = begin(store, TransactionMode::Read).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "SELECT MIN(s.next_due_at) AS next_due_at FROM sqb_trigger_state s JOIN sqb_triggers t ON t.trigger_id=s.trigger_id \ + JOIN sqb_locations l ON l.location_id=t.location_id \ + WHERE s.status='running' AND t.removed_in_snapshot_id IS NULL AND l.archived_at IS NULL", vec![]).await?; + let next: Option> = serde_json::from_value(rows[0]["next_due_at"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if let Some(next) = next { + cycle.wait_for = (next - now) + .to_std() + .unwrap_or(Duration::ZERO) + .min(cycle.wait_for); + } + commit(transaction).await?; + Ok(cycle) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/mod.rs b/crates/sqlbuild-orchestration/src/scheduler/mod.rs new file mode 100644 index 0000000000..d686433c8a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/mod.rs @@ -0,0 +1,9 @@ +//! Fenced scheduler leadership and deterministic decisions over durable due times. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-orchestration/src/scheduler/models.rs b/crates/sqlbuild-orchestration/src/scheduler/models.rs new file mode 100644 index 0000000000..7c3dbfde37 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/models.rs @@ -0,0 +1,150 @@ +//! Values owned by the scheduler, independent of Python and runtime timers. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::declarations::models::{JobDefinition, TriggerDefinition, TriggerStatus}; + +/// A holder must present the exact epoch for every scheduler write. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SchedulerLease { + pub holder: Uuid, + pub epoch: i64, + pub expires_at: DateTime, +} + +/// Process identity survives cycles; the database remains authoritative for leadership. +#[derive(Clone, Debug)] +pub struct SchedulerState { + pub(crate) holder: Uuid, + pub(crate) lease: Option, +} + +impl SchedulerState { + #[must_use] + pub const fn new(holder: Uuid) -> Self { + Self { + holder, + lease: None, + } + } +} + +#[derive(Clone, Debug)] +pub struct SchedulerCycle { + pub leader: bool, + pub evaluated: usize, + pub ticks: usize, + pub wait_for: std::time::Duration, +} + +/// Validated at the native boundary and evaluated as wall-clock times with SQLBuild's DST policy. +#[derive(Clone, Debug)] +pub struct CronSchedule { + pub(crate) expression: croner::Cron, + pub(crate) timezone: chrono_tz::Tz, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum CatchUp { + #[default] + Latest, + /// Process the oldest bounded batch; later due times remain due for the next transaction. + All { max_runs: std::num::NonZeroUsize }, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Overlap { + #[default] + Coalesce, + Queue, + Allow, +} + +impl Overlap { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Coalesce => "coalesce", + Self::Queue => "queue", + Self::Allow => "allow", + } + } +} + +/// Due times come from the persisted calendar or cron expansion, never elapsed sleep time. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TickState { + pub trigger_id: Uuid, + pub enabled: bool, + pub last_tick_for: Option>, + pub due_times: Vec>, + /// First omitted time in a bounded expansion; never advance the cursor past an unrecorded tick. + pub continuation: Option>, + pub catch_up: CatchUp, + pub overlap: Overlap, + /// Only unclaimed waiting work can cover a later tick; executing work cannot cover future data. + pub waiting_run: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "outcome", rename_all = "snake_case")] +pub enum TickDecision { + Requested { run_id: Uuid }, + Covered { covered_by_run_id: Uuid }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct TickAction { + pub tick_id: Uuid, + pub scheduled_for: DateTime, + pub decision: TickDecision, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TickPlan { + pub actions: Vec, + pub last_tick_for: Option>, + pub next_due_at: Option>, +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct DueTrigger { + pub(crate) trigger_id: Uuid, + pub(crate) location_id: Uuid, + pub(crate) target: String, + pub(crate) job_key: String, + pub(crate) snapshot_id: Uuid, + pub(crate) image_id: Option, + pub(crate) definition_json: TriggerDefinition, + pub(crate) job_json: JobDefinition, + pub(crate) started_at: DateTime, + pub(crate) last_tick_for: Option>, + pub(crate) status: TriggerStatus, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TickOutcome { + Requested, + Skipped, + Covered, + Failed, + Deferred, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct TickRecord { + pub tick_id: Uuid, + pub trigger_id: Uuid, + pub scheduled_for: DateTime, + pub evaluated_at: DateTime, + pub epoch: i64, + pub outcome: TickOutcome, + pub run_id: Option, + pub covered_by_run_id: Option, + pub reason: Option, +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests.rs b/crates/sqlbuild-orchestration/src/scheduler/tests.rs new file mode 100644 index 0000000000..e0731c81e8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/tests.rs @@ -0,0 +1,12 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_backlog.rs"] +mod test_backlog; +#[path = "tests/test_cron.rs"] +mod test_cron; +#[path = "tests/test_simulation.rs"] +mod test_simulation; +#[path = "tests/test_ticks.rs"] +mod test_ticks; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs new file mode 100644 index 0000000000..bfc531a1fc --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs @@ -0,0 +1,129 @@ +//! Clock-independent schedule fixtures. + +use std::collections::{BTreeMap, BTreeSet}; + +use chrono::{DateTime, Utc}; +use proptest::prelude::any; +use proptest::test_runner::{Config, TestCaseError, TestCaseResult, TestRunner}; +use proptest::{prop_assert, prop_assert_eq}; +use uuid::Uuid; + +use crate::scheduler::main::plan_ticks::plan_ticks; +use crate::scheduler::models::{CatchUp, Overlap, TickAction, TickDecision, TickPlan, TickState}; +use crate::scheduler::tests::test_types::TickTestCase; + +pub(super) fn time(second: i64) -> DateTime { + DateTime::from_timestamp(second, 0).expect("synthetic schedule time is representable") +} + +pub(super) fn state(test_case: &TickTestCase) -> TickState { + TickState { + trigger_id: Uuid::from_u128(19), + enabled: test_case.enabled, + last_tick_for: test_case.last_second.map(time), + due_times: test_case + .due_seconds + .iter() + .map(|second| time(*second)) + .collect(), + catch_up: test_case.catch_up, + overlap: test_case.overlap, + waiting_run: test_case.active.then_some(Uuid::from_u128(31)), + continuation: None, + } +} + +pub(super) fn request_seconds(plan: &TickPlan) -> Vec { + plan.actions + .iter() + .filter(|action| matches!(action.decision, TickDecision::Requested { .. })) + .map(|action| action.scheduled_for.timestamp()) + .collect() +} + +pub(super) fn covered_seconds(plan: &TickPlan) -> Vec { + plan.actions + .iter() + .filter(|action| matches!(action.decision, TickDecision::Covered { .. })) + .map(|action| action.scheduled_for.timestamp()) + .collect() +} + +fn exercise_history(due: Vec, history: Vec<(i64, bool)>, catch_up: CatchUp) -> TestCaseResult { + let expected: BTreeSet = due.iter().copied().collect(); + let mut state = TickState { + trigger_id: Uuid::from_u128(19), + enabled: true, + last_tick_for: None, + due_times: due.into_iter().map(time).collect(), + catch_up, + overlap: Overlap::Queue, + waiting_run: None, + continuation: None, + }; + let mut committed: BTreeMap = BTreeMap::new(); + for (clock, commit_succeeds) in history { + let plan = plan_ticks(&state, time(clock)) + .map_err(|error| TestCaseError::fail(error.to_string()))?; + let Some(plan) = commit_succeeds.then_some(plan) else { + continue; + }; + for action in plan.actions { + prop_assert!(action.scheduled_for.timestamp() <= clock); + prop_assert!( + committed + .insert(action.scheduled_for.timestamp(), action) + .is_none(), + "a committed tick was emitted twice" + ); + } + state.last_tick_for = plan.last_tick_for; + } + for _ in 0..=expected.len() { + let plan = plan_ticks(&state, time(1001)) + .map_err(|error| TestCaseError::fail(error.to_string()))?; + let Some(_) = plan.actions.first() else { + break; + }; + for action in plan.actions { + prop_assert!( + committed + .insert(action.scheduled_for.timestamp(), action) + .is_none(), + "restart repeated a committed tick" + ); + } + state.last_tick_for = plan.last_tick_for; + } + prop_assert_eq!( + committed.keys().copied().collect::>(), + expected, + "a due tick was lost or invented" + ); + prop_assert_eq!( + committed + .values() + .map(|action| action.tick_id) + .collect::>() + .len(), + committed.len() + ); + Ok(()) +} + +pub(super) fn simulate(catch_up: CatchUp) -> Result<(), String> { + let strategy = ( + proptest::collection::vec(0_i64..1000, 0..128), + proptest::collection::vec((-100_i64..1100, any::()), 0..128), + ); + let mut property_runner = TestRunner::new(Config { + cases: 512, + failure_persistence: None, + ..Config::default() + }); + property_runner + .run(&strategy, |(due, history)| { + exercise_history(due, history, catch_up) + }) + .map_err(|error| error.to_string()) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/test_backlog.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/test_backlog.rs new file mode 100644 index 0000000000..19798dce2a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/test_backlog.rs @@ -0,0 +1,107 @@ +use std::collections::BTreeSet; +use std::num::NonZeroUsize; +use std::time::{Duration, Instant}; + +use crate::declarations::models::{Schedule, TriggerDefinition, TriggerStatus}; +use crate::scheduler::_helpers::expansion::expand_schedule; +use crate::scheduler::constants::MAX_TICKS_PER_BATCH; +use crate::scheduler::main::plan_ticks::plan_ticks; +use crate::scheduler::models::{CatchUp, Overlap, TickDecision, TickState}; +use crate::scheduler::tests::helpers::time; +use crate::scheduler::tests::test_types::BacklogTestCase; +use uuid::Uuid; + +#[test] +fn given_a_backlog_larger_than_two_batches_when_drained_then_latest_is_current_and_every_tick_is_retained() + { + let test_cases = [ + BacklogTestCase { + description: "latest records covered prefixes without requesting an obsolete run", + catch_up: CatchUp::Latest, + expected_latest: 1_200_180, + expected_count: 20_003, + }, + BacklogTestCase { + description: "all drains every requested tick in bounded batches", + catch_up: CatchUp::All { + max_runs: NonZeroUsize::new(MAX_TICKS_PER_BATCH).expect("nonzero bound"), + }, + expected_latest: 1_200_180, + expected_count: 20_003, + }, + ]; + for test_case in test_cases { + let started = Instant::now(); + let trigger = TriggerDefinition { + description: "orders backlog".into(), + job_key: "orders".into(), + starts_at: Some(time(60)), + default_status: TriggerStatus::Running, + catch_up: test_case.catch_up, + schedule: Schedule::Cron { + expression: "* * * * *".into(), + timezone: "UTC".into(), + }, + }; + let mut cursor = None; + let mut recorded: BTreeSet = BTreeSet::new(); + let mut requested: BTreeSet = BTreeSet::new(); + for _ in 0..4 { + let (due_times, continuation) = expand_schedule( + &trigger, + cursor.unwrap_or(time(0)), + time(test_case.expected_latest), + ) + .expect("bounded expansion"); + let plan = plan_ticks( + &TickState { + trigger_id: Uuid::from_u128(19), + enabled: true, + last_tick_for: cursor, + due_times, + continuation, + catch_up: test_case.catch_up, + overlap: Overlap::Queue, + waiting_run: None, + }, + time(test_case.expected_latest), + ) + .expect("bounded plan"); + for action in plan.actions { + recorded.insert(action.scheduled_for.timestamp()); + let TickDecision::Requested { .. } = action.decision else { + continue; + }; + requested.insert(action.scheduled_for.timestamp()); + } + cursor = plan.last_tick_for; + } + assert_eq!( + recorded, + (1..=test_case.expected_count) + .map(|minute| minute as i64 * 60) + .collect(), + "{}", + test_case.description + ); + assert_eq!( + requested.last().copied(), + Some(test_case.expected_latest), + "{}", + test_case.description + ); + let expected_requests = + [test_case.expected_count, 1][usize::from(test_case.catch_up == CatchUp::Latest)]; + assert_eq!( + requested.len(), + expected_requests, + "{}", + test_case.description + ); + assert!( + started.elapsed() < Duration::from_secs(5), + "bounded cron expansion exceeded five seconds: {}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/test_cron.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/test_cron.rs new file mode 100644 index 0000000000..d05d1e0ae1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/test_cron.rs @@ -0,0 +1,158 @@ +use chrono::{DateTime, SecondsFormat}; + +use crate::scheduler::main::next_cron_time::next_cron_time; +use crate::scheduler::main::parse_cron::parse_cron; +use crate::scheduler::main::previous_cron_time::previous_cron_time; +use crate::scheduler::tests::test_types::{CronTestCase, PreviousCronTestCase}; + +#[test] +fn given_cron_and_iana_time_zone_when_crossing_dst_then_utc_ticks_follow_the_explicit_policy() { + let test_cases = [ + CronTestCase { + description: "spring gap ends at 03:00, not 03:30", + expression: "30 2 * * *", + timezone: "America/New_York", + after: "2026-03-08T00:00:00Z", + expected_next: "2026-03-08T07:00:00Z", + }, + CronTestCase { + description: "ambiguous daily time uses the later fold", + expression: "30 1 * * *", + timezone: "America/New_York", + after: "2026-11-01T00:00:00Z", + expected_next: "2026-11-01T06:30:00Z", + }, + CronTestCase { + description: "restarting in the first fold does not skip a smaller future wall time", + expression: "30 1 * * *", + timezone: "America/New_York", + after: "2026-11-01T05:45:00Z", + expected_next: "2026-11-01T06:30:00Z", + }, + CronTestCase { + description: "a persisted later-fold cursor does not repeat the tick", + expression: "30 1 * * *", + timezone: "America/New_York", + after: "2026-11-01T06:30:00Z", + expected_next: "2026-11-02T06:30:00Z", + }, + CronTestCase { + description: "wildcard schedules also select only the later fold", + expression: "* * * * *", + timezone: "America/New_York", + after: "2026-11-01T05:45:00Z", + expected_next: "2026-11-01T06:00:00Z", + }, + CronTestCase { + description: "half-hour spring gap moves to the actual gap end", + expression: "15 2 * * *", + timezone: "Australia/Lord_Howe", + after: "2026-10-03T13:00:00Z", + expected_next: "2026-10-03T15:30:00Z", + }, + CronTestCase { + description: "half-hour autumn fold uses the later 01:45", + expression: "45 1 * * *", + timezone: "Australia/Lord_Howe", + after: "2026-04-04T13:00:00Z", + expected_next: "2026-04-04T15:15:00Z", + }, + CronTestCase { + description: "an entire skipped date moves to the first instant of the next date", + expression: "0 12 * * *", + timezone: "Pacific/Apia", + after: "2011-12-30T00:00:00Z", + expected_next: "2011-12-30T10:00:00Z", + }, + CronTestCase { + description: "POSIX day-of-month and weekday are alternatives", + expression: "0 9 13 * MON", + timezone: "UTC", + after: "2026-10-10T12:00:00Z", + expected_next: "2026-10-12T09:00:00Z", + }, + CronTestCase { + description: "UTC schedules do not introduce local-zone shifts", + expression: "17 6 * * *", + timezone: "UTC", + after: "2026-03-08T06:17:00Z", + expected_next: "2026-03-09T06:17:00Z", + }, + ]; + for test_case in test_cases { + let schedule = + parse_cron(test_case.expression, test_case.timezone).expect("valid synthetic cron"); + let after = DateTime::parse_from_rfc3339(test_case.after) + .expect("valid start timestamp") + .to_utc(); + let observed = next_cron_time(&schedule, after).expect("next cron time exists"); + assert_eq!( + observed.to_rfc3339_opts(SecondsFormat::Secs, true), + test_case.expected_next, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_latest_tick_search_when_crossing_folds_and_gaps_then_it_agrees_with_forward_tick_policy() +{ + let test_cases = [ + PreviousCronTestCase { + description: "first fold excludes every ambiguous minute", + expression: "* * * * *", + timezone: "America/New_York", + at: "2026-11-01T05:45:00Z", + expected_previous: "2026-11-01T04:59:00Z", + }, + PreviousCronTestCase { + description: "second fold includes the later ambiguous instant", + expression: "* * * * *", + timezone: "America/New_York", + at: "2026-11-01T06:15:00Z", + expected_previous: "2026-11-01T06:15:00Z", + }, + PreviousCronTestCase { + description: "daily missing time fires exactly at the gap end", + expression: "30 2 * * *", + timezone: "America/New_York", + at: "2026-03-08T07:00:00Z", + expected_previous: "2026-03-08T07:00:00Z", + }, + PreviousCronTestCase { + description: "before gap end uses the preceding day", + expression: "30 2 * * *", + timezone: "America/New_York", + at: "2026-03-08T06:59:59Z", + expected_previous: "2026-03-07T07:30:00Z", + }, + PreviousCronTestCase { + description: "half-hour gap resolves to its actual end", + expression: "15 2 * * *", + timezone: "Australia/Lord_Howe", + at: "2026-10-03T15:30:00Z", + expected_previous: "2026-10-03T15:30:00Z", + }, + PreviousCronTestCase { + description: "skipped-date cron fires at the first representable instant", + expression: "0 12 * * *", + timezone: "Pacific/Apia", + at: "2011-12-30T10:00:00Z", + expected_previous: "2011-12-30T10:00:00Z", + }, + ]; + for test_case in test_cases { + let cron = parse_cron(test_case.expression, test_case.timezone).expect("valid cron"); + let at = DateTime::parse_from_rfc3339(test_case.at) + .expect("search time") + .to_utc(); + let previous = previous_cron_time(&cron, at).expect("previous scheduled instant"); + assert_eq!( + previous.to_rfc3339_opts(SecondsFormat::Secs, true), + test_case.expected_previous, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/test_simulation.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/test_simulation.rs new file mode 100644 index 0000000000..60e0de2784 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/test_simulation.rs @@ -0,0 +1,32 @@ +use std::num::NonZeroUsize; + +use crate::scheduler::models::CatchUp; +use crate::scheduler::tests::helpers::simulate; +use crate::scheduler::tests::test_types::SimulationTestCase; + +#[test] +fn given_random_crashes_and_clock_jumps_when_restarting_then_every_due_tick_commits_once() { + let test_cases = [ + SimulationTestCase { + description: "latest catch-up keeps covered tick history through crashes", + catch_up: CatchUp::Latest, + expected_success: true, + }, + SimulationTestCase { + description: "bounded all-catch-up drains rather than loses the backlog", + catch_up: CatchUp::All { + max_runs: NonZeroUsize::new(3).expect("nonzero test batch"), + }, + expected_success: true, + }, + ]; + for test_case in test_cases { + let observed = simulate(test_case.catch_up); + assert_eq!( + observed.is_ok(), + test_case.expected_success, + "{}: {observed:?}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/test_ticks.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/test_ticks.rs new file mode 100644 index 0000000000..de1eaeeba9 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/test_ticks.rs @@ -0,0 +1,146 @@ +use std::num::NonZeroUsize; + +use crate::scheduler::main::plan_ticks::plan_ticks; +use crate::scheduler::models::{CatchUp, Overlap}; +use crate::scheduler::tests::helpers::{covered_seconds, request_seconds, state, time}; +use crate::scheduler::tests::test_types::TickTestCase; + +#[test] +fn given_durable_schedule_state_when_time_advances_then_decisions_preserve_due_ticks() { + let test_cases = [ + TickTestCase { + description: "latest catch-up covers earlier ticks and deduplicates calendar entries", + enabled: true, + due_seconds: vec![30, 10, 20, 10, 40], + last_second: None, + now_second: 30, + catch_up: CatchUp::Latest, + overlap: Overlap::Coalesce, + active: false, + expected_requests: vec![30], + expected_covered: vec![10, 20], + expected_last: Some(30), + expected_next: Some(40), + }, + TickTestCase { + description: "all catch-up leaves the unprocessed backlog due instead of skipping it", + enabled: true, + due_seconds: vec![10, 20, 30, 40], + last_second: None, + now_second: 40, + catch_up: CatchUp::All { + max_runs: NonZeroUsize::new(2).expect("nonzero test batch"), + }, + overlap: Overlap::Queue, + active: false, + expected_requests: vec![10, 20], + expected_covered: vec![], + expected_last: Some(20), + expected_next: Some(30), + }, + TickTestCase { + description: "coalesced ticks are covered by the already waiting run", + enabled: true, + due_seconds: vec![10, 20, 30, 40], + last_second: Some(10), + now_second: 30, + catch_up: CatchUp::Latest, + overlap: Overlap::Coalesce, + active: true, + expected_requests: vec![], + expected_covered: vec![20, 30], + expected_last: Some(30), + expected_next: Some(40), + }, + TickTestCase { + description: "allow does not coalesce even when a prior run is active", + enabled: true, + due_seconds: vec![10, 20, 30, 40], + last_second: Some(10), + now_second: 30, + catch_up: CatchUp::All { + max_runs: NonZeroUsize::new(5).expect("nonzero test batch"), + }, + overlap: Overlap::Allow, + active: true, + expected_requests: vec![20, 30], + expected_covered: vec![], + expected_last: Some(30), + expected_next: Some(40), + }, + TickTestCase { + description: "a backward clock jump cannot move the durable cursor backward", + enabled: true, + due_seconds: vec![10, 20, 30, 40], + last_second: Some(30), + now_second: 20, + catch_up: CatchUp::Latest, + overlap: Overlap::Coalesce, + active: false, + expected_requests: vec![], + expected_covered: vec![], + expected_last: Some(30), + expected_next: Some(40), + }, + TickTestCase { + description: "pause retains the cursor and does not advertise an overdue next tick", + enabled: false, + due_seconds: vec![10, 20, 30], + last_second: Some(10), + now_second: 30, + catch_up: CatchUp::Latest, + overlap: Overlap::Coalesce, + active: false, + expected_requests: vec![], + expected_covered: vec![], + expected_last: Some(10), + expected_next: None, + }, + ]; + for test_case in test_cases { + let input = state(&test_case); + let plan = plan_ticks(&input, time(test_case.now_second)).expect("plan valid schedule"); + assert_eq!( + request_seconds(&plan), + test_case.expected_requests, + "{}", + test_case.description + ); + assert_eq!( + covered_seconds(&plan), + test_case.expected_covered, + "{}", + test_case.description + ); + assert_eq!( + plan.last_tick_for.map(|time| time.timestamp()), + test_case.expected_last + ); + assert_eq!( + plan.next_due_at.map(|time| time.timestamp()), + test_case.expected_next + ); + let retried = plan_ticks(&input, time(test_case.now_second)).expect("retry schedule"); + assert_eq!( + plan, retried, + "a retry has exactly the same deterministic tick and run IDs" + ); + let tick_ids: std::collections::BTreeSet<_> = + plan.actions.iter().map(|action| action.tick_id).collect(); + assert_eq!( + tick_ids.len(), + test_case.expected_requests.len() + test_case.expected_covered.len() + ); + let advanced = crate::scheduler::models::TickState { + last_tick_for: plan.last_tick_for, + ..input + }; + let second = + plan_ticks(&advanced, time(test_case.now_second)).expect("plan from committed cursor"); + assert!(second.actions.iter().all(|next| { + plan.actions + .iter() + .all(|before| before.tick_id != next.tick_id) + })); + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/test_types.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/test_types.rs new file mode 100644 index 0000000000..ee92508cab --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/test_types.rs @@ -0,0 +1,52 @@ +//! Inputs make cursor, catch-up, and overlap decisions observably different. + +use crate::scheduler::models::{CatchUp, Overlap}; + +#[derive(Debug)] +pub(super) struct TickTestCase { + pub(super) description: &'static str, + pub(super) enabled: bool, + pub(super) due_seconds: Vec, + pub(super) last_second: Option, + pub(super) now_second: i64, + pub(super) catch_up: CatchUp, + pub(super) overlap: Overlap, + pub(super) active: bool, + pub(super) expected_requests: Vec, + pub(super) expected_covered: Vec, + pub(super) expected_last: Option, + pub(super) expected_next: Option, +} + +#[derive(Debug)] +pub(super) struct CronTestCase { + pub(super) description: &'static str, + pub(super) expression: &'static str, + pub(super) timezone: &'static str, + pub(super) after: &'static str, + pub(super) expected_next: &'static str, +} + +#[derive(Debug)] +pub(super) struct PreviousCronTestCase { + pub(super) description: &'static str, + pub(super) expression: &'static str, + pub(super) timezone: &'static str, + pub(super) at: &'static str, + pub(super) expected_previous: &'static str, +} + +#[derive(Debug)] +pub(super) struct BacklogTestCase { + pub(super) description: &'static str, + pub(super) catch_up: CatchUp, + pub(super) expected_latest: i64, + pub(super) expected_count: usize, +} + +#[derive(Debug)] +pub(super) struct SimulationTestCase { + pub(super) description: &'static str, + pub(super) catch_up: CatchUp, + pub(super) expected_success: bool, +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/content.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/content.rs new file mode 100644 index 0000000000..4ff3f7b1f3 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/content.rs @@ -0,0 +1,72 @@ +//! Canonical content addresses and compressed immutable bodies. + +use std::collections::BTreeMap; +use std::io::{Read, Write}; + +use flate2::{Compression, read::GzDecoder, write::GzEncoder}; +use serde_json::Value; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; + +use crate::snapshots::models::{PreparedNode, SnapshotNode}; +use crate::store::errors::ControlError; + +fn canonical(value: &Value) -> Value { + match value { + Value::Object(values) => { + let sorted: BTreeMap<_, _> = values + .iter() + .map(|(key, value)| (key.clone(), canonical(value))) + .collect(); + Value::Object(sorted.into_iter().collect()) + } + Value::Array(values) => Value::Array(values.iter().map(canonical).collect()), + _ => value.clone(), + } +} + +pub(crate) fn json_bytes(value: &Value) -> Result, ControlError> { + serde_json::to_vec(&canonical(value)).map_err(|error| ControlError::Invalid(error.to_string())) +} + +pub(crate) fn compress(bytes: &[u8]) -> Result, ControlError> { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(bytes)?; + Ok(encoder.finish()?) +} + +pub(crate) fn decompress(value: &Value) -> Result, ControlError> { + let bytes: Vec = serde_json::from_value(value.clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let mut output: Vec = Vec::new(); + GzDecoder::new(bytes.as_slice()).read_to_end(&mut output)?; + Ok(output) +} + +pub(crate) fn prepare_node(node: &SnapshotNode) -> Result { + let mut sources: BTreeMap<&'static str, (String, Vec)> = BTreeMap::new(); + for (kind, text) in [ + ("raw_sql", &node.raw_sql), + ("compiled_sql", &node.compiled_sql), + ("python_source", &node.python_source), + ] { + if let Some(text) = text { + sources.insert( + kind, + (sha256_hex(text.as_bytes()), compress(text.as_bytes())?), + ); + } + } + let source_hashes: BTreeMap<_, _> = sources + .iter() + .map(|(kind, (hash, _))| (*kind, hash)) + .collect(); + let definition = json_bytes( + &serde_json::json!({"kind": node.kind, "definition": node.definition, "sources": source_hashes}), + )?; + Ok(PreparedNode { + kind: node.kind.clone(), + hash: sha256_hex(&definition), + definition: compress(&definition)?, + sources, + }) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs new file mode 100644 index 0000000000..8853fb96b6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod content; +pub(crate) mod preparation; +pub(crate) mod publication; +pub(crate) mod reading; diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/preparation.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/preparation.rs new file mode 100644 index 0000000000..310f723503 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/preparation.rs @@ -0,0 +1,74 @@ +//! Validate and content-address a successful compile before holding a database write lock. + +use std::collections::BTreeMap; + +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; + +use crate::snapshots::_helpers::content::{compress, json_bytes, prepare_node}; +use crate::snapshots::models::{PreparedNode, PreparedSnapshot, SnapshotInput}; +use crate::store::errors::ControlError; + +pub(crate) fn prepare_snapshot(input: &SnapshotInput) -> Result { + if input.target.trim().is_empty() + || input.environment_hash.is_empty() + || input.generation <= 0 + || input + .nodes + .iter() + .any(|(key, node)| key.is_empty() || node.kind.is_empty()) + { + return Err(ControlError::Invalid( + "target, environment, positive generation, and named nodes are required".into(), + )); + } + let mut nodes: BTreeMap = BTreeMap::new(); + for (key, node) in &input.nodes { + nodes.insert(key.clone(), prepare_node(node)?); + } + let hashes: BTreeMap = nodes + .iter() + .map(|(key, node)| (key.clone(), node.hash.clone())) + .collect(); + let documents = prepare_documents(input)?; + let document_hashes: BTreeMap<_, _> = documents + .iter() + .map(|(kind, (hash, _))| (kind, hash)) + .collect(); + let manifest_digest = sha256_hex(&json_bytes(&serde_json::json!({ + "nodes": hashes, "documents": document_hashes, "environment_hash": input.environment_hash, + "image_id": input.image_id, "target": input.target, "sqb_version": input.sqb_version, + "git_revision": input.git_revision, "git_dirty": input.git_dirty, "git_author": input.git_author, + }))?); + Ok(PreparedSnapshot { + nodes, + hashes, + documents, + manifest_digest, + }) +} + +fn prepare_documents( + input: &SnapshotInput, +) -> Result)>, ControlError> { + let mut documents: BTreeMap)> = BTreeMap::new(); + for (kind, value) in &input.documents { + if ![ + "jobs", + "groups", + "triggers", + "roles", + "lineage", + "columns", + "providers", + ] + .contains(&kind.as_str()) + { + return Err(ControlError::Invalid(format!( + "unknown snapshot document {kind}" + ))); + } + let bytes = json_bytes(value)?; + documents.insert(kind.clone(), (sha256_hex(&bytes), compress(&bytes)?)); + } + Ok(documents) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/publication.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/publication.rs new file mode 100644 index 0000000000..eb7b841932 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/publication.rs @@ -0,0 +1,34 @@ +//! Immutable body inserts shared by snapshot publication. + +use std::collections::BTreeMap; + +use uuid::Uuid; + +use crate::snapshots::models::PreparedNode; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn store_nodes( + mut transaction: ControlTransaction, + snapshot_id: Uuid, + nodes: BTreeMap, +) -> Result { + for (key, node) in nodes { + (transaction, _) = query(transaction, + "INSERT INTO sqb_node_definitions (node_hash, body) VALUES ($1, $2) ON CONFLICT DO NOTHING", + vec![Parameter::Text(node.hash.clone()), Parameter::Bytes(node.definition)]).await?; + (transaction, _) = query(transaction, + "INSERT INTO sqb_snapshot_nodes (snapshot_id, node_key, node_kind, node_hash) VALUES ($1, $2, $3, $4)", + vec![Parameter::Id(snapshot_id), Parameter::Text(key.clone()), Parameter::Text(node.kind), Parameter::Text(node.hash)]).await?; + for (kind, (hash, body)) in node.sources { + (transaction, _) = query(transaction, + "INSERT INTO sqb_source_texts (text_hash, body) VALUES ($1, $2) ON CONFLICT DO NOTHING", + vec![Parameter::Text(hash.clone()), Parameter::Bytes(body)]).await?; + (transaction, _) = query(transaction, + "INSERT INTO sqb_snapshot_sources (snapshot_id, node_key, kind, text_hash) VALUES ($1, $2, $3, $4)", + vec![Parameter::Id(snapshot_id), Parameter::Text(key.clone()), Parameter::Text(kind.into()), Parameter::Text(hash)]).await?; + } + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/reading.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/reading.rs new file mode 100644 index 0000000000..91d5c5f6b0 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/reading.rs @@ -0,0 +1,57 @@ +//! Decode stored bodies and attach source text without evaluating it. + +use std::collections::BTreeMap; + +use serde::de::DeserializeOwned; +use serde_json::Value; + +use crate::snapshots::_helpers::content::decompress; +use crate::snapshots::models::SnapshotNode; +use crate::store::errors::ControlError; +use crate::store::types::Row; + +fn decode_body(value: &Value) -> Result { + serde_json::from_slice(&decompress(value)?) + .map_err(|error| ControlError::Invalid(error.to_string())) +} + +pub(crate) fn read_nodes(rows: Vec) -> Result, ControlError> { + read_bodies(rows, "node_key") +} + +fn read_bodies( + rows: Vec, + key: &str, +) -> Result, ControlError> { + rows.into_iter() + .map(|row| { + let body: T = decode_body(&row["body"])?; + Ok((row[key].as_str().unwrap_or_default().to_owned(), body)) + }) + .collect() +} + +pub(crate) fn attach_sources( + mut nodes: BTreeMap, + rows: Vec, +) -> Result, ControlError> { + for row in rows { + let key = row["node_key"].as_str().unwrap_or_default(); + let node = nodes + .get_mut(key) + .ok_or_else(|| ControlError::Invalid("snapshot source has no node".into()))?; + let text = String::from_utf8(decompress(&row["body"])?) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + match row["kind"].as_str() { + Some("raw_sql") => node.raw_sql = Some(text), + Some("compiled_sql") => node.compiled_sql = Some(text), + Some("python_source") => node.python_source = Some(text), + _ => return Err(ControlError::Invalid("unknown stored source kind".into())), + } + } + Ok(nodes) +} + +pub(crate) fn read_documents(rows: Vec) -> Result, ControlError> { + read_bodies(rows, "kind") +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot.rs b/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot.rs new file mode 100644 index 0000000000..b668008c38 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot.rs @@ -0,0 +1,53 @@ +//! Read one consistent generation without locking out snapshot publishers. + +use uuid::Uuid; + +use crate::snapshots::_helpers::reading::{attach_sources, read_documents, read_nodes}; +use crate::snapshots::models::{CurrentSnapshot, SnapshotChange, SnapshotRecord}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// The last successfully published generation, or none before the first successful compile. +pub async fn current_snapshot( + store: &ControlStore, + location_id: Uuid, + target: &str, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query(transaction, + "SELECT s.snapshot_id, s.location_id, s.target, s.generation, s.manifest_digest, s.published_at \ + FROM sqb_current_snapshots c JOIN sqb_snapshots s ON s.snapshot_id=c.snapshot_id WHERE c.location_id=$1 AND c.target=$2", + vec![Parameter::Id(location_id), Parameter::Text(target.into())]).await?; + let Some(row) = rows.into_iter().next() else { + return Ok(None); + }; + let record: SnapshotRecord = decode_row(row)?; + let (transaction, rows) = query(transaction, + "SELECT n.node_key, d.body FROM sqb_snapshot_nodes n JOIN sqb_node_definitions d ON d.node_hash=n.node_hash WHERE n.snapshot_id=$1 ORDER BY n.node_key", + vec![Parameter::Id(record.snapshot_id)]).await?; + let nodes = read_nodes(rows)?; + let (transaction, rows) = query(transaction, + "SELECT s.node_key, s.kind, t.body FROM sqb_snapshot_sources s JOIN sqb_source_texts t ON t.text_hash=s.text_hash WHERE s.snapshot_id=$1", + vec![Parameter::Id(record.snapshot_id)]).await?; + let nodes = attach_sources(nodes, rows)?; + let (transaction, rows) = query(transaction, + "SELECT d.kind, b.body FROM sqb_snapshot_documents d JOIN sqb_documents b ON b.document_hash=d.document_hash WHERE d.snapshot_id=$1", + vec![Parameter::Id(record.snapshot_id)]).await?; + let documents = read_documents(rows)?; + let (transaction, rows) = query(transaction, + "SELECT change_kind, subject_key, before_hash, after_hash FROM sqb_snapshot_changes WHERE snapshot_id=$1 ORDER BY change_seq", + vec![Parameter::Id(record.snapshot_id)]).await?; + let changes: Vec = + rows.into_iter().map(decode_row).collect::>()?; + commit(transaction).await?; + Ok(Some(CurrentSnapshot { + record, + nodes, + documents, + changes, + })) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs new file mode 100644 index 0000000000..da2b1bcca6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs @@ -0,0 +1,2 @@ +pub mod current_snapshot; +pub mod publish_snapshot; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/publish_snapshot.rs b/crates/sqlbuild-orchestration/src/snapshots/main/publish_snapshot.rs new file mode 100644 index 0000000000..0861207ebc --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/publish_snapshot.rs @@ -0,0 +1,118 @@ +//! Publish a complete generation and its current pointer in one transaction. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::Value; +use uuid::Uuid; + +use crate::declarations::main::publish_definitions::publish_definitions; +use crate::declarations::main::snapshot_definitions::snapshot_definitions; +use crate::snapshots::_helpers::preparation::prepare_snapshot; +use crate::snapshots::_helpers::publication::store_nodes; +use crate::snapshots::models::{PreparedSnapshot, SnapshotChange, SnapshotInput, SnapshotRecord}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Reject stale generations without changing the last good snapshot. +pub async fn publish_snapshot( + store: &ControlStore, + input: &SnapshotInput, +) -> Result { + let definitions = snapshot_definitions(input)?; + let PreparedSnapshot { + nodes, + hashes, + documents, + manifest_digest, + } = prepare_snapshot(input)?; + let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, previous) = query(transaction, + "SELECT s.generation FROM sqb_current_snapshots c JOIN sqb_snapshots s ON s.snapshot_id=c.snapshot_id \ + WHERE c.location_id=$1 AND c.target=$2", + vec![Parameter::Id(input.location_id), Parameter::Text(input.target.clone())]).await?; + if previous + .first() + .and_then(|row| row.get("generation")) + .and_then(Value::as_i64) + .is_some_and(|generation| generation >= input.generation) + { + return Err(ControlError::StaleGeneration); + } + let (mut transaction, previous) = query(transaction, + "SELECT n.node_key, n.node_hash FROM sqb_snapshot_nodes n JOIN sqb_current_snapshots c ON c.snapshot_id=n.snapshot_id \ + WHERE c.location_id=$1 AND c.target=$2", + vec![Parameter::Id(input.location_id), Parameter::Text(input.target.clone())]).await?; + let previous: BTreeMap = previous + .into_iter() + .map(|row| { + ( + row["node_key"].as_str().unwrap_or_default().into(), + row["node_hash"].as_str().unwrap_or_default().into(), + ) + }) + .collect(); + let record = SnapshotRecord { + snapshot_id: Uuid::now_v7(), + location_id: input.location_id, + target: input.target.clone(), + generation: input.generation, + manifest_digest, + published_at: transaction.now, + }; + (transaction, _) = query(transaction, + "INSERT INTO sqb_snapshots (snapshot_id, location_id, target, generation, git_revision, git_dirty, git_author, image_id, \ + environment_hash, sqb_version, snapshot_schema_version, manifest_digest, published_at) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,1,$11,$12)", vec![ + Parameter::Id(record.snapshot_id), Parameter::Id(input.location_id), Parameter::Text(input.target.clone()), + Parameter::Integer(input.generation), Parameter::OptionalText(input.git_revision.clone()), Parameter::Boolean(input.git_dirty), + Parameter::OptionalText(input.git_author.clone()), Parameter::OptionalText(input.image_id.clone()), + Parameter::Text(input.environment_hash.clone()), Parameter::Text(input.sqb_version.clone()), + Parameter::Text(record.manifest_digest.clone()), Parameter::Time(record.published_at), + ]).await?; + transaction = store_nodes(transaction, record.snapshot_id, nodes).await?; + for (kind, (hash, body)) in documents { + (transaction, _) = query(transaction, + "INSERT INTO sqb_documents (document_hash, body) VALUES ($1, $2) ON CONFLICT DO NOTHING", + vec![Parameter::Text(hash.clone()), Parameter::Bytes(body)]).await?; + (transaction, _) = query(transaction, + "INSERT INTO sqb_snapshot_documents (snapshot_id, kind, document_hash) VALUES ($1, $2, $3)", + vec![Parameter::Id(record.snapshot_id), Parameter::Text(kind), Parameter::Text(hash)]).await?; + } + let keys: BTreeSet<&String> = previous.keys().chain(hashes.keys()).collect(); + let mut sequence = 0; + for key in keys { + let before = previous.get(key); + let after = hashes.get(key); + if before == after { + continue; + } + let change = SnapshotChange { + change_kind: match (before, after) { + (None, _) => "added", + (_, None) => "removed", + _ => "changed", + } + .into(), + subject_key: key.clone(), + before_hash: before.cloned(), + after_hash: after.cloned(), + }; + (transaction, _) = query(transaction, + "INSERT INTO sqb_snapshot_changes (snapshot_id, change_seq, change_kind, subject_key, before_hash, after_hash) \ + VALUES ($1,$2,$3,$4,$5,$6)", vec![Parameter::Id(record.snapshot_id), Parameter::Integer(sequence), + Parameter::Text(change.change_kind), Parameter::Text(change.subject_key), + Parameter::OptionalText(change.before_hash), Parameter::OptionalText(change.after_hash)]).await?; + sequence += 1; + } + (transaction, _) = query(transaction, + "INSERT INTO sqb_current_snapshots (location_id, target, snapshot_id, since) VALUES ($1,$2,$3,$4) \ + ON CONFLICT (location_id, target) DO UPDATE SET snapshot_id=excluded.snapshot_id, since=excluded.since", + vec![Parameter::Id(input.location_id), Parameter::Text(input.target.clone()), + Parameter::Id(record.snapshot_id), Parameter::Time(record.published_at)]).await?; + transaction = publish_definitions(transaction, &record, definitions).await?; + commit(transaction).await?; + Ok(record) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/mod.rs new file mode 100644 index 0000000000..5359000d16 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod _helpers; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-orchestration/src/snapshots/models.rs b/crates/sqlbuild-orchestration/src/snapshots/models.rs new file mode 100644 index 0000000000..6f1e5da193 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/models.rs @@ -0,0 +1,79 @@ +//! Snapshot inputs are data only: no executable user objects cross into the control plane. + +use std::collections::BTreeMap; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use uuid::Uuid; + +/// A compiled node and its display-only source. SQL and Python are never run by the server. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct SnapshotNode { + pub kind: String, + pub definition: Value, + pub raw_sql: Option, + pub compiled_sql: Option, + pub python_source: Option, +} + +/// A location's successful compile for one target. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct SnapshotInput { + pub location_id: Uuid, + pub target: String, + pub generation: i64, + pub git_revision: Option, + pub git_dirty: bool, + pub git_author: Option, + pub image_id: Option, + pub environment_hash: String, + pub sqb_version: String, + pub nodes: BTreeMap, + pub documents: BTreeMap, +} + +/// Immutable publication metadata. Generations increase independently per location and target. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct SnapshotRecord { + pub snapshot_id: Uuid, + pub location_id: Uuid, + pub target: String, + pub generation: i64, + pub manifest_digest: String, + pub published_at: DateTime, +} + +/// Node-level changes do not duplicate unchanged definitions or source text. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct SnapshotChange { + pub change_kind: String, + pub subject_key: String, + pub before_hash: Option, + pub after_hash: Option, +} + +/// The current successful snapshot, read with a consistent generation and change set. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct CurrentSnapshot { + pub record: SnapshotRecord, + pub nodes: BTreeMap, + pub documents: BTreeMap, + pub changes: Vec, +} + +#[derive(Debug)] +pub(crate) struct PreparedNode { + pub(crate) kind: String, + pub(crate) hash: String, + pub(crate) definition: Vec, + pub(crate) sources: BTreeMap<&'static str, (String, Vec)>, +} + +#[derive(Debug)] +pub(crate) struct PreparedSnapshot { + pub(crate) nodes: BTreeMap, + pub(crate) hashes: BTreeMap, + pub(crate) documents: BTreeMap)>, + pub(crate) manifest_digest: String, +} diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/database.rs b/crates/sqlbuild-orchestration/src/store/_helpers/database.rs new file mode 100644 index 0000000000..433dd5b495 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/_helpers/database.rs @@ -0,0 +1,154 @@ +//! Typed binding and row decoding for the two control-store engines. + +use chrono::{DateTime, SecondsFormat, Utc}; +use rusqlite::types::{Value as SqliteValue, ValueRef}; +use serde_json::Value; +use sqlbuild_recall::schema::models::ColumnType; +use sqlx_core::{column::Column, row::Row as _, type_info::TypeInfo, value::ValueRef as _}; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::models::Parameter; +use crate::store::types::Row; + +pub(crate) fn sqlite_value(parameter: &Parameter) -> SqliteValue { + match parameter { + Parameter::Null(_) => SqliteValue::Null, + Parameter::Text(value) => SqliteValue::Text(value.clone()), + Parameter::OptionalText(value) => { + value.clone().map_or(SqliteValue::Null, SqliteValue::Text) + } + Parameter::Bytes(value) => SqliteValue::Blob(value.clone()), + Parameter::Integer(value) => SqliteValue::Integer(*value), + Parameter::Real(value) => SqliteValue::Real(*value), + Parameter::Json(value) => SqliteValue::Text(value.to_string()), + Parameter::Boolean(value) => SqliteValue::Integer(i64::from(*value)), + Parameter::Id(value) => SqliteValue::Text(value.to_string()), + Parameter::OptionalId(value) => value.map_or(SqliteValue::Null, |value| { + SqliteValue::Text(value.to_string()) + }), + Parameter::Time(value) => { + SqliteValue::Text(value.to_rfc3339_opts(SecondsFormat::Micros, true)) + } + Parameter::OptionalTime(value) => value.map_or(SqliteValue::Null, |value| { + SqliteValue::Text(value.to_rfc3339_opts(SecondsFormat::Micros, true)) + }), + } +} + +pub(crate) fn sqlite_query( + connection: &rusqlite::Connection, + sql: &str, + parameters: &[Parameter], +) -> Result, ControlError> { + let values: Vec = parameters.iter().map(sqlite_value).collect(); + let mut statement = connection.prepare(sql)?; + let columns: Vec = statement + .column_names() + .into_iter() + .map(str::to_owned) + .collect(); + let declared_types: Vec> = statement + .columns() + .into_iter() + .map(|column| column.decl_type().map(str::to_ascii_uppercase)) + .collect(); + for (index, value) in values.into_iter().enumerate() { + let name = format!("${}", index + 1); + let position = statement + .parameter_index(&name)? + .ok_or_else(|| ControlError::Invalid(format!("missing SQL parameter {name}")))?; + statement.raw_bind_parameter(position, value)?; + } + let mut query = statement.raw_query(); + let mut result: Vec = Vec::new(); + while let Some(row) = query.next()? { + let mut mapped = Row::new(); + for (index, name) in columns.iter().enumerate() { + let value = match row.get_ref(index)? { + ValueRef::Null => Value::Null, + ValueRef::Integer(value) if declared_types[index].as_deref() == Some("BOOLEAN") => { + Value::Bool(value != 0) + } + ValueRef::Integer(value) => Value::from(value), + ValueRef::Real(value) => Value::from(value), + ValueRef::Text(value) => Value::from(String::from_utf8_lossy(value).into_owned()), + ValueRef::Blob(value) => Value::from(value.to_vec()), + }; + let value = if declared_types[index].as_deref() == Some("JSON") { + value.as_str().map_or_else( + || Ok(value.clone()), + |text| { + serde_json::from_str(text) + .map_err(|error| ControlError::Invalid(error.to_string())) + }, + )? + } else { + value + }; + mapped.insert(name.clone(), value); + } + result.push(mapped); + } + Ok(result) +} + +pub(crate) async fn postgres_query( + transaction: &mut sqlx_core::transaction::Transaction<'static, sqlx_postgres::Postgres>, + sql: &str, + parameters: Vec, +) -> Result, ControlError> { + let mut query = sqlx_core::query::query(sql); + for parameter in parameters { + query = match parameter { + Parameter::Null(kind) => match kind { + ColumnType::Text => query.bind(None::), + ColumnType::Timestamp => query.bind(None::>), + ColumnType::Json => query.bind(None::), + ColumnType::Integer => query.bind(None::), + ColumnType::Boolean => query.bind(None::), + ColumnType::Real => query.bind(None::), + ColumnType::Blob => query.bind(None::>), + }, + Parameter::Text(value) => query.bind(value), + Parameter::OptionalText(value) => query.bind(value), + Parameter::Bytes(value) => query.bind(value), + Parameter::Integer(value) => query.bind(value), + Parameter::Real(value) => query.bind(value), + Parameter::Json(value) => query.bind(value), + Parameter::Boolean(value) => query.bind(value), + Parameter::Id(value) => query.bind(value), + Parameter::OptionalId(value) => query.bind(value), + Parameter::Time(value) => query.bind(value), + Parameter::OptionalTime(value) => query.bind(value), + }; + } + let mut result: Vec = Vec::new(); + for row in query.fetch_all(&mut **transaction).await? { + let mut mapped = Row::new(); + for (index, column) in row.columns().iter().enumerate() { + let raw = row.try_get_raw(index)?; + let value = if raw.is_null() { + Value::Null + } else { + match column.type_info().name() { + "BOOL" => Value::from(row.try_get::(index)?), + "INT8" => Value::from(row.try_get::(index)?), + "INT4" => Value::from(row.try_get::(index)?), + "FLOAT8" => Value::from(row.try_get::(index)?), + "UUID" => Value::from(row.try_get::(index)?.to_string()), + "TIMESTAMPTZ" => Value::from( + row.try_get::, _>(index)? + .to_rfc3339_opts(SecondsFormat::Micros, true), + ), + "JSONB" | "JSON" => row.try_get::(index)?, + "BYTEA" => Value::from(row.try_get::, _>(index)?), + _ => Value::from(row.try_get::(index)?), + } + }; + mapped.insert(column.name().to_owned(), value); + } + result.push(mapped); + } + Ok(result) +} diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs new file mode 100644 index 0000000000..53420e77dc --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -0,0 +1,13 @@ +//! One schema template, rendered only at the database type boundary. + +use crate::store::models::ControlStore; + +pub(crate) const SCHEMA: &str = include_str!("../schema.sql"); + +pub(crate) fn render(store: &ControlStore, sql: &str) -> String { + let postgres = matches!(store, ControlStore::Postgres(_)); + sql.replace("{uuid}", if postgres { "UUID" } else { "TEXT" }) + .replace("{time}", if postgres { "TIMESTAMPTZ" } else { "TEXT" }) + .replace("{binary}", if postgres { "BYTEA" } else { "BLOB" }) + .replace("{json}", if postgres { "JSONB" } else { "JSON" }) +} diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/store/_helpers/mod.rs new file mode 100644 index 0000000000..08fcf1b81c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod database; +pub(crate) mod migration; +pub(crate) mod open_sqlite; +pub(crate) mod recall_rows; +pub(crate) mod recall_schema; diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/open_sqlite.rs b/crates/sqlbuild-orchestration/src/store/_helpers/open_sqlite.rs new file mode 100644 index 0000000000..9408ebaa14 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/_helpers/open_sqlite.rs @@ -0,0 +1,37 @@ +//! Configure a file-backed SQLite control store without interpreting filenames as URIs. + +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use crate::store::errors::ControlError; + +pub(crate) fn sqlite_path(url: &str) -> Result { + let path = Path::new(url.strip_prefix("sqlite://").unwrap_or(url)).to_path_buf(); + if path.as_os_str().is_empty() + || path == Path::new(":memory:") + || url.contains("://") && !url.starts_with("sqlite://") + { + return Err(ControlError::Invalid( + "control-store URL must name PostgreSQL or a SQLite file".into(), + )); + } + Ok(path) +} + +pub(crate) fn open_sqlite(path: PathBuf) -> Result<(), ControlError> { + if let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + std::fs::create_dir_all(parent)?; + } + let connection = rusqlite::Connection::open_with_flags( + path, + rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE + | rusqlite::OpenFlags::SQLITE_OPEN_CREATE + | rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX, + )?; + connection.busy_timeout(Duration::from_secs(30))?; + connection.execute_batch("PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL")?; + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/recall_rows.rs b/crates/sqlbuild-orchestration/src/store/_helpers/recall_rows.rs new file mode 100644 index 0000000000..e1827a832f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/_helpers/recall_rows.rs @@ -0,0 +1,111 @@ +//! Insert native Recall rows and identify retry conflicts within a control transaction. + +use std::collections::HashSet; + +use chrono::{DateTime, SecondsFormat}; +use sqlbuild_recall::records::models::{RecallRow, RecallValue}; +use sqlbuild_recall::schema::main::postgres_partition_statement::postgres_partition_statement; +use sqlbuild_recall::schema::main::table_columns::table_columns; +use sqlbuild_recall::schema::main::table_spec::table_spec; +use sqlbuild_recall::schema::models::{ColumnType, Table}; +use sqlbuild_recall::store::main::conflict_evidence::conflict_evidence; +use sqlbuild_recall::store::main::relational_insert::relational_insert; +use sqlbuild_recall::store::main::relational_lookup::relational_lookup; + +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{Connection, ControlTransaction, Parameter, RecallAppendOutcome}; + +fn parameters(values: &[(&RecallValue, ColumnType)]) -> Result, ControlError> { + values + .iter() + .map(|(value, kind)| { + Ok(match (value, kind) { + (RecallValue::Null, kind) => Parameter::Null(*kind), + (RecallValue::Integer(value), ColumnType::Boolean) => { + Parameter::Boolean(*value != 0) + } + (RecallValue::Integer(value), _) => Parameter::Integer(*value), + (RecallValue::Real(value), _) => Parameter::Real(*value), + (RecallValue::Text(value), ColumnType::Json) => Parameter::Json( + serde_json::from_str(value) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + ), + (RecallValue::Text(value), ColumnType::Timestamp) => Parameter::Time( + DateTime::parse_from_rfc3339(value) + .map_err(|error| ControlError::Invalid(error.to_string()))? + .to_utc(), + ), + (RecallValue::Text(value), _) => Parameter::Text(value.clone()), + (RecallValue::Blob(value), _) => Parameter::Bytes(value.clone()), + }) + }) + .collect() +} + +pub(crate) fn text<'a>(row: &'a RecallRow, name: &str) -> Result<&'a str, ControlError> { + match row.value(name) { + Some(RecallValue::Text(value)) => Ok(value), + _ => Err(ControlError::Invalid(format!("Recall {name} must be text"))), + } +} + +pub(crate) async fn write_row( + mut transaction: ControlTransaction, + row: &RecallRow, +) -> Result<(ControlTransaction, RecallAppendOutcome), ControlError> { + let postgres = matches!(transaction.connection, Some(Connection::Postgres(_))); + if postgres && row.table != Table::Artifacts { + let date = DateTime::parse_from_rfc3339(text(row, table_spec(row.table).expiry_column)?) + .map_err(|error| ControlError::Invalid(error.to_string()))? + .to_utc() + .date_naive(); + let statement = postgres_partition_statement(row.table, date) + .ok_or_else(|| ControlError::Invalid("Recall partition date is out of range".into()))?; + (transaction, _) = query(transaction, &statement, vec![]).await?; + } + let ingested = RecallValue::Text(transaction.now.to_rfc3339_opts(SecondsFormat::Micros, true)); + let columns: HashSet = table_columns(row.table) + .into_iter() + .map(|column| column.name.to_owned()) + .collect(); + let insert = relational_insert((row, &columns), &ingested, postgres); + let returning = if row.table == Table::Events { + "storage_id" + } else { + "1 AS inserted" + }; + let (transaction, rows) = query( + transaction, + &format!("{} RETURNING {returning}", insert.sql), + parameters(&insert.values)?, + ) + .await?; + if let Some(found) = rows.first() { + return Ok(( + transaction, + RecallAppendOutcome { + inserted: true, + storage_id: found.get("storage_id").and_then(serde_json::Value::as_i64), + conflict: None, + }, + )); + } + let lookup = relational_lookup(row, postgres, true); + let (transaction, rows) = query(transaction, &lookup.sql, parameters(&lookup.values)?).await?; + let found = rows.first().ok_or_else(|| { + ControlError::Invalid("Recall duplicate has no matching stored key".into()) + })?; + let existing = found + .get("content_hash") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| ControlError::Invalid("stored Recall row has no content hash".into()))?; + Ok(( + transaction, + RecallAppendOutcome { + inserted: false, + storage_id: found.get("storage_id").and_then(serde_json::Value::as_i64), + conflict: conflict_evidence(row, existing), + }, + )) +} diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/recall_schema.rs b/crates/sqlbuild-orchestration/src/store/_helpers/recall_schema.rs new file mode 100644 index 0000000000..d63c9b5681 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/_helpers/recall_schema.rs @@ -0,0 +1,13 @@ +//! Embed Recall in the control database so run facts and projections share one transaction. + +use sqlbuild_recall::schema::main::control_table_statements::control_table_statements; +use sqlbuild_recall::schema::main::control_view_statements::control_view_statements; + +use crate::store::models::ControlStore; + +pub(crate) fn recall_schema(store: &ControlStore) -> Vec { + let postgres = matches!(store, ControlStore::Postgres(_)); + let mut statements = control_table_statements(postgres); + statements.extend(control_view_statements(postgres)); + statements +} diff --git a/crates/sqlbuild-orchestration/src/store/errors.rs b/crates/sqlbuild-orchestration/src/store/errors.rs new file mode 100644 index 0000000000..54a8a6d29d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/errors.rs @@ -0,0 +1,22 @@ +//! Control-store failures; callers must not publish a successful state after one. + +/// A rejected control-plane operation or failed durable write. +#[derive(Debug, thiserror::Error)] +pub enum ControlError { + #[error("control store: {0}")] + Database(#[from] sqlx_core::Error), + #[error("control store: {0}")] + Sqlite(#[from] rusqlite::Error), + #[error("control-store worker failed: {0}")] + Worker(#[from] tokio::task::JoinError), + #[error("control-store filesystem: {0}")] + Io(#[from] std::io::Error), + #[error("invalid control-plane data: {0}")] + Invalid(String), + #[error("control-store schema is unsupported: {0}")] + Schema(String), + #[error("snapshot generation is not newer than the current generation")] + StaleGeneration, + #[error("scheduler lease expired or was fenced by another holder")] + Fenced, +} diff --git a/crates/sqlbuild-orchestration/src/store/main/append_recall.rs b/crates/sqlbuild-orchestration/src/store/main/append_recall.rs new file mode 100644 index 0000000000..349fe86d1a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/append_recall.rs @@ -0,0 +1,36 @@ +//! Append Recall facts and retry-conflict evidence to the caller's control transaction. + +use chrono::SecondsFormat; +use sqlbuild_recall::records::main::conflict_row::conflict_row; +use sqlbuild_recall::records::models::{RecallRow, RowEnvelope}; +use sqlbuild_recall::schema::main::table_spec::table_spec; +use sqlbuild_recall::schema::models::Table; + +use crate::store::_helpers::recall_rows::{text, write_row}; +use crate::store::errors::ControlError; +use crate::store::models::{ControlTransaction, RecallAppendOutcome}; + +pub(crate) async fn append_recall( + transaction: ControlTransaction, + row: &RecallRow, +) -> Result<(ControlTransaction, RecallAppendOutcome), ControlError> { + let (mut transaction, outcome) = write_row(transaction, row).await?; + if let Some(evidence) = &outcome.conflict + && row.table != Table::Conflicts + { + let envelope = RowEnvelope { + invocation_id: text(row, "invocation_id")?.into(), + invocation_started_at: text(row, "invocation_started_at")?.into(), + producer_version: text(row, "producer_version")?.into(), + }; + let detected_at = transaction.now.to_rfc3339_opts(SecondsFormat::Micros, true); + let conflict = conflict_row( + &envelope, + (table_spec(row.table).name, &evidence.record_key), + (&evidence.existing_sha256, &evidence.rejected_sha256), + &detected_at, + ); + (transaction, _) = write_row(transaction, &conflict).await?; + } + Ok((transaction, outcome)) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/begin.rs b/crates/sqlbuild-orchestration/src/store/main/begin.rs new file mode 100644 index 0000000000..e97c51e077 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/begin.rs @@ -0,0 +1,59 @@ +//! Serialize durable appends before allocating their storage IDs. + +use std::time::Duration; + +use chrono::Utc; +use rusqlite::OpenFlags; + +use crate::store::errors::ControlError; +use crate::store::main::refresh_clock::refresh_clock; +use crate::store::models::{Connection, ControlStore, ControlTransaction, TransactionMode}; + +pub(crate) async fn begin( + store: &ControlStore, + mode: TransactionMode, +) -> Result { + let connection = match store { + ControlStore::Sqlite(path) => { + let path = path.clone(); + Connection::Sqlite( + tokio::task::spawn_blocking(move || { + let connection = rusqlite::Connection::open_with_flags( + path, + OpenFlags::SQLITE_OPEN_READ_WRITE + | OpenFlags::SQLITE_OPEN_CREATE + | OpenFlags::SQLITE_OPEN_NO_MUTEX, + )?; + connection.busy_timeout(Duration::from_secs(30))?; + connection.execute_batch(if mode == TransactionMode::Write { + "PRAGMA foreign_keys=ON; BEGIN IMMEDIATE" + } else { + "PRAGMA foreign_keys=ON; PRAGMA query_only=ON; BEGIN" + })?; + Ok::<_, ControlError>(connection) + }) + .await??, + ) + } + ControlStore::Postgres(pool) => { + let mut transaction = pool.begin().await?; + if mode == TransactionMode::Write { + sqlx_core::query::query("SELECT pg_advisory_xact_lock(1936810572, 1)") + .execute(&mut *transaction) + .await?; + } else { + sqlx_core::query::query( + "SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY", + ) + .execute(&mut *transaction) + .await?; + } + Connection::Postgres(transaction) + } + }; + let transaction = ControlTransaction { + connection: Some(connection), + now: Utc::now(), + }; + refresh_clock(transaction).await +} diff --git a/crates/sqlbuild-orchestration/src/store/main/commit.rs b/crates/sqlbuild-orchestration/src/store/main/commit.rs new file mode 100644 index 0000000000..f9e3cb46e0 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/commit.rs @@ -0,0 +1,18 @@ +//! Commit facts and their projections together. + +use crate::store::errors::ControlError; +use crate::store::models::{Connection, ControlTransaction}; + +pub(crate) async fn commit(mut transaction: ControlTransaction) -> Result<(), ControlError> { + match transaction + .connection + .take() + .ok_or_else(|| ControlError::Invalid("transaction has already finished".into()))? + { + Connection::Sqlite(connection) => { + tokio::task::spawn_blocking(move || connection.execute_batch("COMMIT")).await??; + } + Connection::Postgres(transaction) => transaction.commit().await?, + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/create_location.rs b/crates/sqlbuild-orchestration/src/store/main/create_location.rs new file mode 100644 index 0000000000..075de8a140 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/create_location.rs @@ -0,0 +1,30 @@ +//! Create a server-owned location identity independent of its mutable label. + +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Register a new location; names are labels, never credentials or stable identifiers. +pub async fn create_location( + store: &ControlStore, + name: &str, + actor: &str, +) -> Result { + if name.trim().is_empty() || actor.trim().is_empty() { + return Err(ControlError::Invalid( + "location name and actor are required".into(), + )); + } + let mut transaction = begin(store, TransactionMode::Write).await?; + let id = Uuid::now_v7(); + let now = transaction.now; + (transaction, _) = query(transaction, + "INSERT INTO sqb_locations (location_id, name, created_by, created_at) VALUES ($1, $2, $3, $4)", + vec![Parameter::Id(id), Parameter::Text(name.into()), Parameter::Text(actor.into()), Parameter::Time(now)]).await?; + commit(transaction).await?; + Ok(id) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/decode_row.rs b/crates/sqlbuild-orchestration/src/store/main/decode_row.rs new file mode 100644 index 0000000000..bc02200fb1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/decode_row.rs @@ -0,0 +1,12 @@ +//! Decode driver-neutral typed records through the store's shared row contract. + +use serde::de::DeserializeOwned; +use serde_json::Value; + +use crate::store::errors::ControlError; +use crate::store::types::Row; + +pub(crate) fn decode_row(row: Row) -> Result { + serde_json::from_value(Value::Object(row.into_iter().collect())) + .map_err(|error| ControlError::Invalid(error.to_string())) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs new file mode 100644 index 0000000000..b5a38030f1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -0,0 +1,70 @@ +//! Forward-only migration under the same lock used by all other control-plane writes. + +use semver::Version; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use uuid::Uuid; + +use crate::store::_helpers::migration::{SCHEMA, render}; +use crate::store::_helpers::recall_schema::recall_schema; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Create the schema once, or reject drift and schemas newer than this server supports. +pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { + let mut transaction = begin(store, TransactionMode::Write).await?; + (transaction, _) = query(transaction, &render(store, + "CREATE TABLE IF NOT EXISTS sqb_schema_migrations (version BIGINT PRIMARY KEY, \ + name TEXT NOT NULL, checksum TEXT NOT NULL, min_server_version TEXT NOT NULL, applied_at {time} NOT NULL)"), vec![]).await?; + let (mut transaction, rows) = query( + transaction, + "SELECT version, checksum, min_server_version FROM sqb_schema_migrations ORDER BY version", + vec![], + ) + .await?; + let current = Version::parse(env!("CARGO_PKG_VERSION")) + .map_err(|error| ControlError::Schema(error.to_string()))?; + let recall_statements = recall_schema(store); + let digest = sha256_hex(format!("{SCHEMA}\n{}", recall_statements.join(";\n")).as_bytes()); + for row in &rows { + let minimum = row + .get("min_server_version") + .and_then(|value| value.as_str()) + .ok_or_else(|| { + ControlError::Schema("migration lacks a minimum server version".into()) + })?; + let minimum = + Version::parse(minimum).map_err(|error| ControlError::Schema(error.to_string()))?; + if row.get("version").and_then(|value| value.as_i64()) != Some(1) + || minimum > current + || row.get("checksum").and_then(|value| value.as_str()) != Some(digest.as_str()) + { + return Err(ControlError::Schema( + "unknown migration, checksum mismatch, or minimum server version not met".into(), + )); + } + } + if rows.is_empty() { + for statement in SCHEMA + .split(';') + .map(str::trim) + .filter(|statement| !statement.is_empty()) + { + (transaction, _) = query(transaction, &render(store, statement), vec![]).await?; + } + for statement in recall_statements { + (transaction, _) = query(transaction, &statement, vec![]).await?; + } + let now = transaction.now; + (transaction, _) = query(transaction, + "INSERT INTO sqb_installation (singleton, installation_id, created_at) VALUES (1, $1, $2)", + vec![Parameter::Id(Uuid::now_v7()), Parameter::Time(now)]).await?; + (transaction, _) = query(transaction, + "INSERT INTO sqb_schema_migrations (version, name, checksum, min_server_version, applied_at) \ + VALUES (1, 'control_store', $1, $2, $3)", + vec![Parameter::Text(digest), Parameter::Text(current.to_string()), Parameter::Time(now)]).await?; + } + commit(transaction).await +} diff --git a/crates/sqlbuild-orchestration/src/store/main/mod.rs b/crates/sqlbuild-orchestration/src/store/main/mod.rs new file mode 100644 index 0000000000..ba1903c103 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/mod.rs @@ -0,0 +1,12 @@ +pub(crate) mod append_recall; +pub(crate) mod begin; +pub(crate) mod commit; +pub mod create_location; +pub(crate) mod decode_row; +pub mod migrate_store; +pub mod open_store; +pub mod publish_recall; +pub(crate) mod query; +pub mod read_recall_events; +pub mod read_recall_view; +pub(crate) mod refresh_clock; diff --git a/crates/sqlbuild-orchestration/src/store/main/open_store.rs b/crates/sqlbuild-orchestration/src/store/main/open_store.rs new file mode 100644 index 0000000000..db8a450f4f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/open_store.rs @@ -0,0 +1,23 @@ +//! Open a control database; no Python or warehouse connection is involved. + +use std::time::Duration; + +use crate::store::_helpers::open_sqlite::{open_sqlite, sqlite_path}; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; + +/// Open PostgreSQL by URL, or SQLite by an explicit filesystem path. +pub async fn open_store(url: &str) -> Result { + if url.starts_with("postgres://") || url.starts_with("postgresql://") { + let pool = sqlx_postgres::PgPoolOptions::new() + .max_connections(10) + .acquire_timeout(Duration::from_secs(30)) + .connect(url) + .await?; + return Ok(ControlStore::Postgres(pool)); + } + let path = sqlite_path(url)?; + let configure_path = path.clone(); + tokio::task::spawn_blocking(move || open_sqlite(configure_path)).await??; + Ok(ControlStore::Sqlite(path)) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/publish_recall.rs b/crates/sqlbuild-orchestration/src/store/main/publish_recall.rs new file mode 100644 index 0000000000..bbe77db9a8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/publish_recall.rs @@ -0,0 +1,25 @@ +//! Worker publication is atomic; scheduler writes use the same append inside their tick transaction. + +use sqlbuild_recall::records::models::RecallRow; + +use crate::store::errors::ControlError; +use crate::store::main::append_recall::append_recall; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{ControlStore, RecallAppendOutcome, TransactionMode}; + +/// Preserve the first copy of each key and record differing retries as Recall conflicts. +pub async fn publish_recall( + store: &ControlStore, + rows: &[RecallRow], +) -> Result, ControlError> { + let mut transaction = begin(store, TransactionMode::Write).await?; + let mut outcomes = Vec::with_capacity(rows.len()); + for row in rows { + let outcome; + (transaction, outcome) = append_recall(transaction, row).await?; + outcomes.push(outcome); + } + commit(transaction).await?; + Ok(outcomes) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/query.rs b/crates/sqlbuild-orchestration/src/store/main/query.rs new file mode 100644 index 0000000000..9e79cdc7a1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/query.rs @@ -0,0 +1,35 @@ +//! Execute inside an owned transaction, without blocking an async runtime on SQLite. + +use crate::store::_helpers::database::{postgres_query, sqlite_query}; +use crate::store::errors::ControlError; +use crate::store::models::{Connection, ControlTransaction, Parameter}; +use crate::store::types::Row; + +pub(crate) async fn query( + mut transaction: ControlTransaction, + sql: &str, + parameters: Vec, +) -> Result<(ControlTransaction, Vec), ControlError> { + let connection = transaction + .connection + .take() + .ok_or_else(|| ControlError::Invalid("transaction has already finished".into()))?; + let rows: Vec = match connection { + Connection::Sqlite(connection) => { + let sql = sql.to_owned(); + let (connection, result) = tokio::task::spawn_blocking(move || { + let result = sqlite_query(&connection, &sql, ¶meters); + (connection, result) + }) + .await?; + transaction.connection = Some(Connection::Sqlite(connection)); + result? + } + Connection::Postgres(mut connection) => { + let result = postgres_query(&mut connection, sql, parameters).await; + transaction.connection = Some(Connection::Postgres(connection)); + result? + } + }; + Ok((transaction, rows)) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/read_recall_events.rs b/crates/sqlbuild-orchestration/src/store/main/read_recall_events.rs new file mode 100644 index 0000000000..ec9a4f533d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/read_recall_events.rs @@ -0,0 +1,28 @@ +//! Stable cursor paging follows serialized committed appends, not producer timestamps. + +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::types::Row; + +/// Page events in storage order; equal producer timestamps and late arrivals remain visible. +pub async fn read_recall_events( + store: &ControlStore, + after: i64, + limit: u32, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query( + transaction, + "SELECT * FROM recall_events WHERE storage_id > $1 ORDER BY storage_id LIMIT $2", + vec![ + Parameter::Integer(after), + Parameter::Integer(i64::from(limit.min(1000))), + ], + ) + .await?; + commit(transaction).await?; + Ok(rows) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/read_recall_view.rs b/crates/sqlbuild-orchestration/src/store/main/read_recall_view.rs new file mode 100644 index 0000000000..4c323fcd1b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/read_recall_view.rs @@ -0,0 +1,46 @@ +//! Only the supported Recall views can be read; no caller-provided SQL is evaluated. + +use sqlbuild_recall::schema::constants::VIEW_NAMES; + +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::types::Row; + +/// The API authorizes the invocation and evidence capability before entering this storage reader. +pub async fn read_recall_view( + store: &ControlStore, + (view, invocation_id): (&str, &str), + limit: u32, +) -> Result, ControlError> { + if !VIEW_NAMES.contains(&view) { + return Err(ControlError::Invalid("unknown Recall view".into())); + } + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, mut rows) = query( + transaction, + &format!("SELECT * FROM \"{view}\" WHERE invocation_id=$1 LIMIT $2"), + vec![ + Parameter::Text(invocation_id.into()), + Parameter::Integer(i64::from(limit.min(1000))), + ], + ) + .await?; + // These PostgreSQL view columns retain bytes: conversion to database text would reject NUL. + // Decode only the display field, leaving captured content and transcripts byte-exact. + if matches!(view, "output_tail" | "artifacts") { + for row in &mut rows { + if let Some(text) = row.get_mut("text") + && text.is_array() + { + let bytes: Vec = serde_json::from_value(text.clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + *text = serde_json::Value::String(String::from_utf8_lossy(&bytes).into_owned()); + } + } + } + commit(transaction).await?; + Ok(rows) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/refresh_clock.rs b/crates/sqlbuild-orchestration/src/store/main/refresh_clock.rs new file mode 100644 index 0000000000..0453de1ae3 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/refresh_clock.rs @@ -0,0 +1,26 @@ +//! Refresh the authoritative database clock after taking transaction locks. + +use chrono::DateTime; + +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{Connection, ControlTransaction}; + +pub(crate) async fn refresh_clock( + transaction: ControlTransaction, +) -> Result { + let sql = match transaction.connection { + Some(Connection::Postgres(_)) => "SELECT clock_timestamp() AS now", + _ => "SELECT strftime('%Y-%m-%dT%H:%M:%f000Z','now') AS now", + }; + let (mut transaction, rows) = query(transaction, sql, vec![]).await?; + let now = rows + .first() + .and_then(|row| row.get("now")) + .and_then(|value| value.as_str()) + .ok_or_else(|| ControlError::Invalid("database clock did not return a timestamp".into()))?; + transaction.now = DateTime::parse_from_rfc3339(now) + .map_err(|error| ControlError::Invalid(error.to_string()))? + .to_utc(); + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/store/mod.rs b/crates/sqlbuild-orchestration/src/store/mod.rs new file mode 100644 index 0000000000..d92242c092 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod _helpers; +pub mod errors; +pub mod main; +pub mod models; +pub(crate) mod types; diff --git a/crates/sqlbuild-orchestration/src/store/models.rs b/crates/sqlbuild-orchestration/src/store/models.rs new file mode 100644 index 0000000000..3c097f41bd --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/models.rs @@ -0,0 +1,59 @@ +//! Native database handles and bound values shared by control-plane operations. + +use std::path::PathBuf; + +use chrono::{DateTime, Utc}; +use sqlbuild_recall::schema::models::ColumnType; +use sqlbuild_recall::store::models::ConflictEvidence; +use uuid::Uuid; + +/// An installation's control store. Workers and locations never receive this handle. +#[derive(Clone, Debug)] +pub enum ControlStore { + Sqlite(PathBuf), + Postgres(sqlx_postgres::PgPool), +} + +/// The first copy wins; duplicate events retain their original storage cursor. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RecallAppendOutcome { + pub inserted: bool, + pub storage_id: Option, + pub conflict: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum TransactionMode { + Read, + Write, +} + +/// An owned transaction; abandoning it rolls back, including when an async task is cancelled. +#[derive(Debug)] +pub(crate) struct ControlTransaction { + pub(crate) connection: Option, + pub(crate) now: DateTime, +} + +#[derive(Debug)] +pub(crate) enum Connection { + Sqlite(rusqlite::Connection), + Postgres(sqlx_core::transaction::Transaction<'static, sqlx_postgres::Postgres>), +} + +/// Parameters keep their PostgreSQL types; SQLite uses their corresponding storage classes. +#[derive(Clone, Debug)] +pub(crate) enum Parameter { + Null(ColumnType), + Text(String), + OptionalText(Option), + Bytes(Vec), + Integer(i64), + Real(f64), + Json(serde_json::Value), + Boolean(bool), + Id(Uuid), + OptionalId(Option), + Time(DateTime), + OptionalTime(Option>), +} diff --git a/crates/sqlbuild-orchestration/src/store/schema.sql b/crates/sqlbuild-orchestration/src/store/schema.sql new file mode 100644 index 0000000000..c4c689ca2e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/schema.sql @@ -0,0 +1,264 @@ +CREATE TABLE sqb_installation ( + singleton BIGINT PRIMARY KEY CHECK (singleton = 1), + installation_id {uuid} NOT NULL, + created_at {time} NOT NULL +); +CREATE TABLE sqb_locations ( + location_id {uuid} PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + created_by TEXT NOT NULL, + created_at {time} NOT NULL, + archived_at {time}, + broken_deploy_policy TEXT NOT NULL DEFAULT 'fallback' + CHECK (broken_deploy_policy IN ('fallback', 'queue')) +); +CREATE TABLE sqb_source_texts ( + text_hash TEXT PRIMARY KEY, + body {binary} NOT NULL +); +CREATE TABLE sqb_node_definitions ( + node_hash TEXT PRIMARY KEY, + body {binary} NOT NULL +); +CREATE TABLE sqb_documents ( + document_hash TEXT PRIMARY KEY, + body {binary} NOT NULL +); +CREATE TABLE sqb_snapshots ( + snapshot_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + generation BIGINT NOT NULL CHECK (generation > 0), + git_revision TEXT, + git_dirty BOOLEAN NOT NULL, + git_author TEXT, + image_id TEXT, + environment_hash TEXT NOT NULL, + sqb_version TEXT NOT NULL, + snapshot_schema_version BIGINT NOT NULL, + manifest_digest TEXT NOT NULL, + published_at {time} NOT NULL, + UNIQUE (location_id, target, generation), + UNIQUE (location_id, target, snapshot_id) +); +CREATE INDEX sqb_snapshots_location_time ON sqb_snapshots(location_id, target, published_at); +CREATE TABLE sqb_snapshot_nodes ( + snapshot_id {uuid} NOT NULL REFERENCES sqb_snapshots(snapshot_id), + node_key TEXT NOT NULL, + node_kind TEXT NOT NULL, + node_hash TEXT NOT NULL REFERENCES sqb_node_definitions(node_hash), + PRIMARY KEY (snapshot_id, node_key) +); +CREATE TABLE sqb_snapshot_sources ( + snapshot_id {uuid} NOT NULL, + node_key TEXT NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('raw_sql', 'compiled_sql', 'python_source')), + text_hash TEXT NOT NULL REFERENCES sqb_source_texts(text_hash), + PRIMARY KEY (snapshot_id, node_key, kind), + FOREIGN KEY (snapshot_id, node_key) REFERENCES sqb_snapshot_nodes(snapshot_id, node_key) +); +CREATE TABLE sqb_snapshot_documents ( + snapshot_id {uuid} NOT NULL REFERENCES sqb_snapshots(snapshot_id), + kind TEXT NOT NULL CHECK (kind IN ('jobs', 'groups', 'triggers', 'roles', 'lineage', 'columns', 'providers')), + document_hash TEXT NOT NULL REFERENCES sqb_documents(document_hash), + PRIMARY KEY (snapshot_id, kind) +); +CREATE TABLE sqb_current_snapshots ( + location_id {uuid} NOT NULL, + target TEXT NOT NULL, + snapshot_id {uuid} NOT NULL, + since {time} NOT NULL, + PRIMARY KEY (location_id, target), + FOREIGN KEY (location_id, target, snapshot_id) + REFERENCES sqb_snapshots(location_id, target, snapshot_id) +); +CREATE TABLE sqb_snapshot_changes ( + snapshot_id {uuid} NOT NULL REFERENCES sqb_snapshots(snapshot_id), + change_seq BIGINT NOT NULL, + change_kind TEXT NOT NULL CHECK (change_kind IN ('added', 'changed', 'removed')), + subject_key TEXT NOT NULL, + before_hash TEXT, + after_hash TEXT, + PRIMARY KEY (snapshot_id, change_seq) +); +CREATE TABLE sqb_scheduler_lease ( + name TEXT PRIMARY KEY CHECK (name = 'scheduler'), + holder {uuid} NOT NULL, + epoch BIGINT NOT NULL CHECK (epoch > 0), + expires_at {time} NOT NULL +); +CREATE TABLE sqb_jobs ( + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + job_key TEXT NOT NULL, + snapshot_id {uuid} NOT NULL REFERENCES sqb_snapshots(snapshot_id), + kind TEXT NOT NULL CHECK (kind IN ('job', 'group')), + definition_json {json} NOT NULL, + overlap_policy TEXT NOT NULL CHECK (overlap_policy IN ('coalesce', 'queue', 'allow')), + PRIMARY KEY (location_id, target, job_key) +); +CREATE TABLE sqb_job_nodes ( + location_id {uuid} NOT NULL, + target TEXT NOT NULL, + job_key TEXT NOT NULL, + node_key TEXT NOT NULL, + PRIMARY KEY (location_id, target, job_key, node_key), + FOREIGN KEY (location_id, target, job_key) REFERENCES sqb_jobs(location_id, target, job_key) +); +CREATE TABLE sqb_triggers ( + trigger_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('schedule', 'sensor')), + name TEXT NOT NULL, + job_key TEXT NOT NULL, + definition_json {json} NOT NULL, + snapshot_id {uuid} NOT NULL REFERENCES sqb_snapshots(snapshot_id), + removed_in_snapshot_id {uuid} REFERENCES sqb_snapshots(snapshot_id), + UNIQUE (location_id, target, kind, name) +); +CREATE TABLE sqb_trigger_state ( + trigger_id {uuid} PRIMARY KEY REFERENCES sqb_triggers(trigger_id), + status TEXT NOT NULL CHECK (status IN ('running', 'paused')), + status_override TEXT CHECK (status_override IN ('running', 'paused')), + started_at {time} NOT NULL, + cursor_json {json}, + last_tick_for {time}, + next_due_at {time}, + consecutive_failures BIGINT NOT NULL DEFAULT 0, + updated_at {time} NOT NULL +); +CREATE INDEX sqb_trigger_state_due ON sqb_trigger_state(next_due_at) WHERE status = 'running'; +CREATE TABLE sqb_trigger_state_events ( + event_id {uuid} PRIMARY KEY, + trigger_id {uuid} NOT NULL REFERENCES sqb_triggers(trigger_id), + occurred_at {time} NOT NULL, + snapshot_id {uuid} NOT NULL REFERENCES sqb_snapshots(snapshot_id), + request_json {json} NOT NULL +); +CREATE INDEX sqb_trigger_state_events_time ON sqb_trigger_state_events(trigger_id, occurred_at, event_id); +CREATE TABLE sqb_ticks ( + tick_id {uuid} PRIMARY KEY, + trigger_id {uuid} NOT NULL REFERENCES sqb_triggers(trigger_id), + scheduled_for {time} NOT NULL, + evaluated_at {time} NOT NULL, + epoch BIGINT NOT NULL, + outcome TEXT NOT NULL CHECK (outcome IN ('requested', 'skipped', 'covered', 'failed', 'deferred')), + run_id {uuid}, + covered_by_run_id {uuid}, + reason TEXT, + details_json {json} NOT NULL, + UNIQUE (trigger_id, scheduled_for) +); +CREATE INDEX sqb_ticks_trigger_time ON sqb_ticks(trigger_id, scheduled_for); +CREATE TABLE sqb_runs ( + run_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + snapshot_id {uuid} NOT NULL REFERENCES sqb_snapshots(snapshot_id), + image_id TEXT, + kind TEXT NOT NULL CHECK (kind IN ('job', 'group', 'adhoc')), + origin TEXT NOT NULL CHECK (origin IN ('manual', 'schedule', 'sensor', 'rerun', 'backfill')), + job_key TEXT, + parent_run_id {uuid} REFERENCES sqb_runs(run_id), + root_run_id {uuid} NOT NULL, + trigger_id {uuid} REFERENCES sqb_triggers(trigger_id), + tick_id {uuid} REFERENCES sqb_ticks(tick_id), + rerun_of_run_id {uuid} REFERENCES sqb_runs(run_id), + requested_by TEXT NOT NULL, + priority BIGINT NOT NULL DEFAULT 0, + selection_json {json} NOT NULL, + request_json {json} NOT NULL, + write_set_digest TEXT NOT NULL, + rewrite_history BOOLEAN NOT NULL DEFAULT FALSE, + status TEXT NOT NULL CHECK (status IN ('queued', 'launched', 'running', 'unresponsive', 'success', 'warn', 'partial', 'failed', 'cancelled', 'presumed_failed')), + status_reason TEXT, + queued_at {time} NOT NULL, + launched_at {time}, + started_at {time}, + ended_at {time}, + cancel_requested_at {time}, + node_counts_json {json} NOT NULL, + last_event_storage_id BIGINT NOT NULL +); +CREATE INDEX sqb_runs_location_time ON sqb_runs(location_id, target, queued_at); +CREATE INDEX sqb_runs_queue ON sqb_runs(priority DESC, queued_at) WHERE status = 'queued'; +CREATE INDEX sqb_runs_active ON sqb_runs(location_id, target, job_key) WHERE status IN ('queued', 'launched', 'running', 'unresponsive'); +CREATE TABLE sqb_run_nodes ( + run_id {uuid} NOT NULL REFERENCES sqb_runs(run_id), + node_key TEXT NOT NULL, + attempt BIGINT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('pending', 'running', 'success', 'warn', 'failed', 'skipped', 'cancelled')), + writes BOOLEAN NOT NULL, + started_at {time}, + ended_at {time}, + rows_affected BIGINT, + error_code TEXT, + skipped_because_of TEXT, + PRIMARY KEY (run_id, node_key, attempt) +); +CREATE INDEX sqb_run_nodes_node ON sqb_run_nodes(node_key, run_id); +CREATE TABLE sqb_write_claims ( + target TEXT NOT NULL, + node_key TEXT NOT NULL, + run_id {uuid} NOT NULL REFERENCES sqb_runs(run_id), + claimed_at {time} NOT NULL, + PRIMARY KEY (target, node_key) +); +CREATE INDEX sqb_write_claims_run ON sqb_write_claims(run_id); +CREATE TABLE sqb_concurrency_slots ( + group_key TEXT NOT NULL, + slot BIGINT NOT NULL CHECK (slot > 0), + run_id {uuid} NOT NULL REFERENCES sqb_runs(run_id), + claimed_at {time} NOT NULL, + PRIMARY KEY (group_key, slot), + UNIQUE (group_key, run_id) +); +CREATE INDEX sqb_concurrency_slots_run ON sqb_concurrency_slots(run_id); +CREATE TABLE sqb_run_liveness ( + run_id {uuid} PRIMARY KEY REFERENCES sqb_runs(run_id), + worker_instance {uuid} NOT NULL, + container_id TEXT, + claimed_at {time} NOT NULL, + last_heartbeat_at {time} NOT NULL +); +CREATE TABLE sqb_principals ( + principal_id {uuid} PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + kind TEXT NOT NULL CHECK (kind IN ('person', 'service_account')), + created_at {time} NOT NULL, + disabled_at {time} +); +CREATE TABLE sqb_access_tokens ( + token_id {uuid} PRIMARY KEY, + secret_hash TEXT NOT NULL, + principal_id {uuid} REFERENCES sqb_principals(principal_id), + location_id {uuid} REFERENCES sqb_locations(location_id), + run_id {uuid} REFERENCES sqb_runs(run_id), + claim_id {uuid}, + scope_json {json} NOT NULL, + created_at {time} NOT NULL, + expires_at {time} NOT NULL, + revoked_at {time}, + CHECK ( + (principal_id IS NOT NULL AND location_id IS NULL AND run_id IS NULL AND claim_id IS NULL) + OR (principal_id IS NULL AND location_id IS NOT NULL AND run_id IS NULL AND claim_id IS NULL) + OR (principal_id IS NULL AND location_id IS NULL AND run_id IS NOT NULL AND claim_id IS NOT NULL) + ) +); +CREATE INDEX sqb_access_tokens_principal ON sqb_access_tokens(principal_id); +CREATE INDEX sqb_access_tokens_location ON sqb_access_tokens(location_id); +CREATE INDEX sqb_access_tokens_run ON sqb_access_tokens(run_id); +CREATE TABLE sqb_security_audit ( + event_id {uuid} PRIMARY KEY, + occurred_at {time} NOT NULL, + actor TEXT NOT NULL, + action TEXT NOT NULL, + granting_role TEXT NOT NULL, + request_id {uuid} NOT NULL, + ip_address TEXT, + scope_json {json} NOT NULL, + before_json {json} NOT NULL, + after_json {json} NOT NULL +); +CREATE INDEX sqb_security_audit_time ON sqb_security_audit(occurred_at, event_id); diff --git a/crates/sqlbuild-orchestration/src/store/types.rs b/crates/sqlbuild-orchestration/src/store/types.rs new file mode 100644 index 0000000000..c1df9d2d41 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/types.rs @@ -0,0 +1,3 @@ +//! Internal result rows use native JSON scalars, with UUIDs and timestamps as strings. + +pub(crate) type Row = std::collections::BTreeMap; diff --git a/crates/sqlbuild-orchestration/tests/api.rs b/crates/sqlbuild-orchestration/tests/api.rs new file mode 100644 index 0000000000..9ee75a0eb6 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api.rs @@ -0,0 +1,12 @@ +#[path = "api/helpers.rs"] +mod helpers; +#[path = "runs/helpers.rs"] +mod run_helpers; +#[path = "store/helpers.rs"] +mod store_helpers; +#[path = "store/test_types.rs"] +mod store_test_types; +#[path = "api/test_contract.rs"] +mod test_contract; +#[path = "api/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/api/helpers.rs b/crates/sqlbuild-orchestration/tests/api/helpers.rs new file mode 100644 index 0000000000..395e1535f5 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/helpers.rs @@ -0,0 +1,52 @@ +use std::time::Duration; + +use sqlbuild_orchestration::api::main::serve_api::serve_api; +use sqlbuild_orchestration::store::models::ControlStore; + +use crate::test_types::TestApi; + +pub(crate) async fn start_api(store: &ControlStore) -> TestApi { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("isolated API socket"); + let url = format!("http://{}", listener.local_addr().expect("API address")); + let (shutdown, signal) = tokio::sync::oneshot::channel(); + let store = store.clone(); + let task = tokio::spawn(serve_api(store, listener, async { + let _ = signal.await; + })); + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(1)) + .build() + .expect("HTTP client"); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if client + .get(format!("{url}/api/openapi.json")) + .send() + .await + .is_ok_and(|response| response.status().is_success()) + { + break; + } + assert!(!task.is_finished(), "API failed during startup"); + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("API becomes ready"); + TestApi { + url, + shutdown, + task, + } +} + +pub(crate) async fn stop_api(api: TestApi) { + api.shutdown.send(()).expect("request API shutdown"); + tokio::time::timeout(Duration::from_secs(2), api.task) + .await + .expect("prompt API shutdown") + .expect("join API server") + .expect("API server stopped cleanly"); +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_contract.rs b/crates/sqlbuild-orchestration/tests/api/test_contract.rs new file mode 100644 index 0000000000..ef7d0d0902 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_contract.rs @@ -0,0 +1,331 @@ +use std::collections::BTreeMap; +use std::time::Duration; + +use chrono::Utc; +use reqwest::StatusCode; +use serde_json::json; +use sqlbuild_orchestration::credentials::main::issue_token::issue_token; +use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; +use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; +use sqlbuild_orchestration::runs::main::change_run::change_run; +use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; +use sqlbuild_orchestration::runs::main::list_runs::list_runs; +use sqlbuild_orchestration::runs::models::{RunAuthority, RunChange, RunChangeRequest, RunStatus}; +use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::snapshots::models::SnapshotRecord; +use sqlbuild_orchestration::store::main::create_location::create_location; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use uuid::Uuid; + +use crate::helpers::{start_api, stop_api}; +use crate::run_helpers::{queue, request, snapshot}; +use crate::store_helpers::{close_test_store, open_test_store, run}; +use crate::store_test_types::Backend; + +#[test] +fn given_a_real_http_server_when_locations_and_workers_report_then_scopes_retries_and_terminal_expiry_are_enforced() + { + for backend in [Backend::Sqlite, Backend::Postgres] { + run(async { + let Some(fixture) = open_test_store(backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location_id = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "inventory", "test-admin") + .await + .expect("another location"); + let audit = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let expiry = Utc::now() + chrono::Duration::hours(1); + let location_token = issue_token( + store, + &audit, + CredentialScope::Location { location_id }, + expiry, + ) + .await + .expect("location credential"); + let api = start_api(store).await; + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(5)) + .build() + .expect("HTTP client"); + let path = format!( + "{}/api/locations/{location_id}/targets/development/snapshots", + api.url + ); + let response = client + .post(&path) + .header("Content-Type", "application/json") + .body("invalid-json") + .send() + .await + .expect("unauthenticated request"); + assert_eq!( + response.status(), + StatusCode::UNAUTHORIZED, + "authentication precedes JSON parsing" + ); + let openapi: serde_json::Value = client + .get(format!("{}/api/openapi.json", api.url)) + .send() + .await + .expect("OpenAPI response") + .json() + .await + .expect("OpenAPI JSON"); + assert_eq!( + openapi["components"]["securitySchemes"]["bearer"]["scheme"], + "bearer" + ); + assert!(openapi["components"]["schemas"]["SnapshotInput"].is_object()); + assert!(openapi["components"]["schemas"]["RunChangeRequest"].is_object()); + let input = snapshot(location_id); + let response = client + .post(&path) + .bearer_auth(&location_token.bearer) + .json(&input) + .send() + .await + .expect("publish through API"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.headers()["cache-control"], "no-store"); + let published: SnapshotRecord = response.json().await.expect("snapshot response"); + let response = client + .post(&path) + .bearer_auth(&location_token.bearer) + .json(&input) + .send() + .await + .expect("stale publish"); + assert_eq!(response.status(), StatusCode::CONFLICT); + let response = client + .post(format!( + "{}/api/locations/{other}/targets/development/snapshots", + api.url + )) + .bearer_auth(&location_token.bearer) + .json(&snapshot(other)) + .send() + .await + .expect("cross-location publish"); + assert_eq!(response.status(), StatusCode::FORBIDDEN); + let current: serde_json::Value = client + .get(&path) + .bearer_auth(&location_token.bearer) + .send() + .await + .expect("current snapshot") + .json() + .await + .expect("snapshot JSON"); + assert_eq!( + current["record"]["snapshot_id"], + published.snapshot_id.to_string() + ); + assert_eq!(current["nodes"]["orders"]["compiled_sql"], "select 1 as id"); + queue( + store, + &request((location_id, published.snapshot_id), 1, 0, &["orders"]), + ) + .await; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let claim = claim_next_run(store, &lease, &BTreeMap::new()) + .await + .expect("claim") + .expect("work"); + let run_id = claim.request.run_id; + let worker = issue_token( + store, + &audit, + CredentialScope::Run { + run_id, + claim_id: claim.claim_id, + }, + expiry, + ) + .await + .expect("worker credential"); + let events_path = format!("{}/api/runs/{run_id}/events", api.url); + let heartbeat_path = format!("{}/api/runs/{run_id}/heartbeat", api.url); + let started = RunChangeRequest { + event_id: Uuid::now_v7(), + run_id, + claim_id: Some(claim.claim_id), + change: RunChange::Started, + }; + assert_eq!( + client + .post(&events_path) + .bearer_auth(&location_token.bearer) + .json(&started) + .send() + .await + .expect("location cannot act as worker") + .status(), + StatusCode::FORBIDDEN + ); + assert_eq!( + client + .get(&path) + .bearer_auth(&worker.bearer) + .send() + .await + .expect("worker cannot read location snapshot") + .status(), + StatusCode::FORBIDDEN + ); + for applied in [true, false] { + let response = client + .post(&events_path) + .bearer_auth(&worker.bearer) + .json(&started) + .send() + .await + .expect("report started"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response + .json::() + .await + .expect("report response"), + json!({"applied": applied}) + ); + } + let wrong_claim = RunChangeRequest { + event_id: Uuid::now_v7(), + claim_id: Some(Uuid::now_v7()), + ..started.clone() + }; + assert_eq!( + client + .post(&events_path) + .bearer_auth(&worker.bearer) + .json(&wrong_claim) + .send() + .await + .expect("wrong claim") + .status(), + StatusCode::FORBIDDEN + ); + assert_eq!( + client + .post(format!("{}/api/runs/{}/heartbeat", api.url, Uuid::now_v7())) + .bearer_auth(&worker.bearer) + .send() + .await + .expect("other run") + .status(), + StatusCode::FORBIDDEN + ); + let cancellation = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::CancelRequested { + actor: "test-admin".into(), + }, + ..started.clone() + }; + assert_eq!( + client + .post(&events_path) + .bearer_auth(&worker.bearer) + .json(&cancellation) + .send() + .await + .expect("worker cannot impersonate admin") + .status(), + StatusCode::BAD_REQUEST + ); + let response = client + .post(&heartbeat_path) + .bearer_auth(&worker.bearer) + .send() + .await + .expect("worker heartbeat"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response + .json::() + .await + .expect("heartbeat response"), + json!({"cancel_requested": false}) + ); + change_run(store, RunAuthority::Operator("test-admin"), &cancellation) + .await + .expect("operator cancellation"); + let response: serde_json::Value = client + .post(&heartbeat_path) + .bearer_auth(&worker.bearer) + .send() + .await + .expect("cancel heartbeat") + .json() + .await + .expect("cancel response"); + assert_eq!(response, json!({"cancel_requested": true})); + let terminal = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::Finished { + outcome: sqlbuild_orchestration::runs::models::RunOutcome::Cancelled, + reason: "worker acknowledged cancellation".into(), + exit_code: Some(143), + }, + ..started + }; + assert_eq!( + client + .post(&events_path) + .bearer_auth(&worker.bearer) + .json(&terminal) + .send() + .await + .expect("worker terminates") + .status(), + StatusCode::OK + ); + assert_eq!( + client + .post(&heartbeat_path) + .bearer_auth(&worker.bearer) + .send() + .await + .expect("terminal credential rejected") + .status(), + StatusCode::UNAUTHORIZED + ); + assert_eq!( + list_runs(store, (location_id, "development"), 100) + .await + .expect("run projection")[0] + .status, + RunStatus::Cancelled + ); + revoke_token(store, &audit, location_token.identity.token_id) + .await + .expect("revoke location token"); + assert_eq!( + client + .get(&path) + .bearer_auth(&location_token.bearer) + .send() + .await + .expect("revocation effective") + .status(), + StatusCode::UNAUTHORIZED + ); + stop_api(api).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs new file mode 100644 index 0000000000..d62cdc25ce --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -0,0 +1,7 @@ +use sqlbuild_orchestration::store::errors::ControlError; + +pub(crate) struct TestApi { + pub(crate) url: String, + pub(crate) shutdown: tokio::sync::oneshot::Sender<()>, + pub(crate) task: tokio::task::JoinHandle>, +} diff --git a/crates/sqlbuild-orchestration/tests/credentials.rs b/crates/sqlbuild-orchestration/tests/credentials.rs new file mode 100644 index 0000000000..a4b17699db --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials.rs @@ -0,0 +1,12 @@ +#[path = "ticks/fault_helpers.rs"] +mod fault_helpers; +#[path = "runs/helpers.rs"] +mod run_helpers; +#[path = "store/helpers.rs"] +mod store_helpers; +#[path = "store/test_types.rs"] +mod store_test_types; +#[path = "credentials/test_contract.rs"] +mod test_contract; +#[path = "ticks/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs new file mode 100644 index 0000000000..6768c12897 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs @@ -0,0 +1,241 @@ +use std::collections::BTreeMap; + +use chrono::{Duration, Utc}; +use sqlbuild_orchestration::credentials::main::authenticate_token::authenticate_token; +use sqlbuild_orchestration::credentials::main::create_principal::create_principal; +use sqlbuild_orchestration::credentials::main::issue_token::issue_token; +use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; +use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope, PrincipalKind}; +use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; +use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::store::main::create_location::create_location; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use sqlbuild_orchestration::store::models::ControlStore; +use uuid::Uuid; + +use crate::fault_helpers::{install_fault, remove_fault}; +use crate::run_helpers::{finish, queue, request, snapshot}; +use crate::store_helpers::{close_store, close_test_store, open_test_store, run, second_store}; +use crate::store_test_types::Backend; +use crate::test_types::TickFaultTestCase; + +#[test] +fn given_scoped_tokens_when_revoked_or_the_run_ends_then_all_replicas_reject_them_and_audit_failure_rolls_back() + { + for backend in [Backend::Sqlite, Backend::Postgres] { + run(async { + let Some(fixture) = open_test_store(backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let replica = second_store(store).await; + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: Some("127.0.0.1".parse().expect("loopback")), + }; + let fault = TickFaultTestCase { + description: "security audit failure", + table: "sqb_security_audit", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected audit failure')", + postgres_body: "RAISE EXCEPTION 'injected audit failure'", + }; + install_fault(store, &fault).await; + assert!( + create_principal( + store, + &context, + ("orders-ci", PrincipalKind::ServiceAccount) + ) + .await + .is_err() + ); + remove_fault(store, &fault).await; + let principal_id = create_principal( + store, + &context, + ("orders-ci", PrincipalKind::ServiceAccount), + ) + .await + .expect("principal creation and audit rolled back together"); + let location_id = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let scopes = [ + CredentialScope::Principal { principal_id }, + CredentialScope::Location { location_id }, + ]; + let expiry = Utc::now() + Duration::hours(1); + for scope in scopes { + let issued = issue_token(store, &context, scope, expiry) + .await + .expect("issue scoped token"); + assert_eq!( + authenticate_token(&replica, &issued.bearer) + .await + .expect("replica verifies"), + Some(issued.identity.clone()) + ); + assert!( + !format!("{issued:?}").contains(&issued.bearer), + "debug output redacts secrets" + ); + let replacement = if issued.bearer.ends_with('0') { + "1" + } else { + "0" + }; + let tampered = + format!("{}{replacement}", &issued.bearer[..issued.bearer.len() - 1]); + for invalid in [ + &tampered, + "", + "not-a-token", + &format!("{}.{}", Uuid::now_v7(), "0".repeat(64)), + ] { + assert!( + authenticate_token(&replica, invalid) + .await + .expect("invalid credential") + .is_none() + ); + } + install_fault(store, &fault).await; + assert!( + revoke_token(store, &context, issued.identity.token_id) + .await + .is_err() + ); + remove_fault(store, &fault).await; + assert!( + authenticate_token(&replica, &issued.bearer) + .await + .expect("failed revocation rolled back") + .is_some() + ); + assert!( + revoke_token(store, &context, issued.identity.token_id) + .await + .expect("revoke") + ); + assert!( + !revoke_token(&replica, &context, issued.identity.token_id) + .await + .expect("retry revoke") + ); + assert!( + authenticate_token(&replica, &issued.bearer) + .await + .expect("revoked token") + .is_none() + ); + assert!( + issue_token(store, &context, scope, Utc::now() - Duration::seconds(1)) + .await + .is_err() + ); + let stored: Vec = match store { + ControlStore::Sqlite(path) => { + let connection = + rusqlite::Connection::open(path).expect("inspect durable tokens"); + connection + .prepare("SELECT secret_hash FROM sqb_access_tokens") + .expect("token hashes") + .query_map([], |row| row.get(0)) + .expect("read token hashes") + .collect::>() + .expect("hashes") + } + ControlStore::Postgres(pool) => sqlx_core::query_scalar::query_scalar( + "SELECT secret_hash FROM sqb_access_tokens", + ) + .fetch_all(pool) + .await + .expect("read durable token hashes"), + }; + assert!( + stored + .iter() + .all(|hash| hash.len() == 64 && hash != &issued.bearer) + ); + } + for scope in [ + CredentialScope::Principal { + principal_id: Uuid::now_v7(), + }, + CredentialScope::Location { + location_id: Uuid::now_v7(), + }, + CredentialScope::Run { + run_id: Uuid::now_v7(), + claim_id: Uuid::now_v7(), + }, + ] { + assert!( + issue_token(store, &context, scope, expiry).await.is_err(), + "inactive scopes cannot acquire credentials" + ); + } + let published = publish_snapshot(store, &snapshot(location_id)) + .await + .expect("snapshot"); + queue( + store, + &request((location_id, published.snapshot_id), 1, 0, &["orders"]), + ) + .await; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let claim = claim_next_run(store, &lease, &BTreeMap::new()) + .await + .expect("claim") + .expect("work"); + let scope = CredentialScope::Run { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }; + let issued = issue_token(store, &context, scope, expiry) + .await + .expect("mint claimed-run token"); + assert_eq!( + authenticate_token(&replica, &issued.bearer) + .await + .expect("worker token"), + Some(issued.identity.clone()) + ); + assert!( + issue_token( + store, + &context, + CredentialScope::Run { + run_id: claim.request.run_id, + claim_id: Uuid::now_v7() + }, + expiry + ) + .await + .is_err() + ); + finish(store, &lease, &claim).await; + assert!( + authenticate_token(&replica, &issued.bearer) + .await + .expect("terminal token expires immediately") + .is_none() + ); + assert!( + issue_token(store, &context, scope, expiry).await.is_err(), + "a terminal claim cannot mint a new token" + ); + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/recall.rs b/crates/sqlbuild-orchestration/tests/recall.rs new file mode 100644 index 0000000000..274b96b9ed --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/recall.rs @@ -0,0 +1,12 @@ +#[path = "recall/helpers.rs"] +mod helpers; +#[path = "store/helpers.rs"] +mod store_helpers; +#[path = "store/test_types.rs"] +mod store_test_types; +#[path = "recall/test_contract.rs"] +mod test_contract; +#[path = "recall/test_types.rs"] +mod test_types; +#[path = "recall/test_views.rs"] +mod test_views; diff --git a/crates/sqlbuild-orchestration/tests/recall/helpers.rs b/crates/sqlbuild-orchestration/tests/recall/helpers.rs new file mode 100644 index 0000000000..72e824e82a --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/recall/helpers.rs @@ -0,0 +1,137 @@ +use serde_json::json; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use sqlbuild_orchestration::store::main::publish_recall::publish_recall; +use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; +use sqlbuild_recall::records::main::artifact_rows::artifact_rows; +use sqlbuild_recall::records::main::event_object_row::event_object_row; +use sqlbuild_recall::records::models::{ArtifactInput, RecallRow, RowEnvelope}; + +use crate::store_helpers::{close_store, close_test_store, open_test_store, second_store}; +use crate::store_test_types::Backend; + +pub(crate) fn event(id: &str, occurred_at: &str, duration_ms: f64) -> RecallRow { + event_object_row(&RowEnvelope { invocation_id: "orders-build".into(), + invocation_started_at: "2026-03-01T00:30:00+01:00".into(), producer_version: "test".into() }, + json!({ "event_id": id, "event_type": "invocation_completed", "schema_version": 1, + "occurred_at": occurred_at, "producer": "worker", "invocation_sequence": 1, + "payload": { "duration_ms": duration_ms, "exit_code": 0, "nested": [null, true, "orders"] } }) + .as_object().expect("event object")).expect("valid lifecycle event") +} + +pub(crate) async fn recall_contract(backend: Backend) -> Option> { + let fixture = open_test_store(backend).await?; + let store = &fixture.store; + migrate_store(store) + .await + .expect("migrate control and Recall schema"); + let second = second_store(store).await; + let first = event("orders-first", "2026-03-01T00:00:00Z", 3.25); + let first_batch = [first.clone()]; + let (left, right) = tokio::join!( + publish_recall(store, &first_batch), + publish_recall(&second, &first_batch) + ); + let left = left.expect("first writer").remove(0); + let right = right.expect("second writer").remove(0); + assert_ne!(left.inserted, right.inserted, "exactly one writer inserts"); + assert_eq!( + left.storage_id, right.storage_id, + "duplicates preserve cursor" + ); + let cursor = left.storage_id.expect("event storage cursor"); + let conflict = publish_recall(store, &[event("orders-first", "2026-03-01T00:00:00Z", 7.5)]) + .await + .expect("record conflicting retry") + .remove(0); + assert!( + conflict.conflict.is_some(), + "different content records conflict" + ); + let later = event("orders-later-arrival", "2026-02-01T00:00:00Z", 2.5); + let malformed = event("orders-malformed", "2026-04-01T00:00:00Z", 1.0); + let mut malformed = malformed; + malformed.values.retain(|(name, _)| *name != "event_id"); + assert!( + publish_recall(store, &[later.clone(), malformed]) + .await + .is_err(), + "whole malformed batch rolls back" + ); + assert!( + read_recall_events(store, cursor, 100) + .await + .expect("rolled-back page") + .is_empty() + ); + publish_recall(store, &[later]) + .await + .expect("retry rolled-back event"); + let page = read_recall_events(store, cursor, 1) + .await + .expect("late arrival page"); + assert_eq!(page.len(), 1); + assert_eq!( + page[0]["event_id"], "orders-later-arrival", + "page follows commit order, not producer time" + ); + let rows = read_recall_events(store, 0, 100).await.expect("all events"); + assert_eq!( + rows[0]["duration_ms"], + json!(3.25), + "first content wins with fractional precision" + ); + let payload = rows[0]["payload"].as_str().map_or_else( + || rows[0]["payload"].clone(), + |text| serde_json::from_str(text).expect("JSON payload"), + ); + assert_eq!(payload["nested"], json!([null, true, "orders"])); + let artifact = ArtifactInput { + digest: "sha256:orders".into(), + kind: "sql".into(), + media_type: "application/octet-stream".into(), + content: vec![0, 255, 1, 0], + seen_at: "2026-01-01T00:00:00Z".into(), + ref_id: "orders".into(), + role: "input".into(), + statement_id: None, + resource_id: None, + }; + let first_artifact = artifact_rows( + &RowEnvelope { + invocation_id: "first-artifact".into(), + invocation_started_at: artifact.seen_at.clone(), + producer_version: "test".into(), + }, + &artifact, + ); + publish_recall(store, &first_artifact) + .await + .expect("first binary artifact"); + let artifact = ArtifactInput { + seen_at: "2026-04-01T00:00:00Z".into(), + ..artifact + }; + let second_artifact = artifact_rows( + &RowEnvelope { + invocation_id: "second-artifact".into(), + invocation_started_at: artifact.seen_at.clone(), + producer_version: "test".into(), + }, + &artifact, + ); + let outcomes = publish_recall(store, &second_artifact) + .await + .expect("same artifact across months"); + assert!( + !outcomes[0].inserted && outcomes[0].conflict.is_none(), + "artifact identity is global, not partition-local" + ); + assert!(outcomes[1].inserted, "each invocation keeps its reference"); + let ids = rows + .iter() + .map(|row| row["event_id"].as_str().expect("event ID").to_owned()) + .collect(); + close_store(second).await; + close_test_store(fixture).await; + Some(ids) +} diff --git a/crates/sqlbuild-orchestration/tests/recall/test_contract.rs b/crates/sqlbuild-orchestration/tests/recall/test_contract.rs new file mode 100644 index 0000000000..d8d8bf5629 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/recall/test_contract.rs @@ -0,0 +1,28 @@ +use crate::helpers::recall_contract; +use crate::store_helpers::run; +use crate::store_test_types::Backend; +use crate::test_types::RecallTestCase; + +#[test] +fn given_native_recall_facts_when_retried_and_paged_then_content_and_commit_order_are_preserved() { + let test_cases = [ + RecallTestCase { + description: "SQLite Recall transactions", + backend: Backend::Sqlite, + expected_event_ids: vec!["orders-first", "orders-later-arrival"], + }, + RecallTestCase { + description: "PostgreSQL Recall transactions", + backend: Backend::Postgres, + expected_event_ids: vec!["orders-first", "orders-later-arrival"], + }, + ]; + for test_case in test_cases { + let observed = run(recall_contract(test_case.backend)); + assert!( + observed.is_none_or(|ids| ids == test_case.expected_event_ids), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/tests/recall/test_types.rs b/crates/sqlbuild-orchestration/tests/recall/test_types.rs new file mode 100644 index 0000000000..8ecb5f4f63 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/recall/test_types.rs @@ -0,0 +1,13 @@ +use crate::store_test_types::Backend; + +pub(crate) struct RecallTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_event_ids: Vec<&'static str>, +} + +pub(crate) struct RecallViewTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_complete: i64, +} diff --git a/crates/sqlbuild-orchestration/tests/recall/test_views.rs b/crates/sqlbuild-orchestration/tests/recall/test_views.rs new file mode 100644 index 0000000000..0dd0f229a5 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/recall/test_views.rs @@ -0,0 +1,192 @@ +use serde_json::json; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use sqlbuild_orchestration::store::main::publish_recall::publish_recall; +use sqlbuild_orchestration::store::main::read_recall_view::read_recall_view; +use sqlbuild_recall::records::main::artifact_rows::artifact_rows; +use sqlbuild_recall::records::main::delivery_row::delivery_row; +use sqlbuild_recall::records::main::event_object_row::event_object_row; +use sqlbuild_recall::records::main::heartbeat_row::heartbeat_row; +use sqlbuild_recall::records::main::invocation_row::invocation_row; +use sqlbuild_recall::records::main::output_row::output_row; +use sqlbuild_recall::records::models::{ + ArtifactInput, DeliveryCounts, InvocationContext, OutputChunk, OutputStream, RowEnvelope, +}; + +use crate::store_helpers::{close_test_store, open_test_store, run}; +use crate::store_test_types::Backend; +use crate::test_types::RecallViewTestCase; + +#[test] +fn given_native_context_output_and_audits_when_reading_views_then_both_databases_preserve_their_meaning() + { + let test_cases = [ + RecallViewTestCase { + description: "SQLite supported views", + backend: Backend::Sqlite, + expected_complete: 1, + }, + RecallViewTestCase { + description: "PostgreSQL supported views", + backend: Backend::Postgres, + expected_complete: 1, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate view store"); + let envelope = RowEnvelope { + invocation_id: "orders-output".into(), + invocation_started_at: "2026-01-02T00:00:00Z".into(), + producer_version: "test".into(), + }; + let context = InvocationContext { + command: Some("build".into()), + selection: vec!["orders".into()], + external_context: Some( + json!({"integration": {"name": "orders", "retry_number": 2}}).to_string(), + ), + ..InvocationContext::default() + }; + let mut rows = vec![ + invocation_row(&envelope, &context), + heartbeat_row( + &envelope, + ("2026-01-02T00:00:01Z", "2026-01-02T00:00:31Z"), + (None, None), + (Some("build"), None), + ), + ]; + for (sequence, (stream, bytes)) in [ + (OutputStream::Stdout, b"orders\x00\xff".as_slice()), + (OutputStream::Stderr, b"warning\n".as_slice()), + (OutputStream::Stdout, b"\xc3\x28 tail".as_slice()), + ] + .into_iter() + .enumerate() + { + rows.push(output_row( + &envelope, + &OutputChunk { + sequence: sequence as u64, + stream, + occurred_at: "2026-01-02T00:00:02Z".into(), + data: bytes.to_vec(), + byte_offset: 0, + dropped_before: 0, + omitted: None, + }, + )); + } + rows.push(delivery_row( + &envelope, + "output", + ( + &DeliveryCounts { + accepted: 3, + delivered: 3, + ..DeliveryCounts::default() + }, + true, + ), + "2026-01-02T00:00:03Z", + )); + for (event_id, event_type, payload) in [ + ( + "orders-audit", + "audit_completed", + json!({"audit_name":"unique_orders", "violation_count": 7, "measured_value": 3.5, "target_name":"orders", "evidence_truncated": true}), + ), + ( + "orders-complete", + "invocation_completed", + json!({"duration_ms": 12.75, "exit_code": 0}), + ), + ] { + rows.push(event_object_row(&envelope, json!({"event_id": event_id, "event_type": event_type, "schema_version": 1, "occurred_at": "2026-01-02T00:00:03Z", "producer": "worker", "invocation_sequence": 1, "payload": payload}).as_object().expect("event object")).expect("lifecycle fact")); + } + rows.extend(artifact_rows( + &envelope, + &ArtifactInput { + digest: "sha256:orders-display".into(), + kind: "sql".into(), + media_type: "text/plain".into(), + content: b"orders\x00\xff".to_vec(), + seen_at: envelope.invocation_started_at.clone(), + ref_id: "orders-statement".into(), + role: "input".into(), + statement_id: None, + resource_id: None, + }, + )); + publish_recall(store, &rows) + .await + .expect("write view inputs"); + let output = + read_recall_view(store, ("output_invocations", &envelope.invocation_id), 100) + .await + .expect("output accounting"); + assert_eq!(output[0]["complete"], json!(test_case.expected_complete)); + assert_eq!(output[0]["chunks"], json!(3)); + assert_eq!(output[0]["missing_sequences"], json!(0)); + let transcript = + read_recall_view(store, ("output_transcripts", &envelope.invocation_id), 100) + .await + .expect("binary transcript"); + assert_eq!( + transcript[0]["stdout"], + json!([b"orders\x00\xff".as_slice(), b"\xc3\x28 tail".as_slice()].concat()) + ); + assert_eq!( + transcript[0]["combined"], + json!( + [ + b"orders\x00\xff".as_slice(), + b"warning\n".as_slice(), + b"\xc3\x28 tail".as_slice() + ] + .concat() + ) + ); + let invocation = read_recall_view(store, ("invocations", &envelope.invocation_id), 100) + .await + .expect("invocation projection"); + assert_eq!(invocation[0]["status"], "completed"); + assert_eq!(invocation[0]["duration_ms"], json!(12.75)); + assert_eq!(invocation[0]["integration_retry"], json!(2)); + let audit = read_recall_view(store, ("audit_results", &envelope.invocation_id), 100) + .await + .expect("audit projection"); + assert_eq!(audit[0]["violation_count"], json!(7)); + assert_eq!(audit[0]["measured_value"], json!(3.5)); + assert_eq!(audit[0]["evidence_truncated"], json!(1)); + let tail = read_recall_view(store, ("output_tail", &envelope.invocation_id), 100) + .await + .expect("lossy output display"); + assert_eq!( + tail.iter() + .find(|row| row["sequence"] == 0) + .expect("first chunk")["text"], + json!("orders\u{0}\u{fffd}"), + "{}", + test_case.description + ); + assert_eq!( + tail.iter() + .find(|row| row["sequence"] == 2) + .expect("last chunk")["text"], + json!("\u{fffd}( tail") + ); + let artifacts = read_recall_view(store, ("artifacts", &envelope.invocation_id), 100) + .await + .expect("lossy artifact display"); + assert_eq!(artifacts[0]["text"], json!("orders\u{0}\u{fffd}")); + assert_eq!(artifacts[0]["content"], json!(b"orders\x00\xff".to_vec())); + assert_eq!(artifacts[0]["available"], json!(1)); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs.rs b/crates/sqlbuild-orchestration/tests/runs.rs new file mode 100644 index 0000000000..f2b42df0c2 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs.rs @@ -0,0 +1,18 @@ +#[path = "ticks/fault_helpers.rs"] +mod fault_helpers; +#[path = "runs/helpers.rs"] +mod helpers; +#[path = "store/helpers.rs"] +mod store_helpers; +#[path = "store/test_types.rs"] +mod store_test_types; +#[path = "runs/test_claims.rs"] +mod test_claims; +#[path = "runs/test_coalesce.rs"] +mod test_coalesce; +#[path = "runs/test_faults.rs"] +mod test_faults; +#[path = "runs/test_lifecycle.rs"] +mod test_lifecycle; +#[path = "ticks/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/runs/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/helpers.rs new file mode 100644 index 0000000000..e98f62b1df --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/helpers.rs @@ -0,0 +1,118 @@ +//! Run fixtures exercise native queues and Recall against isolated real databases. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::json; +use sqlbuild_orchestration::runs::main::change_run::change_run; +use sqlbuild_orchestration::runs::main::request_run::request_run; +use sqlbuild_orchestration::runs::models::{ + RunAuthority, RunChange, RunChangeRequest, RunClaim, RunKind, RunOrigin, RunOutcome, RunRequest, +}; +use sqlbuild_orchestration::scheduler::models::{Overlap, SchedulerLease}; +use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; +use sqlbuild_orchestration::store::models::ControlStore; +use uuid::Uuid; + +pub(crate) fn snapshot(location_id: Uuid) -> SnapshotInput { + SnapshotInput { + location_id, + target: "development".into(), + generation: 1, + git_revision: None, + git_dirty: false, + git_author: None, + image_id: Some("sha256:synthetic-orders-image".into()), + environment_hash: "orders-environment".into(), + sqb_version: env!("CARGO_PKG_VERSION").into(), + nodes: ["customers", "inventory", "orders", "products"] + .into_iter() + .map(|name| { + ( + name.into(), + SnapshotNode { + kind: "model".into(), + definition: json!({"materialization": "table"}), + raw_sql: Some("select 1 as id".into()), + compiled_sql: Some("select 1 as id".into()), + python_source: None, + }, + ) + }) + .collect(), + documents: BTreeMap::new(), + } +} + +pub(crate) fn request( + (location_id, snapshot_id): (Uuid, Uuid), + id: u128, + priority: i64, + nodes: &[&str], +) -> RunRequest { + let selection: BTreeSet = nodes.iter().map(|name| (*name).into()).collect(); + let run_id = Uuid::from_u128(id); + RunRequest { + run_id, + location_id, + target: "development".into(), + snapshot_id, + image_id: Some("sha256:synthetic-orders-image".into()), + kind: RunKind::Adhoc, + origin: RunOrigin::Manual, + job_key: None, + parent_run_id: None, + root_run_id: run_id, + trigger_id: None, + tick_id: None, + rerun_of_run_id: None, + requested_by: "test-operator".into(), + priority, + write_set: selection.clone(), + selection, + settings: BTreeMap::new(), + concurrency_groups: BTreeSet::new(), + overlap: Overlap::Allow, + } +} + +pub(crate) async fn queue(store: &ControlStore, request: &RunRequest) { + assert!( + request_run(store, request) + .await + .expect("queue pinned request") + ); + assert!( + !request_run(store, request) + .await + .expect("retry pinned request") + ); + let mut changed = request.clone(); + changed.priority += 1; + assert!( + request_run(store, &changed).await.is_err(), + "a reused run ID cannot change its request" + ); +} + +pub(crate) async fn finish(store: &ControlStore, lease: &SchedulerLease, claim: &RunClaim) { + let request = RunChangeRequest { + event_id: Uuid::new_v5(&claim.claim_id, b"test-finished"), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change: RunChange::Finished { + outcome: RunOutcome::Failed, + reason: "synthetic worker exit".into(), + exit_code: Some(1), + }, + }; + assert!( + change_run(store, RunAuthority::Scheduler(lease), &request) + .await + .expect("terminate claimed work") + ); + assert!( + !change_run(store, RunAuthority::Scheduler(lease), &request) + .await + .expect("retry terminal fact") + ); +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_claims.rs b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs new file mode 100644 index 0000000000..4489f0322d --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs @@ -0,0 +1,252 @@ +use std::collections::BTreeMap; +use std::num::NonZeroU32; + +use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; +use sqlbuild_orchestration::runs::main::list_runs::list_runs; +use sqlbuild_orchestration::runs::models::RunStatus; +use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::store::main::create_location::create_location; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; +use uuid::Uuid; + +use crate::helpers::{finish, queue, request, snapshot}; +use crate::store_helpers::{close_store, close_test_store, open_test_store, run, second_store}; +use crate::store_test_types::Backend; + +#[test] +fn given_overlapping_priority_queues_when_two_launchers_claim_then_conflicts_rollback_and_terminal_facts_release_claims() + { + for backend in [Backend::Sqlite, Backend::Postgres] { + run(async { + let Some(fixture) = open_test_store(backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate run store"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("create location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("publish snapshot"); + let scope = (location, published.snapshot_id); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + for input in [ + request(scope, 1, 1, &["orders"]), + request(scope, 2, 3, &["orders"]), + request(scope, 3, 2, &["products"]), + ] { + queue(store, &input).await; + } + let second = second_store(store).await; + let limits = BTreeMap::new(); + let (first, other) = tokio::join!( + claim_next_run(store, &lease, &limits), + claim_next_run(&second, &lease, &limits) + ); + let first = first.expect("first claimant").expect("claim available"); + let other = other + .expect("second claimant") + .expect("independent claim available"); + let claimed: std::collections::BTreeSet<_> = + [first.request.run_id, other.request.run_id] + .into_iter() + .collect(); + assert_eq!( + claimed, + [Uuid::from_u128(2), Uuid::from_u128(3)] + .into_iter() + .collect(), + "higher priority wins the shared write set" + ); + assert!( + claim_next_run(store, &lease, &limits) + .await + .expect("blocked run stays queued") + .is_none() + ); + queue(store, &request(scope, 4, 9, &["customers", "orders"])).await; + queue(store, &request(scope, 5, 0, &["customers"])).await; + let independent = claim_next_run(store, &lease, &limits) + .await + .expect("try partially conflicting candidate") + .expect("independent work proceeds"); + assert_eq!( + independent.request.run_id, + Uuid::from_u128(5), + "customers was not leaked by the failed customers/orders claim" + ); + let orders = [&first, &other] + .into_iter() + .find(|claim| claim.request.run_id == Uuid::from_u128(2)) + .expect("orders owner"); + finish(store, &lease, orders).await; + finish(store, &lease, &independent).await; + let next = claim_next_run(store, &lease, &limits) + .await + .expect("terminal cleanup committed") + .expect("next priority run claims both nodes"); + assert_eq!(next.request.run_id, Uuid::from_u128(4)); + let events = read_recall_events(store, 0, 100) + .await + .expect("read immutable run stream"); + assert_eq!( + events + .iter() + .filter(|event| event["event_type"] == "run_claimed") + .count(), + 4 + ); + assert_eq!( + events + .iter() + .filter(|event| event["event_type"] == "run_terminated") + .count(), + 2 + ); + let runs = list_runs(store, (location, "development"), 100) + .await + .expect("read atomic projections"); + assert_eq!( + runs.iter() + .filter(|run| run.status == RunStatus::Failed) + .count(), + 2 + ); + close_store(second).await; + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_named_group_limits_when_capacity_is_lowered_then_active_high_slots_still_count_and_terminal_cleanup_unblocks_work() + { + for backend in [Backend::Sqlite, Backend::Postgres] { + run(async { + let Some(fixture) = open_test_store(backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate run store"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("create location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("publish snapshot"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let mut limits = BTreeMap::from([( + "warehouse".into(), + NonZeroU32::new(2).expect("nonzero limit"), + )]); + for id in 1..=3 { + let mut input = request((location, published.snapshot_id), id, 0, &["orders"]); + input.write_set.clear(); + input.concurrency_groups.insert("warehouse".into()); + queue(store, &input).await; + } + assert!( + claim_next_run(store, &lease, &BTreeMap::new()) + .await + .is_err(), + "unknown groups cannot silently become unlimited" + ); + let first = claim_next_run(store, &lease, &limits) + .await + .expect("first slot") + .expect("available"); + let second = claim_next_run(store, &lease, &limits) + .await + .expect("second slot") + .expect("available"); + assert_eq!(first.request.run_id, Uuid::from_u128(1)); + assert_eq!(second.request.run_id, Uuid::from_u128(2)); + assert!( + claim_next_run(store, &lease, &limits) + .await + .expect("group full") + .is_none() + ); + finish(store, &lease, &first).await; + limits.insert( + "warehouse".into(), + NonZeroU32::new(1).expect("nonzero lowered limit"), + ); + assert!( + claim_next_run(store, &lease, &limits) + .await + .expect("high slot counts after limit reduction") + .is_none() + ); + finish(store, &lease, &second).await; + let third = claim_next_run(store, &lease, &limits) + .await + .expect("released group") + .expect("third queued run"); + assert_eq!(third.request.run_id, Uuid::from_u128(3)); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_more_than_one_page_of_blocked_runs_when_claiming_then_lower_priority_independent_work_is_not_starved() + { + for backend in [Backend::Sqlite, Backend::Postgres] { + run(async { + let Some(fixture) = open_test_store(backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("snapshot"); + let scope = (location, published.snapshot_id); + let mut lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::new(); + queue(store, &request(scope, 1, 0, &["orders"])).await; + claim_next_run(store, &lease, &limits) + .await + .expect("claim blocker") + .expect("available"); + for id in 2..=104 { + queue(store, &request(scope, id, 0, &["orders"])).await; + if id % 20 == 0 { + lease = sqlbuild_orchestration::scheduler::main::renew_lease::renew_lease( + store, &lease, + ) + .await + .expect("keep leader live while preparing backlog"); + } + } + queue(store, &request(scope, 105, -1, &["customers"])).await; + let started = std::time::Instant::now(); + let independent = claim_next_run(store, &lease, &limits) + .await + .expect("scan past a full blocked page") + .expect("independent work"); + assert_eq!(independent.request.run_id, Uuid::from_u128(105)); + assert!( + started.elapsed() < std::time::Duration::from_secs(5), + "claim scan exceeded five seconds on {backend:?}" + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs b/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs new file mode 100644 index 0000000000..c05d357583 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs @@ -0,0 +1,103 @@ +use std::collections::BTreeMap; + +use serde_json::json; +use sqlbuild_orchestration::declarations::main::list_triggers::list_triggers; +use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; +use sqlbuild_orchestration::runs::main::list_runs::list_runs; +use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::scheduler::main::evaluate_trigger::evaluate_trigger; +use sqlbuild_orchestration::scheduler::models::TickDecision; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::store::main::create_location::create_location; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use uuid::Uuid; + +use crate::helpers::snapshot; +use crate::store_helpers::{close_test_store, open_test_store, run}; +use crate::store_test_types::Backend; + +#[test] +fn given_coalescing_triggers_when_a_previous_run_has_been_claimed_then_new_ticks_wait_in_a_new_run_instead_of_covering_executing_work() + { + for backend in [Backend::Sqlite, Backend::Postgres] { + run(async { + let Some(fixture) = open_test_store(backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + input.documents.insert( + "jobs".into(), + json!({"orders": {"description": "Build orders", "selection": ["orders"]}}), + ); + input.documents.insert("triggers".into(), json!({"daily_orders": {"description": "Refresh orders", "job_key": "orders", "starts_at": "2020-01-01T00:00:00Z", + "schedule": {"kind": "calendar", "times": ["2020-01-01T00:00:00Z"], "refresh_at": "2030-01-01T00:00:00Z"}}})); + let published = publish_snapshot(store, &input) + .await + .expect("first calendar publication"); + let trigger = list_triggers(store, location, "development") + .await + .expect("trigger")[0] + .trigger_id; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + evaluate_trigger(store, &lease, trigger) + .await + .expect("first tick"); + let claim = claim_next_run(store, &lease, &BTreeMap::new()) + .await + .expect("claim first run") + .expect("scheduled run"); + assert_eq!(claim.request.snapshot_id, published.snapshot_id); + input.generation = 2; + input.documents.get_mut("triggers").expect("triggers")["daily_orders"]["schedule"]["times"] = + json!(["2020-01-01T00:00:00Z", "2020-01-02T00:00:00Z"]); + publish_snapshot(store, &input) + .await + .expect("refresh calendar"); + let plan = evaluate_trigger(store, &lease, trigger) + .await + .expect("second tick") + .expect("active trigger"); + assert_eq!(plan.actions.len(), 1); + let TickDecision::Requested { run_id: waiting } = plan.actions[0].decision else { + panic!("future work must not be covered by an already claimed run"); + }; + assert_ne!(waiting, claim.request.run_id); + input.generation = 3; + input.documents.get_mut("triggers").expect("triggers")["daily_orders"]["schedule"]["times"] = + json!([ + "2020-01-01T00:00:00Z", + "2020-01-02T00:00:00Z", + "2020-01-03T00:00:00Z" + ]); + publish_snapshot(store, &input) + .await + .expect("next calendar refresh"); + let plan = evaluate_trigger(store, &lease, trigger) + .await + .expect("third tick") + .expect("active trigger"); + assert_eq!( + plan.actions[0].decision, + TickDecision::Covered { + covered_by_run_id: waiting + } + ); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("one executing and one waiting") + .len(), + 2 + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_faults.rs b/crates/sqlbuild-orchestration/tests/runs/test_faults.rs new file mode 100644 index 0000000000..7ab8721043 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_faults.rs @@ -0,0 +1,162 @@ +use std::collections::BTreeMap; +use std::num::NonZeroU32; + +use sqlbuild_orchestration::runs::main::change_run::change_run; +use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; +use sqlbuild_orchestration::runs::main::heartbeat_run::heartbeat_run; +use sqlbuild_orchestration::runs::main::list_runs::list_runs; +use sqlbuild_orchestration::runs::models::{RunAuthority, RunChange, RunChangeRequest}; +use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::main::create_location::create_location; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; +use uuid::Uuid; + +use crate::fault_helpers::{expire_lease, install_fault, remove_fault}; +use crate::helpers::{queue, request, snapshot}; +use crate::store_helpers::{close_test_store, open_test_store, run}; +use crate::store_test_types::Backend; +use crate::test_types::TickFaultTestCase; + +#[test] +fn given_failures_during_claim_publication_when_retried_then_claims_slots_liveness_facts_and_projection_commit_together() + { + for backend in [Backend::Sqlite, Backend::Postgres] { + run(async { + let Some(fixture) = open_test_store(backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("snapshot"); + let mut input = request( + (location, published.snapshot_id), + 1, + 1, + &["customers", "orders"], + ); + input.concurrency_groups.insert("warehouse".into()); + queue(store, &input).await; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::from([( + "warehouse".into(), + NonZeroU32::new(1).expect("positive limit"), + )]); + let facts = read_recall_events(store, 0, 100) + .await + .expect("requested and queued facts"); + let projected = list_runs(store, (location, "development"), 100) + .await + .expect("queued projection"); + for table in [ + "sqb_write_claims", + "sqb_concurrency_slots", + "sqb_run_liveness", + "recall_events", + "sqb_runs", + ] { + let point = TickFaultTestCase { + description: "claim publication failure", + table, + operation: if table == "sqb_runs" { + "UPDATE" + } else { + "INSERT" + }, + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected claim failure')", + postgres_body: "RAISE EXCEPTION 'injected claim failure'", + }; + install_fault(store, &point).await; + assert!( + claim_next_run(store, &lease, &limits).await.is_err(), + "{} at {table}", + point.description + ); + remove_fault(store, &point).await; + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("rolled back claim fact"), + facts + ); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("rolled back claim projection"), + projected + ); + } + let expiry = TickFaultTestCase { + description: "claim fence expires before commit", + table: "sqb_run_liveness", + operation: "INSERT", + condition: "", + sqlite_body: "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + postgres_body: "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + }; + install_fault(store, &expiry).await; + assert!( + matches!( + claim_next_run(store, &lease, &limits).await, + Err(ControlError::Fenced) + ), + "{}", + expiry.description + ); + remove_fault(store, &expiry).await; + let claim = claim_next_run(store, &lease, &limits) + .await + .expect("retry complete claim") + .expect("no leaked claims or slots"); + assert_eq!(claim.request, input); + assert!( + claim_next_run(store, &lease, &limits) + .await + .expect("already claimed run is not relaunched") + .is_none() + ); + expire_lease(store).await; + let next = acquire_lease(store, Uuid::now_v7()) + .await + .expect("take over") + .expect("new leader"); + let launched = RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: input.run_id, + claim_id: Some(claim.claim_id), + change: RunChange::Launched { + container_id: Some("synthetic-orders-worker".into()), + }, + }; + assert!( + matches!( + change_run(store, RunAuthority::Scheduler(&lease), &launched).await, + Err(ControlError::Fenced) + ), + "a former leader cannot acknowledge a launch after takeover" + ); + assert!( + change_run(store, RunAuthority::Scheduler(&next), &launched) + .await + .expect("successor reconciles the existing claim") + ); + assert!( + !heartbeat_run(store, (input.run_id, claim.claim_id)) + .await + .expect("workers remain alive through scheduler failover") + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs new file mode 100644 index 0000000000..7ed2c6874a --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs @@ -0,0 +1,427 @@ +use std::collections::BTreeMap; + +use sqlbuild_orchestration::runs::main::change_run::change_run; +use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; +use sqlbuild_orchestration::runs::main::heartbeat_run::heartbeat_run; +use sqlbuild_orchestration::runs::main::list_run_nodes::list_run_nodes; +use sqlbuild_orchestration::runs::main::list_runs::list_runs; +use sqlbuild_orchestration::runs::models::{ + NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, +}; +use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::store::main::create_location::create_location; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; +use uuid::Uuid; + +use crate::fault_helpers::{install_fault, remove_fault}; +use crate::helpers::{queue, request, snapshot}; +use crate::store_helpers::{close_test_store, open_test_store, run}; +use crate::store_test_types::Backend; +use crate::test_types::TickFaultTestCase; + +#[test] +fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_facts_outcomes_and_claims_are_atomic() + { + for backend in [Backend::Sqlite, Backend::Postgres] { + run(async { + let Some(fixture) = open_test_store(backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("snapshot"); + let scope = (location, published.snapshot_id); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + queue(store, &request(scope, 1, 2, &["customers", "orders"])).await; + queue(store, &request(scope, 2, 1, &["customers", "orders"])).await; + let limits = BTreeMap::new(); + let claim = claim_next_run(store, &lease, &limits) + .await + .expect("claim") + .expect("available"); + let worker = RunAuthority::Worker { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }; + let started = RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change: RunChange::Started, + }; + let other_worker = RunAuthority::Worker { + run_id: Uuid::from_u128(2), + claim_id: Uuid::now_v7(), + }; + assert!(change_run(store, other_worker, &started).await.is_err()); + assert!( + change_run(store, RunAuthority::Operator("test-operator"), &started) + .await + .is_err() + ); + assert!( + change_run(store, worker, &started) + .await + .expect("worker can start before launcher acknowledgement") + ); + assert!( + !change_run(store, worker, &started) + .await + .expect("identical retry") + ); + let launched = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::Launched { + container_id: Some("synthetic-orders-worker".into()), + }, + ..started.clone() + }; + change_run(store, RunAuthority::Scheduler(&lease), &launched) + .await + .expect("late launcher acknowledgement"); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("runs") + .iter() + .find(|run| run.run_id == claim.request.run_id) + .expect("started run") + .status, + RunStatus::Running + ); + let success = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::Finished { + outcome: RunOutcome::Success, + reason: "worker complete".into(), + exit_code: Some(0), + }, + ..started.clone() + }; + assert!( + change_run(store, worker, &success).await.is_err(), + "unfinished nodes cannot be marked successful" + ); + for (node, outcome) in [ + ("customers", NodeOutcome::Success), + ("orders", NodeOutcome::Failed), + ] { + let node_start = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::NodeStarted { + node_key: node.into(), + }, + ..started.clone() + }; + change_run(store, worker, &node_start) + .await + .expect("node started"); + let completed = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::NodeFinished { + node_key: node.into(), + outcome, + rows_affected: Some(17), + error_code: (outcome == NodeOutcome::Failed).then(|| "E100".into()), + skipped_because_of: None, + }, + ..started.clone() + }; + if node == "customers" { + let before_nodes = list_run_nodes(store, started.run_id) + .await + .expect("node before fault"); + let before_events = read_recall_events(store, 0, 100) + .await + .expect("events before node fault"); + for table in ["sqb_run_nodes", "sqb_runs"] { + let point = TickFaultTestCase { + description: "node projection and counter failure", + table, + operation: "UPDATE", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected node failure')", + postgres_body: "RAISE EXCEPTION 'injected node failure'", + }; + install_fault(store, &point).await; + assert!( + change_run(store, worker, &completed).await.is_err(), + "{} at {table}", + point.description + ); + remove_fault(store, &point).await; + assert_eq!( + list_run_nodes(store, started.run_id) + .await + .expect("node rollback"), + before_nodes + ); + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("node fact rollback"), + before_events + ); + } + } + assert!( + change_run(store, worker, &completed) + .await + .expect("node completed") + ); + assert!( + !change_run(store, worker, &completed) + .await + .expect("node report retry") + ); + } + assert!( + change_run(store, worker, &success).await.is_err(), + "mixed node outcomes are partial, not success" + ); + let partial = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::Finished { + outcome: RunOutcome::Partial, + reason: "independent customer work completed".into(), + exit_code: Some(1), + }, + ..started.clone() + }; + let before = read_recall_events(store, 0, 100) + .await + .expect("before terminal"); + for table in [ + "recall_events", + "sqb_runs", + "sqb_write_claims", + "sqb_run_liveness", + ] { + let point = TickFaultTestCase { + description: "terminal transaction failure", + table, + operation: if table == "recall_events" { + "INSERT" + } else if table.starts_with("sqb_run") && table != "sqb_run_liveness" { + "UPDATE" + } else { + "DELETE" + }, + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected terminal failure')", + postgres_body: "RAISE EXCEPTION 'injected terminal failure'", + }; + install_fault(store, &point).await; + assert!( + change_run(store, worker, &partial).await.is_err(), + "{} at {table} must roll back the complete transition", + point.description + ); + remove_fault(store, &point).await; + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("rolled back event"), + before + ); + assert!( + claim_next_run(store, &lease, &limits) + .await + .expect("claims retained") + .is_none() + ); + } + let unresponsive = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::Unresponsive, + ..started.clone() + }; + change_run(store, RunAuthority::Scheduler(&lease), &unresponsive) + .await + .expect("monitor records loss of heartbeat"); + assert!( + !heartbeat_run(store, (started.run_id, claim.claim_id)) + .await + .expect("heartbeat recovers the worker") + ); + assert!( + heartbeat_run(store, (started.run_id, Uuid::now_v7())) + .await + .is_err(), + "another claim cannot report this run alive" + ); + let cancel = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::CancelRequested { + actor: "test-operator".into(), + }, + ..started.clone() + }; + assert!( + change_run(store, worker, &cancel).await.is_err(), + "workers cannot impersonate operators" + ); + change_run(store, RunAuthority::Operator("test-operator"), &cancel) + .await + .expect("record cancellation"); + assert!( + heartbeat_run(store, (started.run_id, claim.claim_id)) + .await + .expect("heartbeat reports pending cancellation") + ); + assert!( + claim_next_run(store, &lease, &limits) + .await + .expect("cancel request does not release an executing run's claims") + .is_none() + ); + assert!( + change_run(store, worker, &partial) + .await + .expect("terminal report atomically releases claims") + ); + assert!( + !change_run(store, worker, &partial) + .await + .expect("terminal retry after liveness cleanup") + ); + let mut changed = partial.clone(); + changed.change = RunChange::Finished { + outcome: RunOutcome::Success, + reason: "changed retry".into(), + exit_code: Some(0), + }; + assert!( + change_run(store, worker, &changed).await.is_err(), + "same event ID cannot change its content" + ); + assert!( + heartbeat_run(store, (started.run_id, claim.claim_id)) + .await + .is_err(), + "terminal runs cannot be revived by a late heartbeat" + ); + let projected = list_runs(store, (location, "development"), 100) + .await + .expect("terminal projection") + .into_iter() + .find(|run| run.run_id == started.run_id) + .expect("original run"); + assert_eq!(projected.status, RunStatus::Partial); + assert_eq!( + projected.node_counts_json, + BTreeMap::from([("success".into(), 1), ("failed".into(), 1)]) + ); + let nodes = list_run_nodes(store, started.run_id) + .await + .expect("node evidence"); + assert_eq!(nodes[0].node_key, "customers"); + assert_eq!(nodes[0].status, "success"); + assert_eq!(nodes[0].rows_affected, Some(17)); + assert_eq!(nodes[1].error_code.as_deref(), Some("E100")); + let following = claim_next_run(store, &lease, &limits) + .await + .expect("claim following run") + .expect("released capacity"); + assert_eq!(following.request.run_id, Uuid::from_u128(2)); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_unclaimed_work_when_an_operator_cancels_then_a_terminal_fact_prevents_launch_and_preserves_the_actor() + { + for backend in [Backend::Sqlite, Backend::Postgres] { + run(async { + let Some(fixture) = open_test_store(backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("snapshot"); + queue( + store, + &request((location, published.snapshot_id), 1, 0, &["orders"]), + ) + .await; + let request = RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: Uuid::from_u128(1), + claim_id: None, + change: RunChange::CancelRequested { + actor: "test-operator".into(), + }, + }; + assert!( + change_run(store, RunAuthority::Operator("another-operator"), &request) + .await + .is_err() + ); + assert!( + change_run(store, RunAuthority::Operator("test-operator"), &request) + .await + .expect("cancel before launch") + ); + assert!( + !change_run(store, RunAuthority::Operator("test-operator"), &request) + .await + .expect("retry cancellation") + ); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + assert!( + claim_next_run(store, &lease, &BTreeMap::new()) + .await + .expect("cancelled work cannot launch") + .is_none() + ); + let projected = list_runs(store, (location, "development"), 100) + .await + .expect("cancelled projection") + .remove(0); + assert_eq!(projected.status, RunStatus::Cancelled); + assert_eq!( + projected.node_counts_json, + BTreeMap::from([("cancelled".into(), 1)]) + ); + let events = read_recall_events(store, 0, 100) + .await + .expect("immutable cancellation facts"); + assert_eq!( + events + .iter() + .map(|row| row["event_type"].as_str().expect("type")) + .collect::>(), + [ + "run_requested", + "run_queued", + "run_cancel_requested", + "run_terminated" + ] + ); + assert_eq!( + events[2]["payload"]["change"]["change"]["actor"], + "test-operator" + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler.rs b/crates/sqlbuild-orchestration/tests/scheduler.rs new file mode 100644 index 0000000000..159c963ed1 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler.rs @@ -0,0 +1,10 @@ +#[path = "scheduler/helpers.rs"] +mod helpers; +#[path = "store/helpers.rs"] +mod store_helpers; +#[path = "store/test_types.rs"] +mod store_test_types; +#[path = "scheduler/test_contract.rs"] +mod test_contract; +#[path = "scheduler/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs new file mode 100644 index 0000000000..f1711e4bfc --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs @@ -0,0 +1,91 @@ +//! Contention and expiry exercise public APIs against the real databases, not fake clocks. + +use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::scheduler::main::renew_lease::renew_lease; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use sqlbuild_orchestration::store::models::ControlStore; +use uuid::Uuid; + +use crate::store_helpers::{close_store, close_test_store, open_test_store, second_store}; +use crate::store_test_types::Backend; + +async fn expire_lease(store: &ControlStore) { + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + panic!("expected SQLite fixture"); + }; + rusqlite::Connection::open(path) + .expect("open fault injection connection") + .execute( + "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + [], + ) + .expect("expire SQLite scheduler lease"); + return; + }; + sqlx_core::query::query( + "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + ) + .execute(pool) + .await + .expect("expire PostgreSQL scheduler lease"); +} + +pub(crate) async fn leadership_contract(backend: Backend) -> Option { + let test_store = open_test_store(backend).await?; + let store = &test_store.store; + migrate_store(store) + .await + .expect("migrate leadership store"); + let second = second_store(store).await; + let (first, other) = tokio::join!( + acquire_lease(store, Uuid::now_v7()), + acquire_lease(&second, Uuid::now_v7()) + ); + let first = first.expect("first contender completes"); + let other = other.expect("second contender completes"); + assert_ne!( + first.is_some(), + other.is_some(), + "exactly one concurrent contender is elected" + ); + let lease = first.or(other).expect("one elected holder"); + let renewed = renew_lease(&second, &lease) + .await + .expect("live epoch renews"); + assert_eq!(renewed.epoch, lease.epoch); + assert!(renewed.expires_at >= lease.expires_at); + assert!( + acquire_lease(store, lease.holder) + .await + .expect("duplicate acquisition completes") + .is_none() + ); + let mut forged = lease.clone(); + forged.holder = Uuid::now_v7(); + assert!(matches!( + renew_lease(store, &forged).await, + Err(ControlError::Fenced) + )); + expire_lease(store).await; + assert!( + matches!(renew_lease(store, &lease).await, Err(ControlError::Fenced)), + "even an uncontested expired holder must reacquire a new epoch" + ); + let successor = acquire_lease(&second, Uuid::now_v7()) + .await + .expect("acquire expired lease") + .expect("successor elected"); + assert!(matches!( + renew_lease(store, &lease).await, + Err(ControlError::Fenced) + )); + renew_lease(store, &successor) + .await + .expect("successor owns the current fence"); + let epoch = successor.epoch; + close_store(second).await; + close_test_store(test_store).await; + Some(epoch) +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/test_contract.rs b/crates/sqlbuild-orchestration/tests/scheduler/test_contract.rs new file mode 100644 index 0000000000..084f5fbb10 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/test_contract.rs @@ -0,0 +1,28 @@ +use crate::helpers::leadership_contract; +use crate::store_helpers::run; +use crate::store_test_types::Backend; +use crate::test_types::LeadershipTestCase; + +#[test] +fn given_two_schedulers_when_leadership_expires_then_only_the_new_epoch_can_renew() { + let test_cases = [ + LeadershipTestCase { + description: "SQLite contenders and takeover", + backend: Backend::Sqlite, + expected_epoch: 2, + }, + LeadershipTestCase { + description: "PostgreSQL contenders and takeover", + backend: Backend::Postgres, + expected_epoch: 2, + }, + ]; + for test_case in test_cases { + let observed = run(leadership_contract(test_case.backend)); + assert!( + observed.is_none_or(|epoch| epoch == test_case.expected_epoch), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs b/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs new file mode 100644 index 0000000000..c7db9ff190 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs @@ -0,0 +1,10 @@ +//! Leadership scenarios run independently on both supported engines. + +use crate::store_test_types::Backend; + +#[derive(Debug)] +pub(crate) struct LeadershipTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_epoch: i64, +} diff --git a/crates/sqlbuild-orchestration/tests/snapshots.rs b/crates/sqlbuild-orchestration/tests/snapshots.rs new file mode 100644 index 0000000000..a554a78fc6 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/snapshots.rs @@ -0,0 +1,10 @@ +#[path = "snapshots/test_contract.rs"] +mod contract; +#[path = "snapshots/helpers.rs"] +mod helpers; +#[path = "store/helpers.rs"] +mod store_helpers; +#[path = "store/test_types.rs"] +mod store_test_types; +#[path = "snapshots/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs new file mode 100644 index 0000000000..aaefbd5da1 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -0,0 +1,294 @@ +//! Real control-store integration coverage; both engines exercise the same public contract. + +use std::collections::BTreeMap; + +use serde_json::json; +use sqlbuild_orchestration::snapshots::main::current_snapshot::current_snapshot; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::main::create_location::create_location; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use sqlbuild_orchestration::store::main::open_store::open_store; +use sqlbuild_orchestration::store::models::ControlStore; +use uuid::Uuid; + +use crate::store_helpers::{close_store, close_test_store, open_test_store, second_store}; +use crate::store_test_types::Backend; +use crate::test_types::SchemaRejectionCase; + +fn snapshot(location_id: Uuid) -> SnapshotInput { + SnapshotInput { + location_id, + target: "development".into(), + generation: 1, + git_revision: None, + git_dirty: true, + git_author: None, + image_id: None, + environment_hash: "synthetic-environment".into(), + sqb_version: env!("CARGO_PKG_VERSION").into(), + nodes: BTreeMap::from([ + ( + "customers".into(), + SnapshotNode { + kind: "model".into(), + definition: json!({"materialization": "table", "columns": ["customer_id"]}), + raw_sql: Some("select 1 as customer_id".into()), + compiled_sql: Some("select 1 as customer_id".into()), + python_source: None, + }, + ), + ( + "orders".into(), + SnapshotNode { + kind: "model".into(), + definition: json!({"dependencies": ["customers"], "materialization": "view"}), + raw_sql: Some("select customer_id from ref('customers')".into()), + compiled_sql: Some("select customer_id from main.customers".into()), + python_source: Some("# display only: orders".into()), + }, + ), + ]), + documents: BTreeMap::from([("lineage".into(), json!({"orders": ["customers"]}))]), + } +} + +async fn exercise_snapshot_contract(store: &ControlStore) -> i64 { + let second = second_store(store).await; + let (first, second_migration) = tokio::join!(migrate_store(store), migrate_store(&second)); + first.expect("first migration succeeds"); + second_migration.expect("concurrent migration succeeds"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("create test location"); + assert!( + current_snapshot(store, location, "development") + .await + .expect("read empty target") + .is_none() + ); + let mut input = snapshot(location); + let initial = publish_snapshot(store, &input) + .await + .expect("publish initial snapshot"); + let loaded = current_snapshot(store, location, "development") + .await + .expect("read initial snapshot") + .expect("initial snapshot exists"); + assert_eq!(loaded.record, initial); + assert_eq!(loaded.nodes, input.nodes); + assert_eq!(loaded.documents, input.documents); + assert_eq!( + loaded + .changes + .iter() + .map(|change| (change.subject_key.as_str(), change.change_kind.as_str())) + .collect::>(), + [("customers", "added"), ("orders", "added")] + ); + + // Container insertion order and the generation do not affect the content address. + input.generation = 2; + input + .nodes + .get_mut("orders") + .expect("orders node exists") + .definition = json!({"materialization": "view", "dependencies": ["customers"]}); + let unchanged = publish_snapshot(store, &input) + .await + .expect("publish reordered definition"); + assert_eq!(unchanged.manifest_digest, initial.manifest_digest); + assert!( + current_snapshot(store, location, "development") + .await + .expect("read unchanged snapshot") + .expect("unchanged snapshot exists") + .changes + .is_empty() + ); + assert!(matches!( + publish_snapshot(store, &input).await, + Err(ControlError::StaleGeneration) + )); + + input.generation = 3; + input + .nodes + .get_mut("orders") + .expect("orders node exists") + .compiled_sql = Some("select customer_id + 5 from main.customers".into()); + let changed = publish_snapshot(store, &input) + .await + .expect("publish changed SQL"); + assert_ne!(changed.manifest_digest, unchanged.manifest_digest); + let loaded = current_snapshot(store, location, "development") + .await + .expect("read changed snapshot") + .expect("changed snapshot exists"); + assert_eq!(loaded.nodes, input.nodes); + assert_eq!(loaded.changes.len(), 1); + assert_eq!(loaded.changes[0].subject_key, "orders"); + assert_eq!(loaded.changes[0].change_kind, "changed"); + assert_ne!(loaded.changes[0].before_hash, loaded.changes[0].after_hash); + + // One target's generation does not suppress another target's first publication. + input.target = "testing".into(); + input.generation = 1; + publish_snapshot(store, &input) + .await + .expect("publish another target"); + assert_eq!( + current_snapshot(store, location, "development") + .await + .expect("read development target") + .expect("development target exists") + .record + .generation, + 3 + ); + assert_eq!( + current_snapshot(store, location, "testing") + .await + .expect("read testing target") + .expect("testing target exists") + .record + .generation, + 1 + ); + + input.target = "development".into(); + input.generation = 4; + input.nodes.remove("orders"); + publish_snapshot(store, &input) + .await + .expect("publish node removal"); + let loaded = current_snapshot(store, location, "development") + .await + .expect("read node removal") + .expect("snapshot exists after removal"); + assert_eq!(loaded.changes.len(), 1); + assert_eq!(loaded.changes[0].change_kind, "removed"); + assert_eq!(loaded.changes[0].subject_key, "orders"); + assert!(loaded.changes[0].after_hash.is_none()); + + // The newer compiler wins regardless of which publisher takes the transaction lock first. + let mut older = input.clone(); + older.generation = 5; + input.generation = 6; + input + .nodes + .get_mut("customers") + .expect("customers node exists") + .compiled_sql = Some("select 6 as customer_id".into()); + let (old_result, new_result) = tokio::join!( + publish_snapshot(store, &older), + publish_snapshot(&second, &input) + ); + assert!(old_result.is_ok() || matches!(old_result, Err(ControlError::StaleGeneration))); + new_result.expect("new generation wins"); + let loaded = current_snapshot(store, location, "development") + .await + .expect("read winning snapshot") + .expect("winning snapshot exists"); + assert_eq!(loaded.record.generation, 6); + assert_eq!(loaded.nodes, input.nodes); + close_store(second).await; + loaded.record.generation +} + +pub(super) async fn snapshot_contract(backend: Backend) -> Option { + let test_store = open_test_store(backend).await?; + let generation = exercise_snapshot_contract(&test_store.store).await; + verify_sqlite_deduplication(&test_store.store); + close_test_store(test_store).await; + Some(generation) +} + +fn verify_sqlite_deduplication(store: &ControlStore) { + let ControlStore::Sqlite(path) = store else { + return; + }; + let connection = rusqlite::Connection::open(path).expect("inspect control store"); + let definitions: i64 = connection + .query_row("SELECT count(*) FROM sqb_node_definitions", [], |row| { + row.get(0) + }) + .expect("count immutable definitions"); + // Two initial nodes, one changed source, and one changed customer; unchanged targets reuse them. + assert_eq!(definitions, 4); + let sources: i64 = connection + .query_row("SELECT count(*) FROM sqb_source_texts", [], |row| { + row.get(0) + }) + .expect("count shared sources"); + assert_eq!(sources, 6); +} + +pub(super) async fn sqlite_failed_publish_and_schema_floor() -> i64 { + let directory = tempfile::tempdir().expect("create test directory"); + let path = directory.path().join("control.sqlite3"); + let store = open_store(path.to_str().expect("UTF-8 test path")) + .await + .expect("open test store"); + migrate_store(&store).await.expect("migrate test store"); + let location = create_location(&store, "orders", "test-admin") + .await + .expect("create test location"); + let mut input = snapshot(location); + let good = publish_snapshot(&store, &input) + .await + .expect("publish good snapshot"); + let connection = rusqlite::Connection::open(&path).expect("open fault injection connection"); + // Fail after immutable bodies were inserted but before switching the current pointer. + connection.execute_batch("CREATE TRIGGER fail_snapshot_switch BEFORE UPDATE ON sqb_current_snapshots BEGIN SELECT RAISE(ABORT, 'synthetic publication failure'); END").expect("install publication failure"); + input.generation = 2; + input + .nodes + .get_mut("orders") + .expect("orders node exists") + .compiled_sql = Some("select 99".into()); + assert!(publish_snapshot(&store, &input).await.is_err()); + assert_eq!( + current_snapshot(&store, location, "development") + .await + .expect("read preserved snapshot") + .expect("last good snapshot exists") + .record, + good + ); + let count: i64 = connection + .query_row("SELECT count(*) FROM sqb_snapshots", [], |row| row.get(0)) + .expect("count committed snapshots"); + assert_eq!(count, 1); + connection + .execute_batch("DROP TRIGGER fail_snapshot_switch") + .expect("remove injected failure"); + let recovered = publish_snapshot(&store, &input) + .await + .expect("retry publication"); + for case in [ + SchemaRejectionCase { + description: "minimum server version is newer", + update_sql: "UPDATE sqb_schema_migrations SET min_server_version='9999.0.0'", + }, + SchemaRejectionCase { + description: "migration checksum changed", + update_sql: "UPDATE sqb_schema_migrations SET min_server_version='0.0.0', checksum='different'", + }, + SchemaRejectionCase { + description: "migration version is unknown", + update_sql: "UPDATE sqb_schema_migrations SET version=2", + }, + ] { + connection + .execute_batch(case.update_sql) + .expect("apply synthetic schema incompatibility"); + assert!( + matches!(migrate_store(&store).await, Err(ControlError::Schema(_))), + "{}", + case.description + ); + } + recovered.generation +} diff --git a/crates/sqlbuild-orchestration/tests/snapshots/test_contract.rs b/crates/sqlbuild-orchestration/tests/snapshots/test_contract.rs new file mode 100644 index 0000000000..8096938089 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/snapshots/test_contract.rs @@ -0,0 +1,48 @@ +use crate::helpers::{snapshot_contract, sqlite_failed_publish_and_schema_floor}; +use crate::store_helpers::run; +use crate::store_test_types::Backend; +use crate::test_types::{RecoveryTestCase, SnapshotTestCase}; + +#[test] +fn given_control_store_when_snapshots_publish_then_generations_sources_and_changes_are_consistent() +{ + let test_cases = [ + SnapshotTestCase { + description: "SQLite snapshot publication", + backend: Backend::Sqlite, + expected_generation: 6, + }, + SnapshotTestCase { + description: "PostgreSQL snapshot publication", + backend: Backend::Postgres, + expected_generation: 6, + }, + ]; + for test_case in test_cases { + let Some(observed) = run(snapshot_contract(test_case.backend)) else { + continue; + }; + assert_eq!( + observed, test_case.expected_generation, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_sqlite_when_a_publish_fails_then_last_good_snapshot_survives_and_schema_drift_is_rejected() +{ + let test_cases = [RecoveryTestCase { + description: "atomic rollback and recovery", + expected_generation: 2, + }]; + for test_case in test_cases { + let observed = run(sqlite_failed_publish_and_schema_floor()); + assert_eq!( + observed, test_case.expected_generation, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/tests/snapshots/test_types.rs b/crates/sqlbuild-orchestration/tests/snapshots/test_types.rs new file mode 100644 index 0000000000..cd4379f770 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/snapshots/test_types.rs @@ -0,0 +1,22 @@ +//! Cases that alter a disposable schema to verify startup rejection. + +use crate::store_test_types::Backend; + +#[derive(Debug)] +pub(super) struct SchemaRejectionCase { + pub(super) description: &'static str, + pub(super) update_sql: &'static str, +} + +#[derive(Debug)] +pub(super) struct SnapshotTestCase { + pub(super) description: &'static str, + pub(super) backend: Backend, + pub(super) expected_generation: i64, +} + +#[derive(Debug)] +pub(super) struct RecoveryTestCase { + pub(super) description: &'static str, + pub(super) expected_generation: i64, +} diff --git a/crates/sqlbuild-orchestration/tests/store/helpers.rs b/crates/sqlbuild-orchestration/tests/store/helpers.rs new file mode 100644 index 0000000000..07aafd8e98 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/helpers.rs @@ -0,0 +1,96 @@ +//! Real-database setup shared by storage, scheduling, and queue contracts. + +use sqlbuild_orchestration::store::main::open_store::open_store; +use sqlbuild_orchestration::store::models::ControlStore; +use uuid::Uuid; + +use crate::store_test_types::{Backend, TestStore}; + +pub(crate) fn run(future: impl std::future::Future) -> T { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("create control-store test runtime") + .block_on(future) +} + +pub(crate) async fn open_test_store(backend: Backend) -> Option { + let Backend::Postgres = backend else { + let directory = tempfile::tempdir().expect("create isolated SQLite directory"); + let store = open_store( + directory + .path() + .join("control.sqlite3") + .to_str() + .expect("UTF-8 test path"), + ) + .await + .expect("open isolated SQLite store"); + return Some(TestStore { + store, + directory: Some(directory), + postgres: None, + }); + }; + let Ok(url) = std::env::var("SQLBUILD_TEST_CONTROL_POSTGRES_URL") else { + assert!( + std::env::var_os("SQLBUILD_REQUIRE_POSTGRES_TESTS").is_none(), + "PostgreSQL test URL is required" + ); + eprintln!("PostgreSQL integration requires SQLBUILD_TEST_CONTROL_POSTGRES_URL"); + return None; + }; + let admin = sqlx_postgres::PgPool::connect(&url) + .await + .expect("connect to test PostgreSQL"); + let schema = format!("sqb_test_{}", Uuid::now_v7().simple()); + sqlx_core::query::query(&format!("CREATE SCHEMA {schema}")) + .execute(&admin) + .await + .expect("create isolated test schema"); + let separator = ['?', '&'][usize::from(url.contains('?'))]; + let store = open_store(&format!("{url}{separator}options=-csearch_path%3D{schema}")) + .await + .expect("open isolated PostgreSQL store"); + Some(TestStore { + store, + directory: None, + postgres: Some((admin, schema)), + }) +} + +pub(crate) async fn close_test_store(test_store: TestStore) { + close_store(test_store.store).await; + let Some((admin, schema)) = test_store.postgres else { + return; + }; + sqlx_core::query::query(&format!("DROP SCHEMA {schema} CASCADE")) + .execute(&admin) + .await + .expect("remove isolated test schema"); + admin.close().await; + drop(test_store.directory); +} + +pub(crate) async fn close_store(store: ControlStore) { + let ControlStore::Postgres(pool) = store else { + return; + }; + pool.close().await; +} + +pub(crate) async fn second_store(store: &ControlStore) -> ControlStore { + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + panic!("expected SQLite fixture"); + }; + return open_store(path.to_str().expect("UTF-8 test path")) + .await + .expect("open independent SQLite handle"); + }; + ControlStore::Postgres( + sqlx_postgres::PgPool::connect_with(pool.connect_options().as_ref().clone()) + .await + .expect("open independent PostgreSQL pool"), + ) +} diff --git a/crates/sqlbuild-orchestration/tests/store/test_types.rs b/crates/sqlbuild-orchestration/tests/store/test_types.rs new file mode 100644 index 0000000000..55a1a65092 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/test_types.rs @@ -0,0 +1,16 @@ +//! Store fixtures retain their isolated SQLite directory or PostgreSQL schema until cleanup. + +use sqlbuild_orchestration::store::models::ControlStore; + +#[derive(Clone, Copy, Debug)] +pub(crate) enum Backend { + Sqlite, + Postgres, +} + +#[derive(Debug)] +pub(crate) struct TestStore { + pub(crate) store: ControlStore, + pub(crate) directory: Option, + pub(crate) postgres: Option<(sqlx_postgres::PgPool, String)>, +} diff --git a/crates/sqlbuild-orchestration/tests/ticks.rs b/crates/sqlbuild-orchestration/tests/ticks.rs new file mode 100644 index 0000000000..00bfc0cbb6 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/ticks.rs @@ -0,0 +1,16 @@ +#[path = "ticks/fault_helpers.rs"] +mod fault_helpers; +#[path = "ticks/helpers.rs"] +mod helpers; +#[path = "store/helpers.rs"] +mod store_helpers; +#[path = "store/test_types.rs"] +mod store_test_types; +#[path = "ticks/test_contract.rs"] +mod test_contract; +#[path = "ticks/test_loop.rs"] +mod test_loop; +#[path = "ticks/test_triggers.rs"] +mod test_triggers; +#[path = "ticks/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/ticks/fault_helpers.rs b/crates/sqlbuild-orchestration/tests/ticks/fault_helpers.rs new file mode 100644 index 0000000000..180b1babce --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/ticks/fault_helpers.rs @@ -0,0 +1,66 @@ +use sqlbuild_orchestration::store::models::ControlStore; + +use crate::test_types::TickFaultTestCase; + +pub(crate) async fn install_fault(store: &ControlStore, point: &TickFaultTestCase) { + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + rusqlite::Connection::open(path) + .expect("fault connection") + .execute_batch(&format!( + "CREATE TRIGGER sqb_test_fault BEFORE {} ON {} {} BEGIN {}; END;", + point.operation, point.table, point.condition, point.sqlite_body + )) + .expect("install SQLite transaction fault"); + return; + }; + sqlx_core::query::query(&format!("CREATE FUNCTION sqb_test_fault() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN {}; RETURN NEW; END $$", point.postgres_body)) + .execute(pool).await.expect("install PostgreSQL fault function"); + sqlx_core::query::query(&format!("CREATE TRIGGER sqb_test_fault BEFORE {} ON {} FOR EACH ROW {} EXECUTE FUNCTION sqb_test_fault()", + point.operation, point.table, point.condition.replace("WHEN ", "WHEN (") + &[")", ""][usize::from(point.condition.is_empty())])) + .execute(pool).await.expect("install PostgreSQL transaction fault"); +} + +pub(crate) async fn remove_fault(store: &ControlStore, point: &TickFaultTestCase) { + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + rusqlite::Connection::open(path) + .expect("fault connection") + .execute_batch("DROP TRIGGER sqb_test_fault") + .expect("remove SQLite transaction fault"); + return; + }; + sqlx_core::query::query(&format!("DROP TRIGGER sqb_test_fault ON {}", point.table)) + .execute(pool) + .await + .expect("remove PostgreSQL fault trigger"); + sqlx_core::query::query("DROP FUNCTION sqb_test_fault()") + .execute(pool) + .await + .expect("remove PostgreSQL fault function"); +} + +pub(crate) async fn expire_lease(store: &ControlStore) { + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + rusqlite::Connection::open(path) + .expect("fault connection") + .execute_batch( + "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + ) + .expect("expire SQLite lease"); + return; + }; + sqlx_core::query::query( + "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + ) + .execute(pool) + .await + .expect("expire PostgreSQL lease"); +} diff --git a/crates/sqlbuild-orchestration/tests/ticks/helpers.rs b/crates/sqlbuild-orchestration/tests/ticks/helpers.rs new file mode 100644 index 0000000000..183b105cde --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/ticks/helpers.rs @@ -0,0 +1,240 @@ +use std::collections::BTreeMap; + +use serde_json::json; +use sqlbuild_orchestration::declarations::main::list_triggers::list_triggers; +use sqlbuild_orchestration::runs::main::list_runs::list_runs; +use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::scheduler::main::evaluate_trigger::evaluate_trigger; +use sqlbuild_orchestration::scheduler::main::list_ticks::list_ticks; +use sqlbuild_orchestration::scheduler::models::TickOutcome; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::main::create_location::create_location; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; +use uuid::Uuid; + +use crate::fault_helpers::{expire_lease, install_fault, remove_fault}; +use crate::store_helpers::{close_store, close_test_store, open_test_store, second_store}; +use crate::store_test_types::Backend; +use crate::test_types::TickFaultTestCase; + +pub(crate) fn snapshot(location_id: Uuid) -> SnapshotInput { + SnapshotInput { + location_id, + target: "development".into(), + generation: 1, + git_revision: None, + git_dirty: false, + git_author: None, + image_id: Some("sha256:synthetic-orders-image".into()), + environment_hash: "orders-environment".into(), + sqb_version: env!("CARGO_PKG_VERSION").into(), + nodes: BTreeMap::from([( + "orders".into(), + SnapshotNode { + kind: "model".into(), + definition: json!({"materialization": "table"}), + raw_sql: Some("select 1 as order_id".into()), + compiled_sql: Some("select 1 as order_id".into()), + python_source: None, + }, + )]), + documents: BTreeMap::from([ + ( + "jobs".into(), + json!({"orders": {"description": "Build orders", "selection": ["orders"]}}), + ), + ( + "triggers".into(), + json!({"daily_orders": {"description": "Refresh orders", "job_key": "orders", "starts_at": "2020-01-01T00:00:00Z", + "schedule": {"kind": "calendar", "times": ["2020-01-01T00:00:00Z", "2020-01-02T00:00:00Z", "2020-01-03T00:00:00Z", "2030-01-01T00:00:00Z"], "refresh_at": "2029-01-01T00:00:00Z"}}}), + ), + ]), + } +} + +pub(crate) async fn transaction_contract(backend: Backend) -> Option<(usize, usize)> { + let fixture = open_test_store(backend).await?; + let store = &fixture.store; + migrate_store(store).await.expect("migrate tick store"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("create location"); + publish_snapshot(store, &snapshot(location)) + .await + .expect("publish job and trigger"); + let trigger_id = list_triggers(store, location, "development") + .await + .expect("published triggers")[0] + .trigger_id; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("acquire lease") + .expect("elected"); + let second = second_store(store).await; + for point in [ + TickFaultTestCase { + description: "tick insertion", + table: "sqb_ticks", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected tick failure')", + postgres_body: "RAISE EXCEPTION 'injected tick failure'", + }, + TickFaultTestCase { + description: "requested event", + table: "recall_events", + operation: "INSERT", + condition: "WHEN NEW.event_type='run_requested'", + sqlite_body: "SELECT RAISE(ABORT,'injected requested failure')", + postgres_body: "RAISE EXCEPTION 'injected requested failure'", + }, + TickFaultTestCase { + description: "queued event", + table: "recall_events", + operation: "INSERT", + condition: "WHEN NEW.event_type='run_queued'", + sqlite_body: "SELECT RAISE(ABORT,'injected queued failure')", + postgres_body: "RAISE EXCEPTION 'injected queued failure'", + }, + TickFaultTestCase { + description: "run projection", + table: "sqb_runs", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected run failure')", + postgres_body: "RAISE EXCEPTION 'injected run failure'", + }, + TickFaultTestCase { + description: "node projection", + table: "sqb_run_nodes", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected node failure')", + postgres_body: "RAISE EXCEPTION 'injected node failure'", + }, + TickFaultTestCase { + description: "cursor commit", + table: "sqb_trigger_state", + operation: "UPDATE", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected cursor failure')", + postgres_body: "RAISE EXCEPTION 'injected cursor failure'", + }, + TickFaultTestCase { + description: "lease expires during append", + table: "sqb_runs", + operation: "INSERT", + condition: "", + sqlite_body: "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + postgres_body: "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + }, + ] { + install_fault(store, &point).await; + let failed = evaluate_trigger(store, &lease, trigger_id).await; + assert!(failed.is_err(), "{} must abort", point.description); + remove_fault(store, &point).await; + assert!( + list_ticks(&second, trigger_id, 100) + .await + .expect("rolled-back ticks") + .is_empty(), + "{}", + point.description + ); + assert!( + list_runs(&second, (location, "development"), 100) + .await + .expect("rolled-back runs") + .is_empty(), + "{}", + point.description + ); + assert!( + read_recall_events(&second, 0, 100) + .await + .expect("rolled-back events") + .is_empty(), + "{}", + point.description + ); + assert!( + list_triggers(&second, location, "development") + .await + .expect("rolled-back cursor")[0] + .last_tick_for + .is_none(), + "{}", + point.description + ); + } + let (left, right) = tokio::join!( + evaluate_trigger(store, &lease, trigger_id), + evaluate_trigger(&second, &lease, trigger_id) + ); + let first_count = left + .expect("first scheduler") + .expect("active trigger") + .actions + .len(); + let second_count = right + .expect("second scheduler") + .expect("active trigger") + .actions + .len(); + assert_eq!( + first_count + second_count, + 3, + "restarts never repeat committed ticks" + ); + let ticks = list_ticks(store, trigger_id, 100) + .await + .expect("committed ticks"); + let runs = list_runs(store, (location, "development"), 100) + .await + .expect("committed run"); + let events = read_recall_events(store, 0, 100) + .await + .expect("committed events"); + assert_eq!( + events + .iter() + .map(|row| row["event_type"].as_str().expect("event type")) + .collect::>(), + ["run_requested", "run_queued"] + ); + assert_eq!(ticks[0].outcome, TickOutcome::Requested); + assert!( + ticks[1..] + .iter() + .all(|tick| tick.outcome == TickOutcome::Covered + && tick.covered_by_run_id == Some(runs[0].run_id)) + ); + assert_eq!( + runs[0].last_event_storage_id, + events[1]["storage_id"].as_i64().expect("event cursor") + ); + expire_lease(store).await; + let successor = acquire_lease(&second, Uuid::now_v7()) + .await + .expect("takeover") + .expect("new leader"); + assert!(matches!( + evaluate_trigger(store, &lease, trigger_id).await, + Err(ControlError::Fenced) + )); + assert!( + evaluate_trigger(&second, &successor, trigger_id) + .await + .expect("failover catch-up") + .expect("active trigger") + .actions + .is_empty() + ); + let counts = (ticks.len(), runs.len()); + close_store(second).await; + close_test_store(fixture).await; + Some(counts) +} diff --git a/crates/sqlbuild-orchestration/tests/ticks/test_contract.rs b/crates/sqlbuild-orchestration/tests/ticks/test_contract.rs new file mode 100644 index 0000000000..7d733d45be --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/ticks/test_contract.rs @@ -0,0 +1,32 @@ +use crate::helpers::transaction_contract; +use crate::store_helpers::run; +use crate::store_test_types::Backend; +use crate::test_types::TickTransactionTestCase; + +#[test] +fn given_two_schedulers_and_faults_between_writes_when_restarting_then_ticks_runs_recall_and_cursor_commit_together() + { + let test_cases = [ + TickTransactionTestCase { + description: "SQLite fenced tick transaction", + backend: Backend::Sqlite, + expected_ticks: 3, + expected_runs: 1, + }, + TickTransactionTestCase { + description: "PostgreSQL fenced tick transaction", + backend: Backend::Postgres, + expected_ticks: 3, + expected_runs: 1, + }, + ]; + for test_case in test_cases { + let observed = run(transaction_contract(test_case.backend)); + assert!( + observed + .is_none_or(|counts| counts == (test_case.expected_ticks, test_case.expected_runs)), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/tests/ticks/test_loop.rs b/crates/sqlbuild-orchestration/tests/ticks/test_loop.rs new file mode 100644 index 0000000000..34b691b8fa --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/ticks/test_loop.rs @@ -0,0 +1,98 @@ +use std::time::Duration; + +use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler; +use sqlbuild_orchestration::scheduler::main::scheduler_cycle::scheduler_cycle; +use sqlbuild_orchestration::scheduler::models::SchedulerState; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::store::main::create_location::create_location; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use uuid::Uuid; + +use crate::fault_helpers::expire_lease; +use crate::helpers::snapshot; +use crate::store_helpers::{close_store, close_test_store, open_test_store, run, second_store}; +use crate::store_test_types::Backend; +use crate::test_types::TickTransactionTestCase; + +#[test] +fn given_embedded_and_standalone_schedulers_when_taking_over_then_only_one_drains_due_work_and_idle_shutdown_is_prompt() + { + let test_cases = [ + TickTransactionTestCase { + description: "SQLite scheduler loop", + backend: Backend::Sqlite, + expected_ticks: 3, + expected_runs: 1, + }, + TickTransactionTestCase { + description: "PostgreSQL scheduler loop", + backend: Backend::Postgres, + expected_ticks: 3, + expected_runs: 1, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate loop store"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + publish_snapshot(store, &snapshot(location)) + .await + .expect("publish due schedule"); + let second = second_store(store).await; + let mut first_state = SchedulerState::new(Uuid::now_v7()); + let mut second_state = SchedulerState::new(Uuid::now_v7()); + let first = scheduler_cycle(store, &mut first_state) + .await + .expect("initial cycle"); + assert!(first.leader, "{}", test_case.description); + assert_eq!(first.evaluated, 1); + assert_eq!(first.ticks, test_case.expected_ticks); + assert_eq!(first.wait_for, Duration::from_secs(5)); + let follower = scheduler_cycle(&second, &mut second_state) + .await + .expect("follower cycle"); + assert!(!follower.leader); + assert_eq!(follower.ticks, 0); + expire_lease(store).await; + let successor = scheduler_cycle(&second, &mut second_state) + .await + .expect("takeover cycle"); + assert!(successor.leader); + assert_eq!(successor.ticks, 0, "takeover never repeats durable work"); + assert!( + !scheduler_cycle(store, &mut first_state) + .await + .expect("fenced cycle") + .leader + ); + let runs = sqlbuild_orchestration::runs::main::list_runs::list_runs( + store, + (location, "development"), + 100, + ) + .await + .expect("queued run"); + assert_eq!(runs.len(), test_case.expected_runs); + let (stop, shutdown) = tokio::sync::watch::channel(false); + let (_changed, changes) = tokio::sync::watch::channel(()); + let mut running = Box::pin(run_scheduler(&second, second_state, shutdown, changes)); + tokio::select! { + result = &mut running => panic!("loop ended before shutdown: {result:?}"), + _ = tokio::time::sleep(Duration::from_millis(50)) => {}, + } + stop.send(true).expect("signal shutdown"); + tokio::time::timeout(Duration::from_secs(2), running) + .await + .expect("idle shutdown does not wait for the five-second timer") + .expect("scheduler exits cleanly"); + close_store(second).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/ticks/test_triggers.rs b/crates/sqlbuild-orchestration/tests/ticks/test_triggers.rs new file mode 100644 index 0000000000..4429dae9fb --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/ticks/test_triggers.rs @@ -0,0 +1,195 @@ +use sqlbuild_orchestration::declarations::main::change_trigger::change_trigger; +use sqlbuild_orchestration::declarations::main::list_trigger_events::list_trigger_events; +use sqlbuild_orchestration::declarations::main::list_triggers::list_triggers; +use sqlbuild_orchestration::declarations::models::{ + TriggerChange, TriggerChangeRequest, TriggerStatus, +}; +use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::scheduler::main::evaluate_trigger::evaluate_trigger; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::store::main::create_location::create_location; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use uuid::Uuid; + +use crate::fault_helpers::{install_fault, remove_fault}; +use crate::helpers::snapshot; +use crate::store_helpers::{close_test_store, open_test_store, run}; +use crate::store_test_types::Backend; +use crate::test_types::{TickFaultTestCase, TriggerControlsTestCase}; + +#[test] +fn given_operator_changes_when_retried_and_republished_then_facts_and_effective_state_remain_consistent() + { + let test_cases = [ + TriggerControlsTestCase { + description: "SQLite operator facts", + backend: Backend::Sqlite, + expected_events: 5, + }, + TriggerControlsTestCase { + description: "PostgreSQL operator facts", + backend: Backend::Postgres, + expected_events: 5, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate trigger store"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + publish_snapshot(store, &input) + .await + .expect("initial publish"); + let trigger_id = list_triggers(store, location, "development") + .await + .expect("trigger")[0] + .trigger_id; + let pause = TriggerChangeRequest { + event_id: Uuid::now_v7(), + trigger_id, + actor: "orders-operator".into(), + reason: "maintenance".into(), + change: TriggerChange::Pause, + }; + assert!(change_trigger(store, &pause).await.expect("pause")); + let paused = list_triggers(store, location, "development") + .await + .expect("paused trigger") + .remove(0); + assert_eq!( + paused.status, + TriggerStatus::Paused, + "{}", + test_case.description + ); + assert!(paused.next_due_at.is_none()); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("election") + .expect("lease"); + assert!( + evaluate_trigger(store, &lease, trigger_id) + .await + .expect("paused evaluation") + .is_none() + ); + let enable = TriggerChangeRequest { + event_id: Uuid::now_v7(), + change: TriggerChange::Enable, + ..pause.clone() + }; + let fault = TickFaultTestCase { + description: "operator event and projection", + table: "sqb_trigger_state", + operation: "UPDATE", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected state failure')", + postgres_body: "RAISE EXCEPTION 'injected state failure'", + }; + install_fault(store, &fault).await; + assert!(change_trigger(store, &enable).await.is_err()); + remove_fault(store, &fault).await; + assert_eq!( + list_trigger_events(store, trigger_id, 100) + .await + .expect("rolled-back fact") + .len(), + 1 + ); + assert!(change_trigger(store, &enable).await.expect("retry enable")); + assert!( + !change_trigger(store, &pause) + .await + .expect("retry old pause is inert") + ); + let conflict = TriggerChangeRequest { + change: TriggerChange::ResetDefault, + ..pause.clone() + }; + assert!( + change_trigger(store, &conflict).await.is_err(), + "same event ID cannot change content" + ); + input.generation = 2; + input + .documents + .get_mut("triggers") + .expect("trigger document")["daily_orders"]["default_status"] = + serde_json::json!("paused"); + publish_snapshot(store, &input) + .await + .expect("publish a changed code default"); + assert_eq!( + list_triggers(store, location, "development") + .await + .expect("retained override")[0] + .status, + TriggerStatus::Running + ); + let reset = TriggerChangeRequest { + event_id: Uuid::now_v7(), + change: TriggerChange::ResetDefault, + ..pause.clone() + }; + change_trigger(store, &reset) + .await + .expect("restore new code default"); + assert_eq!( + list_triggers(store, location, "development") + .await + .expect("default restored")[0] + .status, + TriggerStatus::Paused + ); + let time = chrono::DateTime::parse_from_rfc3339("2020-01-02T00:00:00Z") + .expect("cursor") + .to_utc(); + let cursor = TriggerChangeRequest { + event_id: Uuid::now_v7(), + change: TriggerChange::SetCursor { time }, + ..pause.clone() + }; + change_trigger(store, &cursor) + .await + .expect("set durable cursor"); + assert_eq!( + list_triggers(store, location, "development") + .await + .expect("cursor set")[0] + .last_tick_for, + Some(time) + ); + let reset_cursor = TriggerChangeRequest { + event_id: Uuid::now_v7(), + change: TriggerChange::ResetCursor, + ..pause.clone() + }; + change_trigger(store, &reset_cursor) + .await + .expect("reset cursor"); + assert!( + list_triggers(store, location, "development") + .await + .expect("cursor reset")[0] + .last_tick_for + .is_none() + ); + let events = list_trigger_events(store, trigger_id, 100) + .await + .expect("operator history"); + assert_eq!(events.len(), test_case.expected_events); + assert!( + events + .iter() + .all(|event| event.request_json.actor == "orders-operator") + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/ticks/test_types.rs b/crates/sqlbuild-orchestration/tests/ticks/test_types.rs new file mode 100644 index 0000000000..05ee9cd1d5 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/ticks/test_types.rs @@ -0,0 +1,23 @@ +use crate::store_test_types::Backend; + +pub(crate) struct TickTransactionTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_ticks: usize, + pub(crate) expected_runs: usize, +} + +pub(crate) struct TickFaultTestCase { + pub(crate) description: &'static str, + pub(crate) table: &'static str, + pub(crate) operation: &'static str, + pub(crate) condition: &'static str, + pub(crate) sqlite_body: &'static str, + pub(crate) postgres_body: &'static str, +} + +pub(crate) struct TriggerControlsTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_events: usize, +} diff --git a/crates/sqlbuild-recall/Cargo.toml b/crates/sqlbuild-recall/Cargo.toml index 5738b05067..cfabda69e4 100644 --- a/crates/sqlbuild-recall/Cargo.toml +++ b/crates/sqlbuild-recall/Cargo.toml @@ -9,6 +9,7 @@ rust-version.workspace = true [dependencies] blake3.workspace = true +chrono.workspace = true rusqlite.workspace = true serde_json.workspace = true sha2.workspace = true diff --git a/crates/sqlbuild-recall/src/clickhouse/_helpers/literals.rs b/crates/sqlbuild-recall/src/clickhouse/_helpers/literals.rs index 9863904c39..784b5e1cd3 100644 --- a/crates/sqlbuild-recall/src/clickhouse/_helpers/literals.rs +++ b/crates/sqlbuild-recall/src/clickhouse/_helpers/literals.rs @@ -1,6 +1,7 @@ //! ClickHouse literals and key expressions for the store's own queries. use crate::clickhouse::constants::TIMESTAMP_PATTERN; +use crate::records::_helpers::rows; use crate::records::main::format_timestamp::format_timestamp; use crate::records::main::parse_timestamp::parse_timestamp; use crate::records::main::timestamp_micros::timestamp_micros; @@ -62,10 +63,6 @@ pub(crate) fn key_text(value: Option<&RecallValue>, column_type: ColumnType) -> Some(RecallValue::Text(text)) if column_type == ColumnType::Timestamp => { parse_timestamp(text).map_or_else(|| text.clone(), format_timestamp) } - Some(RecallValue::Text(text)) => text.clone(), - Some(RecallValue::Integer(number)) => number.to_string(), - Some(RecallValue::Real(number)) => number.to_string(), - Some(RecallValue::Blob(_)) => "".to_owned(), - Some(RecallValue::Null) | None => String::new(), + _ => rows::key_text(value), } } diff --git a/crates/sqlbuild-recall/src/records/_helpers/rows.rs b/crates/sqlbuild-recall/src/records/_helpers/rows.rs index 5ad2768b37..f5b60dc1e5 100644 --- a/crates/sqlbuild-recall/src/records/_helpers/rows.rs +++ b/crates/sqlbuild-recall/src/records/_helpers/rows.rs @@ -23,6 +23,17 @@ pub(crate) fn boolean(value: bool) -> RecallValue { RecallValue::Integer(i64::from(value)) } +/// The human-readable part of a conflict key, without dialect-specific timestamp normalization. +pub(crate) fn key_text(value: Option<&RecallValue>) -> String { + match value { + Some(RecallValue::Text(text)) => text.clone(), + Some(RecallValue::Integer(number)) => number.to_string(), + Some(RecallValue::Real(number)) => number.to_string(), + Some(RecallValue::Blob(_)) => "".to_owned(), + Some(RecallValue::Null) | None => String::new(), + } +} + /// `values` plus the envelope columns and `content_hash`; an artifact's digest covers only its content, since every invocation that sees it writes the same row. pub(crate) fn finish_row( table: Table, diff --git a/crates/sqlbuild-recall/src/schema/_helpers/mod.rs b/crates/sqlbuild-recall/src/schema/_helpers/mod.rs index 275383fa1c..e40b47cb19 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/mod.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/mod.rs @@ -1,4 +1,6 @@ pub(crate) mod clickhouse_views; +pub(crate) mod postgres_views; +pub(crate) mod relational_tables; +pub(crate) mod relational_views; pub(crate) mod rendering; -pub(crate) mod sqlite_views; pub(crate) mod version_row; diff --git a/crates/sqlbuild-recall/src/schema/_helpers/postgres_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/postgres_views.rs new file mode 100644 index 0000000000..98515a0e19 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/_helpers/postgres_views.rs @@ -0,0 +1,91 @@ +//! PostgreSQL expressions for the shared relational Recall view definitions. + +use crate::schema::_helpers::relational_views::relational_view_bodies; + +fn json_fields(mut body: String) -> String { + for (field, cast) in [ + ("audit_name", ""), + ("audit_definition_name", ""), + ("target_name", ""), + ("target_database", ""), + ("target_schema", ""), + ("attached_target_kind", ""), + ("attached_target_name", ""), + ("attached_column_name", ""), + ("severity", ""), + ("violation_count", "::BIGINT"), + ("measured_value", "::DOUBLE PRECISION"), + ("evaluation_mode", ""), + ("execution_error", ""), + ("evidence_count", "::BIGINT"), + ("evidence_truncated", "::BOOLEAN::INTEGER"), + ("sample_digest", ""), + ("error_message", ""), + ("signal_number", "::BIGINT"), + ("cost_status", ""), + ] { + body = body.replace( + &format!("json_extract(payload, '$.{field}')"), + &format!("(payload ->> '{field}'){cast}"), + ); + } + for field in ["name", "job_name", "run_id", "retry_number"] { + let cast = if field == "retry_number" { + "::BIGINT" + } else { + "" + }; + body = body.replace( + &format!("json_extract(context.external_context, '$.integration.{field}')"), + &format!("(context.external_context #>> '{{integration,{field}}}'){cast}"), + ); + } + body.replace( + "json_valid(context.external_context)", + "context.external_context IS NOT NULL", + ) + .replace( + "json_array_length(context.selection)", + "jsonb_array_length(context.selection)", + ) +} + +fn postgres_body(body: String) -> String { + let mut body = json_fields(body) + .replace("strftime('%Y-%m-%dT%H:%M:%fZ', 'now')", "CURRENT_TIMESTAMP") + .replace("MAX(artifact.digest IS NOT NULL)", "MAX(CAST(artifact.digest IS NOT NULL AS INTEGER))") + .replace("artifact.digest IS NOT NULL AS available", "CAST(artifact.digest IS NOT NULL AS INTEGER) AS available") + .replace("MAX(\n COALESCE(received.max_sequence", "GREATEST(\n COALESCE(received.max_sequence") + .replace("MAX(COALESCE(received.queue_drops", "GREATEST(COALESCE(received.queue_drops") + .replace("COALESCE(context.recall_filter_active, 0)", "COALESCE(CAST(context.recall_filter_active AS INTEGER), 0)") + .replace("COALESCE(delivery.flush_complete, 0)", "COALESCE(CAST(delivery.flush_complete AS INTEGER), 0)") + // PostgreSQL text cannot hold NUL or malformed UTF-8. Keep display bytes until the reader decodes them. + .replace("CAST(data AS TEXT)", "data") + .replace("CAST(artifact.content AS TEXT)", "artifact.content") + .replace(")\nGROUP BY", ") AS source_rows\nGROUP BY") + .replace(")\n GROUP BY", ") AS source_rows\n GROUP BY") + .replace(")\nWHERE position", ") AS source_rows\nWHERE position") + .replace("ROUND(\n (unixepoch(last_heartbeat_at, 'subsec') - unixepoch(started_at, 'subsec')) * 1000.0, 3\n )", + "CAST(ROUND(CAST(EXTRACT(EPOCH FROM (last_heartbeat_at - started_at)) * 1000 AS NUMERIC), 3) AS DOUBLE PRECISION)"); + for expression in [ + "byte_count", + "CASE WHEN stream = 'stdout' THEN byte_count ELSE 0 END", + "CASE WHEN stream = 'stderr' THEN byte_count ELSE 0 END", + "dropped_before", + "COALESCE(omitted_sequences, 1)", + "COALESCE(omitted_bytes, 0)", + ] { + body = body.replace( + &format!("SUM({expression})"), + &format!("CAST(SUM({expression}) AS BIGINT)"), + ); + } + body +} + +pub(crate) fn postgres_view_bodies() -> Vec<(&'static str, String)> { + relational_view_bodies(true) + .into_iter() + .map(|(name, body)| (name, postgres_body(body))) + .collect() +} diff --git a/crates/sqlbuild-recall/src/schema/_helpers/relational_tables.rs b/crates/sqlbuild-recall/src/schema/_helpers/relational_tables.rs new file mode 100644 index 0000000000..3930030bd3 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/_helpers/relational_tables.rs @@ -0,0 +1,69 @@ +//! One logical schema and index list for SQLite and PostgreSQL. + +use crate::schema::_helpers::rendering::{all_columns, sqlite_identifier, sqlite_type}; +use crate::schema::models::{ColumnType, TableSpec}; + +pub(crate) fn postgres_type(column: ColumnType) -> &'static str { + match column { + ColumnType::Text => "TEXT", + ColumnType::Timestamp => "TIMESTAMPTZ", + ColumnType::Json => "JSONB", + ColumnType::Integer => "BIGINT", + ColumnType::Boolean => "BOOLEAN", + ColumnType::Real => "DOUBLE PRECISION", + ColumnType::Blob => "BYTEA", + } +} + +pub(crate) fn relational_table(spec: &TableSpec, postgres: bool, control: bool) -> Vec { + let mut definitions: Vec = all_columns(spec) + .iter() + .map(|column| { + let not_null = if column.nullable { "" } else { " NOT NULL" }; + let kind = if postgres { + postgres_type(column.column_type) + } else if control && column.column_type == ColumnType::Json { + "JSON" + } else { + sqlite_type(column.column_type) + }; + format!(" {} {kind}{not_null}", sqlite_identifier(column.name)) + }) + .collect(); + if control && spec.name == "recall_events" { + definitions.push(if postgres { + " storage_id BIGINT GENERATED ALWAYS AS IDENTITY".into() + } else { + " storage_id INTEGER PRIMARY KEY AUTOINCREMENT".into() + }); + } + let mut key: Vec<&str> = spec.key.to_vec(); + let partition = if postgres && spec.name != "recall_artifacts" { + if !key.contains(&spec.expiry_column) { + key.push(spec.expiry_column); + } + format!( + " PARTITION BY RANGE ({})", + sqlite_identifier(spec.expiry_column) + ) + } else { + String::new() + }; + let key: Vec = key.into_iter().map(sqlite_identifier).collect(); + definitions.push(format!(" UNIQUE ({})", key.join(", "))); + let mut statements = vec![format!( + "CREATE TABLE IF NOT EXISTS {} (\n{}\n){partition}", + sqlite_identifier(spec.name), + definitions.join(",\n") + )]; + for (suffix, columns) in spec.sqlite_indexes { + let columns: Vec = columns.iter().map(|name| sqlite_identifier(name)).collect(); + statements.push(format!( + "CREATE INDEX IF NOT EXISTS {} ON {} ({})", + sqlite_identifier(&format!("{}_{suffix}", spec.name)), + sqlite_identifier(spec.name), + columns.join(", ") + )); + } + statements +} diff --git a/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs similarity index 92% rename from crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs rename to crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs index 8d4e1c2f81..015c491b5e 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs @@ -1,4 +1,4 @@ -//! The bodies of the supported views in SQLite. +//! Shared relational view definitions, with explicit dialect expressions for binary transcripts. use crate::schema::constants::OUTPUT_TAIL_RECORDS; @@ -212,18 +212,29 @@ SELECT END AS complete FROM counted"; -const OUTPUT_TRANSCRIPTS: &str = "SELECT - invocation_id, - MIN(invocation_started_at) AS invocation_started_at, - CAST(COALESCE( - group_concat(CASE WHEN stream = 'stdout' THEN data END, x'' ORDER BY sequence), x'' - ) AS BLOB) AS stdout, - CAST(COALESCE( - group_concat(CASE WHEN stream = 'stderr' THEN data END, x'' ORDER BY sequence), x'' - ) AS BLOB) AS stderr, - CAST(COALESCE(group_concat(data, x'' ORDER BY sequence), x'') AS BLOB) AS combined -FROM recall_output -GROUP BY invocation_id"; +fn output_transcripts(postgres: bool) -> String { + let columns: Vec = [ + ("CASE WHEN stream = 'stdout' THEN data END", "stdout"), + ("CASE WHEN stream = 'stderr' THEN data END", "stderr"), + ("data", "combined"), + ] + .into_iter() + .map(|(expression, alias)| { + let value = if postgres { + format!("COALESCE(string_agg({expression}, ''::BYTEA ORDER BY sequence), ''::BYTEA)") + } else { + format!( + "CAST(COALESCE(group_concat({expression}, x'' ORDER BY sequence), x'') AS BLOB)" + ) + }; + format!(" {value} AS {alias}") + }) + .collect(); + format!( + "SELECT invocation_id, MIN(invocation_started_at) AS invocation_started_at,\n{}\nFROM recall_output GROUP BY invocation_id", + columns.join(",\n") + ) +} const EVENTS: &str = "SELECT * FROM recall_events"; @@ -414,15 +425,15 @@ WHERE position <= {OUTPUT_TAIL_RECORDS}" ) } -/// Each supported view's name and SQLite body, in creation order. -pub(crate) fn sqlite_view_bodies() -> Vec<(&'static str, String)> { +/// Each supported view's name and relational body, in creation order. +pub(crate) fn relational_view_bodies(postgres: bool) -> Vec<(&'static str, String)> { vec![ ("invocations", invocations()), ("statements", STATEMENTS.to_owned()), ("audit_results", AUDIT_RESULTS.to_owned()), ("output_invocations", OUTPUT_INVOCATIONS.to_owned()), ("output_tail", output_tail()), - ("output_transcripts", OUTPUT_TRANSCRIPTS.to_owned()), + ("output_transcripts", output_transcripts(postgres)), ("events", EVENTS.to_owned()), ("diagnostics", DIAGNOSTICS.to_owned()), ("artifacts", ARTIFACTS.to_owned()), diff --git a/crates/sqlbuild-recall/src/schema/main/control_table_statements.rs b/crates/sqlbuild-recall/src/schema/main/control_table_statements.rs new file mode 100644 index 0000000000..db06b995c6 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/main/control_table_statements.rs @@ -0,0 +1,14 @@ +//! Control stores preserve Recall's logical columns and add a committed-order event cursor. + +use crate::schema::_helpers::relational_tables::relational_table; +use crate::schema::constants::ALL_TABLES; +use crate::schema::main::table_spec::table_spec; + +/// PostgreSQL partitions invocation facts; globally deduplicated artifacts remain unpartitioned. +#[must_use] +pub fn control_table_statements(postgres: bool) -> Vec { + ALL_TABLES + .iter() + .flat_map(|table| relational_table(table_spec(*table), postgres, true)) + .collect() +} diff --git a/crates/sqlbuild-recall/src/schema/main/control_view_statements.rs b/crates/sqlbuild-recall/src/schema/main/control_view_statements.rs new file mode 100644 index 0000000000..7b27ec1a2f --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/main/control_view_statements.rs @@ -0,0 +1,34 @@ +//! Control stores expose Recall's supported views on both engines. + +use crate::schema::_helpers::postgres_views::postgres_view_bodies; +use crate::schema::_helpers::rendering::sqlite_identifier; +use crate::schema::main::view_statements::view_statements; +use crate::schema::models::{DdlOptions, Dialect}; + +/// PostgreSQL validates each view at migration time; SQLite retains its existing view contract. +#[must_use] +pub fn control_view_statements(postgres: bool) -> Vec { + if postgres { + postgres_view_bodies() + .into_iter() + .map(|(name, body)| { + format!( + "CREATE OR REPLACE VIEW {} AS\n{body}", + sqlite_identifier(name) + ) + }) + .collect() + } else { + view_statements( + Dialect::Sqlite, + &DdlOptions { + database: None, + retention_days: 30, + cluster: None, + }, + ) + .into_iter() + .map(|(_, statement)| statement) + .collect() + } +} diff --git a/crates/sqlbuild-recall/src/schema/main/mod.rs b/crates/sqlbuild-recall/src/schema/main/mod.rs index e2251bd838..879cacea23 100644 --- a/crates/sqlbuild-recall/src/schema/main/mod.rs +++ b/crates/sqlbuild-recall/src/schema/main/mod.rs @@ -1,8 +1,11 @@ pub mod add_column_statement; pub mod clickhouse_relation; +pub mod control_table_statements; +pub mod control_view_statements; pub mod create_database_statement; pub mod ddl_script; pub mod ddl_statements; +pub mod postgres_partition_statement; pub mod render_script; pub mod retention_fix_statements; pub mod schema_age; diff --git a/crates/sqlbuild-recall/src/schema/main/postgres_partition_statement.rs b/crates/sqlbuild-recall/src/schema/main/postgres_partition_statement.rs new file mode 100644 index 0000000000..971a3ae792 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/main/postgres_partition_statement.rs @@ -0,0 +1,24 @@ +//! Monthly partition bounds are calculated from a typed date, never interpolated user SQL. + +use chrono::{Datelike, Months, NaiveDate}; + +use crate::schema::_helpers::rendering::sqlite_identifier; +use crate::schema::main::table_spec::table_spec; +use crate::schema::models::Table; + +/// A replay uses the same timestamp partition and therefore the same deterministic unique key. +#[must_use] +pub fn postgres_partition_statement(table: Table, date: NaiveDate) -> Option { + if table == Table::Artifacts { + return None; + } + let first = date.with_day(1)?; + let next = first.checked_add_months(Months::new(1))?; + let spec = table_spec(table); + let partition = format!("{}_{}", spec.name, first.format("%Y%m")); + Some(format!( + "CREATE TABLE IF NOT EXISTS {} PARTITION OF {} FOR VALUES FROM ('{first}T00:00:00Z') TO ('{next}T00:00:00Z')", + sqlite_identifier(&partition), + sqlite_identifier(spec.name) + )) +} diff --git a/crates/sqlbuild-recall/src/schema/main/table_statements.rs b/crates/sqlbuild-recall/src/schema/main/table_statements.rs index 621f8e1f12..7829997b9b 100644 --- a/crates/sqlbuild-recall/src/schema/main/table_statements.rs +++ b/crates/sqlbuild-recall/src/schema/main/table_statements.rs @@ -1,48 +1,14 @@ //! `table_statements`: idempotent `CREATE TABLE` and index statements of every Recall table. +use crate::schema::_helpers::relational_tables::relational_table; use crate::schema::_helpers::rendering::{ all_columns, clickhouse_engine, clickhouse_identifier, clickhouse_table, clickhouse_ttl, - clickhouse_type, on_cluster, sqlite_identifier, sqlite_type, + clickhouse_type, on_cluster, }; use crate::schema::constants::ALL_TABLES; use crate::schema::main::table_spec::table_spec; use crate::schema::models::{DdlOptions, Dialect, TableSpec}; -fn sqlite_table(spec: &TableSpec) -> Vec { - let mut definitions: Vec = all_columns(spec) - .iter() - .map(|column| { - let not_null: &str = if column.nullable { "" } else { " NOT NULL" }; - format!( - " {} {}{not_null}", - sqlite_identifier(column.name), - sqlite_type(column.column_type) - ) - }) - .collect(); - let key: Vec = spec - .key - .iter() - .map(|name| sqlite_identifier(name)) - .collect(); - definitions.push(format!(" UNIQUE ({})", key.join(", "))); - let mut statements: Vec = vec![format!( - "CREATE TABLE IF NOT EXISTS {} (\n{}\n)", - sqlite_identifier(spec.name), - definitions.join(",\n") - )]; - for (suffix, columns) in spec.sqlite_indexes { - let columns: Vec = columns.iter().map(|name| sqlite_identifier(name)).collect(); - statements.push(format!( - "CREATE INDEX IF NOT EXISTS {} ON {} ({})", - sqlite_identifier(&format!("{}_{suffix}", spec.name)), - sqlite_identifier(spec.name), - columns.join(", ") - )); - } - statements -} - fn clickhouse_table_statement(spec: &TableSpec, options: &DdlOptions) -> String { let columns: Vec = all_columns(spec) .iter() @@ -76,7 +42,7 @@ pub fn table_statements(dialect: Dialect, options: &DdlOptions) -> Vec { .flat_map(|table| { let spec: &TableSpec = table_spec(*table); match dialect { - Dialect::Sqlite => sqlite_table(spec), + Dialect::Sqlite => relational_table(spec, false, false), Dialect::ClickHouse => vec![clickhouse_table_statement(spec, options)], } }) diff --git a/crates/sqlbuild-recall/src/schema/main/view_statements.rs b/crates/sqlbuild-recall/src/schema/main/view_statements.rs index ce34504a3c..8527aae75c 100644 --- a/crates/sqlbuild-recall/src/schema/main/view_statements.rs +++ b/crates/sqlbuild-recall/src/schema/main/view_statements.rs @@ -1,15 +1,15 @@ //! `view_statements`: the statements that create the supported views. use crate::schema::_helpers::clickhouse_views::clickhouse_view_bodies; +use crate::schema::_helpers::relational_views::relational_view_bodies; use crate::schema::_helpers::rendering::{clickhouse_table, on_cluster, sqlite_identifier}; -use crate::schema::_helpers::sqlite_views::sqlite_view_bodies; use crate::schema::models::{DdlOptions, Dialect}; /// Each view's name and creation statement. SQLite views are created only when missing; the store drops them first when the schema version rises. ClickHouse views are replaced. #[must_use] pub fn view_statements(dialect: Dialect, options: &DdlOptions) -> Vec<(&'static str, String)> { match dialect { - Dialect::Sqlite => sqlite_view_bodies() + Dialect::Sqlite => relational_view_bodies(false) .into_iter() .map(|(name, body)| { ( diff --git a/crates/sqlbuild-recall/src/store/_helpers/mod.rs b/crates/sqlbuild-recall/src/store/_helpers/mod.rs index 3344c9a183..fab996484f 100644 --- a/crates/sqlbuild-recall/src/store/_helpers/mod.rs +++ b/crates/sqlbuild-recall/src/store/_helpers/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod column_types; pub(crate) mod identity; pub(crate) mod quarantine; +pub(crate) mod relational; pub(crate) mod rows; pub(crate) mod schema; pub(crate) mod sqlite; diff --git a/crates/sqlbuild-recall/src/store/_helpers/relational.rs b/crates/sqlbuild-recall/src/store/_helpers/relational.rs new file mode 100644 index 0000000000..becce06752 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/_helpers/relational.rs @@ -0,0 +1,12 @@ +//! Typed placeholders shared by native SQLite and control-plane transactions. + +use crate::schema::_helpers::relational_tables::postgres_type; +use crate::schema::models::ColumnType; + +pub(crate) fn placeholder(index: usize, kind: ColumnType, postgres: bool) -> String { + if postgres { + format!("CAST(${index} AS {})", postgres_type(kind)) + } else { + format!("${index}") + } +} diff --git a/crates/sqlbuild-recall/src/store/_helpers/rows.rs b/crates/sqlbuild-recall/src/store/_helpers/rows.rs index 5fed0c3900..a253f2ce78 100644 --- a/crates/sqlbuild-recall/src/store/_helpers/rows.rs +++ b/crates/sqlbuild-recall/src/store/_helpers/rows.rs @@ -5,6 +5,7 @@ use std::collections::{HashMap, HashSet}; use rusqlite::{Connection, OptionalExtension}; +use crate::records::_helpers::rows::key_text; use crate::records::main::conflict_row::conflict_row; use crate::records::models::{RecallRow, RecallValue, RowEnvelope}; use crate::schema::main::table_spec::table_spec; @@ -12,18 +13,11 @@ use crate::schema::models::Table; use crate::store::_helpers::schema::stored_columns; use crate::store::_helpers::sqlite::{SqlValue, checked}; use crate::store::errors::StoreError; +use crate::store::main::conflict_evidence::conflict_evidence; +use crate::store::main::relational_insert::relational_insert; +use crate::store::main::relational_lookup::relational_lookup; use crate::store::models::{ConflictEvidence, InsertOutcome}; -fn key_text(value: Option<&RecallValue>) -> String { - match value { - Some(RecallValue::Text(text)) => text.clone(), - Some(RecallValue::Integer(number)) => number.to_string(), - Some(RecallValue::Real(number)) => number.to_string(), - Some(RecallValue::Blob(_)) => "".to_owned(), - Some(RecallValue::Null) | None => String::new(), - } -} - fn text_of(row: &RecallRow, column: &str) -> String { key_text(row.value(column)) } @@ -34,28 +28,17 @@ fn write_row( (row, columns): (&RecallRow, &HashSet), ingested_at: &str, ) -> Result, StoreError> { - let spec = table_spec(row.table); let ingested: RecallValue = RecallValue::Text(ingested_at.to_owned()); - let stored: Vec<&(&str, RecallValue)> = row + let insert = relational_insert((row, columns), &ingested, false); + let parameters: Vec> = insert .values .iter() - .filter(|(name, _)| columns.contains(*name)) + .map(|(value, _)| SqlValue(value)) .collect(); - let mut names: Vec<&str> = stored.iter().map(|(name, _)| *name).collect(); - names.push("ingested_at"); - let placeholders: Vec = (1..=names.len()).map(|index| format!("?{index}")).collect(); - let sql = format!( - "INSERT INTO \"{}\" ({}) VALUES ({}) ON CONFLICT DO NOTHING", - spec.name, - names.join(", "), - placeholders.join(", ") - ); - let parameters: Vec> = stored - .iter() - .map(|(_, value)| SqlValue(value)) - .chain(std::iter::once(SqlValue(&ingested))) - .collect(); - let mut statement = checked(connection.prepare_cached(&sql), "prepare Recall insert")?; + let mut statement = checked( + connection.prepare_cached(&insert.sql), + "prepare Recall insert", + )?; let changed: usize = checked( statement.execute(rusqlite::params_from_iter(parameters.iter())), "write Recall rows", @@ -63,24 +46,16 @@ fn write_row( if changed > 0 { return Ok(None); } - let condition: Vec = spec - .key - .iter() - .enumerate() - .map(|(index, name)| format!("{name} = ?{}", index + 1)) - .collect(); - let lookup = format!( - "SELECT content_hash FROM \"{}\" WHERE {}", - spec.name, - condition.join(" AND ") - ); - let null = RecallValue::Null; - let key_values: Vec> = spec - .key + let lookup = relational_lookup(row, false, false); + let key_values: Vec> = lookup + .values .iter() - .map(|name| SqlValue(row.value(name).unwrap_or(&null))) + .map(|(value, _)| SqlValue(value)) .collect(); - let mut statement = checked(connection.prepare_cached(&lookup), "prepare Recall lookup")?; + let mut statement = checked( + connection.prepare_cached(&lookup.sql), + "prepare Recall lookup", + )?; let existing: Option = checked( statement .query_row(rusqlite::params_from_iter(key_values.iter()), |found| { @@ -89,21 +64,7 @@ fn write_row( .optional(), "read stored Recall row", )?; - let rejected: &str = row.content_hash().unwrap_or_default(); - match existing { - Some(existing) if existing != rejected => Ok(Some(ConflictEvidence { - table: row.table, - record_key: spec - .key - .iter() - .map(|name| key_text(row.value(name))) - .collect::>() - .join(":"), - existing_sha256: existing, - rejected_sha256: rejected.to_owned(), - })), - _ => Ok(None), - } + Ok(existing.and_then(|existing| conflict_evidence(row, &existing))) } /// Write `rows` on the current transaction; conflicts become `recall_conflicts` rows. diff --git a/crates/sqlbuild-recall/src/store/main/conflict_evidence.rs b/crates/sqlbuild-recall/src/store/main/conflict_evidence.rs new file mode 100644 index 0000000000..a343086c2e --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/conflict_evidence.rs @@ -0,0 +1,23 @@ +//! Duplicate rows preserve the first content and expose the rejected digest. + +use crate::records::_helpers::rows::key_text; +use crate::records::models::RecallRow; +use crate::schema::main::table_spec::table_spec; +use crate::store::models::ConflictEvidence; + +/// Evidence only when a stored row's content differs, independent of the storage driver. +#[must_use] +pub fn conflict_evidence(row: &RecallRow, existing: &str) -> Option { + let rejected = row.content_hash().unwrap_or_default(); + (existing != rejected).then(|| ConflictEvidence { + table: row.table, + record_key: table_spec(row.table) + .key + .iter() + .map(|name| key_text(row.value(name))) + .collect::>() + .join(":"), + existing_sha256: existing.to_owned(), + rejected_sha256: rejected.to_owned(), + }) +} diff --git a/crates/sqlbuild-recall/src/store/main/mod.rs b/crates/sqlbuild-recall/src/store/main/mod.rs index 69ccb1cf7e..78f9b89028 100644 --- a/crates/sqlbuild-recall/src/store/main/mod.rs +++ b/crates/sqlbuild-recall/src/store/main/mod.rs @@ -1,5 +1,6 @@ pub mod checkpoint_sqlite_store; pub mod chosen_invocation; +pub mod conflict_evidence; pub mod describe_store; pub mod ensure_sqlite_schema; pub mod insert_rows; @@ -10,5 +11,7 @@ pub mod open_sqlite_store; pub mod quarantine_store; pub mod query_store; pub mod read_output_transcript; +pub mod relational_insert; +pub mod relational_lookup; pub mod resolve_invocation; pub mod store_location_problem; diff --git a/crates/sqlbuild-recall/src/store/main/relational_insert.rs b/crates/sqlbuild-recall/src/store/main/relational_insert.rs new file mode 100644 index 0000000000..ef63916e65 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/relational_insert.rs @@ -0,0 +1,48 @@ +//! One insert contract for standalone SQLite Recall and transactional control stores. + +use std::collections::HashSet; + +use crate::records::models::{RecallRow, RecallValue}; +use crate::schema::_helpers::rendering::sqlite_identifier; +use crate::schema::main::table_columns::table_columns; +use crate::schema::main::table_spec::table_spec; +use crate::schema::models::ColumnType; +use crate::store::_helpers::relational::placeholder; +use crate::store::models::RecallStatement; + +/// Ignore fields absent from an older additive schema; ingestion time is assigned by the store. +#[must_use] +pub fn relational_insert<'a>( + (row, columns): (&'a RecallRow, &HashSet), + ingested_at: &'a RecallValue, + postgres: bool, +) -> RecallStatement<'a> { + let metadata = table_columns(row.table); + let mut names = Vec::new(); + let mut values = Vec::new(); + for (name, value) in &row.values { + if let Some(column) = metadata.iter().find(|column| column.name == *name) + && columns.contains(*name) + && *name != "ingested_at" + { + names.push(sqlite_identifier(name)); + values.push((value, column.column_type)); + } + } + names.push("ingested_at".into()); + values.push((ingested_at, ColumnType::Timestamp)); + let placeholders: Vec = values + .iter() + .enumerate() + .map(|(index, (_, kind))| placeholder(index + 1, *kind, postgres)) + .collect(); + RecallStatement { + sql: format!( + "INSERT INTO {} ({}) VALUES ({}) ON CONFLICT DO NOTHING", + sqlite_identifier(table_spec(row.table).name), + names.join(", "), + placeholders.join(", ") + ), + values, + } +} diff --git a/crates/sqlbuild-recall/src/store/main/relational_lookup.rs b/crates/sqlbuild-recall/src/store/main/relational_lookup.rs new file mode 100644 index 0000000000..d1291fdb4c --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/relational_lookup.rs @@ -0,0 +1,48 @@ +//! Duplicate detection uses the same stable keys as physical unique constraints. + +use crate::records::models::{RecallRow, RecallValue}; +use crate::schema::_helpers::rendering::sqlite_identifier; +use crate::schema::main::table_columns::table_columns; +use crate::schema::main::table_spec::table_spec; +use crate::schema::models::Table; +use crate::store::_helpers::relational::placeholder; +use crate::store::models::RecallStatement; + +/// Control event lookups return their durable cursor too; standalone stores have no cursor column. +#[must_use] +pub fn relational_lookup(row: &RecallRow, postgres: bool, control: bool) -> RecallStatement<'_> { + let spec = table_spec(row.table); + let mut keys = spec.key.to_vec(); + if postgres && row.table != Table::Artifacts && !keys.contains(&spec.expiry_column) { + keys.push(spec.expiry_column); + } + let metadata = table_columns(row.table); + let mut values = Vec::new(); + let mut conditions = Vec::new(); + for key in keys { + let kind = metadata + .iter() + .find(|column| column.name == key) + .expect("Recall key columns belong to their schema") + .column_type; + values.push((row.value(key).unwrap_or(&RecallValue::Null), kind)); + conditions.push(format!( + "{} = {}", + sqlite_identifier(key), + placeholder(values.len(), kind, postgres) + )); + } + let cursor = if control && row.table == Table::Events { + ", storage_id" + } else { + "" + }; + RecallStatement { + sql: format!( + "SELECT content_hash{cursor} FROM {} WHERE {}", + sqlite_identifier(spec.name), + conditions.join(" AND ") + ), + values, + } +} diff --git a/crates/sqlbuild-recall/src/store/models.rs b/crates/sqlbuild-recall/src/store/models.rs index a92d4ead30..61b2f27614 100644 --- a/crates/sqlbuild-recall/src/store/models.rs +++ b/crates/sqlbuild-recall/src/store/models.rs @@ -4,7 +4,14 @@ use std::path::PathBuf; use std::time::{Duration, SystemTime}; use crate::records::models::{RecallValue, RowEnvelope}; -use crate::schema::models::Table; +use crate::schema::models::{ColumnType, Table}; + +/// A relational insert or lookup and its borrowed, typed parameters. +#[derive(Clone, Debug, PartialEq)] +pub struct RecallStatement<'a> { + pub sql: String, + pub values: Vec<(&'a RecallValue, ColumnType)>, +} /// One open store file. Only one thread uses a store. #[derive(Debug)] diff --git a/fensu.toml b/fensu.toml index 7e2ae7561a..2f97466395 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1617,6 +1617,7 @@ crate_names = [ "sqlbuild-discovery", "sqlbuild-host", "sqlbuild-model-config", + "sqlbuild-orchestration", "sqlbuild-project", "sqlbuild-pysource", "sqlbuild-python", From a07e6ffa155d3044d1211ada247631d4b7e7f474 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 30 Oct 2026 23:11:15 +0000 Subject: [PATCH 02/59] feat: checkpoint pinned worker execution verification Add claim-scoped execution inputs and native snapshot drift validation. Incomplete orchestration backend checkpoint; worker launch and the remaining control-plane integration are not implemented yet. Targeted SQLite/PostgreSQL tests pass; full static readiness is pending. Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../src/api/_helpers/runs.rs | 21 ++- .../src/api/main/serve_api.rs | 1 + .../sqlbuild-orchestration/src/api/models.rs | 8 +- .../src/runs/main/mod.rs | 1 + .../src/runs/main/request_run.rs | 6 +- .../src/runs/main/run_execution.rs | 58 ++++++ .../sqlbuild-orchestration/src/runs/models.rs | 24 ++- .../src/scheduler/models.rs | 2 +- .../src/snapshots/_helpers/content.rs | 28 ++- .../src/snapshots/main/mod.rs | 1 + .../src/snapshots/main/verify_execution.rs | 50 ++++++ .../src/snapshots/models.rs | 7 + .../tests/api/test_contract.rs | 170 +++++++++++++++++- .../tests/api/test_types.rs | 9 + 14 files changed, 363 insertions(+), 23 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/runs/main/run_execution.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/main/verify_execution.rs diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs b/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs index 419ec2681a..ef93a4a15a 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs @@ -10,9 +10,28 @@ use crate::api::models::{HeartbeatResponse, MutationResponse}; use crate::credentials::models::{CredentialScope, TokenIdentity}; use crate::runs::main::change_run::change_run; use crate::runs::main::heartbeat_run::heartbeat_run; -use crate::runs::models::{RunAuthority, RunChangeRequest}; +use crate::runs::main::run_execution::run_execution; +use crate::runs::models::{RunAuthority, RunChangeRequest, RunExecution}; use crate::store::models::ControlStore; +#[utoipa::path(get, path = "/api/runs/{run_id}/execution", params(("run_id" = Uuid, Path)), + responses((status = 200, body = RunExecution), (status = 401, description = "Inactive claim or invalid credential"), + (status = 403, description = "Run not authorized")), security(("bearer" = [])))] +pub(crate) async fn execution( + State(store): State, + Extension(identity): Extension, + Path(run_id): Path, +) -> Result, ApiError> { + let claim_id = require_worker(&identity, run_id)?; + let execution = run_execution(&store, (run_id, claim_id)) + .await? + .ok_or_else(|| ApiError { + status: StatusCode::UNAUTHORIZED, + message: "run claim is no longer active".into(), + })?; + Ok(Json(execution)) +} + #[utoipa::path(post, path = "/api/runs/{run_id}/events", params(("run_id" = Uuid, Path)), request_body = RunChangeRequest, responses((status = 200, body = MutationResponse), (status = 400, body = crate::api::models::ApiErrorBody), diff --git a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs index 51d3b48cf4..99927066e8 100644 --- a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs +++ b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs @@ -22,6 +22,7 @@ pub async fn serve_api( "/api/locations/{location_id}/targets/{target}/snapshots", get(snapshots::current).post(snapshots::publish), ) + .route("/api/runs/{run_id}/execution", get(runs::execution)) .route("/api/runs/{run_id}/events", post(runs::report)) .route("/api/runs/{run_id}/heartbeat", post(runs::heartbeat)) .layer(DefaultBodyLimit::max(64 * 1024 * 1024)) diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index 3c3590a1ba..e7db400f75 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -5,7 +5,13 @@ use serde::{Deserialize, Serialize}; use crate::api::_helpers::{runs, snapshots}; #[derive(Debug, utoipa::OpenApi)] -#[openapi(paths(snapshots::publish, snapshots::current, runs::report, runs::heartbeat))] +#[openapi(paths( + snapshots::publish, + snapshots::current, + runs::execution, + runs::report, + runs::heartbeat +))] pub(crate) struct ControlApi; #[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] diff --git a/crates/sqlbuild-orchestration/src/runs/main/mod.rs b/crates/sqlbuild-orchestration/src/runs/main/mod.rs index e3e4c9a4d4..c19b71681a 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/mod.rs @@ -5,3 +5,4 @@ pub mod list_run_nodes; pub mod list_runs; pub(crate) mod queue_run; pub mod request_run; +pub mod run_execution; diff --git a/crates/sqlbuild-orchestration/src/runs/main/request_run.rs b/crates/sqlbuild-orchestration/src/runs/main/request_run.rs index eb55625e5d..82616a6ee2 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/request_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/request_run.rs @@ -41,11 +41,13 @@ pub async fn request_run(store: &ControlStore, request: &RunRequest) -> Result Result Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query( + transaction, + "SELECT r.request_json, s.environment_hash, s.sqb_version FROM sqb_runs r \ + JOIN sqb_run_liveness l ON l.run_id=r.run_id \ + JOIN sqb_snapshots s ON s.snapshot_id=r.snapshot_id \ + WHERE r.run_id=$1 AND l.worker_instance=$2", + vec![Parameter::Id(run_id), Parameter::Id(claim_id)], + ) + .await?; + let Some(row) = rows.into_iter().next() else { + commit(transaction).await?; + return Ok(None); + }; + let metadata: ExecutionMetadata = decode_row(row)?; + let (transaction, rows) = query( + transaction, + "SELECT node_key, node_hash FROM sqb_snapshot_nodes WHERE snapshot_id=$1", + vec![Parameter::Id(metadata.request_json.snapshot_id)], + ) + .await?; + let node_hashes: BTreeMap = rows + .into_iter() + .map(|row| { + let fields: BTreeMap = decode_row(row)?; + Ok((fields["node_key"].clone(), fields["node_hash"].clone())) + }) + .collect::>()?; + commit(transaction).await?; + Ok(Some(RunExecution { + claim: RunClaim { + claim_id, + request: metadata.request_json, + }, + environment_hash: metadata.environment_hash, + sqb_version: metadata.sqb_version, + node_hashes, + })) +} diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index 270381a7f4..636dc6891a 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -6,7 +6,7 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use uuid::Uuid; -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(rename_all = "snake_case")] pub enum RunKind { Job, @@ -25,7 +25,7 @@ impl RunKind { } } -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(rename_all = "snake_case")] pub enum RunOrigin { Manual, @@ -64,7 +64,7 @@ pub enum RunStatus { } /// The authorized selection and pinned execution inputs are immutable once requested. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] pub struct RunRequest { pub run_id: Uuid, pub location_id: Uuid, @@ -141,12 +141,28 @@ pub struct RunNodeRecord { } /// The database owns the claim before a launcher sees the pinned request. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] pub struct RunClaim { pub claim_id: Uuid, pub request: RunRequest, } +/// Workers compile locally and compare against this immutable snapshot, not the current deploy. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct RunExecution { + pub claim: RunClaim, + pub environment_hash: String, + pub sqb_version: String, + pub node_hashes: BTreeMap, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct ExecutionMetadata { + pub(crate) request_json: RunRequest, + pub(crate) environment_hash: String, + pub(crate) sqb_version: String, +} + /// Terminal outcomes cannot be confused with a request to change an active status. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(rename_all = "snake_case")] diff --git a/crates/sqlbuild-orchestration/src/scheduler/models.rs b/crates/sqlbuild-orchestration/src/scheduler/models.rs index 7c3dbfde37..a3a0c2ccad 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/models.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/models.rs @@ -55,7 +55,7 @@ pub enum CatchUp { All { max_runs: std::num::NonZeroUsize }, } -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(rename_all = "snake_case")] pub enum Overlap { #[default] diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/content.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/content.rs index 4ff3f7b1f3..3ce8bbd7c3 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/_helpers/content.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/content.rs @@ -7,7 +7,7 @@ use flate2::{Compression, read::GzDecoder, write::GzEncoder}; use serde_json::Value; use sqlbuild_recall::records::main::sha256_hex::sha256_hex; -use crate::snapshots::models::{PreparedNode, SnapshotNode}; +use crate::snapshots::models::{NodeContent, PreparedNode, SnapshotNode}; use crate::store::errors::ControlError; fn canonical(value: &Value) -> Value { @@ -42,18 +42,15 @@ pub(crate) fn decompress(value: &Value) -> Result, ControlError> { Ok(output) } -pub(crate) fn prepare_node(node: &SnapshotNode) -> Result { - let mut sources: BTreeMap<&'static str, (String, Vec)> = BTreeMap::new(); +pub(crate) fn node_content(node: &SnapshotNode) -> Result, ControlError> { + let mut sources: BTreeMap<&'static str, (String, &str)> = BTreeMap::new(); for (kind, text) in [ ("raw_sql", &node.raw_sql), ("compiled_sql", &node.compiled_sql), ("python_source", &node.python_source), ] { if let Some(text) = text { - sources.insert( - kind, - (sha256_hex(text.as_bytes()), compress(text.as_bytes())?), - ); + sources.insert(kind, (sha256_hex(text.as_bytes()), text)); } } let source_hashes: BTreeMap<_, _> = sources @@ -63,10 +60,23 @@ pub(crate) fn prepare_node(node: &SnapshotNode) -> Result Result { + let content = node_content(node)?; + let sources = content + .sources + .into_iter() + .map(|(kind, (hash, text))| Ok((kind, (hash, compress(text.as_bytes())?)))) + .collect::>()?; Ok(PreparedNode { kind: node.kind.clone(), - hash: sha256_hex(&definition), - definition: compress(&definition)?, + hash: sha256_hex(&content.definition), + definition: compress(&content.definition)?, sources, }) } diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs index da2b1bcca6..88c2bf1eba 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs @@ -1,2 +1,3 @@ pub mod current_snapshot; pub mod publish_snapshot; +pub mod verify_execution; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/verify_execution.rs b/crates/sqlbuild-orchestration/src/snapshots/main/verify_execution.rs new file mode 100644 index 0000000000..d4d2c00236 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/verify_execution.rs @@ -0,0 +1,50 @@ +//! Refuse execution when a worker's local compilation differs from the pinned publication. + +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; + +use crate::runs::models::RunExecution; +use crate::snapshots::_helpers::content::node_content; +use crate::snapshots::models::SnapshotInput; +use crate::store::errors::ControlError; + +/// Display documents and publication generations are not execution inputs. +pub fn verify_execution( + input: &SnapshotInput, + expected: &RunExecution, +) -> Result<(), ControlError> { + let request = &expected.claim.request; + if input.location_id != request.location_id || input.target != request.target { + return Err(ControlError::Invalid( + "worker location or target differs from the pinned run".into(), + )); + } + if input.image_id != request.image_id { + return Err(ControlError::Invalid( + "worker image differs from the pinned run".into(), + )); + } + if input.environment_hash != expected.environment_hash { + return Err(ControlError::Invalid( + "worker environment differs from the pinned snapshot".into(), + )); + } + if input.sqb_version != expected.sqb_version { + return Err(ControlError::Invalid( + "worker SQLBuild version differs from the pinned snapshot".into(), + )); + } + if input.nodes.len() != expected.node_hashes.len() { + return Err(ControlError::Invalid( + "worker node set differs from the pinned snapshot".into(), + )); + } + for (key, node) in &input.nodes { + let hash = sha256_hex(&node_content(node)?.definition); + if expected.node_hashes.get(key) != Some(&hash) { + return Err(ControlError::Invalid( + "worker node content differs from the pinned snapshot".into(), + )); + } + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/models.rs b/crates/sqlbuild-orchestration/src/snapshots/models.rs index 6f1e5da193..20d474c674 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/models.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/models.rs @@ -62,6 +62,13 @@ pub struct CurrentSnapshot { pub changes: Vec, } +/// Canonical identity is shared by publication and worker verification; only publication compresses. +#[derive(Debug)] +pub(crate) struct NodeContent<'a> { + pub(crate) definition: Vec, + pub(crate) sources: BTreeMap<&'static str, (String, &'a str)>, +} + #[derive(Debug)] pub(crate) struct PreparedNode { pub(crate) kind: String, diff --git a/crates/sqlbuild-orchestration/tests/api/test_contract.rs b/crates/sqlbuild-orchestration/tests/api/test_contract.rs index ef7d0d0902..53d9903ae5 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contract.rs @@ -10,9 +10,13 @@ use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope} use sqlbuild_orchestration::runs::main::change_run::change_run; use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; use sqlbuild_orchestration::runs::main::list_runs::list_runs; -use sqlbuild_orchestration::runs::models::{RunAuthority, RunChange, RunChangeRequest, RunStatus}; +use sqlbuild_orchestration::runs::main::request_run::request_run; +use sqlbuild_orchestration::runs::models::{ + RunAuthority, RunChange, RunChangeRequest, RunExecution, RunStatus, +}; use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::snapshots::models::SnapshotRecord; +use sqlbuild_orchestration::snapshots::main::verify_execution::verify_execution; +use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotRecord}; use sqlbuild_orchestration::store::main::create_location::create_location; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use uuid::Uuid; @@ -21,13 +25,26 @@ use crate::helpers::{start_api, stop_api}; use crate::run_helpers::{queue, request, snapshot}; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; +use crate::test_types::ApiContractTestCase; #[test] fn given_a_real_http_server_when_locations_and_workers_report_then_scopes_retries_and_terminal_expiry_are_enforced() { - for backend in [Backend::Sqlite, Backend::Postgres] { + let test_cases = [ + ApiContractTestCase { + description: "SQLite worker and location HTTP contract", + backend: Backend::Sqlite, + expected_status: RunStatus::Cancelled, + }, + ApiContractTestCase { + description: "PostgreSQL worker and location HTTP contract", + backend: Backend::Postgres, + expected_status: RunStatus::Cancelled, + }, + ]; + for test_case in test_cases { run(async { - let Some(fixture) = open_test_store(backend).await else { + let Some(fixture) = open_test_store(test_case.backend).await else { return; }; let store = &fixture.store; @@ -88,6 +105,7 @@ fn given_a_real_http_server_when_locations_and_workers_report_then_scopes_retrie ); assert!(openapi["components"]["schemas"]["SnapshotInput"].is_object()); assert!(openapi["components"]["schemas"]["RunChangeRequest"].is_object()); + assert!(openapi["components"]["schemas"]["RunExecution"].is_object()); let input = snapshot(location_id); let response = client .post(&path) @@ -132,6 +150,18 @@ fn given_a_real_http_server_when_locations_and_workers_report_then_scopes_retrie published.snapshot_id.to_string() ); assert_eq!(current["nodes"]["orders"]["compiled_sql"], "select 1 as id"); + for image_id in [None, Some("sha256:unpublished-orders-image".into())] { + let mut mismatched = + request((location_id, published.snapshot_id), 1, 0, &["orders"]); + mismatched.image_id = image_id; + assert!(request_run(store, &mismatched).await.is_err()); + } + assert!( + list_runs(store, (location_id, "development"), 100) + .await + .expect("rejected requests do not queue") + .is_empty() + ); queue( store, &request((location_id, published.snapshot_id), 1, 0, &["orders"]), @@ -159,6 +189,109 @@ fn given_a_real_http_server_when_locations_and_workers_report_then_scopes_retrie .expect("worker credential"); let events_path = format!("{}/api/runs/{run_id}/events", api.url); let heartbeat_path = format!("{}/api/runs/{run_id}/heartbeat", api.url); + let execution_path = format!("{}/api/runs/{run_id}/execution", api.url); + let response = client + .get(&execution_path) + .bearer_auth(&worker.bearer) + .send() + .await + .expect("worker execution contract"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.headers()["cache-control"], "no-store"); + let execution: RunExecution = response.json().await.expect("pinned execution"); + assert_eq!(execution.claim, claim); + assert_eq!(execution.environment_hash, input.environment_hash); + assert_eq!(execution.sqb_version, input.sqb_version); + assert_eq!( + execution.node_hashes.keys().collect::>(), + input.nodes.keys().collect::>(), + "dependency verification includes nodes outside the run selection" + ); + verify_execution(&input, &execution).expect("matching worker compilation"); + for field in ["target", "image_id", "environment_hash", "sqb_version"] { + let mut changed = json!(input); + changed[field] = json!("different"); + let changed: SnapshotInput = + serde_json::from_value(changed).expect("changed worker inputs"); + assert!(verify_execution(&changed, &execution).is_err(), "{field}"); + } + for field in [ + "kind", + "definition", + "raw_sql", + "compiled_sql", + "python_source", + ] { + let mut changed = json!(input); + changed["nodes"]["inventory"][field] = json!("different"); + let changed: SnapshotInput = + serde_json::from_value(changed).expect("changed dependency"); + assert!(verify_execution(&changed, &execution).is_err(), "{field}"); + } + let mut renamed = input.clone(); + let node = renamed.nodes.remove("inventory").expect("dependency"); + renamed.nodes.insert("renamed_inventory".into(), node); + assert!(verify_execution(&renamed, &execution).is_err()); + renamed.nodes.remove("renamed_inventory"); + assert!(verify_execution(&renamed, &execution).is_err()); + let mut display_only = input.clone(); + display_only.generation += 1; + display_only.git_author = Some("Example Author".into()); + display_only.git_dirty = true; + display_only + .documents + .insert("columns".into(), json!({"orders": ["id"]})); + verify_execution(&display_only, &execution) + .expect("display metadata does not change executable code"); + for (url, token) in [ + (execution_path.clone(), &location_token.bearer), + ( + format!("{}/api/runs/{}/execution", api.url, Uuid::now_v7()), + &worker.bearer, + ), + ] { + assert_eq!( + client + .get(url) + .bearer_auth(token) + .send() + .await + .expect("execution scope denial") + .status(), + StatusCode::FORBIDDEN + ); + } + let mut redeploy = input.clone(); + redeploy.generation += 1; + redeploy.environment_hash = "updated-orders-environment".into(); + redeploy.image_id = Some("sha256:updated-orders-image".into()); + redeploy.sqb_version = "99.0.0".into(); + redeploy + .nodes + .get_mut("orders") + .expect("orders") + .compiled_sql = Some("select 2 as id".into()); + assert_eq!( + client + .post(&path) + .bearer_auth(&location_token.bearer) + .json(&redeploy) + .send() + .await + .expect("publish redeploy") + .status(), + StatusCode::OK + ); + let still_pinned: RunExecution = client + .get(&execution_path) + .bearer_auth(&worker.bearer) + .send() + .await + .expect("execution after redeploy") + .json() + .await + .expect("old execution contract"); + assert_eq!(still_pinned, execution); let started = RunChangeRequest { event_id: Uuid::now_v7(), run_id, @@ -304,13 +437,40 @@ fn given_a_real_http_server_when_locations_and_workers_report_then_scopes_retrie .status(), StatusCode::UNAUTHORIZED ); + assert_eq!( + client + .get(&execution_path) + .bearer_auth(&worker.bearer) + .send() + .await + .expect("terminal execution credential rejected") + .status(), + StatusCode::UNAUTHORIZED + ); assert_eq!( list_runs(store, (location_id, "development"), 100) .await .expect("run projection")[0] .status, - RunStatus::Cancelled + test_case.expected_status, + "{}", + test_case.description ); + redeploy.generation += 1; + redeploy.image_id = None; + let local: SnapshotRecord = client + .post(&path) + .bearer_auth(&location_token.bearer) + .json(&redeploy) + .send() + .await + .expect("publish local snapshot without a container image") + .json() + .await + .expect("local snapshot response"); + let mut local_request = request((location_id, local.snapshot_id), 2, 0, &["orders"]); + local_request.image_id = None; + queue(store, &local_request).await; revoke_token(store, &audit, location_token.identity.token_id) .await .expect("revoke location token"); diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index d62cdc25ce..c3231b7060 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -1,5 +1,14 @@ +use sqlbuild_orchestration::runs::models::RunStatus; use sqlbuild_orchestration::store::errors::ControlError; +use crate::store_test_types::Backend; + +pub(crate) struct ApiContractTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_status: RunStatus, +} + pub(crate) struct TestApi { pub(crate) url: String, pub(crate) shutdown: tokio::sync::oneshot::Sender<()>, From ff4477858b1b2a3bb0a53489d3bbc3327bbebca3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 30 Oct 2026 23:27:25 +0000 Subject: [PATCH 03/59] feat: checkpoint authenticated worker lifecycle client Verify pinned inputs before start and expose retryable run reports and cancellation heartbeats over the authenticated API. Disable redirects for credentialed requests and cover the lifecycle against SQLite and PostgreSQL. Keep the orchestration PR explicitly incomplete pending actual launchers and the remaining backend scope. Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- crates/sqlbuild-orchestration/Cargo.toml | 2 +- crates/sqlbuild-orchestration/src/lib.rs | 1 + .../src/scheduler/_helpers/expansion.rs | 29 +- .../src/scheduler/models.rs | 3 + .../src/worker/_helpers/client.rs | 49 +++ .../src/worker/_helpers/mod.rs | 2 + .../src/worker/_helpers/response.rs | 15 + .../src/worker/errors.rs | 15 + .../src/worker/main/connect_worker.rs | 54 +++ .../src/worker/main/mod.rs | 2 + .../src/worker/main/start_worker.rs | 28 ++ .../sqlbuild-orchestration/src/worker/mod.rs | 4 + .../src/worker/models.rs | 11 + crates/sqlbuild-orchestration/tests/api.rs | 2 + .../tests/api/test_types.rs | 5 + .../tests/api/test_worker.rs | 356 ++++++++++++++++++ 16 files changed, 562 insertions(+), 16 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/worker/_helpers/client.rs create mode 100644 crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/worker/_helpers/response.rs create mode 100644 crates/sqlbuild-orchestration/src/worker/errors.rs create mode 100644 crates/sqlbuild-orchestration/src/worker/main/connect_worker.rs create mode 100644 crates/sqlbuild-orchestration/src/worker/main/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/worker/main/start_worker.rs create mode 100644 crates/sqlbuild-orchestration/src/worker/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/worker/models.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/test_worker.rs diff --git a/crates/sqlbuild-orchestration/Cargo.toml b/crates/sqlbuild-orchestration/Cargo.toml index 7e9e614a65..4e31942af6 100644 --- a/crates/sqlbuild-orchestration/Cargo.toml +++ b/crates/sqlbuild-orchestration/Cargo.toml @@ -14,6 +14,7 @@ chrono-tz.workspace = true croner.workspace = true flate2.workspace = true getrandom.workspace = true +reqwest.workspace = true rusqlite.workspace = true semver.workspace = true serde.workspace = true @@ -29,7 +30,6 @@ uuid.workspace = true [dev-dependencies] proptest.workspace = true -reqwest.workspace = true tempfile.workspace = true [lints] diff --git a/crates/sqlbuild-orchestration/src/lib.rs b/crates/sqlbuild-orchestration/src/lib.rs index d1bc86c3ef..89689306a8 100644 --- a/crates/sqlbuild-orchestration/src/lib.rs +++ b/crates/sqlbuild-orchestration/src/lib.rs @@ -9,3 +9,4 @@ pub mod runs; pub mod scheduler; pub mod snapshots; pub mod store; +pub mod worker; diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs index a3af3bf96e..598797bfc5 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs @@ -9,30 +9,29 @@ use crate::scheduler::constants::MAX_TICKS_PER_BATCH; use crate::scheduler::main::next_cron_time::next_cron_time; use crate::scheduler::main::parse_cron::parse_cron; use crate::scheduler::main::previous_cron_time::previous_cron_time; -use crate::scheduler::models::CatchUp; +use crate::scheduler::models::{CatchUp, ScheduleExpansion}; use crate::store::errors::ControlError; pub(crate) fn expand_schedule( trigger: &TriggerDefinition, from: DateTime, now: DateTime, -) -> Result<(Vec>, Option>), ControlError> { - let Schedule::Cron { - expression, - timezone, - } = &trigger.schedule - else { - let Schedule::Calendar { times, .. } = &trigger.schedule else { - unreachable!() - }; - return Ok(( - times.range((Excluded(from), Unbounded)).copied().collect(), - None, - )); +) -> Result { + let (expression, timezone) = match &trigger.schedule { + Schedule::Cron { + expression, + timezone, + } => (expression, timezone), + Schedule::Calendar { times, .. } => { + return Ok(( + times.range((Excluded(from), Unbounded)).copied().collect(), + None, + )); + } }; let cron = parse_cron(expression, timezone)?; let latest = trigger.catch_up == CatchUp::Latest; - let mut due = Vec::new(); + let mut due: Vec> = Vec::new(); let mut cursor = from; for _ in 0..(MAX_TICKS_PER_BATCH - usize::from(latest)) { cursor = next_cron_time(&cron, cursor)?; diff --git a/crates/sqlbuild-orchestration/src/scheduler/models.rs b/crates/sqlbuild-orchestration/src/scheduler/models.rs index a3a0c2ccad..b219340219 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/models.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/models.rs @@ -46,6 +46,9 @@ pub struct CronSchedule { pub(crate) timezone: chrono_tz::Tz, } +/// Expanded wall-clock times and the first omitted time, if the batch was bounded. +pub(crate) type ScheduleExpansion = (Vec>, Option>); + #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum CatchUp { diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/client.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/client.rs new file mode 100644 index 0000000000..cbac454f9b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/client.rs @@ -0,0 +1,49 @@ +//! A worker's authenticated HTTP requests carry only its own run and claim. + +use uuid::Uuid; + +use crate::api::models::{HeartbeatResponse, MutationResponse}; +use crate::runs::models::{RunChange, RunChangeRequest, RunExecution}; +use crate::worker::_helpers::response::response; +use crate::worker::errors::WorkerError; +use crate::worker::models::WorkerClient; + +impl WorkerClient { + /// Fetch the launcher's immutable request, not the location's latest deployment. + pub async fn execution(&self) -> Result { + let execution: RunExecution = + response(self.client.get(format!("{}execution", self.run_url))).await?; + if execution.claim.request.run_id != self.run_id + || execution.claim.claim_id != self.claim_id + { + return Err(WorkerError::Configuration( + "execution response does not match the worker's run and claim", + )); + } + Ok(execution) + } + + /// Call with the same event ID and payload when retrying an uncertain response. + pub async fn report(&self, event_id: Uuid, change: RunChange) -> Result { + let request = RunChangeRequest { + event_id, + run_id: self.run_id, + claim_id: Some(self.claim_id), + change, + }; + let result: MutationResponse = response( + self.client + .post(format!("{}events", self.run_url)) + .json(&request), + ) + .await?; + Ok(result.applied) + } + + /// Run independently from warehouse execution; cancellation is a cooperative request. + pub async fn heartbeat(&self) -> Result { + let result: HeartbeatResponse = + response(self.client.post(format!("{}heartbeat", self.run_url))).await?; + Ok(result.cancel_requested) + } +} diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs new file mode 100644 index 0000000000..57d94ff592 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod client; +pub(crate) mod response; diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/response.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/response.rs new file mode 100644 index 0000000000..e4da90cca5 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/response.rs @@ -0,0 +1,15 @@ +//! Do not echo response bodies: server diagnostics can contain user configuration. + +use serde::de::DeserializeOwned; + +use crate::worker::errors::WorkerError; + +pub(crate) async fn response( + request: reqwest::RequestBuilder, +) -> Result { + let response = request.send().await.map_err(reqwest::Error::without_url)?; + if !response.status().is_success() { + return Err(WorkerError::Rejected(response.status())); + } + Ok(response.json().await.map_err(reqwest::Error::without_url)?) +} diff --git a/crates/sqlbuild-orchestration/src/worker/errors.rs b/crates/sqlbuild-orchestration/src/worker/errors.rs new file mode 100644 index 0000000000..30a6a18f8e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/errors.rs @@ -0,0 +1,15 @@ +//! Transport failures are distinct from rejected pinned execution inputs. + +#[derive(Debug, thiserror::Error)] +pub enum WorkerError { + #[error("invalid worker connection: {0}")] + Configuration(&'static str), + #[error("worker API request failed: {0}")] + Transport(#[from] reqwest::Error), + #[error("worker API rejected the request with HTTP {0}")] + Rejected(reqwest::StatusCode), + #[error(transparent)] + Verification(#[from] crate::store::errors::ControlError), + #[error("run cancellation was requested before execution")] + Cancelled, +} diff --git a/crates/sqlbuild-orchestration/src/worker/main/connect_worker.rs b/crates/sqlbuild-orchestration/src/worker/main/connect_worker.rs new file mode 100644 index 0000000000..f1c543d1dc --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/main/connect_worker.rs @@ -0,0 +1,54 @@ +//! Bound worker requests have deadlines and must never follow credential-bearing redirects. + +use std::time::Duration; + +use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderValue}; +use uuid::Uuid; + +use crate::worker::errors::WorkerError; +use crate::worker::models::WorkerClient; + +fn worker_url(server_url: &str, run_id: Uuid) -> Result { + let mut url = reqwest::Url::parse(server_url) + .map_err(|_| WorkerError::Configuration("server URL must be an absolute HTTP URL"))?; + if !matches!(url.scheme(), "http" | "https") + || url.host_str().is_none() + || !url.username().is_empty() + || url.password().is_some() + || url.query().is_some() + || url.fragment().is_some() + { + return Err(WorkerError::Configuration( + "server URL must use HTTP or HTTPS without credentials, query or fragment", + )); + } + let prefix = url.path().trim_end_matches('/'); + url.set_path(&format!("{prefix}/api/runs/{run_id}/")); + Ok(url) +} + +/// Connect to an API origin or path prefix using the launcher's run-scoped bearer. +pub fn connect_worker( + server_url: &str, + bearer: &str, + (run_id, claim_id): (Uuid, Uuid), +) -> Result { + let run_url = worker_url(server_url, run_id)?; + let mut authorization = HeaderValue::from_str(&format!("Bearer {bearer}")) + .map_err(|_| WorkerError::Configuration("run bearer is not a valid HTTP credential"))?; + authorization.set_sensitive(true); + let mut headers = HeaderMap::new(); + headers.insert(AUTHORIZATION, authorization); + let client = reqwest::Client::builder() + .default_headers(headers) + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(30)) + .connect_timeout(Duration::from_secs(10)) + .build()?; + Ok(WorkerClient { + client, + run_url, + run_id, + claim_id, + }) +} diff --git a/crates/sqlbuild-orchestration/src/worker/main/mod.rs b/crates/sqlbuild-orchestration/src/worker/main/mod.rs new file mode 100644 index 0000000000..05f5586133 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/main/mod.rs @@ -0,0 +1,2 @@ +pub mod connect_worker; +pub mod start_worker; diff --git a/crates/sqlbuild-orchestration/src/worker/main/start_worker.rs b/crates/sqlbuild-orchestration/src/worker/main/start_worker.rs new file mode 100644 index 0000000000..c5710f3e31 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/main/start_worker.rs @@ -0,0 +1,28 @@ +//! Verify the baseline publication before any run overrides or warehouse mutation. + +use uuid::Uuid; + +use crate::runs::models::{RunChange, RunExecution}; +use crate::snapshots::main::verify_execution::verify_execution; +use crate::snapshots::models::SnapshotInput; +use crate::worker::errors::WorkerError; +use crate::worker::models::WorkerClient; + +/// Verify the baseline compile before planning, cloning, applying run settings or executing. +pub async fn start_worker( + client: &WorkerClient, + baseline: &SnapshotInput, +) -> Result { + let expected = client.execution().await?; + verify_execution(baseline, &expected)?; + if client.heartbeat().await? { + return Err(WorkerError::Cancelled); + } + client + .report( + Uuid::new_v5(&expected.claim.claim_id, b"worker-started"), + RunChange::Started, + ) + .await?; + Ok(expected) +} diff --git a/crates/sqlbuild-orchestration/src/worker/mod.rs b/crates/sqlbuild-orchestration/src/worker/mod.rs new file mode 100644 index 0000000000..59f273cbec --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod _helpers; +pub mod errors; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-orchestration/src/worker/models.rs b/crates/sqlbuild-orchestration/src/worker/models.rs new file mode 100644 index 0000000000..813b4b61e3 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/models.rs @@ -0,0 +1,11 @@ +//! A worker owns only an API credential, never a control-store connection. + +use uuid::Uuid; + +#[derive(Clone, Debug)] +pub struct WorkerClient { + pub(crate) client: reqwest::Client, + pub(crate) run_url: reqwest::Url, + pub(crate) run_id: Uuid, + pub(crate) claim_id: Uuid, +} diff --git a/crates/sqlbuild-orchestration/tests/api.rs b/crates/sqlbuild-orchestration/tests/api.rs index 9ee75a0eb6..14ce5705f3 100644 --- a/crates/sqlbuild-orchestration/tests/api.rs +++ b/crates/sqlbuild-orchestration/tests/api.rs @@ -10,3 +10,5 @@ mod store_test_types; mod test_contract; #[path = "api/test_types.rs"] mod test_types; +#[path = "api/test_worker.rs"] +mod test_worker; diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index c3231b7060..5ebd106605 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -9,6 +9,11 @@ pub(crate) struct ApiContractTestCase { pub(crate) expected_status: RunStatus, } +pub(crate) struct WorkerRedirectTestCase { + pub(crate) description: &'static str, + pub(crate) expected_status: reqwest::StatusCode, +} + pub(crate) struct TestApi { pub(crate) url: String, pub(crate) shutdown: tokio::sync::oneshot::Sender<()>, diff --git a/crates/sqlbuild-orchestration/tests/api/test_worker.rs b/crates/sqlbuild-orchestration/tests/api/test_worker.rs new file mode 100644 index 0000000000..dd61d4724c --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_worker.rs @@ -0,0 +1,356 @@ +use std::collections::BTreeMap; + +use axum::Router; +use axum::response::Redirect; +use axum::routing::get; +use chrono::Utc; +use sqlbuild_orchestration::credentials::main::issue_token::issue_token; +use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; +use sqlbuild_orchestration::runs::main::change_run::change_run; +use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; +use sqlbuild_orchestration::runs::main::list_runs::list_runs; +use sqlbuild_orchestration::runs::models::{ + NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, +}; +use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::main::create_location::create_location; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use sqlbuild_orchestration::worker::errors::WorkerError; +use sqlbuild_orchestration::worker::main::connect_worker::connect_worker; +use sqlbuild_orchestration::worker::main::start_worker::start_worker; +use uuid::Uuid; + +use crate::helpers::{start_api, stop_api}; +use crate::run_helpers::{queue, request, snapshot}; +use crate::store_helpers::{close_test_store, open_test_store, run}; +use crate::store_test_types::Backend; +use crate::test_types::{ApiContractTestCase, TestApi, WorkerRedirectTestCase}; + +#[test] +fn given_a_claimed_run_when_a_worker_connects_then_verification_precedes_start_and_reports_are_retryable() + { + let test_cases = [ + ApiContractTestCase { + description: "SQLite worker HTTP lifecycle", + backend: Backend::Sqlite, + expected_status: RunStatus::Success, + }, + ApiContractTestCase { + description: "PostgreSQL worker HTTP lifecycle", + backend: Backend::Postgres, + expected_status: RunStatus::Success, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let baseline = snapshot(location); + let publication = publish_snapshot(store, &baseline).await.expect("publish"); + queue( + store, + &request((location, publication.snapshot_id), 1, 0, &["orders"]), + ) + .await; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("acquire") + .expect("leader"); + let claim = claim_next_run(store, &lease, &BTreeMap::new()) + .await + .expect("claim") + .expect("work"); + let identity = (claim.request.run_id, claim.claim_id); + let token = issue_token( + store, + &AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }, + CredentialScope::Run { + run_id: identity.0, + claim_id: identity.1, + }, + Utc::now() + chrono::Duration::hours(1), + ) + .await + .expect("worker token"); + let api = start_api(store).await; + let worker = connect_worker(&api.url, &token.bearer, identity).expect("worker client"); + assert!( + !format!("{worker:?}").contains(&token.bearer), + "{}", + test_case.description + ); + let wrong_claim = connect_worker(&api.url, &token.bearer, (identity.0, Uuid::now_v7())) + .expect("different local claim"); + assert!( + matches!( + wrong_claim.execution().await, + Err(WorkerError::Configuration(_)) + ), + "{}", + test_case.description + ); + let mut drifted = baseline.clone(); + drifted + .nodes + .get_mut("inventory") + .expect("unselected dependency") + .raw_sql = Some("select 2 as id".into()); + assert!( + matches!( + start_worker(&worker, &drifted).await, + Err(WorkerError::Verification(_)) + ), + "{}", + test_case.description + ); + let runs = list_runs(store, (location, "development"), 10) + .await + .expect("unstarted"); + assert_eq!( + runs[0].status, + RunStatus::Queued, + "{}", + test_case.description + ); + change_run( + store, + RunAuthority::Scheduler(&lease), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: identity.0, + claim_id: Some(identity.1), + change: RunChange::Launched { container_id: None }, + }, + ) + .await + .expect("record process launch"); + let execution = start_worker(&worker, &baseline) + .await + .expect("verified start"); + assert_eq!(execution.claim, claim, "{}", test_case.description); + assert_eq!( + start_worker(&worker, &baseline).await.expect("retry start"), + execution, + "{}", + test_case.description + ); + assert!( + !worker.heartbeat().await.expect("heartbeat"), + "{}", + test_case.description + ); + let started = Uuid::now_v7(); + assert!( + worker + .report( + started, + RunChange::NodeStarted { + node_key: "orders".into() + } + ) + .await + .expect("node start"), + "{}", + test_case.description + ); + assert!( + !worker + .report( + started, + RunChange::NodeStarted { + node_key: "orders".into() + } + ) + .await + .expect("retry node start"), + "{}", + test_case.description + ); + worker + .report( + Uuid::now_v7(), + RunChange::NodeFinished { + node_key: "orders".into(), + outcome: NodeOutcome::Success, + rows_affected: Some(7), + error_code: None, + skipped_because_of: None, + }, + ) + .await + .expect("node outcome"); + worker + .report( + Uuid::now_v7(), + RunChange::Finished { + outcome: RunOutcome::Success, + reason: "warehouse work completed".into(), + exit_code: Some(0), + }, + ) + .await + .expect("finish"); + let runs = list_runs(store, (location, "development"), 10) + .await + .expect("completed"); + assert_eq!( + runs[0].status, test_case.expected_status, + "{}", + test_case.description + ); + assert!( + matches!( + worker.heartbeat().await, + Err(WorkerError::Rejected(reqwest::StatusCode::UNAUTHORIZED)) + ), + "{}", + test_case.description + ); + queue( + store, + &request((location, publication.snapshot_id), 2, 0, &["orders"]), + ) + .await; + let cancelled_claim = claim_next_run(store, &lease, &BTreeMap::new()) + .await + .expect("claim another run") + .expect("queued work"); + let cancelled_identity = (cancelled_claim.request.run_id, cancelled_claim.claim_id); + let cancelled_token = issue_token( + store, + &AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }, + CredentialScope::Run { + run_id: cancelled_identity.0, + claim_id: cancelled_identity.1, + }, + Utc::now() + chrono::Duration::hours(1), + ) + .await + .expect("another worker credential"); + change_run( + store, + RunAuthority::Scheduler(&lease), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: cancelled_identity.0, + claim_id: Some(cancelled_identity.1), + change: RunChange::Launched { container_id: None }, + }, + ) + .await + .expect("another launched process"); + change_run( + store, + RunAuthority::Operator("test-operator"), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: cancelled_identity.0, + claim_id: Some(cancelled_identity.1), + change: RunChange::CancelRequested { + actor: "test-operator".into(), + }, + }, + ) + .await + .expect("cancel before execution"); + let cancelled_worker = + connect_worker(&api.url, &cancelled_token.bearer, cancelled_identity) + .expect("cancelled worker connection"); + assert!( + matches!( + start_worker(&cancelled_worker, &baseline).await, + Err(WorkerError::Cancelled) + ), + "{}", + test_case.description + ); + let runs = list_runs(store, (location, "development"), 10) + .await + .expect("cancelled run has not started"); + let cancelled_run = runs + .iter() + .find(|run| run.run_id == cancelled_identity.0) + .expect("cancelled projection"); + assert_eq!( + cancelled_run.status, + RunStatus::Launched, + "{}", + test_case.description + ); + assert!( + cancelled_run.started_at.is_none(), + "{}", + test_case.description + ); + stop_api(api).await; + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_a_credentialed_worker_when_the_server_redirects_then_the_redirect_is_not_followed() { + let test_cases = [WorkerRedirectTestCase { + description: "a run bearer must not be forwarded by an API redirect", + expected_status: reqwest::StatusCode::TEMPORARY_REDIRECT, + }]; + for test_case in test_cases { + run(async { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("isolated redirect server"); + let url = format!("http://{}", listener.local_addr().expect("address")); + let application = Router::new().route( + "/api/runs/{run_id}/execution", + get(|| async { Redirect::temporary("/must-not-receive-a-credential") }), + ); + let (shutdown, signal) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(async move { + axum::serve(listener, application) + .with_graceful_shutdown(async { + let _ = signal.await; + }) + .await + .map_err(ControlError::from) + }); + let api = TestApi { + url, + shutdown, + task, + }; + let client = connect_worker( + &api.url, + "synthetic-worker-bearer", + (Uuid::now_v7(), Uuid::now_v7()), + ) + .expect("worker client"); + let Err(WorkerError::Rejected(status)) = client.execution().await else { + panic!("{}", test_case.description); + }; + assert_eq!( + status, test_case.expected_status, + "{}", + test_case.description + ); + stop_api(api).await; + }); + } +} From 6c49ca50fadc7f71e0c25b03086c7eca48fa49e4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 30 Oct 2026 23:54:04 +0000 Subject: [PATCH 04/59] fix: scope orchestration test helpers to their consumers Separate lease expiry, replica connections, terminal run fixtures and fault cases so each integration binary imports only the support it uses. Preserve SQLite and PostgreSQL transaction coverage. Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../tests/credentials.rs | 6 +++- .../tests/credentials/test_contract.rs | 6 ++-- crates/sqlbuild-orchestration/tests/recall.rs | 2 ++ .../tests/recall/helpers.rs | 3 +- crates/sqlbuild-orchestration/tests/runs.rs | 8 ++++- .../tests/runs/helpers.rs | 30 ++----------------- .../tests/runs/terminal_helpers.rs | 30 +++++++++++++++++++ .../tests/runs/test_claims.rs | 6 ++-- .../tests/runs/test_faults.rs | 3 +- .../sqlbuild-orchestration/tests/scheduler.rs | 4 +++ .../tests/scheduler/helpers.rs | 27 ++--------------- .../sqlbuild-orchestration/tests/snapshots.rs | 2 ++ .../tests/snapshots/helpers.rs | 3 +- .../tests/store/fault_types.rs | 10 +++++++ .../tests/store/helpers.rs | 16 ---------- .../tests/store/lease_helpers.rs | 24 +++++++++++++++ .../tests/store/replica_helpers.rs | 15 ++++++++++ crates/sqlbuild-orchestration/tests/ticks.rs | 6 ++++ .../tests/ticks/fault_helpers.rs | 27 ++--------------- .../tests/ticks/helpers.rs | 6 ++-- .../tests/ticks/test_loop.rs | 5 ++-- .../tests/ticks/test_types.rs | 10 +------ 22 files changed, 135 insertions(+), 114 deletions(-) create mode 100644 crates/sqlbuild-orchestration/tests/runs/terminal_helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/store/fault_types.rs create mode 100644 crates/sqlbuild-orchestration/tests/store/lease_helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/store/replica_helpers.rs diff --git a/crates/sqlbuild-orchestration/tests/credentials.rs b/crates/sqlbuild-orchestration/tests/credentials.rs index a4b17699db..c40545cc9e 100644 --- a/crates/sqlbuild-orchestration/tests/credentials.rs +++ b/crates/sqlbuild-orchestration/tests/credentials.rs @@ -1,12 +1,16 @@ #[path = "ticks/fault_helpers.rs"] mod fault_helpers; +#[path = "store/replica_helpers.rs"] +mod replica_helpers; #[path = "runs/helpers.rs"] mod run_helpers; #[path = "store/helpers.rs"] mod store_helpers; #[path = "store/test_types.rs"] mod store_test_types; +#[path = "runs/terminal_helpers.rs"] +mod terminal_helpers; #[path = "credentials/test_contract.rs"] mod test_contract; -#[path = "ticks/test_types.rs"] +#[path = "store/fault_types.rs"] mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs index 6768c12897..9b0ff859f2 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs @@ -15,9 +15,11 @@ use sqlbuild_orchestration::store::models::ControlStore; use uuid::Uuid; use crate::fault_helpers::{install_fault, remove_fault}; -use crate::run_helpers::{finish, queue, request, snapshot}; -use crate::store_helpers::{close_store, close_test_store, open_test_store, run, second_store}; +use crate::replica_helpers::second_store; +use crate::run_helpers::{queue, request, snapshot}; +use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; use crate::store_test_types::Backend; +use crate::terminal_helpers::finish; use crate::test_types::TickFaultTestCase; #[test] diff --git a/crates/sqlbuild-orchestration/tests/recall.rs b/crates/sqlbuild-orchestration/tests/recall.rs index 274b96b9ed..3261be8a4e 100644 --- a/crates/sqlbuild-orchestration/tests/recall.rs +++ b/crates/sqlbuild-orchestration/tests/recall.rs @@ -1,5 +1,7 @@ #[path = "recall/helpers.rs"] mod helpers; +#[path = "store/replica_helpers.rs"] +mod replica_helpers; #[path = "store/helpers.rs"] mod store_helpers; #[path = "store/test_types.rs"] diff --git a/crates/sqlbuild-orchestration/tests/recall/helpers.rs b/crates/sqlbuild-orchestration/tests/recall/helpers.rs index 72e824e82a..ce7de75ac3 100644 --- a/crates/sqlbuild-orchestration/tests/recall/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/recall/helpers.rs @@ -6,7 +6,8 @@ use sqlbuild_recall::records::main::artifact_rows::artifact_rows; use sqlbuild_recall::records::main::event_object_row::event_object_row; use sqlbuild_recall::records::models::{ArtifactInput, RecallRow, RowEnvelope}; -use crate::store_helpers::{close_store, close_test_store, open_test_store, second_store}; +use crate::replica_helpers::second_store; +use crate::store_helpers::{close_store, close_test_store, open_test_store}; use crate::store_test_types::Backend; pub(crate) fn event(id: &str, occurred_at: &str, duration_ms: f64) -> RecallRow { diff --git a/crates/sqlbuild-orchestration/tests/runs.rs b/crates/sqlbuild-orchestration/tests/runs.rs index f2b42df0c2..627c22a308 100644 --- a/crates/sqlbuild-orchestration/tests/runs.rs +++ b/crates/sqlbuild-orchestration/tests/runs.rs @@ -2,10 +2,16 @@ mod fault_helpers; #[path = "runs/helpers.rs"] mod helpers; +#[path = "store/lease_helpers.rs"] +mod lease_helpers; +#[path = "store/replica_helpers.rs"] +mod replica_helpers; #[path = "store/helpers.rs"] mod store_helpers; #[path = "store/test_types.rs"] mod store_test_types; +#[path = "runs/terminal_helpers.rs"] +mod terminal_helpers; #[path = "runs/test_claims.rs"] mod test_claims; #[path = "runs/test_coalesce.rs"] @@ -14,5 +20,5 @@ mod test_coalesce; mod test_faults; #[path = "runs/test_lifecycle.rs"] mod test_lifecycle; -#[path = "ticks/test_types.rs"] +#[path = "store/fault_types.rs"] mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/runs/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/helpers.rs index e98f62b1df..586e14c21e 100644 --- a/crates/sqlbuild-orchestration/tests/runs/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/runs/helpers.rs @@ -3,12 +3,9 @@ use std::collections::{BTreeMap, BTreeSet}; use serde_json::json; -use sqlbuild_orchestration::runs::main::change_run::change_run; use sqlbuild_orchestration::runs::main::request_run::request_run; -use sqlbuild_orchestration::runs::models::{ - RunAuthority, RunChange, RunChangeRequest, RunClaim, RunKind, RunOrigin, RunOutcome, RunRequest, -}; -use sqlbuild_orchestration::scheduler::models::{Overlap, SchedulerLease}; +use sqlbuild_orchestration::runs::models::{RunKind, RunOrigin, RunRequest}; +use sqlbuild_orchestration::scheduler::models::Overlap; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; use sqlbuild_orchestration::store::models::ControlStore; use uuid::Uuid; @@ -93,26 +90,3 @@ pub(crate) async fn queue(store: &ControlStore, request: &RunRequest) { "a reused run ID cannot change its request" ); } - -pub(crate) async fn finish(store: &ControlStore, lease: &SchedulerLease, claim: &RunClaim) { - let request = RunChangeRequest { - event_id: Uuid::new_v5(&claim.claim_id, b"test-finished"), - run_id: claim.request.run_id, - claim_id: Some(claim.claim_id), - change: RunChange::Finished { - outcome: RunOutcome::Failed, - reason: "synthetic worker exit".into(), - exit_code: Some(1), - }, - }; - assert!( - change_run(store, RunAuthority::Scheduler(lease), &request) - .await - .expect("terminate claimed work") - ); - assert!( - !change_run(store, RunAuthority::Scheduler(lease), &request) - .await - .expect("retry terminal fact") - ); -} diff --git a/crates/sqlbuild-orchestration/tests/runs/terminal_helpers.rs b/crates/sqlbuild-orchestration/tests/runs/terminal_helpers.rs new file mode 100644 index 0000000000..68e8742d32 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/terminal_helpers.rs @@ -0,0 +1,30 @@ +use sqlbuild_orchestration::runs::main::change_run::change_run; +use sqlbuild_orchestration::runs::models::{ + RunAuthority, RunChange, RunChangeRequest, RunClaim, RunOutcome, +}; +use sqlbuild_orchestration::scheduler::models::SchedulerLease; +use sqlbuild_orchestration::store::models::ControlStore; +use uuid::Uuid; + +pub(crate) async fn finish(store: &ControlStore, lease: &SchedulerLease, claim: &RunClaim) { + let request = RunChangeRequest { + event_id: Uuid::new_v5(&claim.claim_id, b"test-finished"), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change: RunChange::Finished { + outcome: RunOutcome::Failed, + reason: "synthetic worker exit".into(), + exit_code: Some(1), + }, + }; + assert!( + change_run(store, RunAuthority::Scheduler(lease), &request) + .await + .expect("terminate claimed work") + ); + assert!( + !change_run(store, RunAuthority::Scheduler(lease), &request) + .await + .expect("retry terminal fact") + ); +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_claims.rs b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs index 4489f0322d..b97dc931e5 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_claims.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs @@ -11,9 +11,11 @@ use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; use uuid::Uuid; -use crate::helpers::{finish, queue, request, snapshot}; -use crate::store_helpers::{close_store, close_test_store, open_test_store, run, second_store}; +use crate::helpers::{queue, request, snapshot}; +use crate::replica_helpers::second_store; +use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; use crate::store_test_types::Backend; +use crate::terminal_helpers::finish; #[test] fn given_overlapping_priority_queues_when_two_launchers_claim_then_conflicts_rollback_and_terminal_facts_release_claims() diff --git a/crates/sqlbuild-orchestration/tests/runs/test_faults.rs b/crates/sqlbuild-orchestration/tests/runs/test_faults.rs index 7ab8721043..a0f076c366 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_faults.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_faults.rs @@ -14,8 +14,9 @@ use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; use uuid::Uuid; -use crate::fault_helpers::{expire_lease, install_fault, remove_fault}; +use crate::fault_helpers::{install_fault, remove_fault}; use crate::helpers::{queue, request, snapshot}; +use crate::lease_helpers::expire_lease; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; use crate::test_types::TickFaultTestCase; diff --git a/crates/sqlbuild-orchestration/tests/scheduler.rs b/crates/sqlbuild-orchestration/tests/scheduler.rs index 159c963ed1..be400b8236 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler.rs @@ -1,5 +1,9 @@ #[path = "scheduler/helpers.rs"] mod helpers; +#[path = "store/lease_helpers.rs"] +mod lease_helpers; +#[path = "store/replica_helpers.rs"] +mod replica_helpers; #[path = "store/helpers.rs"] mod store_helpers; #[path = "store/test_types.rs"] diff --git a/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs index f1711e4bfc..8cac2b9924 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs @@ -4,34 +4,13 @@ use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; use sqlbuild_orchestration::scheduler::main::renew_lease::renew_lease; use sqlbuild_orchestration::store::errors::ControlError; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; -use sqlbuild_orchestration::store::models::ControlStore; use uuid::Uuid; -use crate::store_helpers::{close_store, close_test_store, open_test_store, second_store}; +use crate::lease_helpers::expire_lease; +use crate::replica_helpers::second_store; +use crate::store_helpers::{close_store, close_test_store, open_test_store}; use crate::store_test_types::Backend; -async fn expire_lease(store: &ControlStore) { - let ControlStore::Postgres(pool) = store else { - let ControlStore::Sqlite(path) = store else { - panic!("expected SQLite fixture"); - }; - rusqlite::Connection::open(path) - .expect("open fault injection connection") - .execute( - "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", - [], - ) - .expect("expire SQLite scheduler lease"); - return; - }; - sqlx_core::query::query( - "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", - ) - .execute(pool) - .await - .expect("expire PostgreSQL scheduler lease"); -} - pub(crate) async fn leadership_contract(backend: Backend) -> Option { let test_store = open_test_store(backend).await?; let store = &test_store.store; diff --git a/crates/sqlbuild-orchestration/tests/snapshots.rs b/crates/sqlbuild-orchestration/tests/snapshots.rs index a554a78fc6..5dcb1cd22c 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots.rs @@ -2,6 +2,8 @@ mod contract; #[path = "snapshots/helpers.rs"] mod helpers; +#[path = "store/replica_helpers.rs"] +mod replica_helpers; #[path = "store/helpers.rs"] mod store_helpers; #[path = "store/test_types.rs"] diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index aaefbd5da1..b1533caba3 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -13,7 +13,8 @@ use sqlbuild_orchestration::store::main::open_store::open_store; use sqlbuild_orchestration::store::models::ControlStore; use uuid::Uuid; -use crate::store_helpers::{close_store, close_test_store, open_test_store, second_store}; +use crate::replica_helpers::second_store; +use crate::store_helpers::{close_store, close_test_store, open_test_store}; use crate::store_test_types::Backend; use crate::test_types::SchemaRejectionCase; diff --git a/crates/sqlbuild-orchestration/tests/store/fault_types.rs b/crates/sqlbuild-orchestration/tests/store/fault_types.rs new file mode 100644 index 0000000000..eb9e82bd2f --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/fault_types.rs @@ -0,0 +1,10 @@ +//! Transaction failure points shared by credential, tick and run contracts. + +pub(crate) struct TickFaultTestCase { + pub(crate) description: &'static str, + pub(crate) table: &'static str, + pub(crate) operation: &'static str, + pub(crate) condition: &'static str, + pub(crate) sqlite_body: &'static str, + pub(crate) postgres_body: &'static str, +} diff --git a/crates/sqlbuild-orchestration/tests/store/helpers.rs b/crates/sqlbuild-orchestration/tests/store/helpers.rs index 07aafd8e98..f93f44f0f5 100644 --- a/crates/sqlbuild-orchestration/tests/store/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/helpers.rs @@ -78,19 +78,3 @@ pub(crate) async fn close_store(store: ControlStore) { }; pool.close().await; } - -pub(crate) async fn second_store(store: &ControlStore) -> ControlStore { - let ControlStore::Postgres(pool) = store else { - let ControlStore::Sqlite(path) = store else { - panic!("expected SQLite fixture"); - }; - return open_store(path.to_str().expect("UTF-8 test path")) - .await - .expect("open independent SQLite handle"); - }; - ControlStore::Postgres( - sqlx_postgres::PgPool::connect_with(pool.connect_options().as_ref().clone()) - .await - .expect("open independent PostgreSQL pool"), - ) -} diff --git a/crates/sqlbuild-orchestration/tests/store/lease_helpers.rs b/crates/sqlbuild-orchestration/tests/store/lease_helpers.rs new file mode 100644 index 0000000000..8e17cfce86 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/lease_helpers.rs @@ -0,0 +1,24 @@ +//! Expire the persisted lease without introducing a fake control-store clock. + +use sqlbuild_orchestration::store::models::ControlStore; + +pub(crate) async fn expire_lease(store: &ControlStore) { + match store { + ControlStore::Sqlite(path) => { + rusqlite::Connection::open(path) + .expect("open lease expiry connection") + .execute_batch( + "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + ) + .expect("expire SQLite scheduler lease"); + } + ControlStore::Postgres(pool) => { + sqlx_core::query::query( + "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + ) + .execute(pool) + .await + .expect("expire PostgreSQL scheduler lease"); + } + } +} diff --git a/crates/sqlbuild-orchestration/tests/store/replica_helpers.rs b/crates/sqlbuild-orchestration/tests/store/replica_helpers.rs new file mode 100644 index 0000000000..853ba75df8 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/replica_helpers.rs @@ -0,0 +1,15 @@ +use sqlbuild_orchestration::store::main::open_store::open_store; +use sqlbuild_orchestration::store::models::ControlStore; + +pub(crate) async fn second_store(store: &ControlStore) -> ControlStore { + match store { + ControlStore::Sqlite(path) => open_store(path.to_str().expect("UTF-8 test path")) + .await + .expect("open independent SQLite handle"), + ControlStore::Postgres(pool) => ControlStore::Postgres( + sqlx_postgres::PgPool::connect_with(pool.connect_options().as_ref().clone()) + .await + .expect("open independent PostgreSQL pool"), + ), + } +} diff --git a/crates/sqlbuild-orchestration/tests/ticks.rs b/crates/sqlbuild-orchestration/tests/ticks.rs index 00bfc0cbb6..abf2590859 100644 --- a/crates/sqlbuild-orchestration/tests/ticks.rs +++ b/crates/sqlbuild-orchestration/tests/ticks.rs @@ -1,7 +1,13 @@ #[path = "ticks/fault_helpers.rs"] mod fault_helpers; +#[path = "store/fault_types.rs"] +mod fault_types; #[path = "ticks/helpers.rs"] mod helpers; +#[path = "store/lease_helpers.rs"] +mod lease_helpers; +#[path = "store/replica_helpers.rs"] +mod replica_helpers; #[path = "store/helpers.rs"] mod store_helpers; #[path = "store/test_types.rs"] diff --git a/crates/sqlbuild-orchestration/tests/ticks/fault_helpers.rs b/crates/sqlbuild-orchestration/tests/ticks/fault_helpers.rs index 180b1babce..9c973c0fb1 100644 --- a/crates/sqlbuild-orchestration/tests/ticks/fault_helpers.rs +++ b/crates/sqlbuild-orchestration/tests/ticks/fault_helpers.rs @@ -13,14 +13,14 @@ pub(crate) async fn install_fault(store: &ControlStore, point: &TickFaultTestCas "CREATE TRIGGER sqb_test_fault BEFORE {} ON {} {} BEGIN {}; END;", point.operation, point.table, point.condition, point.sqlite_body )) - .expect("install SQLite transaction fault"); + .expect(point.description); return; }; sqlx_core::query::query(&format!("CREATE FUNCTION sqb_test_fault() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN {}; RETURN NEW; END $$", point.postgres_body)) .execute(pool).await.expect("install PostgreSQL fault function"); sqlx_core::query::query(&format!("CREATE TRIGGER sqb_test_fault BEFORE {} ON {} FOR EACH ROW {} EXECUTE FUNCTION sqb_test_fault()", - point.operation, point.table, point.condition.replace("WHEN ", "WHEN (") + &[")", ""][usize::from(point.condition.is_empty())])) - .execute(pool).await.expect("install PostgreSQL transaction fault"); + point.operation, point.table, point.condition.replace("WHEN ", "WHEN (") + [")", ""][usize::from(point.condition.is_empty())])) + .execute(pool).await.expect(point.description); } pub(crate) async fn remove_fault(store: &ControlStore, point: &TickFaultTestCase) { @@ -43,24 +43,3 @@ pub(crate) async fn remove_fault(store: &ControlStore, point: &TickFaultTestCase .await .expect("remove PostgreSQL fault function"); } - -pub(crate) async fn expire_lease(store: &ControlStore) { - let ControlStore::Postgres(pool) = store else { - let ControlStore::Sqlite(path) = store else { - unreachable!() - }; - rusqlite::Connection::open(path) - .expect("fault connection") - .execute_batch( - "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", - ) - .expect("expire SQLite lease"); - return; - }; - sqlx_core::query::query( - "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", - ) - .execute(pool) - .await - .expect("expire PostgreSQL lease"); -} diff --git a/crates/sqlbuild-orchestration/tests/ticks/helpers.rs b/crates/sqlbuild-orchestration/tests/ticks/helpers.rs index 183b105cde..0ce57192b9 100644 --- a/crates/sqlbuild-orchestration/tests/ticks/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/ticks/helpers.rs @@ -15,8 +15,10 @@ use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; use uuid::Uuid; -use crate::fault_helpers::{expire_lease, install_fault, remove_fault}; -use crate::store_helpers::{close_store, close_test_store, open_test_store, second_store}; +use crate::fault_helpers::{install_fault, remove_fault}; +use crate::lease_helpers::expire_lease; +use crate::replica_helpers::second_store; +use crate::store_helpers::{close_store, close_test_store, open_test_store}; use crate::store_test_types::Backend; use crate::test_types::TickFaultTestCase; diff --git a/crates/sqlbuild-orchestration/tests/ticks/test_loop.rs b/crates/sqlbuild-orchestration/tests/ticks/test_loop.rs index 34b691b8fa..d847e5546a 100644 --- a/crates/sqlbuild-orchestration/tests/ticks/test_loop.rs +++ b/crates/sqlbuild-orchestration/tests/ticks/test_loop.rs @@ -8,9 +8,10 @@ use sqlbuild_orchestration::store::main::create_location::create_location; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use uuid::Uuid; -use crate::fault_helpers::expire_lease; use crate::helpers::snapshot; -use crate::store_helpers::{close_store, close_test_store, open_test_store, run, second_store}; +use crate::lease_helpers::expire_lease; +use crate::replica_helpers::second_store; +use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; use crate::store_test_types::Backend; use crate::test_types::TickTransactionTestCase; diff --git a/crates/sqlbuild-orchestration/tests/ticks/test_types.rs b/crates/sqlbuild-orchestration/tests/ticks/test_types.rs index 05ee9cd1d5..9559e8a92f 100644 --- a/crates/sqlbuild-orchestration/tests/ticks/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/ticks/test_types.rs @@ -1,3 +1,4 @@ +pub(crate) use crate::fault_types::TickFaultTestCase; use crate::store_test_types::Backend; pub(crate) struct TickTransactionTestCase { @@ -7,15 +8,6 @@ pub(crate) struct TickTransactionTestCase { pub(crate) expected_runs: usize, } -pub(crate) struct TickFaultTestCase { - pub(crate) description: &'static str, - pub(crate) table: &'static str, - pub(crate) operation: &'static str, - pub(crate) condition: &'static str, - pub(crate) sqlite_body: &'static str, - pub(crate) postgres_body: &'static str, -} - pub(crate) struct TriggerControlsTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, From 39c0faa63fdc2ffa6664b8934a808e74498029ab Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 00:03:36 +0000 Subject: [PATCH 05/59] feat: derive orchestration snapshots from native compiler records Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- Cargo.lock | 2 + .../sql/edge_cases/edge_next_quantity.sql | 8 + .../seeds/edge_cases/edge_quantity_steps.csv | 3 + .../seeds/edge_cases/edge_quantity_steps.yml | 7 + crates/sqlbuild-python/Cargo.toml | 2 + .../bindings/_helpers/compiled_project/mod.rs | 4 + .../_helpers/compiled_project/registration.rs | 3 +- .../_helpers/compiled_project/snapshot.rs | 104 +++++++ .../compiled_project/snapshot_boundary.rs | 183 +++++++++++++ .../compiled_project/snapshot_models.rs | 73 +++++ .../compiled_project/snapshot_resources.rs | 140 ++++++++++ .../_helpers/compiled_project/values.rs | 11 + .../compiled_views/compiled_model_view.rs | 30 +- .../_helpers/compiled_views/function_views.rs | 5 +- .../_helpers/compiled_views/seed_views.rs | 2 +- .../src/bindings/_helpers/native_host/mod.rs | 1 + .../native_host/orchestration_environment.rs | 50 ++++ src/sqlbuild/_native.pyi | 11 + .../compiler/compiled_project/_test_types.py | 7 + .../test_orchestration_snapshot.py | 256 ++++++++++++++++++ 20 files changed, 887 insertions(+), 15 deletions(-) create mode 100644 benchmarks/duckshop/project/functions/sql/edge_cases/edge_next_quantity.sql create mode 100644 benchmarks/duckshop/project/seeds/edge_cases/edge_quantity_steps.csv create mode 100644 benchmarks/duckshop/project/seeds/edge_cases/edge_quantity_steps.yml create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_boundary.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_resources.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_environment.rs create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py diff --git a/Cargo.lock b/Cargo.lock index cef38f434c..92db47c581 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2624,6 +2624,7 @@ dependencies = [ "rustc-hash", "serde", "serde_json", + "sha2", "signal-hook", "sqlbuild-analysis", "sqlbuild-attachments", @@ -2635,6 +2636,7 @@ dependencies = [ "sqlbuild-discovery", "sqlbuild-host", "sqlbuild-model-config", + "sqlbuild-orchestration", "sqlbuild-project", "sqlbuild-pysource", "sqlbuild-recall", diff --git a/benchmarks/duckshop/project/functions/sql/edge_cases/edge_next_quantity.sql b/benchmarks/duckshop/project/functions/sql/edge_cases/edge_next_quantity.sql new file mode 100644 index 0000000000..c4fa936dea --- /dev/null +++ b/benchmarks/duckshop/project/functions/sql/edge_cases/edge_next_quantity.sql @@ -0,0 +1,8 @@ +FUNCTION ( + description 'Advance an order quantity in the inherited function namespace.', + tags [edge_case], + arguments (quantity INTEGER), + returns INTEGER, +); + +SELECT quantity + 1 diff --git a/benchmarks/duckshop/project/seeds/edge_cases/edge_quantity_steps.csv b/benchmarks/duckshop/project/seeds/edge_cases/edge_quantity_steps.csv new file mode 100644 index 0000000000..9e70ca2ea3 --- /dev/null +++ b/benchmarks/duckshop/project/seeds/edge_cases/edge_quantity_steps.csv @@ -0,0 +1,3 @@ +quantity +2 +7 diff --git a/benchmarks/duckshop/project/seeds/edge_cases/edge_quantity_steps.yml b/benchmarks/duckshop/project/seeds/edge_cases/edge_quantity_steps.yml new file mode 100644 index 0000000000..de890aa0b8 --- /dev/null +++ b/benchmarks/duckshop/project/seeds/edge_cases/edge_quantity_steps.yml @@ -0,0 +1,7 @@ +seeds: + - name: edge_quantity_steps + description: Quantities whose seed destination inherits the target namespace. + tags: [edge_case] + columns: + - name: quantity + type: INTEGER diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index 661bd3d6a1..be0cc75d7c 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -24,6 +24,7 @@ rusqlite.workspace = true rustc-hash.workspace = true serde.workspace = true serde_json.workspace = true +sha2.workspace = true signal-hook.workspace = true sqlbuild-analysis.workspace = true sqlbuild-attachments.workspace = true @@ -35,6 +36,7 @@ sqlbuild-cursor.workspace = true sqlbuild-discovery.workspace = true sqlbuild-host.workspace = true sqlbuild-model-config.workspace = true +sqlbuild-orchestration.workspace = true sqlbuild-project.workspace = true sqlbuild-pysource.workspace = true sqlbuild-recall.workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs index bbd4e1a61a..1fb46a0eb0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs @@ -3,4 +3,8 @@ pub(crate) mod project; pub(crate) mod registration; pub(crate) mod rows; pub(crate) mod schema_facts; +pub(crate) mod snapshot; +pub(crate) mod snapshot_boundary; +pub(crate) mod snapshot_models; +pub(crate) mod snapshot_resources; pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs index fd619ffe5f..c342479217 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs @@ -6,5 +6,6 @@ use crate::bindings::_helpers::{compiled_views, input_loops}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { crate::bindings::_helpers::compiled_project::project::register(module)?; input_loops::registration::register(module)?; - compiled_views::registration::register(module) + compiled_views::registration::register(module)?; + crate::bindings::_helpers::compiled_project::snapshot::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs new file mode 100644 index 0000000000..1edaf8fcd4 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs @@ -0,0 +1,104 @@ +//! Project metadata stays native until its immutable snapshot is sent over HTTP. + +use std::collections::BTreeMap; +use std::path::PathBuf; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult, Python}; +use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use serde_json::Value; +use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; + +use crate::bindings::_helpers::compiled_project::snapshot_models::model_nodes; +use crate::bindings::_helpers::compiled_project::snapshot_resources::resource_nodes; +use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; +use crate::bindings::_helpers::native_host::orchestration_environment::environment_hash; +use crate::bindings::_helpers::records::fields::field; + +/// A data-only publication; this object never contains adapters, connections or user callables. +#[pyclass(frozen, module = "sqlbuild._native")] +pub(crate) struct NativeOrchestrationSnapshot { + pub(crate) input: SnapshotInput, +} + +#[pymethods] +impl NativeOrchestrationSnapshot { + /// Serialize only at the public boundary; native transports can use the retained input. + fn to_json(&self, py: Python<'_>) -> PyResult { + py.detach(|| serde_json::to_string(&self.input)) + .map_err(|error| PyValueError::new_err(error.to_string())) + } +} + +/// Compile catalog nodes from their owners, never from Python model or graph projections. +#[pyfunction] +pub(crate) fn build_orchestration_snapshot( + py: Python<'_>, + project: PyRef<'_, NativeProjectView>, + metadata_json: &str, + project_dir: PathBuf, +) -> PyResult { + let mut input: SnapshotInput = serde_json::from_str(metadata_json) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + if !input.nodes.is_empty() { + return Err(PyValueError::new_err( + "snapshot nodes must come from the compiled project", + )); + } + if project + .effective_target_name() + .is_some_and(|target| target != input.target) + { + return Err(PyValueError::new_err( + "snapshot target differs from the compiled project", + )); + } + for diagnostic in project.diagnostics(py).bind(py).try_iter()? { + let severity: String = field(&field(&diagnostic?, "severity")?, "value")?.extract()?; + if severity == "error" { + return Err(PyValueError::new_err( + "cannot publish a project with compiler errors", + )); + } + } + input.sqb_version = env!("CARGO_PKG_VERSION").into(); + input.environment_hash = environment_hash(py, &project_dir)?; + let owner = project.owner().bind(py).borrow(); + input.nodes = model_nodes(py, &project, &owner, input.nodes)?; + input.nodes = resource_nodes(py, &project, &owner, input.nodes)?; + Ok(NativeOrchestrationSnapshot { input }) +} + +/// Duplicate resource identities must fail, rather than silently replacing an earlier node. +pub(crate) fn insert_node( + mut nodes: BTreeMap, + key: String, + node: SnapshotNode, +) -> PyResult> { + if nodes.insert(key, node).is_some() { + return Err(PyValueError::new_err("duplicate snapshot node identity")); + } + Ok(nodes) +} + +pub(crate) fn dependency_keys(deps: &[(String, String)]) -> Vec { + deps.iter() + .map(|(kind, name)| format!("{kind}:{name}")) + .collect() +} + +pub(crate) fn destination(value: &sqlbuild_project::models::DestinationFact) -> Value { + serde_json::json!({ + "database": value.database, + "schema": value.schema, + "name": value.name, + "qualified_name": value.qualified_name, + "logical_database": value.logical_database, + "logical_schema": value.logical_schema, + }) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(build_orchestration_snapshot, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_boundary.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_boundary.rs new file mode 100644 index 0000000000..8eaa09494a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_boundary.rs @@ -0,0 +1,183 @@ +//! Discovery files and authored test payloads remain the Python host's boundary. + +use std::collections::BTreeMap; +use std::fs::File; +use std::io::Read; +use std::path::PathBuf; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use serde_json::{Map, Value, json}; +use sha2::{Digest, Sha256}; +use sqlbuild_orchestration::snapshots::models::SnapshotNode; + +use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::compiled_project::snapshot::{destination, insert_node}; +use crate::bindings::_helpers::compiled_project::values::plain_value; +use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; +use crate::bindings::_helpers::compiled_views::seed_views::{NativeCompiledSeed, SeedViews}; +use crate::bindings::_helpers::records::fields::{field, path_posix}; + +/// Read only declared metadata fields, never callable attributes or connection settings. +pub(crate) fn metadata(value: &Bound<'_, PyAny>, names: &[&str]) -> PyResult { + let mut result = Map::new(); + for name in names { + result.insert((*name).into(), plain_value(&field(value, name)?)?); + } + Ok(Value::Object(result)) +} + +pub(crate) fn relative_path(value: &Bound<'_, PyAny>) -> PyResult { + path_posix(&field(value, "relative_path")?) +} + +/// Source locations are local compiler diagnostics, not relocatable column metadata. +pub(crate) fn columns(value: &Bound<'_, PyAny>) -> PyResult> { + value + .try_iter()? + .map(|column| { + metadata( + &column?, + &["name", "type", "nullable", "description", "meta"], + ) + }) + .collect() +} + +fn file_digest(value: &Bound<'_, PyAny>) -> PyResult { + let path: PathBuf = field(value, "file_path")?.extract()?; + value.py().detach(|| { + let mut file = File::open(path)?; + let mut hash = Sha256::new(); + let mut buffer = [0_u8; 64 * 1024]; + loop { + let length = file.read(&mut buffer)?; + if length == 0 { + return Ok(format!("{:x}", hash.finalize())); + } + hash.update(&buffer[..length]); + } + }) +} + +pub(crate) fn boundary_nodes( + py: Python<'_>, + project: &NativeProjectView, + owner: &NativeCompiledProject, + mut nodes: BTreeMap, +) -> PyResult> { + let seed_views = project.seeds(py); + for (seed, view) in owner.seeds().iter().zip(seed_views.bind(py).try_iter()?) { + let effective = view? + .cast::()? + .get() + .current_destination(py)?; + let parts = seed.parts(); + let seed_file = parts.seed_file.bind(py); + nodes = insert_node( + nodes, + format!("seed:{}", seed.name), + SnapshotNode { + kind: "seed".into(), + definition: json!({ + "name": seed.name, + "path": relative_path(seed_file)?, + "content_hash": file_digest(seed_file)?, + "destination": destination(&effective), + "schema": metadata(parts.schema_entry.bind(py), &[ + "tags", "description", "meta", "csv_settings", + ])?, + "columns": columns(&field(parts.schema_entry.bind(py), "columns")?)?, + }), + raw_sql: None, + compiled_sql: None, + python_source: None, + }, + )?; + } + for (kind, objects, names) in [ + ( + "sql_test", + project.sql_tests(py), + ["mode", "payload", "parameter_schema", "parameter_values"].as_slice(), + ), + ( + "sql_scenario", + project.sql_scenarios(py), + ["authored_ctes", "expected_ctes", "assertion_ctes"].as_slice(), + ), + ] { + for item in objects.bind(py).try_iter()? { + let item = item?; + let key = field(&item, "key")?; + let name: String = field(&key, "name")?.extract()?; + let resource_type: String = field(&key, "resource_type")?.str()?.extract()?; + let definition = metadata(&item, names)?; + nodes = insert_node( + nodes, + format!("{resource_type}:{name}"), + SnapshotNode { + kind: kind.into(), + definition: json!({"name": name, "test": definition}), + raw_sql: None, + compiled_sql: Some(field(&item, "sql_body")?.extract()?), + python_source: None, + }, + )?; + } + } + for (kind, objects) in [ + ("loader", project.loader_functions(py)), + ("hook", project.hook_functions(py)), + ("materialization", project.materialization_files(py)), + ] { + for item in objects.bind(py).try_iter()? { + let item = item?; + let name: String = field(&item, "name")?.extract()?; + let path: PathBuf = field(&item, "file_path")?.extract()?; + let source = py.detach(|| std::fs::read_to_string(path))?; + nodes = insert_node( + nodes, + format!("{kind}:{name}"), + SnapshotNode { + kind: kind.into(), + definition: json!({"name": name, "path": relative_path(&item)?}), + raw_sql: None, + compiled_sql: None, + python_source: Some(source), + }, + )?; + } + } + for item in project.sql_hook_files(py).bind(py).try_iter()? { + let item = item?; + let name: String = field(&item, "name")?.extract()?; + nodes = insert_node( + nodes, + format!("sql_hook:{name}"), + SnapshotNode { + kind: "sql_hook".into(), + definition: json!({"name": name, "path": relative_path(&item)?}), + raw_sql: Some(field(&item, "contents")?.extract()?), + compiled_sql: Some(field(&item, "sql_body")?.extract()?), + python_source: None, + }, + )?; + } + let macros = project.loaded_macros(py); + for (key, item) in macros.bind(py).cast::()?.iter() { + let name: String = key.extract()?; + nodes = insert_node( + nodes, + format!("macro:{name}"), + SnapshotNode { + kind: "macro".into(), + definition: json!({"name": name, "path": relative_path(&item)?}), + raw_sql: None, + compiled_sql: None, + python_source: Some(field(&item, "raw_source")?.extract()?), + }, + )?; + } + Ok(nodes) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs new file mode 100644 index 0000000000..13fa7aca8c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs @@ -0,0 +1,73 @@ +//! Model catalog data comes directly from the compiler's retained facts and records. + +use std::collections::BTreeMap; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{PyResult, Python}; +use pyo3::types::PyTupleMethods; +use serde_json::{Value, json}; +use sqlbuild_analysis::compiled_project::main::model_facts::model_facts; +use sqlbuild_orchestration::snapshots::models::SnapshotNode; + +use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::compiled_project::snapshot::{ + dependency_keys, destination, insert_node, +}; +use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; +use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; + +pub(crate) fn model_nodes( + py: Python<'_>, + project: &NativeProjectView, + owner: &NativeCompiledProject, + mut nodes: BTreeMap, +) -> PyResult> { + for model in project.models(py).bind(py).iter() { + let model = model.cast::()?.get(); + let (name, _, _) = model.file_texts(py)?; + let facts = model_facts(owner.facts(), &name) + .ok_or_else(|| PyValueError::new_err("snapshot model has no retained facts"))?; + if facts.config.encoding_error.is_some() { + return Err(PyValueError::new_err( + "model config cannot be represented in snapshot metadata", + )); + } + let mut definition = model.read(py, |input, _| { + let record = input + .compiled + .as_ref() + .ok_or_else(|| PyValueError::new_err("snapshot model has not been assembled"))?; + let columns: Vec = record + .analysis + .inferred + .as_deref() + .unwrap_or_default() + .iter() + .map(|(name, data_type, nullability)| { + json!({"name": name, "type": data_type, "nullability": nullability}) + }) + .collect(); + Ok(json!({ + "name": facts.name, + "path": facts.relative_path, + "config": facts.config.values, + "materialization": facts.config.materialized, + "deps": dependency_keys(&record.deps), + "columns": columns, + })) + })?; + definition["destination"] = model.read_destination(py, |value| Ok(destination(value)))?; + nodes = insert_node( + nodes, + format!("model:{}", facts.name), + SnapshotNode { + kind: "model".into(), + definition, + raw_sql: Some(facts.authored_sql.to_string()), + compiled_sql: Some(facts.query_sql.to_string()), + python_source: None, + }, + )?; + } + Ok(nodes) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_resources.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_resources.rs new file mode 100644 index 0000000000..213bcc9739 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_resources.rs @@ -0,0 +1,140 @@ +//! Sources, functions and audits are read from their retained native records. + +use std::collections::BTreeMap; + +use pyo3::prelude::{PyAnyMethods, PyResult, Python}; +use serde_json::{Value, json}; +use sqlbuild_orchestration::snapshots::models::SnapshotNode; + +use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::compiled_project::snapshot::{ + dependency_keys, destination, insert_node, +}; +use crate::bindings::_helpers::compiled_project::snapshot_boundary::{ + boundary_nodes, columns, metadata, relative_path, +}; +use crate::bindings::_helpers::compiled_project::values::plain_value; +use crate::bindings::_helpers::compiled_views::function_views::NativeCompiledFunction; +use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; + +pub(crate) fn resource_nodes( + py: Python<'_>, + project: &NativeProjectView, + owner: &NativeCompiledProject, + mut nodes: BTreeMap, +) -> PyResult> { + for source in owner.sources() { + let attached = &source.attached; + nodes = insert_node( + nodes, + format!("source:{}", attached.name), + SnapshotNode { + kind: "source".into(), + definition: json!({ + "name": attached.name, + "path": relative_path(attached.host.source_file.bind(py))?, + "database": attached.database, + "schema": attached.schema, + "table": attached.table, + "expression": attached.expression, + "contract": attached.contract, + "description": attached.description, + "namespace": source.namespace, + "columns": columns(attached.host.columns.bind(py).as_any())?, + "meta": plain_value(attached.host.meta.bind(py))?, + "entry": metadata(attached.host.entry.bind(py), &[ + "loader", "write_strategy", "cursor_column", "unique_key", + "final", "load_batch_size", "freshness", + ])?, + }), + raw_sql: None, + compiled_sql: attached.expression.clone(), + python_source: None, + }, + )?; + } + let function_views = project.functions(py); + for (function, view) in owner + .functions() + .iter() + .zip(function_views.bind(py).try_iter()?) + { + let effective = view? + .cast::()? + .get() + .current_destinations(py)? + .0; + let attached = &function.attached; + let kind = attached.node_type(); + let python = attached.python.as_ref().map(|runtime| { + json!({ + "version": runtime.runtime_version, + "entry_point": runtime.entry_point, + "packages": runtime.packages, + }) + }); + nodes = insert_node( + nodes, + format!("{kind}:{}", attached.name), + SnapshotNode { + kind: kind.into(), + definition: json!({ + "name": attached.name, + "path": attached.relative_path, + "arguments": attached.arguments, + "returns": attached.returns, + "return_columns": attached.return_columns, + "destination": destination(&effective), + "python": python, + "tags": attached.tags, + "description": attached.description, + "deps": dependency_keys(function.deps.as_deref().unwrap_or_default()), + }), + raw_sql: None, + compiled_sql: attached.python.is_none().then(|| attached.body.clone()), + python_source: attached.python.is_some().then(|| attached.body.clone()), + }, + )?; + } + for audit in owner.audits() { + let attached = &audit.attached; + let target: Option = attached.target.as_ref().map( + |target| json!({"kind": target.kind, "name": target.name, "column": target.column}), + ); + nodes = insert_node( + nodes, + format!("audit:{}", attached.compiled_name()), + SnapshotNode { + kind: "audit".into(), + definition: json!({ + "name": attached.compiled_name(), + "definition_name": attached.definition_name, + "measurement": attached.measurement, + "evidence_sql": attached.evidence_sql, + "target": target, + "severity": attached.severity, + "run_scope": attached.run_scope, + "always_run": attached.always_run, + "description": attached.description, + "policy": metadata(attached.host.audit_block.bind(py), &[ + "evaluation_mode", "measurement_contract", + ])?, + "thresholds": plain_value( + attached.host.thresholds.bind(py), + )?, + "minimum_samples": plain_value( + attached.host.minimum_samples.bind(py), + )?, + "evidence_limit": plain_value( + attached.host.evidence_limit.bind(py), + )?, + "deps": dependency_keys(audit.scope_deps.as_deref().unwrap_or_default()), + }), + raw_sql: None, + compiled_sql: Some(attached.sql_body.clone()), + python_source: None, + }, + )?; + } + boundary_nodes(py, project, owner, nodes) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs index 90a9f140d2..2a551bbd39 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs @@ -9,6 +9,7 @@ use serde_json::{Map, Number, Value}; use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::models::NativeRecord; const DATACLASS_FIELDS: &str = "__dataclass_fields__"; const INSTANCE_DICT: &str = "__dict__"; @@ -49,6 +50,16 @@ pub(crate) fn plain_value(value: &Bound<'_, PyAny>) -> PyResult { if let Ok(mapping) = value.downcast::() { return object_value(mapping); } + if let Ok(record) = value.cast::() { + let record = record.get(); + let mut entries = Vec::with_capacity(record.values.len()); + for (field, item) in record.spec.fields.iter().zip(&record.values) { + if !field.name.starts_with(PRIVATE_PREFIX) { + entries.push((field.name.to_owned(), plain_value(item.bind(value.py()))?)); + } + } + return Ok(sorted_object(entries)); + } object_like_value(value) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs index 441e7d723d..a83881c415 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs @@ -260,6 +260,22 @@ impl NativeCompiledModel { self.read(py, |record, py| read(compiled_of(record)?, py)) } + /// Read the effective destination, including namespace overlays, without a Python projection. + pub(crate) fn read_destination( + &self, + py: Python<'_>, + read: impl FnOnce(&DestinationFact) -> PyResult, + ) -> PyResult { + self.compiled(py, |compiled, _| { + read( + self.overlay + .destination + .as_ref() + .unwrap_or(&compiled.destination), + ) + }) + } + /// This view's typed analysis as the owner's model facts read it, and the model's name. pub(crate) fn analysis_fact(&self, py: Python<'_>) -> PyResult<(String, ModelAnalysisFact)> { self.read(py, |record, _| { @@ -552,16 +568,11 @@ impl NativeCompiledModel { #[getter] pub(crate) fn destination(&self, py: Python<'_>) -> PyResult> { - self.compiled(py, |compiled, py| { + self.read_destination(py, |destination| { self.projections .destination .get_or_try_init(py, || { let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - let destination: &DestinationFact = self - .overlay - .destination - .as_ref() - .unwrap_or(&compiled.destination); relation_location(&classes.compiled, destination) }) .map(|destination| destination.clone_ref(py)) @@ -612,12 +623,7 @@ impl NativeCompiledModel { /// `(database, schema, name)` of the model's destination. #[getter] pub(crate) fn destination_row(&self, py: Python<'_>) -> PyResult { - self.compiled(py, |compiled, _| { - let destination: &DestinationFact = self - .overlay - .destination - .as_ref() - .unwrap_or(&compiled.destination); + self.read_destination(py, |destination| { Ok(( destination.database.clone(), destination.schema.clone(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs index 7be4f6bee8..af49d0e3f8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs @@ -436,7 +436,10 @@ impl NativeCompiledFunction { } /// The destination and fingerprint destination, with target defaults once applied. - fn current_destinations(&self, py: Python<'_>) -> PyResult<(DestinationFact, DestinationFact)> { + pub(crate) fn current_destinations( + &self, + py: Python<'_>, + ) -> PyResult<(DestinationFact, DestinationFact)> { if let Some(defaulted) = &self.defaulted { return Ok(defaulted.clone()); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs index b6f857e5f5..2d070f9535 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs @@ -237,7 +237,7 @@ impl NativeCompiledSeed { }) } - fn current_destination(&self, py: Python<'_>) -> PyResult { + pub(crate) fn current_destination(&self, py: Python<'_>) -> PyResult { if let Some(defaulted) = &self.defaulted { return Ok(defaulted.clone()); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index 8f9ded74e2..637d9ef489 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -4,6 +4,7 @@ pub(crate) mod connections; pub(crate) mod header_projection; pub(crate) mod host; pub(crate) mod model_lookup; +pub(crate) mod orchestration_environment; pub(crate) mod registration; pub(crate) mod results; pub(crate) mod sqlite_connection; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_environment.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_environment.rs new file mode 100644 index 0000000000..fac1f3356d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_environment.rs @@ -0,0 +1,50 @@ +//! Read the location interpreter's installed dependencies, never its credentials or paths. + +use std::path::Path; + +use pyo3::prelude::{PyAnyMethods, PyResult, Python}; +use sha2::{Digest, Sha256}; + +/// A lockfile binds declared dependencies; without one, bind installed distribution versions. +pub(crate) fn environment_hash(py: Python<'_>, project_dir: &Path) -> PyResult { + let mut hash = Sha256::new(); + let version = py.version_info(); + hash.update(format!( + "sqlbuild:{}\npython:{}.{}.{}{}\n", + env!("CARGO_PKG_VERSION"), + version.major, + version.minor, + version.patch, + version.suffix.unwrap_or_default(), + )); + match py.detach(|| std::fs::read(project_dir.join("uv.lock"))) { + Ok(lockfile) => { + hash.update(b"uv.lock\n"); + hash.update(lockfile); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + let mut packages = Vec::new(); + for distribution in py + .import("importlib.metadata")? + .call_method0("distributions")? + .try_iter()? + { + let distribution = distribution?; + let name: String = distribution + .getattr("metadata")? + .get_item("Name")? + .extract()?; + let version: String = distribution.getattr("version")?.extract()?; + packages.push((name.replace(['_', '.'], "-").to_ascii_lowercase(), version)); + } + packages.sort(); + hash.update(b"installed-packages\n"); + hash.update( + serde_json::to_vec(&packages) + .map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string()))?, + ); + } + Err(error) => return Err(error.into()), + } + Ok(format!("{:x}", hash.finalize())) +} diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 4a78cb2ea7..3452f57cab 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1738,6 +1738,17 @@ class NativeCompiledAudit: @property def meta(self) -> dict[str, object] | None: ... +class NativeOrchestrationSnapshot: + """A compiled data-only publication retained natively until transport.""" + + def to_json(self) -> str: ... + +def build_orchestration_snapshot( + project: CompiledProjectView, + metadata_json: str, + project_dir: Path, +) -> NativeOrchestrationSnapshot: ... + class NativeProjectView: @property def run_id(self) -> str: ... diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index bfb3ca0eab..2d7ea4bbb5 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -190,3 +190,10 @@ class QuotedHookBoundaryTestCase: description: str argument: str expected_sql: str + + +@dataclass(frozen=True) +class SnapshotCatalogTestCase: + description: str + files: dict[str, str] + expected_kinds: frozenset[str] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py new file mode 100644 index 0000000000..629fc0bdc1 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -0,0 +1,256 @@ +"""Real compilation produces immutable, relocatable control-plane catalog data.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any +from uuid import UUID + +import pytest + +from sqlbuild._native import ( + NativeOrchestrationSnapshot, + build_compiled_project, + build_orchestration_snapshot, + discover_project_inputs, +) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.types import CompiledProjectView +from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( + SnapshotCatalogTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( + FUNCTION_PROJECT_FILES, + PARITY_FILES, + TYPE_PROOF_FILES, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + +METADATA: dict[str, Any] = { + "location_id": str(UUID(int=1)), + "target": "development", + "generation": 1, + "git_revision": None, + "git_dirty": False, + "git_author": None, + "image_id": None, + "environment_hash": "test-environment", + "sqb_version": "not-the-installed-version", + "nodes": {}, + "documents": {}, +} + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotCatalogTestCase( + "model and seed catalog", + TYPE_PROOF_FILES + | { + "seeds/payment_methods.csv": PARITY_FILES["seeds/payment_methods.csv"], + "seeds/payment_methods.yml": PARITY_FILES["seeds/payment_methods.yml"], + "models/payments.sql": ( + "MODEL (description 'Payment methods.');\n" + 'SELECT method FROM __seed("payment_methods")\n' + ), + }, + frozenset({"model", "seed"}), + ), + SnapshotCatalogTestCase( + "attached audits", + TYPE_PROOF_FILES + | { + "models/orders.sql": ( + "MODEL (description 'Orders.', columns (order_id (audits [not_null])));\n" + 'SELECT order_id, amount FROM __ref("stg_orders")\n' + ), + }, + frozenset({"model", "audit"}), + ), + SnapshotCatalogTestCase( + "SQL and Python functions", + { + path: value + for path, value in FUNCTION_PROJECT_FILES.items() + if path != "enums/status.sql" + } + | {"functions/sql/_enums/status.sql": FUNCTION_PROJECT_FILES["enums/status.sql"]}, + frozenset({"model", "udf", "table_fn"}), + ), + SnapshotCatalogTestCase( + "Python hooks", + TYPE_PROOF_FILES + | { + "hooks/python/notifications.py": ( + "from sqlbuild.hooks import hook\n\n" + "@hook\ndef notify(ctx, channel):\n" + ' """Notify support."""\n return None\n' + ), + "models/orders.sql": ( + "MODEL (description 'Orders.', pre_hooks [python('notify', channel: 'support')]);\n" + 'SELECT order_id, amount FROM __ref("stg_orders")\n' + ), + }, + frozenset({"model", "hook"}), + ), + SnapshotCatalogTestCase( + "SQL hooks", + TYPE_PROOF_FILES + | { + "hooks/sql/record_load.sql": "HOOK (description 'Record load.');\nSELECT 3 AS load_count\n", + "models/orders.sql": ( + "MODEL (description 'Orders.', pre_hooks [sql('record_load')]);\n" + 'SELECT order_id FROM __ref("stg_orders")\n' + ), + }, + frozenset({"model", "sql_hook"}), + ), + SnapshotCatalogTestCase( + "source catalog", + TYPE_PROOF_FILES + | { + "sources/orders.yml": ( + "sources:\n - name: raw_orders\n description: Raw orders.\n" + " schema: raw\n table: orders\n" + " columns:\n - name: order_id\n type: INTEGER\n" + ), + }, + frozenset({"model", "source"}), + ), + SnapshotCatalogTestCase( + "DuckShop inherited seed and function namespaces", + TYPE_PROOF_FILES + | { + path: (Path("benchmarks/duckshop/project") / path).read_text() + for path in ( + "seeds/edge_cases/edge_quantity_steps.csv", + "seeds/edge_cases/edge_quantity_steps.yml", + "functions/sql/edge_cases/edge_next_quantity.sql", + ) + }, + frozenset({"model", "seed", "udf"}), + ), + ], + ids=lambda case: case.description, +) +def test_given_real_projects_when_building_snapshots_then_catalogs_are_stable_and_relocatable( + test_case: SnapshotCatalogTestCase, + tmp_path: Path, +) -> None: + publications: list[dict[str, Any]] = [] + metadata: str = json.dumps(METADATA) + for directory in (tmp_path / "original", tmp_path / "relocated"): + write_files(project_dir=directory, files=test_case.files) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=directory), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + assert not [item for item in project.diagnostics if item.severity == "error"] + snapshot: NativeOrchestrationSnapshot = build_orchestration_snapshot( + project, metadata, directory + ) + publication: dict[str, Any] = json.loads(snapshot.to_json()) + publications.append(publication) + assert publication["sqb_version"] != "not-the-installed-version" + assert publication["environment_hash"] != "test-environment" + assert test_case.expected_kinds <= {node["kind"] for node in publication["nodes"].values()} + assert str(directory) not in snapshot.to_json() + for model in project.models: + node: dict[str, Any] = publication["nodes"][f"model:{model.name}"] + assert node["compiled_sql"] == model.query_sql + assert ( + node["definition"]["destination"]["qualified_name"] + == model.destination.qualified_name + ) + assert set(node["definition"]["deps"]) == { + f"{dependency.resource_type}:{dependency.name}" for dependency in model.deps + } + for function in project.functions: + assert ( + publication["nodes"][f"{function.key.resource_type}:{function.name}"]["definition"][ + "destination" + ]["qualified_name"] + == function.destination.qualified_name + ) + for seed in project.seeds: + assert ( + publication["nodes"][f"seed:{seed.name}"]["definition"]["destination"][ + "qualified_name" + ] + == seed.destination.qualified_name + ) + assert publications[0] == publications[1] + + +def test_given_invalid_compilation_or_forged_nodes_when_publishing_then_no_snapshot_is_emitted( + tmp_path: Path, +) -> None: + write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + with pytest.raises(ValueError, match="nodes must come from the compiled project"): + build_orchestration_snapshot( + project, + json.dumps( + METADATA + | { + "nodes": { + "model:forged": { + "kind": "model", + "definition": {}, + "raw_sql": None, + "compiled_sql": "select 2", + "python_source": None, + } + } + } + ), + tmp_path, + ) + (tmp_path / "models/orders.sql").write_text("MODEL ();\nSELECT 1 AS order_id\n") + broken: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + assert any(item.severity == "error" for item in broken.diagnostics) + with pytest.raises(ValueError, match="cannot publish a project with compiler errors"): + build_orchestration_snapshot(broken, json.dumps(METADATA), tmp_path) + + +def test_given_a_dependency_lock_when_it_changes_then_environment_identity_changes_not_node_content( + tmp_path: Path, +) -> None: + write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + installed: dict[str, Any] = json.loads( + build_orchestration_snapshot(project, json.dumps(METADATA), tmp_path).to_json() + ) + (tmp_path / "uv.lock").write_text("version = 1\n# dependency revision one\n") + locked: dict[str, Any] = json.loads( + build_orchestration_snapshot(project, json.dumps(METADATA), tmp_path).to_json() + ) + (tmp_path / "uv.lock").write_text("version = 1\n# dependency revision two\n") + changed: dict[str, Any] = json.loads( + build_orchestration_snapshot(project, json.dumps(METADATA), tmp_path).to_json() + ) + assert ( + len( + {installed["environment_hash"], locked["environment_hash"], changed["environment_hash"]} + ) + == 3 + ) + assert installed["nodes"] == locked["nodes"] == changed["nodes"] + assert changed == json.loads( + build_orchestration_snapshot(project, json.dumps(METADATA), tmp_path).to_json() + ) From c3e0437c029a1581db43f09331392335271dfecf Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 00:19:40 +0000 Subject: [PATCH 06/59] refactor: clarify scheduler storage ownership and regression cases Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../src/identifier_text/main/mod.rs | 1 + .../identifier_text/main/quote_identifier.rs | 6 + .../src/runs/_helpers/claims.rs | 2 +- .../src/runs/_helpers/nodes.rs | 7 +- .../src/runs/constants.rs | 4 + .../src/runs/main/change_run.rs | 3 +- crates/sqlbuild-orchestration/src/runs/mod.rs | 1 + .../src/scheduler/_helpers/expansion.rs | 3 +- .../src/scheduler/main/run_scheduler.rs | 3 +- .../src/scheduler/main/scheduler_cycle.rs | 8 +- .../src/scheduler/mod.rs | 1 + .../src/scheduler/models.rs | 3 - .../src/scheduler/types.rs | 6 + .../src/store/main/read_recall_view.rs | 4 +- crates/sqlbuild-orchestration/tests/runs.rs | 4 +- .../tests/runs/test_claims.rs | 101 ++++++++++--- .../tests/runs/test_coalesce.rs | 21 ++- .../tests/runs/test_faults.rs | 42 ++++-- .../tests/runs/test_lifecycle.rs | 134 +++++++++++------- .../tests/runs/test_types.rs | 44 ++++++ .../tests/ticks/test_loop.rs | 13 +- .../_helpers/compiled_project/snapshot.rs | 3 +- .../native_host/orchestration_environment.rs | 2 +- .../src/clickhouse/_helpers/literals.rs | 3 +- .../src/clickhouse/constants.rs | 2 - .../src/clickhouse/main/encode_rows.rs | 3 +- .../src/records/_helpers/rows.rs | 11 -- crates/sqlbuild-recall/src/records/models.rs | 14 ++ .../src/schema/_helpers/postgres_views.rs | 14 +- .../src/schema/_helpers/relational_tables.rs | 23 +-- .../src/schema/_helpers/relational_views.rs | 48 +++---- .../src/schema/_helpers/rendering.rs | 5 - .../sqlbuild-recall/src/schema/constants.rs | 5 +- .../src/schema/main/add_column_statement.rs | 5 +- .../schema/main/control_view_statements.rs | 3 +- .../main/postgres_partition_statement.rs | 2 +- .../src/schema/main/sqlite_retention_sql.rs | 3 +- .../src/schema/main/upgrade_statements.rs | 3 +- .../src/schema/main/view_statements.rs | 4 +- crates/sqlbuild-recall/src/schema/models.rs | 16 +++ .../src/store/_helpers/relational.rs | 3 +- .../src/store/_helpers/rows.rs | 4 +- .../src/store/main/conflict_evidence.rs | 8 +- .../src/store/main/relational_insert.rs | 12 +- .../src/store/main/relational_lookup.rs | 18 ++- fensu.toml | 12 ++ .../compiler/compiled_project/_test_types.py | 17 +++ .../test_orchestration_snapshot.py | 84 +++++++---- 48 files changed, 491 insertions(+), 247 deletions(-) create mode 100644 crates/sqlbuild-core/src/identifier_text/main/quote_identifier.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/constants.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/types.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/test_types.rs diff --git a/crates/sqlbuild-core/src/identifier_text/main/mod.rs b/crates/sqlbuild-core/src/identifier_text/main/mod.rs index a29dc793b1..b9d09a0ef5 100644 --- a/crates/sqlbuild-core/src/identifier_text/main/mod.rs +++ b/crates/sqlbuild-core/src/identifier_text/main/mod.rs @@ -1 +1,2 @@ pub mod is_lowercase_identifier; +pub mod quote_identifier; diff --git a/crates/sqlbuild-core/src/identifier_text/main/quote_identifier.rs b/crates/sqlbuild-core/src/identifier_text/main/quote_identifier.rs new file mode 100644 index 0000000000..efd258cbc0 --- /dev/null +++ b/crates/sqlbuild-core/src/identifier_text/main/quote_identifier.rs @@ -0,0 +1,6 @@ +//! Standard SQL double-quoted identifiers, including embedded quote escaping. + +#[must_use] +pub fn quote_identifier(name: &str) -> String { + format!("\"{}\"", name.replace('"', "\"\"")) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/claims.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/claims.rs index 2b65b19559..8e69205b5f 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/claims.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/claims.rs @@ -9,6 +9,7 @@ use crate::store::errors::ControlError; use crate::store::main::query::query; use crate::store::models::{ControlTransaction, Parameter}; +/// Occupied slots above a lowered limit still count against the run's concurrency group. pub(crate) async fn reserve_claims( mut transaction: ControlTransaction, request: &RunRequest, @@ -62,7 +63,6 @@ pub(crate) async fn reserve_claims( .iter() .filter_map(|row| row.get("slot").and_then(serde_json::Value::as_i64)) .collect(); - // Lowering a limit also counts occupied slots above the new maximum. if occupied.len() >= limit as usize { return Ok((transaction, false)); } diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/nodes.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/nodes.rs index cb8d430a89..8ae44b1a94 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/nodes.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/nodes.rs @@ -4,6 +4,7 @@ use std::collections::BTreeMap; use serde_json::json; +use crate::runs::constants::{NODE_PENDING, NODE_RUNNING}; use crate::runs::models::{RunChange, RunChangeRequest}; use crate::store::errors::ControlError; use crate::store::main::query::query; @@ -15,7 +16,7 @@ pub(crate) async fn project_node( counts: &BTreeMap, ) -> Result { let (node, status, rows, error, skipped) = match &request.change { - RunChange::NodeStarted { node_key } => (node_key, "running", None, None, None), + RunChange::NodeStarted { node_key } => (node_key, NODE_RUNNING, None, None, None), RunChange::NodeFinished { node_key, outcome, @@ -47,7 +48,7 @@ pub(crate) async fn project_node( .and_then(|row| row.get("status")) .and_then(serde_json::Value::as_str) .ok_or_else(|| ControlError::Invalid("node is outside the frozen run selection".into()))?; - if previous != "pending" && (previous != "running" || status == "running") { + if previous != NODE_PENDING && (previous != NODE_RUNNING || status == NODE_RUNNING) { return Err(ControlError::Invalid( "node change is inconsistent with its current outcome".into(), )); @@ -67,7 +68,7 @@ pub(crate) async fn project_node( let (transaction, _) = query(transaction, "UPDATE sqb_run_nodes SET status=$1, started_at=CASE WHEN $1='running' THEN $2 ELSE started_at END, \ ended_at=$3, rows_affected=$4, error_code=$5, skipped_because_of=$6 WHERE run_id=$7 AND node_key=$8 AND attempt=1", - vec![Parameter::Text(status.into()), Parameter::Time(now), Parameter::OptionalTime((status != "running").then_some(now)), + vec![Parameter::Text(status.into()), Parameter::Time(now), Parameter::OptionalTime((status != NODE_RUNNING).then_some(now)), rows.map_or(Parameter::Null(sqlbuild_recall::schema::models::ColumnType::Integer), Parameter::Integer), Parameter::OptionalText(error), Parameter::OptionalText(skipped), Parameter::Id(request.run_id), Parameter::Text(node.clone())]).await?; let (transaction, _) = query( diff --git a/crates/sqlbuild-orchestration/src/runs/constants.rs b/crates/sqlbuild-orchestration/src/runs/constants.rs new file mode 100644 index 0000000000..d92a2e052a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/constants.rs @@ -0,0 +1,4 @@ +//! Nonterminal node states; terminal states come from NodeOutcome. + +pub(crate) const NODE_PENDING: &str = "pending"; +pub(crate) const NODE_RUNNING: &str = "running"; diff --git a/crates/sqlbuild-orchestration/src/runs/main/change_run.rs b/crates/sqlbuild-orchestration/src/runs/main/change_run.rs index f3884cb69f..458ae8ae4a 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/change_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/change_run.rs @@ -14,7 +14,7 @@ use crate::store::main::decode_row::decode_row; use crate::store::main::query::query; use crate::store::models::{ControlStore, Parameter, TransactionMode}; -/// Authority comes from authenticated API state, not client-supplied actor or scope fields. +/// Use authenticated authority and record unclaimed cancellation as terminal in the same transaction. pub async fn change_run( store: &ControlStore, authority: RunAuthority<'_>, @@ -67,7 +67,6 @@ pub async fn change_run( ) .await?; let mut transaction = project_change(transaction, request, &state).await?; - // No process can exist before a durable claim. Cancel queued work without leaving it inert forever. if matches!(request.change, RunChange::CancelRequested { .. }) && state.worker_instance.is_none() { diff --git a/crates/sqlbuild-orchestration/src/runs/mod.rs b/crates/sqlbuild-orchestration/src/runs/mod.rs index 5359000d16..f147865646 100644 --- a/crates/sqlbuild-orchestration/src/runs/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod _helpers; +pub(crate) mod constants; pub mod main; pub mod models; diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs index 598797bfc5..8ba7d7e422 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs @@ -9,7 +9,8 @@ use crate::scheduler::constants::MAX_TICKS_PER_BATCH; use crate::scheduler::main::next_cron_time::next_cron_time; use crate::scheduler::main::parse_cron::parse_cron; use crate::scheduler::main::previous_cron_time::previous_cron_time; -use crate::scheduler::models::{CatchUp, ScheduleExpansion}; +use crate::scheduler::models::CatchUp; +use crate::scheduler::types::ScheduleExpansion; use crate::store::errors::ControlError; pub(crate) fn expand_schedule( diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs b/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs index 4bf8917b6f..29c8180b65 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs @@ -15,7 +15,8 @@ pub async fn run_scheduler( mut changes: watch::Receiver<()>, ) -> Result<(), ControlError> { while !*shutdown.borrow() { - let cycle = scheduler_cycle(store, &mut state).await?; + let (next_state, cycle) = scheduler_cycle(store, state).await?; + state = next_state; tokio::select! { _ = tokio::time::sleep(cycle.wait_for) => {}, result = changes.changed() => { if result.is_err() { break; } }, diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/scheduler_cycle.rs b/crates/sqlbuild-orchestration/src/scheduler/main/scheduler_cycle.rs index 9e6edf9a32..011d28e303 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/scheduler_cycle.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/scheduler_cycle.rs @@ -18,8 +18,8 @@ use crate::store::models::{ControlStore, Parameter, TransactionMode}; pub async fn scheduler_cycle( store: &ControlStore, - state: &mut SchedulerState, -) -> Result { + mut state: SchedulerState, +) -> Result<(SchedulerState, SchedulerCycle), ControlError> { let renewal = match state.lease.as_ref() { Some(lease) => renew_lease(store, lease).await.map(Some), None => acquire_lease(store, state.holder).await, @@ -36,7 +36,7 @@ pub async fn scheduler_cycle( wait_for: Duration::from_secs(RENEW_INTERVAL_SECONDS), }; let Some(mut lease) = state.lease.clone() else { - return Ok(cycle); + return Ok((state, cycle)); }; let transaction = begin(store, TransactionMode::Read).await?; let now = transaction.now; @@ -71,5 +71,5 @@ pub async fn scheduler_cycle( .min(cycle.wait_for); } commit(transaction).await?; - Ok(cycle) + Ok((state, cycle)) } diff --git a/crates/sqlbuild-orchestration/src/scheduler/mod.rs b/crates/sqlbuild-orchestration/src/scheduler/mod.rs index d686433c8a..3978df0a4a 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/mod.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/mod.rs @@ -4,6 +4,7 @@ pub(crate) mod _helpers; pub mod constants; pub mod main; pub mod models; +pub(crate) mod types; #[cfg(test)] mod tests; diff --git a/crates/sqlbuild-orchestration/src/scheduler/models.rs b/crates/sqlbuild-orchestration/src/scheduler/models.rs index b219340219..a3a0c2ccad 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/models.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/models.rs @@ -46,9 +46,6 @@ pub struct CronSchedule { pub(crate) timezone: chrono_tz::Tz, } -/// Expanded wall-clock times and the first omitted time, if the batch was bounded. -pub(crate) type ScheduleExpansion = (Vec>, Option>); - #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum CatchUp { diff --git a/crates/sqlbuild-orchestration/src/scheduler/types.rs b/crates/sqlbuild-orchestration/src/scheduler/types.rs new file mode 100644 index 0000000000..274b453740 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/types.rs @@ -0,0 +1,6 @@ +//! Compound scheduler return values. + +use chrono::{DateTime, Utc}; + +/// Expanded wall-clock times and the first omitted time, if the batch was bounded. +pub(crate) type ScheduleExpansion = (Vec>, Option>); diff --git a/crates/sqlbuild-orchestration/src/store/main/read_recall_view.rs b/crates/sqlbuild-orchestration/src/store/main/read_recall_view.rs index 4c323fcd1b..a07da01efb 100644 --- a/crates/sqlbuild-orchestration/src/store/main/read_recall_view.rs +++ b/crates/sqlbuild-orchestration/src/store/main/read_recall_view.rs @@ -9,7 +9,7 @@ use crate::store::main::query::query; use crate::store::models::{ControlStore, Parameter, TransactionMode}; use crate::store::types::Row; -/// The API authorizes the invocation and evidence capability before entering this storage reader. +/// Read authorized evidence, decoding only display text while preserving byte-exact captured content. pub async fn read_recall_view( store: &ControlStore, (view, invocation_id): (&str, &str), @@ -28,8 +28,6 @@ pub async fn read_recall_view( ], ) .await?; - // These PostgreSQL view columns retain bytes: conversion to database text would reject NUL. - // Decode only the display field, leaving captured content and transcripts byte-exact. if matches!(view, "output_tail" | "artifacts") { for row in &mut rows { if let Some(text) = row.get_mut("text") diff --git a/crates/sqlbuild-orchestration/tests/runs.rs b/crates/sqlbuild-orchestration/tests/runs.rs index 627c22a308..10a2813d22 100644 --- a/crates/sqlbuild-orchestration/tests/runs.rs +++ b/crates/sqlbuild-orchestration/tests/runs.rs @@ -1,5 +1,7 @@ #[path = "ticks/fault_helpers.rs"] mod fault_helpers; +#[path = "store/fault_types.rs"] +mod fault_types; #[path = "runs/helpers.rs"] mod helpers; #[path = "store/lease_helpers.rs"] @@ -20,5 +22,5 @@ mod test_coalesce; mod test_faults; #[path = "runs/test_lifecycle.rs"] mod test_lifecycle; -#[path = "store/fault_types.rs"] +#[path = "runs/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_claims.rs b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs index b97dc931e5..509232d7f2 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_claims.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs @@ -16,13 +16,30 @@ use crate::replica_helpers::second_store; use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; use crate::store_test_types::Backend; use crate::terminal_helpers::finish; +use crate::test_types::{BacklogClaimTestCase, GroupLimitTestCase, PriorityClaimsTestCase}; #[test] fn given_overlapping_priority_queues_when_two_launchers_claim_then_conflicts_rollback_and_terminal_facts_release_claims() { - for backend in [Backend::Sqlite, Backend::Postgres] { + let test_cases = [ + PriorityClaimsTestCase { + description: "SQLite claims prioritize independent write sets and release atomically", + backend: Backend::Sqlite, + expected_initial_claims: [2, 3], + expected_claim_events: 4, + expected_terminal_events: 2, + }, + PriorityClaimsTestCase { + description: "PostgreSQL claims prioritize independent write sets and release atomically", + backend: Backend::Postgres, + expected_initial_claims: [2, 3], + expected_claim_events: 4, + expected_terminal_events: 2, + }, + ]; + for test_case in test_cases { run(async { - let Some(fixture) = open_test_store(backend).await else { + let Some(fixture) = open_test_store(test_case.backend).await else { return; }; let store = &fixture.store; @@ -61,10 +78,13 @@ fn given_overlapping_priority_queues_when_two_launchers_claim_then_conflicts_rol .collect(); assert_eq!( claimed, - [Uuid::from_u128(2), Uuid::from_u128(3)] + test_case + .expected_initial_claims .into_iter() + .map(Uuid::from_u128) .collect(), - "higher priority wins the shared write set" + "{}", + test_case.description ); assert!( claim_next_run(store, &lease, &limits) @@ -102,14 +122,18 @@ fn given_overlapping_priority_queues_when_two_launchers_claim_then_conflicts_rol .iter() .filter(|event| event["event_type"] == "run_claimed") .count(), - 4 + test_case.expected_claim_events, + "{}", + test_case.description ); assert_eq!( events .iter() .filter(|event| event["event_type"] == "run_terminated") .count(), - 2 + test_case.expected_terminal_events, + "{}", + test_case.description ); let runs = list_runs(store, (location, "development"), 100) .await @@ -129,9 +153,21 @@ fn given_overlapping_priority_queues_when_two_launchers_claim_then_conflicts_rol #[test] fn given_named_group_limits_when_capacity_is_lowered_then_active_high_slots_still_count_and_terminal_cleanup_unblocks_work() { - for backend in [Backend::Sqlite, Backend::Postgres] { + let test_cases = [ + GroupLimitTestCase { + description: "SQLite occupied high slots still count after lowering a group limit", + backend: Backend::Sqlite, + expected_unblocked_run: 3, + }, + GroupLimitTestCase { + description: "PostgreSQL occupied high slots still count after lowering a group limit", + backend: Backend::Postgres, + expected_unblocked_run: 3, + }, + ]; + for test_case in test_cases { run(async { - let Some(fixture) = open_test_store(backend).await else { + let Some(fixture) = open_test_store(test_case.backend).await else { return; }; let store = &fixture.store; @@ -194,7 +230,12 @@ fn given_named_group_limits_when_capacity_is_lowered_then_active_high_slots_stil .await .expect("released group") .expect("third queued run"); - assert_eq!(third.request.run_id, Uuid::from_u128(3)); + assert_eq!( + third.request.run_id, + Uuid::from_u128(test_case.expected_unblocked_run), + "{}", + test_case.description + ); close_test_store(fixture).await; }); } @@ -203,9 +244,21 @@ fn given_named_group_limits_when_capacity_is_lowered_then_active_high_slots_stil #[test] fn given_more_than_one_page_of_blocked_runs_when_claiming_then_lower_priority_independent_work_is_not_starved() { - for backend in [Backend::Sqlite, Backend::Postgres] { + let test_cases = [ + BacklogClaimTestCase { + description: "SQLite scans past more than a full page of blocked work", + backend: Backend::Sqlite, + expected_independent_run: 105, + }, + BacklogClaimTestCase { + description: "PostgreSQL scans past more than a full page of blocked work", + backend: Backend::Postgres, + expected_independent_run: 105, + }, + ]; + for test_case in test_cases { run(async { - let Some(fixture) = open_test_store(backend).await else { + let Some(fixture) = open_test_store(test_case.backend).await else { return; }; let store = &fixture.store; @@ -227,15 +280,15 @@ fn given_more_than_one_page_of_blocked_runs_when_claiming_then_lower_priority_in .await .expect("claim blocker") .expect("available"); - for id in 2..=104 { - queue(store, &request(scope, id, 0, &["orders"])).await; - if id % 20 == 0 { - lease = sqlbuild_orchestration::scheduler::main::renew_lease::renew_lease( - store, &lease, - ) - .await - .expect("keep leader live while preparing backlog"); + for batch in (2..=104).collect::>().chunks(20) { + for id in batch { + queue(store, &request(scope, *id, 0, &["orders"])).await; } + lease = sqlbuild_orchestration::scheduler::main::renew_lease::renew_lease( + store, &lease, + ) + .await + .expect("keep leader live while preparing backlog"); } queue(store, &request(scope, 105, -1, &["customers"])).await; let started = std::time::Instant::now(); @@ -243,10 +296,16 @@ fn given_more_than_one_page_of_blocked_runs_when_claiming_then_lower_priority_in .await .expect("scan past a full blocked page") .expect("independent work"); - assert_eq!(independent.request.run_id, Uuid::from_u128(105)); + assert_eq!( + independent.request.run_id, + Uuid::from_u128(test_case.expected_independent_run), + "{}", + test_case.description + ); assert!( started.elapsed() < std::time::Duration::from_secs(5), - "claim scan exceeded five seconds on {backend:?}" + "claim scan exceeded five seconds: {}", + test_case.description ); close_test_store(fixture).await; }); diff --git a/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs b/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs index c05d357583..8c0ea844dd 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs @@ -15,13 +15,26 @@ use uuid::Uuid; use crate::helpers::snapshot; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; +use crate::test_types::CoalescedRunTestCase; #[test] fn given_coalescing_triggers_when_a_previous_run_has_been_claimed_then_new_ticks_wait_in_a_new_run_instead_of_covering_executing_work() { - for backend in [Backend::Sqlite, Backend::Postgres] { + let test_cases = [ + CoalescedRunTestCase { + description: "SQLite coalesces only work that has not been claimed", + backend: Backend::Sqlite, + expected_runs: 2, + }, + CoalescedRunTestCase { + description: "PostgreSQL coalesces only work that has not been claimed", + backend: Backend::Postgres, + expected_runs: 2, + }, + ]; + for test_case in test_cases { run(async { - let Some(fixture) = open_test_store(backend).await else { + let Some(fixture) = open_test_store(test_case.backend).await else { return; }; let store = &fixture.store; @@ -95,7 +108,9 @@ fn given_coalescing_triggers_when_a_previous_run_has_been_claimed_then_new_ticks .await .expect("one executing and one waiting") .len(), - 2 + test_case.expected_runs, + "{}", + test_case.description ); close_test_store(fixture).await; }); diff --git a/crates/sqlbuild-orchestration/tests/runs/test_faults.rs b/crates/sqlbuild-orchestration/tests/runs/test_faults.rs index a0f076c366..3332a4bd5c 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_faults.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_faults.rs @@ -19,14 +19,26 @@ use crate::helpers::{queue, request, snapshot}; use crate::lease_helpers::expire_lease; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; -use crate::test_types::TickFaultTestCase; +use crate::test_types::{ClaimFailureTestCase, TickFaultTestCase}; #[test] fn given_failures_during_claim_publication_when_retried_then_claims_slots_liveness_facts_and_projection_commit_together() { - for backend in [Backend::Sqlite, Backend::Postgres] { + let test_cases = [ + ClaimFailureTestCase { + description: "SQLite rolls back every claim-publication failure window", + backend: Backend::Sqlite, + expected_run: 1, + }, + ClaimFailureTestCase { + description: "PostgreSQL rolls back every claim-publication failure window", + backend: Backend::Postgres, + expected_run: 1, + }, + ]; + for test_case in test_cases { run(async { - let Some(fixture) = open_test_store(backend).await else { + let Some(fixture) = open_test_store(test_case.backend).await else { return; }; let store = &fixture.store; @@ -59,21 +71,17 @@ fn given_failures_during_claim_publication_when_retried_then_claims_slots_livene let projected = list_runs(store, (location, "development"), 100) .await .expect("queued projection"); - for table in [ - "sqb_write_claims", - "sqb_concurrency_slots", - "sqb_run_liveness", - "recall_events", - "sqb_runs", + for (table, operation) in [ + ("sqb_write_claims", "INSERT"), + ("sqb_concurrency_slots", "INSERT"), + ("sqb_run_liveness", "INSERT"), + ("recall_events", "INSERT"), + ("sqb_runs", "UPDATE"), ] { let point = TickFaultTestCase { description: "claim publication failure", table, - operation: if table == "sqb_runs" { - "UPDATE" - } else { - "INSERT" - }, + operation, condition: "", sqlite_body: "SELECT RAISE(ABORT,'injected claim failure')", postgres_body: "RAISE EXCEPTION 'injected claim failure'", @@ -121,6 +129,12 @@ fn given_failures_during_claim_publication_when_retried_then_claims_slots_livene .expect("retry complete claim") .expect("no leaked claims or slots"); assert_eq!(claim.request, input); + assert_eq!( + claim.request.run_id, + Uuid::from_u128(test_case.expected_run), + "{}", + test_case.description + ); assert!( claim_next_run(store, &lease, &limits) .await diff --git a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs index 7ed2c6874a..d61b50e82a 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs @@ -19,14 +19,26 @@ use crate::fault_helpers::{install_fault, remove_fault}; use crate::helpers::{queue, request, snapshot}; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; -use crate::test_types::TickFaultTestCase; +use crate::test_types::{RunLifecycleTestCase, TickFaultTestCase}; #[test] fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_facts_outcomes_and_claims_are_atomic() { - for backend in [Backend::Sqlite, Backend::Postgres] { + let test_cases = [ + RunLifecycleTestCase { + description: "SQLite preserves facts, outcomes and claims through worker failures", + backend: Backend::Sqlite, + expected_status: RunStatus::Partial, + }, + RunLifecycleTestCase { + description: "PostgreSQL preserves facts, outcomes and claims through worker failures", + backend: Backend::Postgres, + expected_status: RunStatus::Partial, + }, + ]; + for test_case in test_cases { run(async { - let Some(fixture) = open_test_store(backend).await else { + let Some(fixture) = open_test_store(test_case.backend).await else { return; }; let store = &fixture.store; @@ -137,42 +149,40 @@ fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_ }, ..started.clone() }; - if node == "customers" { - let before_nodes = list_run_nodes(store, started.run_id) - .await - .expect("node before fault"); - let before_events = read_recall_events(store, 0, 100) - .await - .expect("events before node fault"); - for table in ["sqb_run_nodes", "sqb_runs"] { - let point = TickFaultTestCase { - description: "node projection and counter failure", - table, - operation: "UPDATE", - condition: "", - sqlite_body: "SELECT RAISE(ABORT,'injected node failure')", - postgres_body: "RAISE EXCEPTION 'injected node failure'", - }; - install_fault(store, &point).await; - assert!( - change_run(store, worker, &completed).await.is_err(), - "{} at {table}", - point.description - ); - remove_fault(store, &point).await; - assert_eq!( - list_run_nodes(store, started.run_id) - .await - .expect("node rollback"), - before_nodes - ); - assert_eq!( - read_recall_events(store, 0, 100) - .await - .expect("node fact rollback"), - before_events - ); - } + let before_nodes = list_run_nodes(store, started.run_id) + .await + .expect("node before fault"); + let before_events = read_recall_events(store, 0, 100) + .await + .expect("events before node fault"); + for table in ["sqb_run_nodes", "sqb_runs"] { + let point = TickFaultTestCase { + description: "node projection and counter failure", + table, + operation: "UPDATE", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected node failure')", + postgres_body: "RAISE EXCEPTION 'injected node failure'", + }; + install_fault(store, &point).await; + assert!( + change_run(store, worker, &completed).await.is_err(), + "{} at {table}", + point.description + ); + remove_fault(store, &point).await; + assert_eq!( + list_run_nodes(store, started.run_id) + .await + .expect("node rollback"), + before_nodes + ); + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("node fact rollback"), + before_events + ); } assert!( change_run(store, worker, &completed) @@ -201,22 +211,16 @@ fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_ let before = read_recall_events(store, 0, 100) .await .expect("before terminal"); - for table in [ - "recall_events", - "sqb_runs", - "sqb_write_claims", - "sqb_run_liveness", + for (table, operation) in [ + ("recall_events", "INSERT"), + ("sqb_runs", "UPDATE"), + ("sqb_write_claims", "DELETE"), + ("sqb_run_liveness", "DELETE"), ] { let point = TickFaultTestCase { description: "terminal transaction failure", table, - operation: if table == "recall_events" { - "INSERT" - } else if table.starts_with("sqb_run") && table != "sqb_run_liveness" { - "UPDATE" - } else { - "DELETE" - }, + operation, condition: "", sqlite_body: "SELECT RAISE(ABORT,'injected terminal failure')", postgres_body: "RAISE EXCEPTION 'injected terminal failure'", @@ -317,7 +321,11 @@ fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_ .into_iter() .find(|run| run.run_id == started.run_id) .expect("original run"); - assert_eq!(projected.status, RunStatus::Partial); + assert_eq!( + projected.status, test_case.expected_status, + "{}", + test_case.description + ); assert_eq!( projected.node_counts_json, BTreeMap::from([("success".into(), 1), ("failed".into(), 1)]) @@ -342,9 +350,21 @@ fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_ #[test] fn given_unclaimed_work_when_an_operator_cancels_then_a_terminal_fact_prevents_launch_and_preserves_the_actor() { - for backend in [Backend::Sqlite, Backend::Postgres] { + let test_cases = [ + RunLifecycleTestCase { + description: "SQLite cancellation before launch preserves the operator and terminal fact", + backend: Backend::Sqlite, + expected_status: RunStatus::Cancelled, + }, + RunLifecycleTestCase { + description: "PostgreSQL cancellation before launch preserves the operator and terminal fact", + backend: Backend::Postgres, + expected_status: RunStatus::Cancelled, + }, + ]; + for test_case in test_cases { run(async { - let Some(fixture) = open_test_store(backend).await else { + let Some(fixture) = open_test_store(test_case.backend).await else { return; }; let store = &fixture.store; @@ -397,7 +417,11 @@ fn given_unclaimed_work_when_an_operator_cancels_then_a_terminal_fact_prevents_l .await .expect("cancelled projection") .remove(0); - assert_eq!(projected.status, RunStatus::Cancelled); + assert_eq!( + projected.status, test_case.expected_status, + "{}", + test_case.description + ); assert_eq!( projected.node_counts_json, BTreeMap::from([("cancelled".into(), 1)]) diff --git a/crates/sqlbuild-orchestration/tests/runs/test_types.rs b/crates/sqlbuild-orchestration/tests/runs/test_types.rs new file mode 100644 index 0000000000..61c94312d4 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_types.rs @@ -0,0 +1,44 @@ +//! Expected queue outcomes for each real control-store backend. + +use sqlbuild_orchestration::runs::models::RunStatus; + +pub(crate) use crate::fault_types::TickFaultTestCase; +use crate::store_test_types::Backend; + +pub(crate) struct PriorityClaimsTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_initial_claims: [u128; 2], + pub(crate) expected_claim_events: usize, + pub(crate) expected_terminal_events: usize, +} + +pub(crate) struct GroupLimitTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_unblocked_run: u128, +} + +pub(crate) struct BacklogClaimTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_independent_run: u128, +} + +pub(crate) struct CoalescedRunTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_runs: usize, +} + +pub(crate) struct ClaimFailureTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_run: u128, +} + +pub(crate) struct RunLifecycleTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_status: RunStatus, +} diff --git a/crates/sqlbuild-orchestration/tests/ticks/test_loop.rs b/crates/sqlbuild-orchestration/tests/ticks/test_loop.rs index d847e5546a..2864da8921 100644 --- a/crates/sqlbuild-orchestration/tests/ticks/test_loop.rs +++ b/crates/sqlbuild-orchestration/tests/ticks/test_loop.rs @@ -46,30 +46,31 @@ fn given_embedded_and_standalone_schedulers_when_taking_over_then_only_one_drain .await .expect("publish due schedule"); let second = second_store(store).await; - let mut first_state = SchedulerState::new(Uuid::now_v7()); - let mut second_state = SchedulerState::new(Uuid::now_v7()); - let first = scheduler_cycle(store, &mut first_state) + let first_state = SchedulerState::new(Uuid::now_v7()); + let second_state = SchedulerState::new(Uuid::now_v7()); + let (first_state, first) = scheduler_cycle(store, first_state) .await .expect("initial cycle"); assert!(first.leader, "{}", test_case.description); assert_eq!(first.evaluated, 1); assert_eq!(first.ticks, test_case.expected_ticks); assert_eq!(first.wait_for, Duration::from_secs(5)); - let follower = scheduler_cycle(&second, &mut second_state) + let (second_state, follower) = scheduler_cycle(&second, second_state) .await .expect("follower cycle"); assert!(!follower.leader); assert_eq!(follower.ticks, 0); expire_lease(store).await; - let successor = scheduler_cycle(&second, &mut second_state) + let (second_state, successor) = scheduler_cycle(&second, second_state) .await .expect("takeover cycle"); assert!(successor.leader); assert_eq!(successor.ticks, 0, "takeover never repeats durable work"); assert!( - !scheduler_cycle(store, &mut first_state) + !scheduler_cycle(store, first_state) .await .expect("fenced cycle") + .1 .leader ); let runs = sqlbuild_orchestration::runs::main::list_runs::list_runs( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs index 1edaf8fcd4..b0b0d878fe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs @@ -7,6 +7,7 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult, Python}; use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; use serde_json::Value; +use sqlbuild_core::models::DiagnosticSeverity; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; use crate::bindings::_helpers::compiled_project::snapshot_models::model_nodes; @@ -55,7 +56,7 @@ pub(crate) fn build_orchestration_snapshot( } for diagnostic in project.diagnostics(py).bind(py).try_iter()? { let severity: String = field(&field(&diagnostic?, "severity")?, "value")?.extract()?; - if severity == "error" { + if severity == DiagnosticSeverity::Error.as_str() { return Err(PyValueError::new_err( "cannot publish a project with compiler errors", )); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_environment.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_environment.rs index fac1f3356d..5b49bcaa07 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_environment.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_environment.rs @@ -23,7 +23,7 @@ pub(crate) fn environment_hash(py: Python<'_>, project_dir: &Path) -> PyResult { - let mut packages = Vec::new(); + let mut packages: Vec<(String, String)> = Vec::new(); for distribution in py .import("importlib.metadata")? .call_method0("distributions")? diff --git a/crates/sqlbuild-recall/src/clickhouse/_helpers/literals.rs b/crates/sqlbuild-recall/src/clickhouse/_helpers/literals.rs index 784b5e1cd3..a6934e5168 100644 --- a/crates/sqlbuild-recall/src/clickhouse/_helpers/literals.rs +++ b/crates/sqlbuild-recall/src/clickhouse/_helpers/literals.rs @@ -1,7 +1,6 @@ //! ClickHouse literals and key expressions for the store's own queries. use crate::clickhouse::constants::TIMESTAMP_PATTERN; -use crate::records::_helpers::rows; use crate::records::main::format_timestamp::format_timestamp; use crate::records::main::parse_timestamp::parse_timestamp; use crate::records::main::timestamp_micros::timestamp_micros; @@ -63,6 +62,6 @@ pub(crate) fn key_text(value: Option<&RecallValue>, column_type: ColumnType) -> Some(RecallValue::Text(text)) if column_type == ColumnType::Timestamp => { parse_timestamp(text).map_or_else(|| text.clone(), format_timestamp) } - _ => rows::key_text(value), + _ => value.map_or_else(String::new, RecallValue::key_text), } } diff --git a/crates/sqlbuild-recall/src/clickhouse/constants.rs b/crates/sqlbuild-recall/src/clickhouse/constants.rs index d11b50dc61..2c8f2719df 100644 --- a/crates/sqlbuild-recall/src/clickhouse/constants.rs +++ b/crates/sqlbuild-recall/src/clickhouse/constants.rs @@ -61,8 +61,6 @@ pub(crate) const MAX_DATETIME64_PRECISION: u64 = 9; pub(crate) const MICROS_DIGITS: u64 = 6; /// One decimal digit of timestamp scale. pub(crate) const DECIMAL_SCALE: i64 = 10; -/// The column the store sets when a row is written. -pub(crate) const INGESTED_AT_COLUMN: &str = "ingested_at"; pub(crate) const VARINT_PAYLOAD_BITS: u32 = 7; pub(crate) const VARINT_CONTINUE: u8 = 0x80; pub(crate) const VARINT_MASK: u64 = 0x7f; diff --git a/crates/sqlbuild-recall/src/clickhouse/main/encode_rows.rs b/crates/sqlbuild-recall/src/clickhouse/main/encode_rows.rs index a91f7a0e35..7bd512fa2a 100644 --- a/crates/sqlbuild-recall/src/clickhouse/main/encode_rows.rs +++ b/crates/sqlbuild-recall/src/clickhouse/main/encode_rows.rs @@ -1,9 +1,10 @@ //! `encode_rows`: one table's rows as a `RowBinary` insert block. use crate::clickhouse::_helpers::row_binary::{encoded, scaled}; -use crate::clickhouse::constants::{INGESTED_AT_COLUMN, MICROS_DIGITS}; +use crate::clickhouse::constants::MICROS_DIGITS; use crate::clickhouse::models::{ColumnEncoding, StoredColumn}; use crate::records::models::{RecallRow, RecallValue}; +use crate::schema::constants::INGESTED_AT_COLUMN; /// The column names and `RowBinary` block of `rows`, all of one table, written with `ingested_at`. Values for columns the table lacks are left out; columns the rows lack take their defaults. pub fn encode_rows( diff --git a/crates/sqlbuild-recall/src/records/_helpers/rows.rs b/crates/sqlbuild-recall/src/records/_helpers/rows.rs index f5b60dc1e5..5ad2768b37 100644 --- a/crates/sqlbuild-recall/src/records/_helpers/rows.rs +++ b/crates/sqlbuild-recall/src/records/_helpers/rows.rs @@ -23,17 +23,6 @@ pub(crate) fn boolean(value: bool) -> RecallValue { RecallValue::Integer(i64::from(value)) } -/// The human-readable part of a conflict key, without dialect-specific timestamp normalization. -pub(crate) fn key_text(value: Option<&RecallValue>) -> String { - match value { - Some(RecallValue::Text(text)) => text.clone(), - Some(RecallValue::Integer(number)) => number.to_string(), - Some(RecallValue::Real(number)) => number.to_string(), - Some(RecallValue::Blob(_)) => "".to_owned(), - Some(RecallValue::Null) | None => String::new(), - } -} - /// `values` plus the envelope columns and `content_hash`; an artifact's digest covers only its content, since every invocation that sees it writes the same row. pub(crate) fn finish_row( table: Table, diff --git a/crates/sqlbuild-recall/src/records/models.rs b/crates/sqlbuild-recall/src/records/models.rs index e9da820c87..e99a1cf280 100644 --- a/crates/sqlbuild-recall/src/records/models.rs +++ b/crates/sqlbuild-recall/src/records/models.rs @@ -12,6 +12,20 @@ pub enum RecallValue { Blob(Vec), } +impl RecallValue { + /// Human-readable conflict-key text, before dialect-specific timestamp normalization. + #[must_use] + pub fn key_text(&self) -> String { + match self { + Self::Text(text) => text.clone(), + Self::Integer(number) => number.to_string(), + Self::Real(number) => number.to_string(), + Self::Blob(_) => "".to_owned(), + Self::Null => String::new(), + } + } +} + /// One row bound for a Recall table: named values, without `ingested_at`, which the store sets. #[derive(Clone, Debug, PartialEq)] pub struct RecallRow { diff --git a/crates/sqlbuild-recall/src/schema/_helpers/postgres_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/postgres_views.rs index 98515a0e19..ece90e302a 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/postgres_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/postgres_views.rs @@ -29,12 +29,12 @@ fn json_fields(mut body: String) -> String { &format!("(payload ->> '{field}'){cast}"), ); } - for field in ["name", "job_name", "run_id", "retry_number"] { - let cast = if field == "retry_number" { - "::BIGINT" - } else { - "" - }; + for (field, cast) in [ + ("name", ""), + ("job_name", ""), + ("run_id", ""), + ("retry_number", "::BIGINT"), + ] { body = body.replace( &format!("json_extract(context.external_context, '$.integration.{field}')"), &format!("(context.external_context #>> '{{integration,{field}}}'){cast}"), @@ -50,6 +50,7 @@ fn json_fields(mut body: String) -> String { ) } +/// Keep display bytes until readers decode them: PostgreSQL text rejects NUL and malformed UTF-8. fn postgres_body(body: String) -> String { let mut body = json_fields(body) .replace("strftime('%Y-%m-%dT%H:%M:%fZ', 'now')", "CURRENT_TIMESTAMP") @@ -59,7 +60,6 @@ fn postgres_body(body: String) -> String { .replace("MAX(COALESCE(received.queue_drops", "GREATEST(COALESCE(received.queue_drops") .replace("COALESCE(context.recall_filter_active, 0)", "COALESCE(CAST(context.recall_filter_active AS INTEGER), 0)") .replace("COALESCE(delivery.flush_complete, 0)", "COALESCE(CAST(delivery.flush_complete AS INTEGER), 0)") - // PostgreSQL text cannot hold NUL or malformed UTF-8. Keep display bytes until the reader decodes them. .replace("CAST(data AS TEXT)", "data") .replace("CAST(artifact.content AS TEXT)", "artifact.content") .replace(")\nGROUP BY", ") AS source_rows\nGROUP BY") diff --git a/crates/sqlbuild-recall/src/schema/_helpers/relational_tables.rs b/crates/sqlbuild-recall/src/schema/_helpers/relational_tables.rs index 3930030bd3..5430d20356 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/relational_tables.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/relational_tables.rs @@ -1,19 +1,10 @@ //! One logical schema and index list for SQLite and PostgreSQL. -use crate::schema::_helpers::rendering::{all_columns, sqlite_identifier, sqlite_type}; -use crate::schema::models::{ColumnType, TableSpec}; +use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; -pub(crate) fn postgres_type(column: ColumnType) -> &'static str { - match column { - ColumnType::Text => "TEXT", - ColumnType::Timestamp => "TIMESTAMPTZ", - ColumnType::Json => "JSONB", - ColumnType::Integer => "BIGINT", - ColumnType::Boolean => "BOOLEAN", - ColumnType::Real => "DOUBLE PRECISION", - ColumnType::Blob => "BYTEA", - } -} +use crate::schema::_helpers::rendering::{all_columns, sqlite_type}; +use crate::schema::constants::{ARTIFACTS_TABLE, EVENTS_TABLE}; +use crate::schema::models::{ColumnType, TableSpec}; pub(crate) fn relational_table(spec: &TableSpec, postgres: bool, control: bool) -> Vec { let mut definitions: Vec = all_columns(spec) @@ -21,7 +12,7 @@ pub(crate) fn relational_table(spec: &TableSpec, postgres: bool, control: bool) .map(|column| { let not_null = if column.nullable { "" } else { " NOT NULL" }; let kind = if postgres { - postgres_type(column.column_type) + column.column_type.postgres_type() } else if control && column.column_type == ColumnType::Json { "JSON" } else { @@ -30,7 +21,7 @@ pub(crate) fn relational_table(spec: &TableSpec, postgres: bool, control: bool) format!(" {} {kind}{not_null}", sqlite_identifier(column.name)) }) .collect(); - if control && spec.name == "recall_events" { + if control && spec.name == EVENTS_TABLE.name { definitions.push(if postgres { " storage_id BIGINT GENERATED ALWAYS AS IDENTITY".into() } else { @@ -38,7 +29,7 @@ pub(crate) fn relational_table(spec: &TableSpec, postgres: bool, control: bool) }); } let mut key: Vec<&str> = spec.key.to_vec(); - let partition = if postgres && spec.name != "recall_artifacts" { + let partition = if postgres && spec.name != ARTIFACTS_TABLE.name { if !key.contains(&spec.expiry_column) { key.push(spec.expiry_column); } diff --git a/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs index 015c491b5e..c2bffe4987 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs @@ -212,30 +212,6 @@ SELECT END AS complete FROM counted"; -fn output_transcripts(postgres: bool) -> String { - let columns: Vec = [ - ("CASE WHEN stream = 'stdout' THEN data END", "stdout"), - ("CASE WHEN stream = 'stderr' THEN data END", "stderr"), - ("data", "combined"), - ] - .into_iter() - .map(|(expression, alias)| { - let value = if postgres { - format!("COALESCE(string_agg({expression}, ''::BYTEA ORDER BY sequence), ''::BYTEA)") - } else { - format!( - "CAST(COALESCE(group_concat({expression}, x'' ORDER BY sequence), x'') AS BLOB)" - ) - }; - format!(" {value} AS {alias}") - }) - .collect(); - format!( - "SELECT invocation_id, MIN(invocation_started_at) AS invocation_started_at,\n{}\nFROM recall_output GROUP BY invocation_id", - columns.join(",\n") - ) -} - const EVENTS: &str = "SELECT * FROM recall_events"; const DIAGNOSTICS: &str = "SELECT * FROM recall_diagnostics"; @@ -261,6 +237,30 @@ LEFT JOIN recall_artifacts AS artifact ON artifact.digest = refs.digest"; const CONFLICTS: &str = "SELECT * FROM recall_conflicts"; +fn output_transcripts(postgres: bool) -> String { + let columns: Vec = [ + ("CASE WHEN stream = 'stdout' THEN data END", "stdout"), + ("CASE WHEN stream = 'stderr' THEN data END", "stderr"), + ("data", "combined"), + ] + .into_iter() + .map(|(expression, alias)| { + let value = if postgres { + format!("COALESCE(string_agg({expression}, ''::BYTEA ORDER BY sequence), ''::BYTEA)") + } else { + format!( + "CAST(COALESCE(group_concat({expression}, x'' ORDER BY sequence), x'') AS BLOB)" + ) + }; + format!(" {value} AS {alias}") + }) + .collect(); + format!( + "SELECT invocation_id, MIN(invocation_started_at) AS invocation_started_at,\n{}\nFROM recall_output GROUP BY invocation_id", + columns.join(",\n") + ) +} + fn invocations() -> String { format!( "WITH ids AS ( diff --git a/crates/sqlbuild-recall/src/schema/_helpers/rendering.rs b/crates/sqlbuild-recall/src/schema/_helpers/rendering.rs index 658aef267d..d7f3eb635b 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/rendering.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/rendering.rs @@ -36,11 +36,6 @@ pub(crate) fn clickhouse_type(column: &ColumnSpec) -> String { } } -/// `"name"` for SQLite. -pub(crate) fn sqlite_identifier(name: &str) -> String { - format!("\"{}\"", name.replace('"', "\"\"")) -} - /// `` `name` `` for ClickHouse. pub(crate) fn clickhouse_identifier(name: &str) -> String { format!("`{}`", name.replace('\\', "\\\\").replace('`', "\\`")) diff --git a/crates/sqlbuild-recall/src/schema/constants.rs b/crates/sqlbuild-recall/src/schema/constants.rs index 48e5a69a36..05c9c15d98 100644 --- a/crates/sqlbuild-recall/src/schema/constants.rs +++ b/crates/sqlbuild-recall/src/schema/constants.rs @@ -17,13 +17,16 @@ pub const DDL_SCRIPT_INVOCATION: &str = "ddl-script"; pub const META_MAINTENANCE_COMPONENT: &str = "maintenance"; pub const META_MAINTENANCE_CLAIM_KEY: &str = "claim"; +/// Assigned by storage rather than accepted from a producer's row values. +pub const INGESTED_AT_COLUMN: &str = "ingested_at"; + /// Columns every Recall row carries, after the table's own columns. pub const COMMON_COLUMNS: [ColumnSpec; 6] = [ ColumnSpec::required("invocation_id", ColumnType::Text), ColumnSpec::required("invocation_started_at", ColumnType::Timestamp), ColumnSpec::required("schema_version", ColumnType::Integer), ColumnSpec::required("producer_version", ColumnType::Text), - ColumnSpec::required("ingested_at", ColumnType::Timestamp), + ColumnSpec::required(INGESTED_AT_COLUMN, ColumnType::Timestamp), ColumnSpec::required("content_hash", ColumnType::Text), ]; /// The artifact columns its content digest covers: what it is, not who saw it when. diff --git a/crates/sqlbuild-recall/src/schema/main/add_column_statement.rs b/crates/sqlbuild-recall/src/schema/main/add_column_statement.rs index cea0815488..dc69488373 100644 --- a/crates/sqlbuild-recall/src/schema/main/add_column_statement.rs +++ b/crates/sqlbuild-recall/src/schema/main/add_column_statement.rs @@ -1,8 +1,9 @@ //! `add_column_statement`: the additive upgrade of one table by one column. +use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; + use crate::schema::_helpers::rendering::{ - clickhouse_identifier, clickhouse_table, clickhouse_type, on_cluster, sqlite_identifier, - sqlite_type, + clickhouse_identifier, clickhouse_table, clickhouse_type, on_cluster, sqlite_type, }; use crate::schema::models::{ColumnSpec, DdlOptions, Dialect, TableSpec}; diff --git a/crates/sqlbuild-recall/src/schema/main/control_view_statements.rs b/crates/sqlbuild-recall/src/schema/main/control_view_statements.rs index 7b27ec1a2f..826dd58587 100644 --- a/crates/sqlbuild-recall/src/schema/main/control_view_statements.rs +++ b/crates/sqlbuild-recall/src/schema/main/control_view_statements.rs @@ -1,7 +1,8 @@ //! Control stores expose Recall's supported views on both engines. +use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; + use crate::schema::_helpers::postgres_views::postgres_view_bodies; -use crate::schema::_helpers::rendering::sqlite_identifier; use crate::schema::main::view_statements::view_statements; use crate::schema::models::{DdlOptions, Dialect}; diff --git a/crates/sqlbuild-recall/src/schema/main/postgres_partition_statement.rs b/crates/sqlbuild-recall/src/schema/main/postgres_partition_statement.rs index 971a3ae792..4d24b5fbdb 100644 --- a/crates/sqlbuild-recall/src/schema/main/postgres_partition_statement.rs +++ b/crates/sqlbuild-recall/src/schema/main/postgres_partition_statement.rs @@ -1,8 +1,8 @@ //! Monthly partition bounds are calculated from a typed date, never interpolated user SQL. use chrono::{Datelike, Months, NaiveDate}; +use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; -use crate::schema::_helpers::rendering::sqlite_identifier; use crate::schema::main::table_spec::table_spec; use crate::schema::models::Table; diff --git a/crates/sqlbuild-recall/src/schema/main/sqlite_retention_sql.rs b/crates/sqlbuild-recall/src/schema/main/sqlite_retention_sql.rs index 1a44c7b2e4..44b5d5ee02 100644 --- a/crates/sqlbuild-recall/src/schema/main/sqlite_retention_sql.rs +++ b/crates/sqlbuild-recall/src/schema/main/sqlite_retention_sql.rs @@ -1,6 +1,7 @@ //! `sqlite_retention_sql`: the deletes SQLite maintenance runs, all rows of an invocation together. -use crate::schema::_helpers::rendering::sqlite_identifier; +use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; + use crate::schema::constants::{ INVOCATION_TABLES, META_MAINTENANCE_COMPONENT, RETENTION_CANDIDATE_TABLES, }; diff --git a/crates/sqlbuild-recall/src/schema/main/upgrade_statements.rs b/crates/sqlbuild-recall/src/schema/main/upgrade_statements.rs index 0b8c7430e2..913bf845df 100644 --- a/crates/sqlbuild-recall/src/schema/main/upgrade_statements.rs +++ b/crates/sqlbuild-recall/src/schema/main/upgrade_statements.rs @@ -1,6 +1,7 @@ //! `upgrade_statements`: what an administrator runs to bring an observed store up to this build's schema. -use crate::schema::_helpers::rendering::sqlite_identifier; +use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; + use crate::schema::_helpers::version_row::version_row_statement; use crate::schema::constants::{ALL_TABLES, RECALL_SCHEMA_VERSION}; use crate::schema::main::add_column_statement::add_column_statement; diff --git a/crates/sqlbuild-recall/src/schema/main/view_statements.rs b/crates/sqlbuild-recall/src/schema/main/view_statements.rs index 8527aae75c..0351938b81 100644 --- a/crates/sqlbuild-recall/src/schema/main/view_statements.rs +++ b/crates/sqlbuild-recall/src/schema/main/view_statements.rs @@ -1,8 +1,10 @@ //! `view_statements`: the statements that create the supported views. +use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; + use crate::schema::_helpers::clickhouse_views::clickhouse_view_bodies; use crate::schema::_helpers::relational_views::relational_view_bodies; -use crate::schema::_helpers::rendering::{clickhouse_table, on_cluster, sqlite_identifier}; +use crate::schema::_helpers::rendering::{clickhouse_table, on_cluster}; use crate::schema::models::{DdlOptions, Dialect}; /// Each view's name and creation statement. SQLite views are created only when missing; the store drops them first when the schema version rises. ClickHouse views are replaced. diff --git a/crates/sqlbuild-recall/src/schema/models.rs b/crates/sqlbuild-recall/src/schema/models.rs index 022e4f0b84..5da839a462 100644 --- a/crates/sqlbuild-recall/src/schema/models.rs +++ b/crates/sqlbuild-recall/src/schema/models.rs @@ -22,6 +22,22 @@ pub enum ColumnType { Boolean, } +impl ColumnType { + /// The PostgreSQL type used by both DDL and typed query placeholders. + #[must_use] + pub const fn postgres_type(self) -> &'static str { + match self { + Self::Text => "TEXT", + Self::Timestamp => "TIMESTAMPTZ", + Self::Json => "JSONB", + Self::Integer => "BIGINT", + Self::Boolean => "BOOLEAN", + Self::Real => "DOUBLE PRECISION", + Self::Blob => "BYTEA", + } + } +} + /// One column: its name, type and whether it may be null. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct ColumnSpec { diff --git a/crates/sqlbuild-recall/src/store/_helpers/relational.rs b/crates/sqlbuild-recall/src/store/_helpers/relational.rs index becce06752..7122ec88b9 100644 --- a/crates/sqlbuild-recall/src/store/_helpers/relational.rs +++ b/crates/sqlbuild-recall/src/store/_helpers/relational.rs @@ -1,11 +1,10 @@ //! Typed placeholders shared by native SQLite and control-plane transactions. -use crate::schema::_helpers::relational_tables::postgres_type; use crate::schema::models::ColumnType; pub(crate) fn placeholder(index: usize, kind: ColumnType, postgres: bool) -> String { if postgres { - format!("CAST(${index} AS {})", postgres_type(kind)) + format!("CAST(${index} AS {})", kind.postgres_type()) } else { format!("${index}") } diff --git a/crates/sqlbuild-recall/src/store/_helpers/rows.rs b/crates/sqlbuild-recall/src/store/_helpers/rows.rs index a253f2ce78..aa5b4d0b42 100644 --- a/crates/sqlbuild-recall/src/store/_helpers/rows.rs +++ b/crates/sqlbuild-recall/src/store/_helpers/rows.rs @@ -5,7 +5,6 @@ use std::collections::{HashMap, HashSet}; use rusqlite::{Connection, OptionalExtension}; -use crate::records::_helpers::rows::key_text; use crate::records::main::conflict_row::conflict_row; use crate::records::models::{RecallRow, RecallValue, RowEnvelope}; use crate::schema::main::table_spec::table_spec; @@ -19,7 +18,8 @@ use crate::store::main::relational_lookup::relational_lookup; use crate::store::models::{ConflictEvidence, InsertOutcome}; fn text_of(row: &RecallRow, column: &str) -> String { - key_text(row.value(column)) + row.value(column) + .map_or_else(String::new, RecallValue::key_text) } /// Insert one row's values for the columns its table has; `Some(evidence)` when its key holds different content. diff --git a/crates/sqlbuild-recall/src/store/main/conflict_evidence.rs b/crates/sqlbuild-recall/src/store/main/conflict_evidence.rs index a343086c2e..a4c04a92e6 100644 --- a/crates/sqlbuild-recall/src/store/main/conflict_evidence.rs +++ b/crates/sqlbuild-recall/src/store/main/conflict_evidence.rs @@ -1,7 +1,6 @@ //! Duplicate rows preserve the first content and expose the rejected digest. -use crate::records::_helpers::rows::key_text; -use crate::records::models::RecallRow; +use crate::records::models::{RecallRow, RecallValue}; use crate::schema::main::table_spec::table_spec; use crate::store::models::ConflictEvidence; @@ -14,7 +13,10 @@ pub fn conflict_evidence(row: &RecallRow, existing: &str) -> Option>() .join(":"), existing_sha256: existing.to_owned(), diff --git a/crates/sqlbuild-recall/src/store/main/relational_insert.rs b/crates/sqlbuild-recall/src/store/main/relational_insert.rs index ef63916e65..70aa341299 100644 --- a/crates/sqlbuild-recall/src/store/main/relational_insert.rs +++ b/crates/sqlbuild-recall/src/store/main/relational_insert.rs @@ -2,8 +2,10 @@ use std::collections::HashSet; +use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; + use crate::records::models::{RecallRow, RecallValue}; -use crate::schema::_helpers::rendering::sqlite_identifier; +use crate::schema::constants::INGESTED_AT_COLUMN; use crate::schema::main::table_columns::table_columns; use crate::schema::main::table_spec::table_spec; use crate::schema::models::ColumnType; @@ -18,18 +20,18 @@ pub fn relational_insert<'a>( postgres: bool, ) -> RecallStatement<'a> { let metadata = table_columns(row.table); - let mut names = Vec::new(); - let mut values = Vec::new(); + let mut names: Vec = Vec::new(); + let mut values: Vec<(&RecallValue, ColumnType)> = Vec::new(); for (name, value) in &row.values { if let Some(column) = metadata.iter().find(|column| column.name == *name) && columns.contains(*name) - && *name != "ingested_at" + && *name != INGESTED_AT_COLUMN { names.push(sqlite_identifier(name)); values.push((value, column.column_type)); } } - names.push("ingested_at".into()); + names.push(INGESTED_AT_COLUMN.into()); values.push((ingested_at, ColumnType::Timestamp)); let placeholders: Vec = values .iter() diff --git a/crates/sqlbuild-recall/src/store/main/relational_lookup.rs b/crates/sqlbuild-recall/src/store/main/relational_lookup.rs index d1291fdb4c..8d6658a6d2 100644 --- a/crates/sqlbuild-recall/src/store/main/relational_lookup.rs +++ b/crates/sqlbuild-recall/src/store/main/relational_lookup.rs @@ -1,10 +1,11 @@ //! Duplicate detection uses the same stable keys as physical unique constraints. +use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; + use crate::records::models::{RecallRow, RecallValue}; -use crate::schema::_helpers::rendering::sqlite_identifier; use crate::schema::main::table_columns::table_columns; use crate::schema::main::table_spec::table_spec; -use crate::schema::models::Table; +use crate::schema::models::{ColumnType, Table}; use crate::store::_helpers::relational::placeholder; use crate::store::models::RecallStatement; @@ -17,14 +18,11 @@ pub fn relational_lookup(row: &RecallRow, postgres: bool, control: bool) -> Reca keys.push(spec.expiry_column); } let metadata = table_columns(row.table); - let mut values = Vec::new(); - let mut conditions = Vec::new(); - for key in keys { - let kind = metadata - .iter() - .find(|column| column.name == key) - .expect("Recall key columns belong to their schema") - .column_type; + let mut values: Vec<(&RecallValue, ColumnType)> = Vec::new(); + let mut conditions: Vec = Vec::new(); + for column in metadata.iter().filter(|column| keys.contains(&column.name)) { + let key = column.name; + let kind = column.column_type; values.push((row.value(key).unwrap_or(&RecallValue::Null), kind)); conditions.push(format!( "{} = {}", diff --git a/fensu.toml b/fensu.toml index 2f97466395..2338c3e56a 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1564,6 +1564,18 @@ rule = "FPRSS201" path = "crates/sqlbuild-python/src/bindings/models.rs" reason = "CompactAnalysisJob holds a prepared analysis closure, which cannot implement Debug; other models derive it." +[[targets.rust.rule_exceptions]] +rule = "FPRSS201" +path = "crates/sqlbuild-orchestration/src/credentials/models.rs" +symbols = ["IssuedToken"] +reason = "IssuedToken implements Debug manually to redact its bearer credential; deriving Debug would expose the secret." + +[[targets.rust.rule_exceptions]] +rule = "XSB072" +path = "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs" +symbols = ["NativeOrchestrationSnapshot"] +reason = "The pyclass module attribute names sqlbuild._native but executes no Python call. The struct retains only SnapshotInput; binding functions remain checked." + [[targets.rust.rule_exceptions]] rule = "FPRSS201" path = "crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index 2d7ea4bbb5..6f5b1c5053 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -197,3 +197,20 @@ class SnapshotCatalogTestCase: description: str files: dict[str, str] expected_kinds: frozenset[str] + + +@dataclass(frozen=True) +class SnapshotRejectionTestCase: + description: str + files: dict[str, str] + nodes: dict[str, object] + expected_compiler_errors: bool + expected_message: str + + +@dataclass(frozen=True) +class SnapshotEnvironmentTestCase: + description: str + first_lockfile: str + second_lockfile: str + expected_environment_count: int diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 629fc0bdc1..e53269d8a3 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -19,6 +19,8 @@ from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( SnapshotCatalogTestCase, + SnapshotEnvironmentTestCase, + SnapshotRejectionTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( FUNCTION_PROJECT_FILES, @@ -148,7 +150,7 @@ def test_given_real_projects_when_building_snapshots_then_catalogs_are_stable_an adapter=DuckDbAdapter(), resolved_connection={"database": "warehouse", "schema": "main"}, ) - assert not [item for item in project.diagnostics if item.severity == "error"] + assert not any(item.severity == "error" for item in project.diagnostics) snapshot: NativeOrchestrationSnapshot = build_orchestration_snapshot( project, metadata, directory ) @@ -185,46 +187,70 @@ def test_given_real_projects_when_building_snapshots_then_catalogs_are_stable_an assert publications[0] == publications[1] +@pytest.mark.parametrize( + "test_case", + [ + SnapshotRejectionTestCase( + description="a valid compile cannot publish caller-forged nodes", + files=TYPE_PROOF_FILES, + nodes={ + "model:forged": { + "kind": "model", + "definition": {}, + "raw_sql": None, + "compiled_sql": "select 2", + "python_source": None, + } + }, + expected_compiler_errors=False, + expected_message="nodes must come from the compiled project", + ), + SnapshotRejectionTestCase( + description="compiler errors cannot replace the last good publication", + files=TYPE_PROOF_FILES | {"models/orders.sql": "MODEL ();\nSELECT 1 AS order_id\n"}, + nodes={}, + expected_compiler_errors=True, + expected_message="cannot publish a project with compiler errors", + ), + ], + ids=lambda case: case.description, +) def test_given_invalid_compilation_or_forged_nodes_when_publishing_then_no_snapshot_is_emitted( + test_case: SnapshotRejectionTestCase, tmp_path: Path, ) -> None: - write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES) + write_files(project_dir=tmp_path, files=test_case.files) project: CompiledProjectView = build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter=DuckDbAdapter(), resolved_connection={"database": "warehouse", "schema": "main"}, ) - with pytest.raises(ValueError, match="nodes must come from the compiled project"): + assert ( + any(item.severity == "error" for item in project.diagnostics) + == test_case.expected_compiler_errors + ) + with pytest.raises(ValueError, match=test_case.expected_message): build_orchestration_snapshot( project, - json.dumps( - METADATA - | { - "nodes": { - "model:forged": { - "kind": "model", - "definition": {}, - "raw_sql": None, - "compiled_sql": "select 2", - "python_source": None, - } - } - } - ), + json.dumps(METADATA | {"nodes": test_case.nodes}), tmp_path, ) - (tmp_path / "models/orders.sql").write_text("MODEL ();\nSELECT 1 AS order_id\n") - broken: CompiledProjectView = build_compiled_project( - discovered_inputs=discover_project_inputs(project_dir=tmp_path), - adapter=DuckDbAdapter(), - resolved_connection={"database": "warehouse", "schema": "main"}, - ) - assert any(item.severity == "error" for item in broken.diagnostics) - with pytest.raises(ValueError, match="cannot publish a project with compiler errors"): - build_orchestration_snapshot(broken, json.dumps(METADATA), tmp_path) +@pytest.mark.parametrize( + "test_case", + [ + SnapshotEnvironmentTestCase( + description="installed dependencies and two lock revisions identify different environments", + first_lockfile="version = 1\n# dependency revision one\n", + second_lockfile="version = 1\n# dependency revision two\n", + expected_environment_count=3, + ), + ], + ids=lambda case: case.description, +) def test_given_a_dependency_lock_when_it_changes_then_environment_identity_changes_not_node_content( + test_case: SnapshotEnvironmentTestCase, tmp_path: Path, ) -> None: write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES) @@ -236,11 +262,11 @@ def test_given_a_dependency_lock_when_it_changes_then_environment_identity_chang installed: dict[str, Any] = json.loads( build_orchestration_snapshot(project, json.dumps(METADATA), tmp_path).to_json() ) - (tmp_path / "uv.lock").write_text("version = 1\n# dependency revision one\n") + (tmp_path / "uv.lock").write_text(test_case.first_lockfile) locked: dict[str, Any] = json.loads( build_orchestration_snapshot(project, json.dumps(METADATA), tmp_path).to_json() ) - (tmp_path / "uv.lock").write_text("version = 1\n# dependency revision two\n") + (tmp_path / "uv.lock").write_text(test_case.second_lockfile) changed: dict[str, Any] = json.loads( build_orchestration_snapshot(project, json.dumps(METADATA), tmp_path).to_json() ) @@ -248,7 +274,7 @@ def test_given_a_dependency_lock_when_it_changes_then_environment_identity_chang len( {installed["environment_hash"], locked["environment_hash"], changed["environment_hash"]} ) - == 3 + == test_case.expected_environment_count ) assert installed["nodes"] == locked["nodes"] == changed["nodes"] assert changed == json.loads( From e307189c1767cc4337c3509600cce7beabbc80d8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 00:31:49 +0000 Subject: [PATCH 07/59] refactor: separate scheduler policies from transactional coordinators Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../src/native_commands/_helpers/decision.rs | 42 ++++++++++- .../src/native_commands/_helpers/execution.rs | 41 +++++++++++ .../src/native_commands/main/_run_control.rs | 72 ++----------------- .../src/declarations/_helpers/mod.rs | 1 + .../declarations/_helpers/trigger_state.rs | 27 +++++++ .../src/declarations/main/change_trigger.rs | 24 ++----- .../declarations/main/publish_definitions.rs | 29 +++----- .../src/declarations/mod.rs | 1 + .../src/runs/_helpers/changes.rs | 19 +++++ .../src/runs/main/change_run.rs | 13 +--- .../src/scheduler/_helpers/ticks.rs | 32 ++++++++- .../src/scheduler/main/plan_ticks.rs | 32 +-------- .../src/snapshots/_helpers/publication.rs | 39 +++++++++- .../src/snapshots/main/publish_snapshot.rs | 33 ++------- .../src/store/_helpers/migration.rs | 28 ++++++++ .../src/store/main/migrate_store.rs | 24 +------ 16 files changed, 258 insertions(+), 199 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_state.rs diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index 9355478a2d..21c6342413 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -1,7 +1,10 @@ -//! Which parsed invocations run without Python, decided from the parsed values alone. +//! Resolve native invocations and control-service settings from parsed arguments. +use std::net::{IpAddr, SocketAddr}; use std::path::PathBuf; +use sqlbuild_orchestration::store::errors::ControlError; + use crate::argv::constants::{JSON_OPTION, REVIEW_PROG}; use crate::argv::models::{ArgvOutcome, Namespace, Value}; use crate::dispatch::main::invocation_policy::invocation_policy; @@ -36,6 +39,43 @@ pub(crate) fn flag(namespace: &Namespace, name: &str) -> bool { namespace.get(name) == Some(&Value::Bool(true)) } +pub(crate) fn control_address( + namespace: &Namespace, + standalone: bool, +) -> Result, ControlError> { + if standalone { + return Ok(None); + } + let host: IpAddr = text(namespace, "control_host") + .unwrap_or_else(|| "127.0.0.1".into()) + .parse() + .map_err(|_| ControlError::Invalid("--host must be an IP address".into()))?; + let port = match namespace.get("control_port") { + Some(Value::Int(value)) => value + .parse::() + .map_err(|_| ControlError::Invalid("--port must be between 0 and 65535".into()))?, + _ => 8000, + }; + Ok(Some(SocketAddr::new(host, port))) +} + +pub(crate) fn control_database(namespace: &Namespace) -> Result { + let variable = text(namespace, "control_database_env") + .unwrap_or_else(|| "SQLBUILD_CONTROL_DATABASE_URL".into()); + match std::env::var(&variable) { + Ok(value) => Ok(value), + Err(std::env::VarError::NotPresent) => Ok(PathBuf::from( + text(namespace, "project_dir").unwrap_or_else(|| ".".into()), + ) + .join(".sqlbuild/control.sqlite3") + .to_string_lossy() + .into_owned()), + Err(std::env::VarError::NotUnicode(_)) => Err(ControlError::Invalid( + "control database variable must be UTF-8".into(), + )), + } +} + /// The strings of a list attribute, or `None` when it holds anything else. fn texts(namespace: &Namespace, name: &str) -> Option> { match namespace.get(name) { diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 90297029d9..ac2f27678e 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -4,6 +4,10 @@ use std::io; use std::path::{Path, PathBuf}; use std::time::SystemTime; +use sqlbuild_orchestration::store::errors::ControlError; +use tokio::sync::watch; +use tokio::task::JoinSet; + use crate::argv::models::Stream; use crate::compute_logs::constants::DEFAULT_RETENTION_COUNT; use crate::compute_logs::main::abandon_capture::abandon_capture; @@ -25,6 +29,43 @@ use crate::native_commands::models::{ CommandOutput, InstalledPackage, LostStdout, NativeCommand, NativeInvocation, ProjectLocation, }; +async fn shutdown_signal() -> Result<(), ControlError> { + #[cfg(unix)] + { + let mut terminate = + tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?; + tokio::select! { + result = tokio::signal::ctrl_c() => result?, + _ = terminate.recv() => {}, + } + } + #[cfg(not(unix))] + tokio::signal::ctrl_c().await?; + Ok(()) +} + +/// Stop sibling roles on a signal or role failure, then drain their shutdown results. +pub(crate) async fn supervise_control_roles( + mut tasks: JoinSet>, + stop: watch::Sender, +) -> Result<(), ControlError> { + let mut result = tokio::select! { + result = tasks.join_next() => match result { + Some(result) => result.map_err(ControlError::from).and_then(std::convert::identity), + None => Ok(()), + }, + result = shutdown_signal() => result, + }; + stop.send_replace(true); + while let Some(joined) = tasks.join_next().await { + let joined = joined + .map_err(ControlError::from) + .and_then(std::convert::identity); + result = result.and(joined); + } + result +} + /// The capture Python's `dispatch_with_compute_logs` opens; a failure only loses the log. fn open_capture( invocation: &NativeInvocation, diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs index 111fc3a1a1..7de55c4455 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs @@ -1,8 +1,5 @@ //! Supervise native API/scheduler roles together; a failed role stops its siblings. -use std::net::{IpAddr, SocketAddr}; -use std::path::PathBuf; - use sqlbuild_orchestration::api::main::serve_api::serve_api; use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler; use sqlbuild_orchestration::scheduler::models::SchedulerState; @@ -12,8 +9,9 @@ use sqlbuild_orchestration::store::main::open_store::open_store; use tokio::sync::watch; use uuid::Uuid; -use crate::argv::models::{Namespace, Stream, Value}; -use crate::native_commands::_helpers::decision::{flag, text}; +use crate::argv::models::{Namespace, Stream}; +use crate::native_commands::_helpers::decision::{control_address, control_database, flag}; +use crate::native_commands::_helpers::execution::supervise_control_roles; use crate::native_commands::_helpers::terminal::write_output; async fn stopped(mut shutdown: watch::Receiver) { @@ -24,56 +22,12 @@ async fn stopped(mut shutdown: watch::Receiver) { } } -async fn shutdown_signal() -> Result<(), ControlError> { - #[cfg(unix)] - { - let mut terminate = - tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?; - tokio::select! { - result = tokio::signal::ctrl_c() => result?, - _ = terminate.recv() => {}, - } - } - #[cfg(not(unix))] - tokio::signal::ctrl_c().await?; - Ok(()) -} - pub(crate) async fn run_control( namespace: &Namespace, standalone: bool, ) -> Result<(), ControlError> { - let address = if standalone { - None - } else { - let host: IpAddr = text(namespace, "control_host") - .unwrap_or_else(|| "127.0.0.1".into()) - .parse() - .map_err(|_| ControlError::Invalid("--host must be an IP address".into()))?; - let port = match namespace.get("control_port") { - Some(Value::Int(value)) => value - .parse::() - .map_err(|_| ControlError::Invalid("--port must be between 0 and 65535".into()))?, - _ => 8000, - }; - Some(SocketAddr::new(host, port)) - }; - let variable = text(namespace, "control_database_env") - .unwrap_or_else(|| "SQLBUILD_CONTROL_DATABASE_URL".into()); - let url = match std::env::var(&variable) { - Ok(value) => value, - Err(std::env::VarError::NotPresent) => { - PathBuf::from(text(namespace, "project_dir").unwrap_or_else(|| ".".into())) - .join(".sqlbuild/control.sqlite3") - .to_string_lossy() - .into_owned() - } - Err(std::env::VarError::NotUnicode(_)) => { - return Err(ControlError::Invalid( - "control database variable must be UTF-8".into(), - )); - } - }; + let address = control_address(namespace, standalone)?; + let url = control_database(namespace)?; drop(write_output(&[( Stream::Stderr, "Opening control store...\n", @@ -117,19 +71,5 @@ pub(crate) async fn run_control( "Scheduler loop started; leadership is lease-guarded\n", )])); } - let mut result = tokio::select! { - result = tasks.join_next() => match result { - Some(result) => result.map_err(ControlError::from).and_then(std::convert::identity), - None => Ok(()), - }, - result = shutdown_signal() => result, - }; - stop.send_replace(true); - while let Some(joined) = tasks.join_next().await { - let joined = joined - .map_err(ControlError::from) - .and_then(std::convert::identity); - result = result.and(joined); - } - result + supervise_control_roles(tasks, stop).await } diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs new file mode 100644 index 0000000000..40c4be79ad --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod trigger_state; diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_state.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_state.rs new file mode 100644 index 0000000000..fccc6753f0 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_state.rs @@ -0,0 +1,27 @@ +//! Recompute the effective schedule after code publication or an operator override. + +use chrono::{DateTime, TimeDelta, Utc}; + +use crate::declarations::models::{TriggerDefinition, TriggerStatus}; +use crate::scheduler::main::next_schedule_time::next_schedule_time; +use crate::store::errors::ControlError; + +pub(crate) fn trigger_timing( + definition: &TriggerDefinition, + started_at: DateTime, + last_tick_for: Option>, + status_override: Option, +) -> Result<(TriggerStatus, Option>), ControlError> { + let status = status_override.unwrap_or(definition.default_status); + let after = last_tick_for.unwrap_or( + started_at + .checked_sub_signed(TimeDelta::microseconds(1)) + .ok_or_else(|| ControlError::Invalid("trigger start is out of range".into()))?, + ); + let next = if status == TriggerStatus::Running { + next_schedule_time(&definition.schedule, after)? + } else { + None + }; + Ok((status, next)) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs b/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs index e7d2165cf9..4ccd6765e0 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs @@ -1,11 +1,9 @@ //! Apply an operator fact and its projection in one transaction, with retry-safe request identity. -use chrono::TimeDelta; - +use crate::declarations::_helpers::trigger_state::trigger_timing; use crate::declarations::models::{ EditableTriggerState, TriggerChange, TriggerChangeRequest, TriggerStatus, }; -use crate::scheduler::main::next_schedule_time::next_schedule_time; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; @@ -60,20 +58,12 @@ pub async fn change_trigger( TriggerChange::SetCursor { time } => state.last_tick_for = Some(time), TriggerChange::ResetCursor => state.last_tick_for = None, } - let status = state - .status_override - .unwrap_or(state.definition_json.default_status); - let after = state.last_tick_for.unwrap_or( - state - .started_at - .checked_sub_signed(TimeDelta::microseconds(1)) - .ok_or_else(|| ControlError::Invalid("trigger start is out of range".into()))?, - ); - let next = if status == TriggerStatus::Running { - next_schedule_time(&state.definition_json.schedule, after)? - } else { - None - }; + let (status, next) = trigger_timing( + &state.definition_json, + state.started_at, + state.last_tick_for, + state.status_override, + )?; let now = transaction.now; let (transaction, _) = query(transaction, "INSERT INTO sqb_trigger_state_events (event_id, trigger_id, occurred_at, snapshot_id, request_json) VALUES ($1,$2,$3,$4,$5)", diff --git a/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs b/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs index c65b13468f..6110788d5e 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs @@ -1,10 +1,9 @@ //! Publish definition projections inside the current-snapshot transaction. -use chrono::TimeDelta; use uuid::Uuid; -use crate::declarations::models::{Definitions, PreviousTriggerState, TriggerStatus}; -use crate::scheduler::main::next_schedule_time::next_schedule_time; +use crate::declarations::_helpers::trigger_state::trigger_timing; +use crate::declarations::models::{Definitions, PreviousTriggerState}; use crate::snapshots::models::SnapshotRecord; use crate::store::errors::ControlError; use crate::store::main::decode_row::decode_row; @@ -67,24 +66,12 @@ pub(crate) async fn publish_definitions( trigger.starts_at.unwrap_or(snapshot.published_at), |state| state.started_at, ); - let default = trigger.default_status; - let status = state - .as_ref() - .and_then(|state| state.status_override) - .unwrap_or(default); - let after = state - .as_ref() - .and_then(|state| state.last_tick_for) - .unwrap_or( - started_at - .checked_sub_signed(TimeDelta::microseconds(1)) - .ok_or_else(|| ControlError::Invalid("trigger start is out of range".into()))?, - ); - let next = if status == TriggerStatus::Running { - next_schedule_time(&trigger.schedule, after)? - } else { - None - }; + let (status, next) = trigger_timing( + &trigger, + started_at, + state.as_ref().and_then(|state| state.last_tick_for), + state.as_ref().and_then(|state| state.status_override), + )?; (transaction, _) = query(transaction, "INSERT INTO sqb_trigger_state (trigger_id, status, started_at, next_due_at, updated_at) VALUES ($1,$2,$3,$4,$5) \ ON CONFLICT (trigger_id) DO UPDATE SET status=excluded.status, next_due_at=excluded.next_due_at, updated_at=excluded.updated_at", diff --git a/crates/sqlbuild-orchestration/src/declarations/mod.rs b/crates/sqlbuild-orchestration/src/declarations/mod.rs index 5355003ff2..5359000d16 100644 --- a/crates/sqlbuild-orchestration/src/declarations/mod.rs +++ b/crates/sqlbuild-orchestration/src/declarations/mod.rs @@ -1,2 +1,3 @@ +pub(crate) mod _helpers; pub mod main; pub mod models; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/changes.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/changes.rs index ba4b35be6e..e9480d69da 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/changes.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/changes.rs @@ -4,6 +4,25 @@ use crate::runs::models::{ ChangingRun, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, }; use crate::store::errors::ControlError; +use crate::store::types::Row; + +pub(crate) fn validate_retry( + previous: &Row, + request: &RunChangeRequest, + body: &serde_json::Value, +) -> Result<(), ControlError> { + if previous.get("payload") != Some(body) + || previous + .get("invocation_id") + .and_then(serde_json::Value::as_str) + != Some(format!("scheduler/{}", request.run_id).as_str()) + { + return Err(ControlError::Invalid( + "run change event ID already has different content".into(), + )); + } + Ok(()) +} pub(crate) fn validate_authority( authority: RunAuthority<'_>, diff --git a/crates/sqlbuild-orchestration/src/runs/main/change_run.rs b/crates/sqlbuild-orchestration/src/runs/main/change_run.rs index 458ae8ae4a..3bef1033c8 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/change_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/change_run.rs @@ -2,7 +2,7 @@ use serde_json::json; -use crate::runs::_helpers::changes::{validate_authority, validate_change}; +use crate::runs::_helpers::changes::{validate_authority, validate_change, validate_retry}; use crate::runs::_helpers::events::append_run_event; use crate::runs::_helpers::projection::project_change; use crate::runs::models::{ChangingRun, RunAuthority, RunChange, RunChangeRequest, RunOutcome}; @@ -33,16 +33,7 @@ pub async fn change_run( ) .await?; if let Some(previous) = previous.first() { - if previous.get("payload") != Some(&body) - || previous - .get("invocation_id") - .and_then(serde_json::Value::as_str) - != Some(format!("scheduler/{}", request.run_id).as_str()) - { - return Err(ControlError::Invalid( - "run change event ID already has different content".into(), - )); - } + validate_retry(previous, request, &body)?; commit(transaction).await?; return Ok(false); } diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/ticks.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/ticks.rs index 4cb2683e54..a4de72e3d6 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/ticks.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/ticks.rs @@ -3,7 +3,37 @@ use chrono::{DateTime, SecondsFormat, Utc}; use uuid::Uuid; -use crate::scheduler::models::{TickAction, TickDecision}; +use crate::scheduler::models::{CatchUp, Overlap, TickAction, TickDecision, TickState}; + +pub(crate) fn catch_up_actions(state: &TickState, due: &[DateTime]) -> Vec { + let covering_run = (state.overlap == Overlap::Coalesce) + .then_some(state.waiting_run) + .flatten(); + match (covering_run, state.catch_up) { + (Some(run_id), _) => due + .iter() + .map(|time| action(state.trigger_id, *time, Some(run_id))) + .collect(), + (None, CatchUp::All { max_runs }) => due + .iter() + .take(max_runs.get()) + .map(|time| action(state.trigger_id, *time, None)) + .collect(), + (None, CatchUp::Latest) => { + let latest = due.last().copied(); + let run_id = latest.map(|time| requested_run(tick_id(state.trigger_id, time))); + due.iter() + .map(|time| { + action( + state.trigger_id, + *time, + (Some(*time) != latest).then_some(run_id).flatten(), + ) + }) + .collect() + } + } +} pub(crate) fn tick_id(trigger_id: Uuid, scheduled_for: DateTime) -> Uuid { Uuid::new_v5( diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/plan_ticks.rs b/crates/sqlbuild-orchestration/src/scheduler/main/plan_ticks.rs index 6ddee7f820..0e8d05b971 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/plan_ticks.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/plan_ticks.rs @@ -4,9 +4,9 @@ use std::collections::BTreeSet; use chrono::{DateTime, Utc}; -use crate::scheduler::_helpers::ticks::{action, requested_run, tick_id}; +use crate::scheduler::_helpers::ticks::catch_up_actions; use crate::scheduler::constants::MAX_TICKS_PER_BATCH; -use crate::scheduler::models::{CatchUp, Overlap, TickAction, TickPlan, TickState}; +use crate::scheduler::models::{TickPlan, TickState}; use crate::store::errors::ControlError; /// Return facts to append and the cursor to commit atomically with them. @@ -34,33 +34,7 @@ pub fn plan_ticks(state: &TickState, now: DateTime) -> Result = match (covering_run, state.catch_up) { - (Some(run_id), _) => due - .iter() - .map(|time| action(state.trigger_id, *time, Some(run_id))) - .collect(), - (None, CatchUp::All { max_runs }) => due - .iter() - .take(max_runs.get()) - .map(|time| action(state.trigger_id, *time, None)) - .collect(), - (None, CatchUp::Latest) => { - let latest = due.last().copied(); - let run_id = latest.map(|time| requested_run(tick_id(state.trigger_id, time))); - due.iter() - .map(|time| { - action( - state.trigger_id, - *time, - (Some(*time) != latest).then_some(run_id).flatten(), - ) - }) - .collect() - } - }; + let actions = catch_up_actions(state, &due); let last_tick_for = actions .iter() .rev() diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/publication.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/publication.rs index eb7b841932..e05f14da2d 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/_helpers/publication.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/publication.rs @@ -1,10 +1,10 @@ //! Immutable body inserts shared by snapshot publication. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use uuid::Uuid; -use crate::snapshots::models::PreparedNode; +use crate::snapshots::models::{PreparedNode, SnapshotChange}; use crate::store::errors::ControlError; use crate::store::main::query::query; use crate::store::models::{ControlTransaction, Parameter}; @@ -32,3 +32,38 @@ pub(crate) async fn store_nodes( } Ok(transaction) } + +pub(crate) async fn store_changes( + mut transaction: ControlTransaction, + snapshot_id: Uuid, + previous: &BTreeMap, + hashes: &BTreeMap, +) -> Result { + let keys: BTreeSet<&String> = previous.keys().chain(hashes.keys()).collect(); + let mut sequence = 0; + for key in keys { + let before = previous.get(key); + let after = hashes.get(key); + if before == after { + continue; + } + let change = SnapshotChange { + change_kind: match (before, after) { + (None, _) => "added", + (_, None) => "removed", + _ => "changed", + } + .into(), + subject_key: key.clone(), + before_hash: before.cloned(), + after_hash: after.cloned(), + }; + (transaction, _) = query(transaction, + "INSERT INTO sqb_snapshot_changes (snapshot_id, change_seq, change_kind, subject_key, before_hash, after_hash) \ + VALUES ($1,$2,$3,$4,$5,$6)", vec![Parameter::Id(snapshot_id), Parameter::Integer(sequence), + Parameter::Text(change.change_kind), Parameter::Text(change.subject_key), + Parameter::OptionalText(change.before_hash), Parameter::OptionalText(change.after_hash)]).await?; + sequence += 1; + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/publish_snapshot.rs b/crates/sqlbuild-orchestration/src/snapshots/main/publish_snapshot.rs index 0861207ebc..c34cf82ab9 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/publish_snapshot.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/publish_snapshot.rs @@ -1,6 +1,6 @@ //! Publish a complete generation and its current pointer in one transaction. -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeMap; use serde_json::Value; use uuid::Uuid; @@ -8,8 +8,8 @@ use uuid::Uuid; use crate::declarations::main::publish_definitions::publish_definitions; use crate::declarations::main::snapshot_definitions::snapshot_definitions; use crate::snapshots::_helpers::preparation::prepare_snapshot; -use crate::snapshots::_helpers::publication::store_nodes; -use crate::snapshots::models::{PreparedSnapshot, SnapshotChange, SnapshotInput, SnapshotRecord}; +use crate::snapshots::_helpers::publication::{store_changes, store_nodes}; +use crate::snapshots::models::{PreparedSnapshot, SnapshotInput, SnapshotRecord}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; @@ -81,32 +81,7 @@ pub async fn publish_snapshot( "INSERT INTO sqb_snapshot_documents (snapshot_id, kind, document_hash) VALUES ($1, $2, $3)", vec![Parameter::Id(record.snapshot_id), Parameter::Text(kind), Parameter::Text(hash)]).await?; } - let keys: BTreeSet<&String> = previous.keys().chain(hashes.keys()).collect(); - let mut sequence = 0; - for key in keys { - let before = previous.get(key); - let after = hashes.get(key); - if before == after { - continue; - } - let change = SnapshotChange { - change_kind: match (before, after) { - (None, _) => "added", - (_, None) => "removed", - _ => "changed", - } - .into(), - subject_key: key.clone(), - before_hash: before.cloned(), - after_hash: after.cloned(), - }; - (transaction, _) = query(transaction, - "INSERT INTO sqb_snapshot_changes (snapshot_id, change_seq, change_kind, subject_key, before_hash, after_hash) \ - VALUES ($1,$2,$3,$4,$5,$6)", vec![Parameter::Id(record.snapshot_id), Parameter::Integer(sequence), - Parameter::Text(change.change_kind), Parameter::Text(change.subject_key), - Parameter::OptionalText(change.before_hash), Parameter::OptionalText(change.after_hash)]).await?; - sequence += 1; - } + transaction = store_changes(transaction, record.snapshot_id, &previous, &hashes).await?; (transaction, _) = query(transaction, "INSERT INTO sqb_current_snapshots (location_id, target, snapshot_id, since) VALUES ($1,$2,$3,$4) \ ON CONFLICT (location_id, target) DO UPDATE SET snapshot_id=excluded.snapshot_id, since=excluded.since", diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index 53420e77dc..a29a62c75f 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -1,9 +1,37 @@ //! One schema template, rendered only at the database type boundary. +use semver::Version; + +use crate::store::errors::ControlError; use crate::store::models::ControlStore; +use crate::store::types::Row; pub(crate) const SCHEMA: &str = include_str!("../schema.sql"); +pub(crate) fn verify_migrations(rows: &[Row], digest: &str) -> Result { + let current = Version::parse(env!("CARGO_PKG_VERSION")) + .map_err(|error| ControlError::Schema(error.to_string()))?; + for row in rows { + let minimum = row + .get("min_server_version") + .and_then(|value| value.as_str()) + .ok_or_else(|| { + ControlError::Schema("migration lacks a minimum server version".into()) + })?; + let minimum = + Version::parse(minimum).map_err(|error| ControlError::Schema(error.to_string()))?; + if row.get("version").and_then(|value| value.as_i64()) != Some(1) + || minimum > current + || row.get("checksum").and_then(|value| value.as_str()) != Some(digest) + { + return Err(ControlError::Schema( + "unknown migration, checksum mismatch, or minimum server version not met".into(), + )); + } + } + Ok(current) +} + pub(crate) fn render(store: &ControlStore, sql: &str) -> String { let postgres = matches!(store, ControlStore::Postgres(_)); sql.replace("{uuid}", if postgres { "UUID" } else { "TEXT" }) diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index b5a38030f1..fca5fe59c0 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -1,10 +1,9 @@ //! Forward-only migration under the same lock used by all other control-plane writes. -use semver::Version; use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; -use crate::store::_helpers::migration::{SCHEMA, render}; +use crate::store::_helpers::migration::{SCHEMA, render, verify_migrations}; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; use crate::store::main::begin::begin; @@ -24,28 +23,9 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { vec![], ) .await?; - let current = Version::parse(env!("CARGO_PKG_VERSION")) - .map_err(|error| ControlError::Schema(error.to_string()))?; let recall_statements = recall_schema(store); let digest = sha256_hex(format!("{SCHEMA}\n{}", recall_statements.join(";\n")).as_bytes()); - for row in &rows { - let minimum = row - .get("min_server_version") - .and_then(|value| value.as_str()) - .ok_or_else(|| { - ControlError::Schema("migration lacks a minimum server version".into()) - })?; - let minimum = - Version::parse(minimum).map_err(|error| ControlError::Schema(error.to_string()))?; - if row.get("version").and_then(|value| value.as_i64()) != Some(1) - || minimum > current - || row.get("checksum").and_then(|value| value.as_str()) != Some(digest.as_str()) - { - return Err(ControlError::Schema( - "unknown migration, checksum mismatch, or minimum server version not met".into(), - )); - } - } + let current = verify_migrations(&rows, &digest)?; if rows.is_empty() { for statement in SCHEMA .split(';') From 08aac67772bc2d761e4e3b9d078dbf2fd93cafdd Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 00:44:47 +0000 Subject: [PATCH 08/59] feat: observe platform health independently of scheduler leadership Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../src/native_commands/main/_run_control.rs | 14 ++ .../tests/native_commands/helpers.rs | 11 +- .../tests/native_commands/test_control.rs | 10 +- crates/sqlbuild-orchestration/src/lib.rs | 1 + .../src/platform/_helpers/mod.rs | 2 + .../src/platform/_helpers/observations.rs | 57 ++++++ .../src/platform/_helpers/watchdog.rs | 123 +++++++++++++ .../src/platform/main/mod.rs | 1 + .../src/platform/main/run_watchdog.rs | 25 +++ .../src/platform/mod.rs | 3 + .../src/platform/models.rs | 50 ++++++ .../src/runs/_helpers/events.rs | 52 ++---- .../src/store/main/append_control_event.rs | 52 ++++++ .../src/store/main/mod.rs | 1 + .../src/store/models.rs | 12 ++ .../sqlbuild-orchestration/tests/platform.rs | 12 ++ .../tests/platform/helpers.rs | 107 +++++++++++ .../tests/platform/test_types.rs | 26 +++ .../tests/platform/test_watchdog.rs | 166 ++++++++++++++++++ 19 files changed, 677 insertions(+), 48 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/_helpers/observations.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/run_watchdog.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/models.rs create mode 100644 crates/sqlbuild-orchestration/src/store/main/append_control_event.rs create mode 100644 crates/sqlbuild-orchestration/tests/platform.rs create mode 100644 crates/sqlbuild-orchestration/tests/platform/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/platform/test_types.rs create mode 100644 crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs index 7de55c4455..06f74ae5df 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs @@ -1,6 +1,7 @@ //! Supervise native API/scheduler roles together; a failed role stops its siblings. use sqlbuild_orchestration::api::main::serve_api::serve_api; +use sqlbuild_orchestration::platform::main::run_watchdog::run_watchdog; use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler; use sqlbuild_orchestration::scheduler::models::SchedulerState; use sqlbuild_orchestration::store::errors::ControlError; @@ -53,6 +54,19 @@ pub(crate) async fn run_control( listener, stopped(shutdown.clone()), )); + drop(write_output(&[( + Stream::Stderr, + "Starting platform watchdog...\n", + )])); + let watchdog_store = store.clone(); + let watchdog_shutdown = shutdown.clone(); + tasks.spawn(async move { + run_watchdog(&watchdog_store, Uuid::now_v7(), watchdog_shutdown).await + }); + drop(write_output(&[( + Stream::Stderr, + "Platform watchdog started\n", + )])); } if standalone || !flag(namespace, "control_no_scheduler") { drop(write_output(&[(Stream::Stderr, "Starting scheduler...\n")])); diff --git a/crates/sqlbuild-cli/tests/native_commands/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/helpers.rs index 5e4ae4bfde..258d9c319d 100644 --- a/crates/sqlbuild-cli/tests/native_commands/helpers.rs +++ b/crates/sqlbuild-cli/tests/native_commands/helpers.rs @@ -65,16 +65,19 @@ pub(crate) async fn api_ready(control: &mut TestControl) -> String { .expect("native API becomes ready") } -pub(crate) fn leader_count(control: &TestControl) -> Option { +pub(crate) fn health_counts(control: &TestControl) -> Option<(i64, i64)> { let connection = rusqlite::Connection::open_with_flags( control.directory.path().join(".sqlbuild/control.sqlite3"), rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, ) .ok()?; connection - .query_row("SELECT COUNT(*) FROM sqb_scheduler_lease", [], |row| { - row.get(0) - }) + .query_row( + "SELECT (SELECT COUNT(*) FROM sqb_scheduler_lease), \ + (SELECT COUNT(*) FROM recall_events WHERE event_type='platform_heartbeat')", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) .ok() } diff --git a/crates/sqlbuild-cli/tests/native_commands/test_control.rs b/crates/sqlbuild-cli/tests/native_commands/test_control.rs index de16cd7367..b3394df6e3 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_control.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_control.rs @@ -1,7 +1,7 @@ use std::process::Command; use std::time::Duration; -use crate::helpers::{api_ready, isolated_binary, leader_count, start_control}; +use crate::helpers::{api_ready, health_counts, isolated_binary, start_control}; #[test] fn given_a_standalone_binary_when_printing_control_help_or_openapi_then_python_and_database_access_are_not_required() @@ -66,12 +66,14 @@ async fn given_a_standalone_binary_when_serving_with_or_without_a_scheduler_then .expect("protected native API"); assert_eq!(response.status(), reqwest::StatusCode::UNAUTHORIZED); tokio::time::timeout(Duration::from_secs(5), async { - while leader_count(&control) != Some(i64::from(scheduler)) { + while !health_counts(&control).is_some_and(|(leaders, heartbeats)| { + leaders == i64::from(scheduler) && heartbeats > 0 + }) { tokio::time::sleep(Duration::from_millis(20)).await; } }) .await - .expect("embedded scheduler enabled exactly when requested"); + .expect("watchdog records health with or without the optional scheduler"); #[cfg(unix)] crate::helpers::terminate(&mut control).await; } @@ -82,7 +84,7 @@ async fn given_a_standalone_scheduler_when_started_without_python_then_it_migrat { let mut control = start_control(&["scheduler"]); tokio::time::timeout(Duration::from_secs(5), async { - while leader_count(&control) != Some(1) { + while health_counts(&control).map(|(leaders, _)| leaders) != Some(1) { assert!( control .child diff --git a/crates/sqlbuild-orchestration/src/lib.rs b/crates/sqlbuild-orchestration/src/lib.rs index 89689306a8..2546686e46 100644 --- a/crates/sqlbuild-orchestration/src/lib.rs +++ b/crates/sqlbuild-orchestration/src/lib.rs @@ -5,6 +5,7 @@ pub mod api; pub mod credentials; pub mod declarations; +pub mod platform; pub mod runs; pub mod scheduler; pub mod snapshots; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs new file mode 100644 index 0000000000..b604790017 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod observations; +pub(crate) mod watchdog; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/observations.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/observations.rs new file mode 100644 index 0000000000..8f5779d7fb --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/observations.rs @@ -0,0 +1,57 @@ +//! Replicas share a deterministic observation key and a serialized append transaction. + +use chrono::{DurationRound, TimeDelta}; +use serde_json::json; +use uuid::Uuid; + +use crate::platform::models::PlatformEvent; +use crate::store::errors::ControlError; +use crate::store::main::append_control_event::append_control_event; +use crate::store::main::query::query; +use crate::store::models::{ControlEvent, ControlTransaction, Parameter}; + +pub(crate) async fn observe( + transaction: ControlTransaction, + installation_id: Uuid, + event: PlatformEvent, +) -> Result { + let occurred_at = transaction + .now + .duration_trunc(TimeDelta::seconds(30)) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let payload = json!({"platform": event}); + let identity = serde_json::to_vec(&(occurred_at, &event)) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let event_id = Uuid::new_v5(&installation_id, &identity); + let (transaction, previous) = query( + transaction, + "SELECT 1 AS present FROM recall_events WHERE event_id=$1 AND occurred_at=$2 LIMIT 1", + vec![ + Parameter::Text(event_id.to_string()), + Parameter::Time(occurred_at), + ], + ) + .await?; + if !previous.is_empty() { + return Ok(transaction); + } + let (transaction, outcome) = append_control_event( + transaction, + ControlEvent { + event_id, + invocation_id: format!("platform/{installation_id}/{}", occurred_at.date_naive()), + event_type: event.event_type(), + producer: "platform", + occurred_at, + run_id: None, + payload, + }, + ) + .await?; + if outcome.conflict.is_some() { + return Err(ControlError::Invalid( + "platform observation conflicts with a durable fact".into(), + )); + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs new file mode 100644 index 0000000000..1757b07c66 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs @@ -0,0 +1,123 @@ +//! Health scanning releases the writer lock between bounded trigger pages. + +use chrono::TimeDelta; +use tokio::sync::watch; +use uuid::Uuid; + +use crate::declarations::models::Schedule; +use crate::platform::_helpers::observations::observe; +use crate::platform::models::{InstallationHealth, PlatformEvent, WatchdogTrigger}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub(crate) async fn scan_health( + store: &ControlStore, + replica_id: Uuid, + shutdown: &watch::Receiver, +) -> Result<(), ControlError> { + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "SELECT i.installation_id, CASE WHEN l.expires_at>$1 THEN 1 ELSE 0 END AS leader \ + FROM sqb_installation i LEFT JOIN sqb_scheduler_lease l ON l.name='scheduler' WHERE i.singleton=1", + vec![Parameter::Time(now)]).await?; + let health: InstallationHealth = decode_row( + rows.into_iter() + .next() + .ok_or_else(|| ControlError::Invalid("control store has no installation".into()))?, + )?; + let mut transaction = transaction; + if health.leader == 0 { + transaction = observe( + transaction, + health.installation_id, + PlatformEvent::SchedulerNoLeader, + ) + .await?; + } + commit(transaction).await?; + let mut after = None; + while !*shutdown.borrow() { + after = scan_trigger_page(store, health.installation_id, after).await?; + if after.is_none() { + break; + } + tokio::task::yield_now().await; + } + if *shutdown.borrow() { + return Ok(()); + } + let transaction = begin(store, TransactionMode::Write).await?; + commit( + observe( + transaction, + health.installation_id, + PlatformEvent::Heartbeat { replica_id }, + ) + .await?, + ) + .await?; + Ok(()) +} + +async fn scan_trigger_page( + store: &ControlStore, + installation_id: Uuid, + after: Option, +) -> Result, ControlError> { + let cursor = after.map_or("", |_| "AND t.trigger_id>$1"); + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let (mut transaction, rows) = query( + transaction, + &format!( + "SELECT t.trigger_id, t.definition_json, s.next_due_at \ + FROM sqb_triggers t JOIN sqb_trigger_state s ON s.trigger_id=t.trigger_id \ + JOIN sqb_locations l ON l.location_id=t.location_id \ + WHERE s.status='running' AND t.removed_in_snapshot_id IS NULL AND l.archived_at IS NULL \ + {cursor} ORDER BY t.trigger_id LIMIT 100" + ), + after.into_iter().map(Parameter::Id).collect(), + ) + .await?; + let triggers: Vec = + rows.into_iter().map(decode_row).collect::>()?; + let next = triggers.last().map(|trigger| trigger.trigger_id); + for trigger in triggers { + if let Some(scheduled_for) = trigger.next_due_at + && scheduled_for < now - TimeDelta::seconds(30) + { + transaction = observe( + transaction, + installation_id, + PlatformEvent::TickOverdue { + trigger_id: trigger.trigger_id, + scheduled_for, + }, + ) + .await?; + } + if let Schedule::Calendar { times, refresh_at } = trigger.definition_json.schedule + && times + .last() + .is_none_or(|last| *last <= now + TimeDelta::days(1)) + { + transaction = observe( + transaction, + installation_id, + PlatformEvent::CalendarHorizonLow { + trigger_id: trigger.trigger_id, + refresh_at, + horizon_end: times.last().copied(), + }, + ) + .await?; + } + } + commit(transaction).await?; + Ok(next) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/mod.rs b/crates/sqlbuild-orchestration/src/platform/main/mod.rs new file mode 100644 index 0000000000..bb879e253f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/mod.rs @@ -0,0 +1 @@ +pub mod run_watchdog; diff --git a/crates/sqlbuild-orchestration/src/platform/main/run_watchdog.rs b/crates/sqlbuild-orchestration/src/platform/main/run_watchdog.rs new file mode 100644 index 0000000000..d8a72dc0fb --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/run_watchdog.rs @@ -0,0 +1,25 @@ +//! Every server observes health independently; scheduler leadership is not required. + +use std::time::Duration; + +use tokio::sync::watch; +use uuid::Uuid; + +use crate::platform::_helpers::watchdog::scan_health; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; + +pub async fn run_watchdog( + store: &ControlStore, + replica_id: Uuid, + mut shutdown: watch::Receiver, +) -> Result<(), ControlError> { + while !*shutdown.borrow() { + scan_health(store, replica_id, &shutdown).await?; + tokio::select! { + _ = tokio::time::sleep(Duration::from_secs(30)) => {}, + result = shutdown.changed() => { if result.is_err() { break; } }, + } + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/platform/mod.rs b/crates/sqlbuild-orchestration/src/platform/mod.rs new file mode 100644 index 0000000000..5359000d16 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod _helpers; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs new file mode 100644 index 0000000000..0f3bb0325c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -0,0 +1,50 @@ +//! Platform health is a typed stream of observations, independent of user code. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::declarations::models::TriggerDefinition; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum PlatformEvent { + Heartbeat { + replica_id: Uuid, + }, + SchedulerNoLeader, + TickOverdue { + trigger_id: Uuid, + scheduled_for: DateTime, + }, + CalendarHorizonLow { + trigger_id: Uuid, + refresh_at: DateTime, + horizon_end: Option>, + }, +} + +impl PlatformEvent { + #[must_use] + pub const fn event_type(&self) -> &'static str { + match self { + Self::Heartbeat { .. } => "platform_heartbeat", + Self::SchedulerNoLeader => "platform_scheduler_no_leader", + Self::TickOverdue { .. } => "platform_tick_overdue", + Self::CalendarHorizonLow { .. } => "platform_calendar_horizon_low", + } + } +} + +#[derive(Debug, Deserialize)] +pub(crate) struct InstallationHealth { + pub(crate) installation_id: Uuid, + pub(crate) leader: i64, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct WatchdogTrigger { + pub(crate) trigger_id: Uuid, + pub(crate) definition_json: TriggerDefinition, + pub(crate) next_due_at: Option>, +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/events.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/events.rs index 7ef23c279b..cb7b38073c 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/events.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/events.rs @@ -1,15 +1,11 @@ //! All orchestration transitions share Recall's invocation sequence and transaction. -use chrono::SecondsFormat; -use serde_json::{Map, Value, json}; -use sqlbuild_recall::records::main::event_object_row::event_object_row; -use sqlbuild_recall::records::models::RowEnvelope; +use serde_json::Value; use uuid::Uuid; use crate::store::errors::ControlError; -use crate::store::main::append_recall::append_recall; -use crate::store::main::query::query; -use crate::store::models::{ControlTransaction, Parameter}; +use crate::store::main::append_control_event::append_control_event; +use crate::store::models::{ControlEvent, ControlTransaction}; pub(crate) async fn append_run_event( transaction: ControlTransaction, @@ -17,40 +13,16 @@ pub(crate) async fn append_run_event( kind: &str, payload: Value, ) -> Result<(ControlTransaction, i64), ControlError> { - let invocation_id = format!("scheduler/{run_id}"); - let timestamp = transaction.now.to_rfc3339_opts(SecondsFormat::Micros, true); - let (transaction, rows) = query(transaction, - "SELECT invocation_sequence AS last_sequence, invocation_started_at AS started_at FROM recall_events \ - WHERE invocation_id=$1 ORDER BY invocation_sequence DESC LIMIT 1", - vec![Parameter::Text(invocation_id.clone())]).await?; - let previous = rows.first(); - let sequence = previous - .and_then(|row| row.get("last_sequence")) - .and_then(Value::as_i64) - .map_or(Some(0), |value| value.checked_add(1)) - .ok_or_else(|| ControlError::Invalid("run event sequence is exhausted".into()))?; - let envelope = RowEnvelope { - invocation_id, - invocation_started_at: previous - .and_then(|row| row.get("started_at")) - .and_then(Value::as_str) - .unwrap_or(×tamp) - .into(), - producer_version: env!("CARGO_PKG_VERSION").into(), + let event = ControlEvent { + event_id, + invocation_id: format!("scheduler/{run_id}"), + event_type: kind, + producer: "scheduler", + occurred_at: transaction.now, + run_id: Some(run_id), + payload, }; - let event = Map::from_iter([ - ("event_id".into(), json!(event_id)), - ("event_type".into(), json!(kind)), - ("schema_version".into(), json!(1)), - ("occurred_at".into(), json!(timestamp)), - ("producer".into(), json!("scheduler")), - ("invocation_sequence".into(), json!(sequence)), - ("run_id".into(), json!(run_id)), - ("payload".into(), payload), - ]); - let row = event_object_row(&envelope, &event) - .map_err(|error| ControlError::Invalid(error.to_string()))?; - let (transaction, outcome) = append_recall(transaction, &row).await?; + let (transaction, outcome) = append_control_event(transaction, event).await?; if outcome.conflict.is_some() { return Err(ControlError::Invalid( "run event conflicts with a durable fact".into(), diff --git a/crates/sqlbuild-orchestration/src/store/main/append_control_event.rs b/crates/sqlbuild-orchestration/src/store/main/append_control_event.rs new file mode 100644 index 0000000000..3143159449 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/append_control_event.rs @@ -0,0 +1,52 @@ +//! Sequence scheduler and platform facts inside the caller's serialized transaction. + +use chrono::SecondsFormat; +use serde_json::{Map, Value, json}; +use sqlbuild_recall::records::main::event_object_row::event_object_row; +use sqlbuild_recall::records::models::RowEnvelope; + +use crate::store::errors::ControlError; +use crate::store::main::append_recall::append_recall; +use crate::store::main::query::query; +use crate::store::models::{ControlEvent, ControlTransaction, Parameter, RecallAppendOutcome}; + +pub(crate) async fn append_control_event( + transaction: ControlTransaction, + event: ControlEvent<'_>, +) -> Result<(ControlTransaction, RecallAppendOutcome), ControlError> { + let timestamp = event + .occurred_at + .to_rfc3339_opts(SecondsFormat::Micros, true); + let (transaction, rows) = query(transaction, + "SELECT invocation_sequence AS last_sequence, invocation_started_at AS started_at FROM recall_events \ + WHERE invocation_id=$1 ORDER BY invocation_sequence DESC LIMIT 1", + vec![Parameter::Text(event.invocation_id.clone())]).await?; + let previous = rows.first(); + let sequence = previous + .and_then(|row| row.get("last_sequence")) + .and_then(Value::as_i64) + .map_or(Some(0), |value| value.checked_add(1)) + .ok_or_else(|| ControlError::Invalid("run event sequence is exhausted".into()))?; + let envelope = RowEnvelope { + invocation_id: event.invocation_id, + invocation_started_at: previous + .and_then(|row| row.get("started_at")) + .and_then(Value::as_str) + .unwrap_or(×tamp) + .into(), + producer_version: env!("CARGO_PKG_VERSION").into(), + }; + let object = Map::from_iter([ + ("event_id".into(), json!(event.event_id)), + ("event_type".into(), json!(event.event_type)), + ("schema_version".into(), json!(1)), + ("occurred_at".into(), json!(timestamp)), + ("producer".into(), json!(event.producer)), + ("invocation_sequence".into(), json!(sequence)), + ("run_id".into(), json!(event.run_id)), + ("payload".into(), event.payload), + ]); + let row = event_object_row(&envelope, &object) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + append_recall(transaction, &row).await +} diff --git a/crates/sqlbuild-orchestration/src/store/main/mod.rs b/crates/sqlbuild-orchestration/src/store/main/mod.rs index ba1903c103..d22e12f725 100644 --- a/crates/sqlbuild-orchestration/src/store/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/store/main/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod append_control_event; pub(crate) mod append_recall; pub(crate) mod begin; pub(crate) mod commit; diff --git a/crates/sqlbuild-orchestration/src/store/models.rs b/crates/sqlbuild-orchestration/src/store/models.rs index 3c097f41bd..414bca79cd 100644 --- a/crates/sqlbuild-orchestration/src/store/models.rs +++ b/crates/sqlbuild-orchestration/src/store/models.rs @@ -22,6 +22,18 @@ pub struct RecallAppendOutcome { pub conflict: Option, } +/// A control-plane fact shares Recall's durable sequencing and conflict handling. +#[derive(Debug)] +pub(crate) struct ControlEvent<'a> { + pub(crate) event_id: Uuid, + pub(crate) invocation_id: String, + pub(crate) event_type: &'a str, + pub(crate) producer: &'static str, + pub(crate) occurred_at: DateTime, + pub(crate) run_id: Option, + pub(crate) payload: serde_json::Value, +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum TransactionMode { Read, diff --git a/crates/sqlbuild-orchestration/tests/platform.rs b/crates/sqlbuild-orchestration/tests/platform.rs new file mode 100644 index 0000000000..673cc30aef --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/platform.rs @@ -0,0 +1,12 @@ +#[path = "platform/helpers.rs"] +mod helpers; +#[path = "store/replica_helpers.rs"] +mod replica_helpers; +#[path = "store/helpers.rs"] +mod store_helpers; +#[path = "store/test_types.rs"] +mod store_test_types; +#[path = "platform/test_types.rs"] +mod test_types; +#[path = "platform/test_watchdog.rs"] +mod test_watchdog; diff --git a/crates/sqlbuild-orchestration/tests/platform/helpers.rs b/crates/sqlbuild-orchestration/tests/platform/helpers.rs new file mode 100644 index 0000000000..c2ea45fa05 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/platform/helpers.rs @@ -0,0 +1,107 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::time::Duration; + +use chrono::{TimeDelta, Utc}; +use serde_json::json; +use sqlbuild_orchestration::platform::models::PlatformEvent; +use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; +use sqlbuild_orchestration::store::models::ControlStore; +use tokio::task::JoinHandle; +use uuid::Uuid; + +use crate::test_types::Observation; + +pub(crate) fn snapshot(location_id: Uuid) -> SnapshotInput { + let future = Utc::now() + TimeDelta::days(10); + let mut triggers = BTreeMap::new(); + for index in 0..101 { + triggers.insert( + format!("daily_orders_{index}"), + json!({ + "description": "Refresh orders", "job_key": "orders", + "starts_at": "2020-01-01T00:00:00Z", + "schedule": {"kind": "calendar", "times": ["2020-01-01T00:00:00Z", future], + "refresh_at": "2020-01-01T00:00:00Z"} + }), + ); + } + for (name, status) in [("expired_orders", "running"), ("paused_orders", "paused")] { + triggers.insert(name.into(), json!({ + "description": "Refresh orders", "job_key": "orders", "default_status": status, + "starts_at": "2020-01-01T00:00:00Z", + "schedule": {"kind": "calendar", "times": ["2020-01-01T00:00:00Z"], "refresh_at": "2020-01-01T00:00:00Z"} + })); + } + triggers.insert( + "future_orders".into(), + json!({ + "description": "Future orders refresh", "job_key": "orders", + "schedule": {"kind": "calendar", "times": [future], "refresh_at": future} + }), + ); + SnapshotInput { + location_id, + target: "development".into(), + generation: 1, + git_revision: None, + git_dirty: false, + git_author: None, + image_id: None, + environment_hash: "orders-watchdog-environment".into(), + sqb_version: env!("CARGO_PKG_VERSION").into(), + nodes: BTreeMap::from([( + "orders".into(), + SnapshotNode { + kind: "model".into(), + definition: json!({"materialization": "table"}), + raw_sql: Some("select 1 as order_id".into()), + compiled_sql: Some("select 1 as order_id".into()), + python_source: None, + }, + )]), + documents: BTreeMap::from([ + ( + "jobs".into(), + json!({"orders": {"description": "Build orders", "selection": ["orders"]}}), + ), + ("triggers".into(), json!(triggers)), + ]), + } +} + +pub(crate) async fn wait_for_scans( + store: &ControlStore, + replicas: &BTreeSet, + tasks: &[JoinHandle>], +) -> Vec { + tokio::time::timeout(Duration::from_secs(10), async { + let mut cursor = 0; + let mut observations = Vec::new(); + let mut scanned = BTreeSet::new(); + while &scanned != replicas { + assert!( + tasks.iter().all(|task| !task.is_finished()), + "watchdog failed before scanning" + ); + for row in read_recall_events(store, cursor, 1000) + .await + .expect("read health facts") + { + let observation: Observation = + serde_json::from_value(json!(row)).expect("typed platform event"); + cursor = observation.storage_id; + scanned.extend(match observation.payload.platform { + PlatformEvent::Heartbeat { replica_id } => Some(replica_id), + _ => None, + }); + observations.push(observation); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + observations + }) + .await + .expect("both replicas finish a paged scan without a scheduler") +} diff --git a/crates/sqlbuild-orchestration/tests/platform/test_types.rs b/crates/sqlbuild-orchestration/tests/platform/test_types.rs new file mode 100644 index 0000000000..1f1fec3985 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/platform/test_types.rs @@ -0,0 +1,26 @@ +use chrono::{DateTime, Utc}; +use serde::Deserialize; +use sqlbuild_orchestration::platform::models::PlatformEvent; + +use crate::store_test_types::Backend; + +pub(crate) struct WatchdogTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) take_lease: bool, + pub(crate) expected_no_leader: bool, + pub(crate) expected_overdue: usize, + pub(crate) expected_low_horizons: usize, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct Observation { + pub(crate) storage_id: i64, + pub(crate) occurred_at: DateTime, + pub(crate) payload: PlatformPayload, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct PlatformPayload { + pub(crate) platform: PlatformEvent, +} diff --git a/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs b/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs new file mode 100644 index 0000000000..26e40396d0 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs @@ -0,0 +1,166 @@ +use std::collections::BTreeSet; +use std::time::Duration; + +use sqlbuild_orchestration::platform::main::run_watchdog::run_watchdog; +use sqlbuild_orchestration::platform::models::PlatformEvent; +use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::store::main::create_location::create_location; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use uuid::Uuid; + +use crate::helpers::{snapshot, wait_for_scans}; +use crate::replica_helpers::second_store; +use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store_test_types::Backend; +use crate::test_types::WatchdogTestCase; + +#[test] +fn given_two_server_replicas_when_scheduling_is_stalled_then_health_facts_are_durable_deduplicated_and_independent_of_leadership() + { + let test_cases = [ + WatchdogTestCase { + description: "SQLite without a leader", + backend: Backend::Sqlite, + take_lease: false, + expected_no_leader: true, + expected_overdue: 102, + expected_low_horizons: 1, + }, + WatchdogTestCase { + description: "PostgreSQL without a leader", + backend: Backend::Postgres, + take_lease: false, + expected_no_leader: true, + expected_overdue: 102, + expected_low_horizons: 1, + }, + WatchdogTestCase { + description: "SQLite with a frozen leader", + backend: Backend::Sqlite, + take_lease: true, + expected_no_leader: false, + expected_overdue: 102, + expected_low_horizons: 1, + }, + WatchdogTestCase { + description: "PostgreSQL with a frozen leader", + backend: Backend::Postgres, + take_lease: true, + expected_no_leader: false, + expected_overdue: 102, + expected_low_horizons: 1, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate health store"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + input + .documents + .get_mut("triggers") + .expect("trigger document")["removed_orders"] = + input.documents["triggers"]["expired_orders"].clone(); + publish_snapshot(store, &input) + .await + .expect("old definition"); + input.generation = 2; + input + .documents + .get_mut("triggers") + .expect("trigger document") + .as_object_mut() + .expect("trigger map") + .remove("removed_orders"); + publish_snapshot(store, &input) + .await + .expect("current definition"); + if test_case.take_lease { + acquire_lease(store, Uuid::now_v7()) + .await + .expect("acquire lease") + .expect("leader"); + } + let second = second_store(store).await; + let replicas = BTreeSet::from([Uuid::now_v7(), Uuid::now_v7()]); + let (stop, shutdown) = tokio::sync::watch::channel(false); + let tasks: Vec<_> = [store.clone(), second.clone()] + .into_iter() + .zip(replicas.iter()) + .map(|(store, id)| { + let id = *id; + let shutdown = shutdown.clone(); + tokio::spawn(async move { run_watchdog(&store, id, shutdown).await }) + }) + .collect(); + let observations = wait_for_scans(store, &replicas, &tasks).await; + let overdue: BTreeSet<_> = observations + .iter() + .filter_map(|row| match row.payload.platform { + PlatformEvent::TickOverdue { trigger_id, .. } => Some(trigger_id), + _ => None, + }) + .collect(); + assert_eq!( + overdue.len(), + test_case.expected_overdue, + "{}", + test_case.description + ); + let horizons: BTreeSet<_> = observations + .iter() + .filter_map(|row| match row.payload.platform { + PlatformEvent::CalendarHorizonLow { trigger_id, .. } => Some(trigger_id), + _ => None, + }) + .collect(); + assert_eq!( + horizons.len(), + test_case.expected_low_horizons, + "paused and removed calendars are excluded; refresh cadence alone is not a low horizon" + ); + assert_eq!( + observations + .iter() + .any(|row| matches!(row.payload.platform, PlatformEvent::SchedulerNoLeader)), + test_case.expected_no_leader + ); + let logical_keys: BTreeSet<_> = observations + .iter() + .map(|row| { + ( + row.occurred_at, + serde_json::to_string(&row.payload.platform).expect("event identity"), + ) + }) + .collect(); + assert_eq!( + logical_keys.len(), + observations.len(), + "replicas do not duplicate an observation within its window" + ); + assert!( + observations + .iter() + .all(|row| row.occurred_at.timestamp().rem_euclid(30) == 0) + ); + stop.send(true).expect("shutdown both replicas"); + for task in tasks { + tokio::time::timeout(Duration::from_secs(2), task) + .await + .expect("prompt shutdown") + .expect("join watchdog") + .expect("watchdog stopped"); + } + close_store(second).await; + close_test_store(fixture).await; + }); + } +} From f41cb3ebceeaffdf307ffc13aff6632f0bf3f571 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 01:01:11 +0000 Subject: [PATCH 09/59] test: align backend integration contracts with domain ownership Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../tests/native_commands/helpers.rs | 71 ++++-- .../tests/native_commands/test_control.rs | 240 ++++++++++++------ .../tests/native_commands/test_types.rs | 20 ++ .../tests/api/helpers.rs | 19 +- .../tests/credentials.rs | 12 +- .../tests/credentials/helpers.rs | 21 ++ .../tests/credentials/test_contract.rs | 53 ++-- .../tests/credentials/test_types.rs | 8 + .../sqlbuild-orchestration/tests/platform.rs | 2 +- .../tests/platform/helpers.rs | 30 ++- .../tests/platform/test_types.rs | 2 +- .../tests/platform/test_watchdog.rs | 27 +- crates/sqlbuild-orchestration/tests/recall.rs | 14 - crates/sqlbuild-orchestration/tests/runs.rs | 10 +- .../helpers.rs} | 2 + .../tests/runs/test_faults.rs | 6 +- .../tests/runs/test_lifecycle.rs | 6 +- .../tests/runs/test_types.rs | 2 +- .../sqlbuild-orchestration/tests/scheduler.rs | 10 +- .../tests/{ => scheduler}/ticks/helpers.rs | 18 +- .../{ => scheduler}/ticks/test_contract.rs | 6 +- .../tests/{ => scheduler}/ticks/test_loop.rs | 6 +- .../tests/scheduler/ticks/test_ticks.rs | 10 + .../{ => scheduler}/ticks/test_triggers.rs | 8 +- .../tests/{ => scheduler}/ticks/test_types.rs | 4 +- .../sqlbuild-orchestration/tests/snapshots.rs | 2 +- crates/sqlbuild-orchestration/tests/store.rs | 16 ++ .../faults/helpers.rs} | 8 +- .../{fault_types.rs => faults/test_types.rs} | 4 +- .../tests/store/lease_helpers.rs | 24 -- .../tests/store/leases/helpers.rs | 24 ++ .../tests/{ => store}/recall/helpers.rs | 2 + .../tests/{ => store}/recall/test_contract.rs | 2 + .../tests/{ => store}/recall/test_types.rs | 2 + .../tests/{ => store}/recall/test_views.rs | 2 + .../tests/store/replica_helpers.rs | 15 -- .../tests/store/replicas/helpers.rs | 20 ++ crates/sqlbuild-orchestration/tests/ticks.rs | 22 -- 38 files changed, 474 insertions(+), 276 deletions(-) create mode 100644 crates/sqlbuild-orchestration/tests/credentials/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/credentials/test_types.rs delete mode 100644 crates/sqlbuild-orchestration/tests/recall.rs rename crates/sqlbuild-orchestration/tests/runs/{terminal_helpers.rs => termination/helpers.rs} (93%) rename crates/sqlbuild-orchestration/tests/{ => scheduler}/ticks/helpers.rs (96%) rename crates/sqlbuild-orchestration/tests/{ => scheduler}/ticks/test_contract.rs (83%) rename crates/sqlbuild-orchestration/tests/{ => scheduler}/ticks/test_loop.rs (96%) create mode 100644 crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs rename crates/sqlbuild-orchestration/tests/{ => scheduler}/ticks/test_triggers.rs (96%) rename crates/sqlbuild-orchestration/tests/{ => scheduler}/ticks/test_types.rs (78%) create mode 100644 crates/sqlbuild-orchestration/tests/store.rs rename crates/sqlbuild-orchestration/tests/{ticks/fault_helpers.rs => store/faults/helpers.rs} (92%) rename crates/sqlbuild-orchestration/tests/store/{fault_types.rs => faults/test_types.rs} (68%) delete mode 100644 crates/sqlbuild-orchestration/tests/store/lease_helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/store/leases/helpers.rs rename crates/sqlbuild-orchestration/tests/{ => store}/recall/helpers.rs (98%) rename crates/sqlbuild-orchestration/tests/{ => store}/recall/test_contract.rs (93%) rename crates/sqlbuild-orchestration/tests/{ => store}/recall/test_types.rs (84%) rename crates/sqlbuild-orchestration/tests/{ => store}/recall/test_views.rs (99%) delete mode 100644 crates/sqlbuild-orchestration/tests/store/replica_helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/store/replicas/helpers.rs delete mode 100644 crates/sqlbuild-orchestration/tests/ticks.rs diff --git a/crates/sqlbuild-cli/tests/native_commands/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/helpers.rs index 258d9c319d..606246903b 100644 --- a/crates/sqlbuild-cli/tests/native_commands/helpers.rs +++ b/crates/sqlbuild-cli/tests/native_commands/helpers.rs @@ -2,10 +2,10 @@ use std::path::Path; use std::process::{Command, Stdio}; use std::time::Duration; -use crate::test_types::TestControl; +use crate::test_types::{ServiceTestCase, TestControl}; pub(crate) fn isolated_binary(directory: &Path) -> std::path::PathBuf { - let binary = directory.join(if cfg!(windows) { "sqb.exe" } else { "sqb" }); + let binary = directory.join(format!("sqb{}", std::env::consts::EXE_SUFFIX)); std::fs::copy(env!("CARGO_BIN_EXE_sqb"), &binary) .expect("copy standalone binary without Python CLI"); binary @@ -37,29 +37,33 @@ pub(crate) async fn api_ready(control: &mut TestControl) -> String { .build() .expect("HTTP client"); tokio::time::timeout(Duration::from_secs(10), async { - loop { + for _ in 0..500 { assert!( control.child.try_wait().expect("child status").is_none(), "control process exited: {}", std::fs::read_to_string(&control.stderr).unwrap_or_default() ); + tokio::time::sleep(Duration::from_millis(20)).await; let log = std::fs::read_to_string(&control.stderr).expect("read control log"); - if let Some(address) = log + let Some(address) = log .lines() .find_map(|line| line.strip_prefix("Control API ready: ")) - { - let url = format!("http://{address}"); - if client - .get(format!("{url}/api/openapi.json")) - .send() - .await - .is_ok_and(|response| response.status().is_success()) - { - return url; - } - } - tokio::time::sleep(Duration::from_millis(20)).await; + else { + continue; + }; + let url = format!("http://{address}"); + let Ok(response) = client + .get(format!("{url}/api/openapi.json")) + .send() + .await + .and_then(reqwest::Response::error_for_status) + else { + continue; + }; + drop(response); + return url; } + panic!("native API exhausted readiness attempts"); }) .await .expect("native API becomes ready") @@ -81,6 +85,31 @@ pub(crate) fn health_counts(control: &TestControl) -> Option<(i64, i64)> { .ok() } +pub(crate) async fn wait_for_health( + control: &mut TestControl, + case: &ServiceTestCase, +) -> (i64, i64) { + tokio::time::timeout(Duration::from_secs(5), async { + for _ in 0..250 { + assert!( + control.child.try_wait().expect("service status").is_none(), + "service exited: {}", + std::fs::read_to_string(&control.stderr).unwrap_or_default() + ); + tokio::time::sleep(Duration::from_millis(20)).await; + let Some(counts) = health_counts(control).filter(|(leaders, heartbeats)| { + *leaders == case.expected_leaders && *heartbeats >= case.expected_heartbeats + }) else { + continue; + }; + return counts; + } + panic!("control service exhausted health readiness attempts"); + }) + .await + .expect("native control service publishes its health state") +} + #[cfg(unix)] pub(crate) async fn terminate(control: &mut TestControl) { let status = Command::new("kill") @@ -89,12 +118,14 @@ pub(crate) async fn terminate(control: &mut TestControl) { .expect("send SIGTERM"); assert!(status.success()); let status = tokio::time::timeout(Duration::from_secs(5), async { - loop { - if let Some(status) = control.child.try_wait().expect("child exit") { - return status; - } + for _ in 0..250 { tokio::time::sleep(Duration::from_millis(20)).await; + let Some(status) = control.child.try_wait().expect("child exit") else { + continue; + }; + return status; } + panic!("control roles exhausted shutdown attempts"); }) .await .expect("control roles stop promptly"); diff --git a/crates/sqlbuild-cli/tests/native_commands/test_control.rs b/crates/sqlbuild-cli/tests/native_commands/test_control.rs index b3394df6e3..85a2e4e44c 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_control.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_control.rs @@ -1,102 +1,188 @@ use std::process::Command; -use std::time::Duration; -use crate::helpers::{api_ready, health_counts, isolated_binary, start_control}; +use crate::helpers::{api_ready, isolated_binary, start_control, wait_for_health}; +use crate::test_types::{CommandTestCase, OpenApiTestCase, ServiceTestCase}; #[test] -fn given_a_standalone_binary_when_printing_control_help_or_openapi_then_python_and_database_access_are_not_required() +fn given_a_standalone_binary_when_printing_help_or_rejecting_invalid_input_then_python_and_database_access_are_not_required() { - let directory = tempfile::tempdir().expect("isolated CLI directory"); - let binary = isolated_binary(directory.path()); - for arguments in [ - ["server", "--help"], - ["scheduler", "--help"], - ["server", "--openapi"], - ] { + let test_cases = [ + CommandTestCase { + description: "native server help", + arguments: &["server", "--help"], + expected_success: true, + expected_stdout_prefix: "usage: sqb ", + expected_stderr: "", + }, + CommandTestCase { + description: "native scheduler help", + arguments: &["scheduler", "--help"], + expected_success: true, + expected_stdout_prefix: "usage: sqb ", + expected_stderr: "", + }, + CommandTestCase { + description: "invalid port is rejected before store initialization", + arguments: &["server", "--port", "65536"], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control service failed: invalid control-plane data: --port must be between 0 and 65535\n", + }, + ]; + for test_case in test_cases { + let directory = tempfile::tempdir().expect("isolated CLI directory"); + let binary = isolated_binary(directory.path()); let output = Command::new(&binary) - .args(arguments) + .args(test_case.arguments) .current_dir(directory.path()) .env("PATH", "") .env("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://not-opened") .output() .expect("standalone native command"); - assert!( + assert_eq!( output.status.success(), + test_case.expected_success, + "{}", + test_case.description + ); + assert_eq!( + String::from_utf8_lossy(&output.stderr), + test_case.expected_stderr, + "{}", + test_case.description + ); + assert!( + String::from_utf8_lossy(&output.stdout).starts_with(test_case.expected_stdout_prefix), + "{}", + test_case.description + ); + assert!( + !directory.path().join(".sqlbuild").exists(), "{}", - String::from_utf8_lossy(&output.stderr) + test_case.description ); - assert!(output.stderr.is_empty()); - if arguments[1] == "--openapi" { - let document: serde_json::Value = - serde_json::from_slice(&output.stdout).expect("machine-readable OpenAPI"); - assert!(document["paths"]["/api/runs/{run_id}/events"].is_object()); - } else { - assert!(String::from_utf8_lossy(&output.stdout).starts_with("usage: sqb ")); - } } - assert!(!directory.path().join(".sqlbuild").exists()); - let output = Command::new(&binary) - .args(["server", "--port", "65536"]) - .current_dir(directory.path()) - .env("PATH", "") - .output() - .expect("invalid port"); - assert!(!output.status.success()); - assert!(String::from_utf8_lossy(&output.stderr).contains("--port must be between 0 and 65535")); - assert!( - !directory.path().join(".sqlbuild").exists(), - "invalid input cannot initialize a database" - ); } -#[tokio::test] -async fn given_a_standalone_binary_when_serving_with_or_without_a_scheduler_then_migrations_auth_and_shutdown_are_native() +#[test] +fn given_a_standalone_server_when_exporting_openapi_then_the_worker_contract_is_machine_readable_without_python_or_a_database() { - for scheduler in [false, true] { - let mut arguments = vec!["server", "--port", "0"]; - if !scheduler { - arguments.push("--no-scheduler"); - } - let mut control = start_control(&arguments); - let url = api_ready(&mut control).await; - let response = reqwest::Client::new() - .post(format!("{url}/api/runs/{}/heartbeat", uuid::Uuid::now_v7())) - .send() - .await - .expect("protected native API"); - assert_eq!(response.status(), reqwest::StatusCode::UNAUTHORIZED); - tokio::time::timeout(Duration::from_secs(5), async { - while !health_counts(&control).is_some_and(|(leaders, heartbeats)| { - leaders == i64::from(scheduler) && heartbeats > 0 - }) { - tokio::time::sleep(Duration::from_millis(20)).await; - } - }) - .await - .expect("watchdog records health with or without the optional scheduler"); - #[cfg(unix)] - crate::helpers::terminate(&mut control).await; + let test_cases = [OpenApiTestCase { + description: "worker lifecycle endpoint", + expected_path: "/api/runs/{run_id}/events", + }]; + for test_case in test_cases { + let directory = tempfile::tempdir().expect("isolated CLI directory"); + let output = Command::new(isolated_binary(directory.path())) + .args(["server", "--openapi"]) + .current_dir(directory.path()) + .env("PATH", "") + .env("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://not-opened") + .output() + .expect("standalone OpenAPI"); + assert!(output.status.success(), "{}", test_case.description); + assert!(output.stderr.is_empty(), "{}", test_case.description); + let document: serde_json::Value = + serde_json::from_slice(&output.stdout).expect("machine-readable OpenAPI"); + assert!( + document["paths"][test_case.expected_path].is_object(), + "{}", + test_case.description + ); + assert!( + !directory.path().join(".sqlbuild").exists(), + "{}", + test_case.description + ); } } -#[tokio::test] -async fn given_a_standalone_scheduler_when_started_without_python_then_it_migrates_and_takes_the_durable_lease() +#[test] +fn given_a_standalone_binary_when_serving_with_or_without_a_scheduler_then_migrations_auth_and_shutdown_are_native() { - let mut control = start_control(&["scheduler"]); - tokio::time::timeout(Duration::from_secs(5), async { - while health_counts(&control).map(|(leaders, _)| leaders) != Some(1) { + let test_cases = [ + ServiceTestCase { + description: "server without scheduling still observes platform health", + arguments: &["server", "--port", "0", "--no-scheduler"], + expected_leaders: 0, + expected_heartbeats: 1, + }, + ServiceTestCase { + description: "server with embedded scheduling", + arguments: &["server", "--port", "0"], + expected_leaders: 1, + expected_heartbeats: 1, + }, + ]; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("test runtime"); + for test_case in test_cases { + runtime.block_on(async { + let mut control = start_control(test_case.arguments); + let url = api_ready(&mut control).await; + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(5)) + .build() + .expect("bounded HTTP requests"); + let response = client + .post(format!("{url}/api/runs/{}/heartbeat", uuid::Uuid::now_v7())) + .send() + .await + .expect("protected native API"); + assert_eq!( + response.status(), + reqwest::StatusCode::UNAUTHORIZED, + "{}", + test_case.description + ); + let (leaders, heartbeats) = wait_for_health(&mut control, &test_case).await; + assert_eq!( + leaders, test_case.expected_leaders, + "{}", + test_case.description + ); assert!( - control - .child - .try_wait() - .expect("scheduler status") - .is_none() + heartbeats >= test_case.expected_heartbeats, + "{}", + test_case.description ); - tokio::time::sleep(Duration::from_millis(20)).await; - } - }) - .await - .expect("standalone scheduler acquires leadership"); - #[cfg(unix)] - crate::helpers::terminate(&mut control).await; + #[cfg(unix)] + crate::helpers::terminate(&mut control).await; + }); + } +} + +#[test] +fn given_a_standalone_scheduler_when_started_without_python_then_it_migrates_and_takes_the_durable_lease() + { + let test_cases = [ServiceTestCase { + description: "standalone scheduler takes a lease without starting a server watchdog", + arguments: &["scheduler"], + expected_leaders: 1, + expected_heartbeats: 0, + }]; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("test runtime"); + for test_case in test_cases { + runtime.block_on(async { + let mut control = start_control(test_case.arguments); + let (leaders, heartbeats) = wait_for_health(&mut control, &test_case).await; + assert_eq!( + leaders, test_case.expected_leaders, + "{}", + test_case.description + ); + assert_eq!( + heartbeats, test_case.expected_heartbeats, + "{}", + test_case.description + ); + #[cfg(unix)] + crate::helpers::terminate(&mut control).await; + }); + } } diff --git a/crates/sqlbuild-cli/tests/native_commands/test_types.rs b/crates/sqlbuild-cli/tests/native_commands/test_types.rs index 35842e7f61..15974f0788 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_types.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_types.rs @@ -1,6 +1,26 @@ use std::path::PathBuf; use std::process::Child; +pub(crate) struct CommandTestCase { + pub(crate) description: &'static str, + pub(crate) arguments: &'static [&'static str], + pub(crate) expected_success: bool, + pub(crate) expected_stdout_prefix: &'static str, + pub(crate) expected_stderr: &'static str, +} + +pub(crate) struct OpenApiTestCase { + pub(crate) description: &'static str, + pub(crate) expected_path: &'static str, +} + +pub(crate) struct ServiceTestCase { + pub(crate) description: &'static str, + pub(crate) arguments: &'static [&'static str], + pub(crate) expected_leaders: i64, + pub(crate) expected_heartbeats: i64, +} + pub(crate) struct TestControl { pub(crate) child: Child, pub(crate) directory: tempfile::TempDir, diff --git a/crates/sqlbuild-orchestration/tests/api/helpers.rs b/crates/sqlbuild-orchestration/tests/api/helpers.rs index 395e1535f5..4364fba3a3 100644 --- a/crates/sqlbuild-orchestration/tests/api/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/helpers.rs @@ -20,18 +20,21 @@ pub(crate) async fn start_api(store: &ControlStore) -> TestApi { .build() .expect("HTTP client"); tokio::time::timeout(Duration::from_secs(5), async { - loop { - if client + for _ in 0..250 { + assert!(!task.is_finished(), "API failed during startup"); + tokio::time::sleep(Duration::from_millis(20)).await; + let Ok(response) = client .get(format!("{url}/api/openapi.json")) .send() .await - .is_ok_and(|response| response.status().is_success()) - { - break; - } - assert!(!task.is_finished(), "API failed during startup"); - tokio::time::sleep(Duration::from_millis(20)).await; + .and_then(reqwest::Response::error_for_status) + else { + continue; + }; + drop(response); + return; } + panic!("API exhausted readiness attempts"); }) .await .expect("API becomes ready"); diff --git a/crates/sqlbuild-orchestration/tests/credentials.rs b/crates/sqlbuild-orchestration/tests/credentials.rs index c40545cc9e..b31553bc0d 100644 --- a/crates/sqlbuild-orchestration/tests/credentials.rs +++ b/crates/sqlbuild-orchestration/tests/credentials.rs @@ -1,6 +1,10 @@ -#[path = "ticks/fault_helpers.rs"] +#[path = "store/faults/helpers.rs"] mod fault_helpers; -#[path = "store/replica_helpers.rs"] +#[path = "store/faults/test_types.rs"] +mod fault_types; +#[path = "credentials/helpers.rs"] +mod helpers; +#[path = "store/replicas/helpers.rs"] mod replica_helpers; #[path = "runs/helpers.rs"] mod run_helpers; @@ -8,9 +12,9 @@ mod run_helpers; mod store_helpers; #[path = "store/test_types.rs"] mod store_test_types; -#[path = "runs/terminal_helpers.rs"] +#[path = "runs/termination/helpers.rs"] mod terminal_helpers; #[path = "credentials/test_contract.rs"] mod test_contract; -#[path = "store/fault_types.rs"] +#[path = "credentials/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/credentials/helpers.rs b/crates/sqlbuild-orchestration/tests/credentials/helpers.rs new file mode 100644 index 0000000000..e01295e856 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/helpers.rs @@ -0,0 +1,21 @@ +use sqlbuild_orchestration::store::models::ControlStore; + +pub(crate) async fn stored_hashes(store: &ControlStore) -> Vec { + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + let connection = rusqlite::Connection::open(path).expect("inspect durable tokens"); + return connection + .prepare("SELECT secret_hash FROM sqb_access_tokens") + .expect("token hashes") + .query_map([], |row| row.get(0)) + .expect("read token hashes") + .collect::>() + .expect("hashes"); + }; + sqlx_core::query_scalar::query_scalar("SELECT secret_hash FROM sqb_access_tokens") + .fetch_all(pool) + .await + .expect("read durable token hashes") +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs index 9b0ff859f2..4c39180fad 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs @@ -11,23 +11,35 @@ use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; use sqlbuild_orchestration::store::main::create_location::create_location; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; -use sqlbuild_orchestration::store::models::ControlStore; use uuid::Uuid; use crate::fault_helpers::{install_fault, remove_fault}; +use crate::helpers::stored_hashes; use crate::replica_helpers::second_store; use crate::run_helpers::{queue, request, snapshot}; use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; use crate::store_test_types::Backend; use crate::terminal_helpers::finish; -use crate::test_types::TickFaultTestCase; +use crate::test_types::{CredentialTestCase, TransactionFault}; #[test] fn given_scoped_tokens_when_revoked_or_the_run_ends_then_all_replicas_reject_them_and_audit_failure_rolls_back() { - for backend in [Backend::Sqlite, Backend::Postgres] { + let test_cases = [ + CredentialTestCase { + description: "SQLite credential lifecycle and atomic audit", + backend: Backend::Sqlite, + expected_hash_length: 64, + }, + CredentialTestCase { + description: "PostgreSQL credential lifecycle and atomic audit", + backend: Backend::Postgres, + expected_hash_length: 64, + }, + ]; + for test_case in test_cases { run(async { - let Some(fixture) = open_test_store(backend).await else { + let Some(fixture) = open_test_store(test_case.backend).await else { return; }; let store = &fixture.store; @@ -39,7 +51,7 @@ fn given_scoped_tokens_when_revoked_or_the_run_ends_then_all_replicas_reject_the request_id: Uuid::now_v7(), ip_address: Some("127.0.0.1".parse().expect("loopback")), }; - let fault = TickFaultTestCase { + let fault = TransactionFault { description: "security audit failure", table: "sqb_security_audit", operation: "INSERT", @@ -87,11 +99,7 @@ fn given_scoped_tokens_when_revoked_or_the_run_ends_then_all_replicas_reject_the !format!("{issued:?}").contains(&issued.bearer), "debug output redacts secrets" ); - let replacement = if issued.bearer.ends_with('0') { - "1" - } else { - "0" - }; + let replacement = ["0", "1"][usize::from(issued.bearer.ends_with('0'))]; let tampered = format!("{}{replacement}", &issued.bearer[..issued.bearer.len() - 1]); for invalid in [ @@ -141,29 +149,14 @@ fn given_scoped_tokens_when_revoked_or_the_run_ends_then_all_replicas_reject_the .await .is_err() ); - let stored: Vec = match store { - ControlStore::Sqlite(path) => { - let connection = - rusqlite::Connection::open(path).expect("inspect durable tokens"); - connection - .prepare("SELECT secret_hash FROM sqb_access_tokens") - .expect("token hashes") - .query_map([], |row| row.get(0)) - .expect("read token hashes") - .collect::>() - .expect("hashes") - } - ControlStore::Postgres(pool) => sqlx_core::query_scalar::query_scalar( - "SELECT secret_hash FROM sqb_access_tokens", - ) - .fetch_all(pool) - .await - .expect("read durable token hashes"), - }; + let stored = stored_hashes(store).await; assert!( stored .iter() - .all(|hash| hash.len() == 64 && hash != &issued.bearer) + .all(|hash| hash.len() == test_case.expected_hash_length + && hash != &issued.bearer), + "{}", + test_case.description ); } for scope in [ diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_types.rs b/crates/sqlbuild-orchestration/tests/credentials/test_types.rs new file mode 100644 index 0000000000..077ee03ce9 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/test_types.rs @@ -0,0 +1,8 @@ +pub(crate) use crate::fault_types::TransactionFault; +use crate::store_test_types::Backend; + +pub(crate) struct CredentialTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_hash_length: usize, +} diff --git a/crates/sqlbuild-orchestration/tests/platform.rs b/crates/sqlbuild-orchestration/tests/platform.rs index 673cc30aef..7908cbf127 100644 --- a/crates/sqlbuild-orchestration/tests/platform.rs +++ b/crates/sqlbuild-orchestration/tests/platform.rs @@ -1,6 +1,6 @@ #[path = "platform/helpers.rs"] mod helpers; -#[path = "store/replica_helpers.rs"] +#[path = "store/replicas/helpers.rs"] mod replica_helpers; #[path = "store/helpers.rs"] mod store_helpers; diff --git a/crates/sqlbuild-orchestration/tests/platform/helpers.rs b/crates/sqlbuild-orchestration/tests/platform/helpers.rs index c2ea45fa05..310870ecfb 100644 --- a/crates/sqlbuild-orchestration/tests/platform/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/platform/helpers.rs @@ -80,27 +80,31 @@ pub(crate) async fn wait_for_scans( let mut cursor = 0; let mut observations = Vec::new(); let mut scanned = BTreeSet::new(); - while &scanned != replicas { + for _ in 0..1000 { + let Some(_) = replicas.difference(&scanned).next() else { + return observations; + }; assert!( tasks.iter().all(|task| !task.is_finished()), "watchdog failed before scanning" ); - for row in read_recall_events(store, cursor, 1000) + let batch: Vec = read_recall_events(store, cursor, 1000) .await .expect("read health facts") - { - let observation: Observation = - serde_json::from_value(json!(row)).expect("typed platform event"); - cursor = observation.storage_id; - scanned.extend(match observation.payload.platform { - PlatformEvent::Heartbeat { replica_id } => Some(replica_id), - _ => None, - }); - observations.push(observation); - } + .into_iter() + .map(|row| serde_json::from_value(json!(row)).expect("typed platform event")) + .collect(); + cursor = batch.last().map_or(cursor, |row| row.storage_id); + scanned.extend(batch.iter().filter_map(|row| { + let PlatformEvent::Heartbeat { replica_id } = row.payload.platform else { + return None; + }; + Some(replica_id) + })); + observations.extend(batch); tokio::time::sleep(Duration::from_millis(10)).await; } - observations + panic!("watchdog exhausted scan completion attempts"); }) .await .expect("both replicas finish a paged scan without a scheduler") diff --git a/crates/sqlbuild-orchestration/tests/platform/test_types.rs b/crates/sqlbuild-orchestration/tests/platform/test_types.rs index 1f1fec3985..418d6a8cb3 100644 --- a/crates/sqlbuild-orchestration/tests/platform/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/platform/test_types.rs @@ -7,7 +7,7 @@ use crate::store_test_types::Backend; pub(crate) struct WatchdogTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, - pub(crate) take_lease: bool, + pub(crate) leaders: usize, pub(crate) expected_no_leader: bool, pub(crate) expected_overdue: usize, pub(crate) expected_low_horizons: usize, diff --git a/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs b/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs index 26e40396d0..6954c6bfe0 100644 --- a/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs +++ b/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs @@ -22,7 +22,7 @@ fn given_two_server_replicas_when_scheduling_is_stalled_then_health_facts_are_du WatchdogTestCase { description: "SQLite without a leader", backend: Backend::Sqlite, - take_lease: false, + leaders: 0, expected_no_leader: true, expected_overdue: 102, expected_low_horizons: 1, @@ -30,7 +30,7 @@ fn given_two_server_replicas_when_scheduling_is_stalled_then_health_facts_are_du WatchdogTestCase { description: "PostgreSQL without a leader", backend: Backend::Postgres, - take_lease: false, + leaders: 0, expected_no_leader: true, expected_overdue: 102, expected_low_horizons: 1, @@ -38,7 +38,7 @@ fn given_two_server_replicas_when_scheduling_is_stalled_then_health_facts_are_du WatchdogTestCase { description: "SQLite with a frozen leader", backend: Backend::Sqlite, - take_lease: true, + leaders: 1, expected_no_leader: false, expected_overdue: 102, expected_low_horizons: 1, @@ -46,7 +46,7 @@ fn given_two_server_replicas_when_scheduling_is_stalled_then_health_facts_are_du WatchdogTestCase { description: "PostgreSQL with a frozen leader", backend: Backend::Postgres, - take_lease: true, + leaders: 1, expected_no_leader: false, expected_overdue: 102, expected_low_horizons: 1, @@ -82,7 +82,7 @@ fn given_two_server_replicas_when_scheduling_is_stalled_then_health_facts_are_du publish_snapshot(store, &input) .await .expect("current definition"); - if test_case.take_lease { + for _ in 0..test_case.leaders { acquire_lease(store, Uuid::now_v7()) .await .expect("acquire lease") @@ -103,9 +103,11 @@ fn given_two_server_replicas_when_scheduling_is_stalled_then_health_facts_are_du let observations = wait_for_scans(store, &replicas, &tasks).await; let overdue: BTreeSet<_> = observations .iter() - .filter_map(|row| match row.payload.platform { - PlatformEvent::TickOverdue { trigger_id, .. } => Some(trigger_id), - _ => None, + .filter_map(|row| { + let PlatformEvent::TickOverdue { trigger_id, .. } = row.payload.platform else { + return None; + }; + Some(trigger_id) }) .collect(); assert_eq!( @@ -116,9 +118,12 @@ fn given_two_server_replicas_when_scheduling_is_stalled_then_health_facts_are_du ); let horizons: BTreeSet<_> = observations .iter() - .filter_map(|row| match row.payload.platform { - PlatformEvent::CalendarHorizonLow { trigger_id, .. } => Some(trigger_id), - _ => None, + .filter_map(|row| { + let PlatformEvent::CalendarHorizonLow { trigger_id, .. } = row.payload.platform + else { + return None; + }; + Some(trigger_id) }) .collect(); assert_eq!( diff --git a/crates/sqlbuild-orchestration/tests/recall.rs b/crates/sqlbuild-orchestration/tests/recall.rs deleted file mode 100644 index 3261be8a4e..0000000000 --- a/crates/sqlbuild-orchestration/tests/recall.rs +++ /dev/null @@ -1,14 +0,0 @@ -#[path = "recall/helpers.rs"] -mod helpers; -#[path = "store/replica_helpers.rs"] -mod replica_helpers; -#[path = "store/helpers.rs"] -mod store_helpers; -#[path = "store/test_types.rs"] -mod store_test_types; -#[path = "recall/test_contract.rs"] -mod test_contract; -#[path = "recall/test_types.rs"] -mod test_types; -#[path = "recall/test_views.rs"] -mod test_views; diff --git a/crates/sqlbuild-orchestration/tests/runs.rs b/crates/sqlbuild-orchestration/tests/runs.rs index 10a2813d22..5b2f42488c 100644 --- a/crates/sqlbuild-orchestration/tests/runs.rs +++ b/crates/sqlbuild-orchestration/tests/runs.rs @@ -1,18 +1,18 @@ -#[path = "ticks/fault_helpers.rs"] +#[path = "store/faults/helpers.rs"] mod fault_helpers; -#[path = "store/fault_types.rs"] +#[path = "store/faults/test_types.rs"] mod fault_types; #[path = "runs/helpers.rs"] mod helpers; -#[path = "store/lease_helpers.rs"] +#[path = "store/leases/helpers.rs"] mod lease_helpers; -#[path = "store/replica_helpers.rs"] +#[path = "store/replicas/helpers.rs"] mod replica_helpers; #[path = "store/helpers.rs"] mod store_helpers; #[path = "store/test_types.rs"] mod store_test_types; -#[path = "runs/terminal_helpers.rs"] +#[path = "runs/termination/helpers.rs"] mod terminal_helpers; #[path = "runs/test_claims.rs"] mod test_claims; diff --git a/crates/sqlbuild-orchestration/tests/runs/terminal_helpers.rs b/crates/sqlbuild-orchestration/tests/runs/termination/helpers.rs similarity index 93% rename from crates/sqlbuild-orchestration/tests/runs/terminal_helpers.rs rename to crates/sqlbuild-orchestration/tests/runs/termination/helpers.rs index 68e8742d32..9d6c347e0b 100644 --- a/crates/sqlbuild-orchestration/tests/runs/terminal_helpers.rs +++ b/crates/sqlbuild-orchestration/tests/runs/termination/helpers.rs @@ -1,3 +1,5 @@ +//! Terminal facts exercise claim release through the public run lifecycle. + use sqlbuild_orchestration::runs::main::change_run::change_run; use sqlbuild_orchestration::runs::models::{ RunAuthority, RunChange, RunChangeRequest, RunClaim, RunOutcome, diff --git a/crates/sqlbuild-orchestration/tests/runs/test_faults.rs b/crates/sqlbuild-orchestration/tests/runs/test_faults.rs index 3332a4bd5c..6af9167fbb 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_faults.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_faults.rs @@ -19,7 +19,7 @@ use crate::helpers::{queue, request, snapshot}; use crate::lease_helpers::expire_lease; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; -use crate::test_types::{ClaimFailureTestCase, TickFaultTestCase}; +use crate::test_types::{ClaimFailureTestCase, TransactionFault}; #[test] fn given_failures_during_claim_publication_when_retried_then_claims_slots_liveness_facts_and_projection_commit_together() @@ -78,7 +78,7 @@ fn given_failures_during_claim_publication_when_retried_then_claims_slots_livene ("recall_events", "INSERT"), ("sqb_runs", "UPDATE"), ] { - let point = TickFaultTestCase { + let point = TransactionFault { description: "claim publication failure", table, operation, @@ -106,7 +106,7 @@ fn given_failures_during_claim_publication_when_retried_then_claims_slots_livene projected ); } - let expiry = TickFaultTestCase { + let expiry = TransactionFault { description: "claim fence expires before commit", table: "sqb_run_liveness", operation: "INSERT", diff --git a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs index d61b50e82a..0f02f31093 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs @@ -19,7 +19,7 @@ use crate::fault_helpers::{install_fault, remove_fault}; use crate::helpers::{queue, request, snapshot}; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; -use crate::test_types::{RunLifecycleTestCase, TickFaultTestCase}; +use crate::test_types::{RunLifecycleTestCase, TransactionFault}; #[test] fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_facts_outcomes_and_claims_are_atomic() @@ -156,7 +156,7 @@ fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_ .await .expect("events before node fault"); for table in ["sqb_run_nodes", "sqb_runs"] { - let point = TickFaultTestCase { + let point = TransactionFault { description: "node projection and counter failure", table, operation: "UPDATE", @@ -217,7 +217,7 @@ fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_ ("sqb_write_claims", "DELETE"), ("sqb_run_liveness", "DELETE"), ] { - let point = TickFaultTestCase { + let point = TransactionFault { description: "terminal transaction failure", table, operation, diff --git a/crates/sqlbuild-orchestration/tests/runs/test_types.rs b/crates/sqlbuild-orchestration/tests/runs/test_types.rs index 61c94312d4..81471a4f67 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_types.rs @@ -2,7 +2,7 @@ use sqlbuild_orchestration::runs::models::RunStatus; -pub(crate) use crate::fault_types::TickFaultTestCase; +pub(crate) use crate::fault_types::TransactionFault; use crate::store_test_types::Backend; pub(crate) struct PriorityClaimsTestCase { diff --git a/crates/sqlbuild-orchestration/tests/scheduler.rs b/crates/sqlbuild-orchestration/tests/scheduler.rs index be400b8236..7142cff07b 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler.rs @@ -1,8 +1,12 @@ +#[path = "store/faults/helpers.rs"] +mod fault_helpers; +#[path = "store/faults/test_types.rs"] +mod fault_types; #[path = "scheduler/helpers.rs"] mod helpers; -#[path = "store/lease_helpers.rs"] +#[path = "store/leases/helpers.rs"] mod lease_helpers; -#[path = "store/replica_helpers.rs"] +#[path = "store/replicas/helpers.rs"] mod replica_helpers; #[path = "store/helpers.rs"] mod store_helpers; @@ -12,3 +16,5 @@ mod store_test_types; mod test_contract; #[path = "scheduler/test_types.rs"] mod test_types; +#[path = "scheduler/ticks/test_ticks.rs"] +mod ticks; diff --git a/crates/sqlbuild-orchestration/tests/ticks/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs similarity index 96% rename from crates/sqlbuild-orchestration/tests/ticks/helpers.rs rename to crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs index 0ce57192b9..fa51f828f5 100644 --- a/crates/sqlbuild-orchestration/tests/ticks/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs @@ -1,3 +1,5 @@ +//! Trigger and failure fixtures exercise the scheduler's transactional tick boundary. + use std::collections::BTreeMap; use serde_json::json; @@ -20,7 +22,7 @@ use crate::lease_helpers::expire_lease; use crate::replica_helpers::second_store; use crate::store_helpers::{close_store, close_test_store, open_test_store}; use crate::store_test_types::Backend; -use crate::test_types::TickFaultTestCase; +use crate::ticks::test_types::TransactionFault; pub(crate) fn snapshot(location_id: Uuid) -> SnapshotInput { SnapshotInput { @@ -77,7 +79,7 @@ pub(crate) async fn transaction_contract(backend: Backend) -> Option<(usize, usi .expect("elected"); let second = second_store(store).await; for point in [ - TickFaultTestCase { + TransactionFault { description: "tick insertion", table: "sqb_ticks", operation: "INSERT", @@ -85,7 +87,7 @@ pub(crate) async fn transaction_contract(backend: Backend) -> Option<(usize, usi sqlite_body: "SELECT RAISE(ABORT,'injected tick failure')", postgres_body: "RAISE EXCEPTION 'injected tick failure'", }, - TickFaultTestCase { + TransactionFault { description: "requested event", table: "recall_events", operation: "INSERT", @@ -93,7 +95,7 @@ pub(crate) async fn transaction_contract(backend: Backend) -> Option<(usize, usi sqlite_body: "SELECT RAISE(ABORT,'injected requested failure')", postgres_body: "RAISE EXCEPTION 'injected requested failure'", }, - TickFaultTestCase { + TransactionFault { description: "queued event", table: "recall_events", operation: "INSERT", @@ -101,7 +103,7 @@ pub(crate) async fn transaction_contract(backend: Backend) -> Option<(usize, usi sqlite_body: "SELECT RAISE(ABORT,'injected queued failure')", postgres_body: "RAISE EXCEPTION 'injected queued failure'", }, - TickFaultTestCase { + TransactionFault { description: "run projection", table: "sqb_runs", operation: "INSERT", @@ -109,7 +111,7 @@ pub(crate) async fn transaction_contract(backend: Backend) -> Option<(usize, usi sqlite_body: "SELECT RAISE(ABORT,'injected run failure')", postgres_body: "RAISE EXCEPTION 'injected run failure'", }, - TickFaultTestCase { + TransactionFault { description: "node projection", table: "sqb_run_nodes", operation: "INSERT", @@ -117,7 +119,7 @@ pub(crate) async fn transaction_contract(backend: Backend) -> Option<(usize, usi sqlite_body: "SELECT RAISE(ABORT,'injected node failure')", postgres_body: "RAISE EXCEPTION 'injected node failure'", }, - TickFaultTestCase { + TransactionFault { description: "cursor commit", table: "sqb_trigger_state", operation: "UPDATE", @@ -125,7 +127,7 @@ pub(crate) async fn transaction_contract(backend: Backend) -> Option<(usize, usi sqlite_body: "SELECT RAISE(ABORT,'injected cursor failure')", postgres_body: "RAISE EXCEPTION 'injected cursor failure'", }, - TickFaultTestCase { + TransactionFault { description: "lease expires during append", table: "sqb_runs", operation: "INSERT", diff --git a/crates/sqlbuild-orchestration/tests/ticks/test_contract.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_contract.rs similarity index 83% rename from crates/sqlbuild-orchestration/tests/ticks/test_contract.rs rename to crates/sqlbuild-orchestration/tests/scheduler/ticks/test_contract.rs index 7d733d45be..af4492a29d 100644 --- a/crates/sqlbuild-orchestration/tests/ticks/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_contract.rs @@ -1,7 +1,9 @@ -use crate::helpers::transaction_contract; +//! Crash points must not separate a tick from its queued work and durable cursor. + use crate::store_helpers::run; use crate::store_test_types::Backend; -use crate::test_types::TickTransactionTestCase; +use crate::ticks::helpers::transaction_contract; +use crate::ticks::test_types::TickTransactionTestCase; #[test] fn given_two_schedulers_and_faults_between_writes_when_restarting_then_ticks_runs_recall_and_cursor_commit_together() diff --git a/crates/sqlbuild-orchestration/tests/ticks/test_loop.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs similarity index 96% rename from crates/sqlbuild-orchestration/tests/ticks/test_loop.rs rename to crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs index 2864da8921..f9341cf027 100644 --- a/crates/sqlbuild-orchestration/tests/ticks/test_loop.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs @@ -1,3 +1,5 @@ +//! Embedded and standalone schedulers share takeover and shutdown semantics. + use std::time::Duration; use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler; @@ -8,12 +10,12 @@ use sqlbuild_orchestration::store::main::create_location::create_location; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use uuid::Uuid; -use crate::helpers::snapshot; use crate::lease_helpers::expire_lease; use crate::replica_helpers::second_store; use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; use crate::store_test_types::Backend; -use crate::test_types::TickTransactionTestCase; +use crate::ticks::helpers::snapshot; +use crate::ticks::test_types::TickTransactionTestCase; #[test] fn given_embedded_and_standalone_schedulers_when_taking_over_then_only_one_drains_due_work_and_idle_shutdown_is_prompt() diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs new file mode 100644 index 0000000000..3f36d1d4fe --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs @@ -0,0 +1,10 @@ +#[path = "helpers.rs"] +mod helpers; +#[path = "test_contract.rs"] +mod test_contract; +#[path = "test_loop.rs"] +mod test_loop; +#[path = "test_triggers.rs"] +mod test_triggers; +#[path = "test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/ticks/test_triggers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs similarity index 96% rename from crates/sqlbuild-orchestration/tests/ticks/test_triggers.rs rename to crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs index 4429dae9fb..4df7178b12 100644 --- a/crates/sqlbuild-orchestration/tests/ticks/test_triggers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs @@ -1,3 +1,5 @@ +//! Operator trigger facts remain consistent across publication and retry boundaries. + use sqlbuild_orchestration::declarations::main::change_trigger::change_trigger; use sqlbuild_orchestration::declarations::main::list_trigger_events::list_trigger_events; use sqlbuild_orchestration::declarations::main::list_triggers::list_triggers; @@ -12,10 +14,10 @@ use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use uuid::Uuid; use crate::fault_helpers::{install_fault, remove_fault}; -use crate::helpers::snapshot; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; -use crate::test_types::{TickFaultTestCase, TriggerControlsTestCase}; +use crate::ticks::helpers::snapshot; +use crate::ticks::test_types::{TransactionFault, TriggerControlsTestCase}; #[test] fn given_operator_changes_when_retried_and_republished_then_facts_and_effective_state_remain_consistent() @@ -84,7 +86,7 @@ fn given_operator_changes_when_retried_and_republished_then_facts_and_effective_ change: TriggerChange::Enable, ..pause.clone() }; - let fault = TickFaultTestCase { + let fault = TransactionFault { description: "operator event and projection", table: "sqb_trigger_state", operation: "UPDATE", diff --git a/crates/sqlbuild-orchestration/tests/ticks/test_types.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs similarity index 78% rename from crates/sqlbuild-orchestration/tests/ticks/test_types.rs rename to crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs index 9559e8a92f..95953810a5 100644 --- a/crates/sqlbuild-orchestration/tests/ticks/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs @@ -1,4 +1,6 @@ -pub(crate) use crate::fault_types::TickFaultTestCase; +//! Database cases and expected scheduler tick outcomes. + +pub(crate) use crate::fault_types::TransactionFault; use crate::store_test_types::Backend; pub(crate) struct TickTransactionTestCase { diff --git a/crates/sqlbuild-orchestration/tests/snapshots.rs b/crates/sqlbuild-orchestration/tests/snapshots.rs index 5dcb1cd22c..58d1347746 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots.rs @@ -2,7 +2,7 @@ mod contract; #[path = "snapshots/helpers.rs"] mod helpers; -#[path = "store/replica_helpers.rs"] +#[path = "store/replicas/helpers.rs"] mod replica_helpers; #[path = "store/helpers.rs"] mod store_helpers; diff --git a/crates/sqlbuild-orchestration/tests/store.rs b/crates/sqlbuild-orchestration/tests/store.rs new file mode 100644 index 0000000000..5a79851f21 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store.rs @@ -0,0 +1,16 @@ +//! Recall storage integration contracts for SQLite and PostgreSQL. + +#[path = "store/recall/helpers.rs"] +mod helpers; +#[path = "store/replicas/helpers.rs"] +mod replica_helpers; +#[path = "store/helpers.rs"] +mod store_helpers; +#[path = "store/test_types.rs"] +mod store_test_types; +#[path = "store/recall/test_contract.rs"] +mod test_contract; +#[path = "store/recall/test_types.rs"] +mod test_types; +#[path = "store/recall/test_views.rs"] +mod test_views; diff --git a/crates/sqlbuild-orchestration/tests/ticks/fault_helpers.rs b/crates/sqlbuild-orchestration/tests/store/faults/helpers.rs similarity index 92% rename from crates/sqlbuild-orchestration/tests/ticks/fault_helpers.rs rename to crates/sqlbuild-orchestration/tests/store/faults/helpers.rs index 9c973c0fb1..c7d5c81dd5 100644 --- a/crates/sqlbuild-orchestration/tests/ticks/fault_helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/faults/helpers.rs @@ -1,8 +1,10 @@ +//! Database-trigger fault injection at durable publication boundaries. + use sqlbuild_orchestration::store::models::ControlStore; -use crate::test_types::TickFaultTestCase; +use crate::fault_types::TransactionFault; -pub(crate) async fn install_fault(store: &ControlStore, point: &TickFaultTestCase) { +pub(crate) async fn install_fault(store: &ControlStore, point: &TransactionFault) { let ControlStore::Postgres(pool) = store else { let ControlStore::Sqlite(path) = store else { unreachable!() @@ -23,7 +25,7 @@ pub(crate) async fn install_fault(store: &ControlStore, point: &TickFaultTestCas .execute(pool).await.expect(point.description); } -pub(crate) async fn remove_fault(store: &ControlStore, point: &TickFaultTestCase) { +pub(crate) async fn remove_fault(store: &ControlStore, point: &TransactionFault) { let ControlStore::Postgres(pool) = store else { let ControlStore::Sqlite(path) = store else { unreachable!() diff --git a/crates/sqlbuild-orchestration/tests/store/fault_types.rs b/crates/sqlbuild-orchestration/tests/store/faults/test_types.rs similarity index 68% rename from crates/sqlbuild-orchestration/tests/store/fault_types.rs rename to crates/sqlbuild-orchestration/tests/store/faults/test_types.rs index eb9e82bd2f..e25282c3cc 100644 --- a/crates/sqlbuild-orchestration/tests/store/fault_types.rs +++ b/crates/sqlbuild-orchestration/tests/store/faults/test_types.rs @@ -1,6 +1,6 @@ -//! Transaction failure points shared by credential, tick and run contracts. +//! Transaction failure cases shared by credential, tick and run contracts. -pub(crate) struct TickFaultTestCase { +pub(crate) struct TransactionFault { pub(crate) description: &'static str, pub(crate) table: &'static str, pub(crate) operation: &'static str, diff --git a/crates/sqlbuild-orchestration/tests/store/lease_helpers.rs b/crates/sqlbuild-orchestration/tests/store/lease_helpers.rs deleted file mode 100644 index 8e17cfce86..0000000000 --- a/crates/sqlbuild-orchestration/tests/store/lease_helpers.rs +++ /dev/null @@ -1,24 +0,0 @@ -//! Expire the persisted lease without introducing a fake control-store clock. - -use sqlbuild_orchestration::store::models::ControlStore; - -pub(crate) async fn expire_lease(store: &ControlStore) { - match store { - ControlStore::Sqlite(path) => { - rusqlite::Connection::open(path) - .expect("open lease expiry connection") - .execute_batch( - "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", - ) - .expect("expire SQLite scheduler lease"); - } - ControlStore::Postgres(pool) => { - sqlx_core::query::query( - "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", - ) - .execute(pool) - .await - .expect("expire PostgreSQL scheduler lease"); - } - } -} diff --git a/crates/sqlbuild-orchestration/tests/store/leases/helpers.rs b/crates/sqlbuild-orchestration/tests/store/leases/helpers.rs new file mode 100644 index 0000000000..b813d957d3 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/leases/helpers.rs @@ -0,0 +1,24 @@ +//! Expire the persisted lease without introducing a fake control-store clock. + +use sqlbuild_orchestration::store::models::ControlStore; + +pub(crate) async fn expire_lease(store: &ControlStore) { + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + rusqlite::Connection::open(path) + .expect("open lease expiry connection") + .execute_batch( + "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + ) + .expect("expire SQLite scheduler lease"); + return; + }; + sqlx_core::query::query( + "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + ) + .execute(pool) + .await + .expect("expire PostgreSQL scheduler lease"); +} diff --git a/crates/sqlbuild-orchestration/tests/recall/helpers.rs b/crates/sqlbuild-orchestration/tests/store/recall/helpers.rs similarity index 98% rename from crates/sqlbuild-orchestration/tests/recall/helpers.rs rename to crates/sqlbuild-orchestration/tests/store/recall/helpers.rs index ce7de75ac3..2fb0605a1d 100644 --- a/crates/sqlbuild-orchestration/tests/recall/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/recall/helpers.rs @@ -1,3 +1,5 @@ +//! Recall publication fixtures exercise the control store's durable append boundary. + use serde_json::json; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::publish_recall::publish_recall; diff --git a/crates/sqlbuild-orchestration/tests/recall/test_contract.rs b/crates/sqlbuild-orchestration/tests/store/recall/test_contract.rs similarity index 93% rename from crates/sqlbuild-orchestration/tests/recall/test_contract.rs rename to crates/sqlbuild-orchestration/tests/store/recall/test_contract.rs index d8d8bf5629..d0ccf95c46 100644 --- a/crates/sqlbuild-orchestration/tests/recall/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/store/recall/test_contract.rs @@ -1,3 +1,5 @@ +//! Idempotency and commit-order paging through the public store API. + use crate::helpers::recall_contract; use crate::store_helpers::run; use crate::store_test_types::Backend; diff --git a/crates/sqlbuild-orchestration/tests/recall/test_types.rs b/crates/sqlbuild-orchestration/tests/store/recall/test_types.rs similarity index 84% rename from crates/sqlbuild-orchestration/tests/recall/test_types.rs rename to crates/sqlbuild-orchestration/tests/store/recall/test_types.rs index 8ecb5f4f63..481cac0601 100644 --- a/crates/sqlbuild-orchestration/tests/recall/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/store/recall/test_types.rs @@ -1,3 +1,5 @@ +//! Expected Recall storage outcomes for both supported databases. + use crate::store_test_types::Backend; pub(crate) struct RecallTestCase { diff --git a/crates/sqlbuild-orchestration/tests/recall/test_views.rs b/crates/sqlbuild-orchestration/tests/store/recall/test_views.rs similarity index 99% rename from crates/sqlbuild-orchestration/tests/recall/test_views.rs rename to crates/sqlbuild-orchestration/tests/store/recall/test_views.rs index 0dd0f229a5..75a8b9971d 100644 --- a/crates/sqlbuild-orchestration/tests/recall/test_views.rs +++ b/crates/sqlbuild-orchestration/tests/store/recall/test_views.rs @@ -1,3 +1,5 @@ +//! Durable Recall views preserve the published execution records. + use serde_json::json; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::publish_recall::publish_recall; diff --git a/crates/sqlbuild-orchestration/tests/store/replica_helpers.rs b/crates/sqlbuild-orchestration/tests/store/replica_helpers.rs deleted file mode 100644 index 853ba75df8..0000000000 --- a/crates/sqlbuild-orchestration/tests/store/replica_helpers.rs +++ /dev/null @@ -1,15 +0,0 @@ -use sqlbuild_orchestration::store::main::open_store::open_store; -use sqlbuild_orchestration::store::models::ControlStore; - -pub(crate) async fn second_store(store: &ControlStore) -> ControlStore { - match store { - ControlStore::Sqlite(path) => open_store(path.to_str().expect("UTF-8 test path")) - .await - .expect("open independent SQLite handle"), - ControlStore::Postgres(pool) => ControlStore::Postgres( - sqlx_postgres::PgPool::connect_with(pool.connect_options().as_ref().clone()) - .await - .expect("open independent PostgreSQL pool"), - ), - } -} diff --git a/crates/sqlbuild-orchestration/tests/store/replicas/helpers.rs b/crates/sqlbuild-orchestration/tests/store/replicas/helpers.rs new file mode 100644 index 0000000000..22fe1b374b --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/replicas/helpers.rs @@ -0,0 +1,20 @@ +//! Independent database handles for multi-replica integration contracts. + +use sqlbuild_orchestration::store::main::open_store::open_store; +use sqlbuild_orchestration::store::models::ControlStore; + +pub(crate) async fn second_store(store: &ControlStore) -> ControlStore { + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + return open_store(path.to_str().expect("UTF-8 test path")) + .await + .expect("open independent SQLite handle"); + }; + ControlStore::Postgres( + sqlx_postgres::PgPool::connect_with(pool.connect_options().as_ref().clone()) + .await + .expect("open independent PostgreSQL pool"), + ) +} diff --git a/crates/sqlbuild-orchestration/tests/ticks.rs b/crates/sqlbuild-orchestration/tests/ticks.rs deleted file mode 100644 index abf2590859..0000000000 --- a/crates/sqlbuild-orchestration/tests/ticks.rs +++ /dev/null @@ -1,22 +0,0 @@ -#[path = "ticks/fault_helpers.rs"] -mod fault_helpers; -#[path = "store/fault_types.rs"] -mod fault_types; -#[path = "ticks/helpers.rs"] -mod helpers; -#[path = "store/lease_helpers.rs"] -mod lease_helpers; -#[path = "store/replica_helpers.rs"] -mod replica_helpers; -#[path = "store/helpers.rs"] -mod store_helpers; -#[path = "store/test_types.rs"] -mod store_test_types; -#[path = "ticks/test_contract.rs"] -mod test_contract; -#[path = "ticks/test_loop.rs"] -mod test_loop; -#[path = "ticks/test_triggers.rs"] -mod test_triggers; -#[path = "ticks/test_types.rs"] -mod test_types; From a5a1ee9666225492da0898a2bceb0a9c53a016ea Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 01:14:37 +0000 Subject: [PATCH 10/59] feat: monitor silent workers during scheduler cycles Observe heartbeat loss in fenced, bounded transactions while retaining physical write claims. Recover through existing worker heartbeats and continue full batches without an idle delay. Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../src/runs/_helpers/mod.rs | 1 + .../src/runs/_helpers/publish_change.rs | 54 +++ .../src/runs/constants.rs | 3 + .../src/runs/main/change_run.rs | 42 +-- .../src/runs/main/mod.rs | 1 + .../src/runs/main/monitor_runs.rs | 55 +++ .../scheduler/_helpers/drain_due_triggers.rs | 37 ++ .../src/scheduler/_helpers/mod.rs | 1 + .../src/scheduler/main/scheduler_cycle.rs | 32 +- .../src/scheduler/models.rs | 1 + crates/sqlbuild-orchestration/tests/runs.rs | 4 + .../tests/runs/monitor/helpers.rs | 70 ++++ .../tests/runs/test_monitor.rs | 343 ++++++++++++++++++ .../tests/runs/test_types.rs | 13 + 14 files changed, 597 insertions(+), 60 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/publish_change.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/monitor_runs.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/test_monitor.rs diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs index 5994d61750..75a1590516 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs @@ -3,3 +3,4 @@ pub(crate) mod claims; pub(crate) mod events; pub(crate) mod nodes; pub(crate) mod projection; +pub(crate) mod publish_change; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/publish_change.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/publish_change.rs new file mode 100644 index 0000000000..18ba3f5835 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/publish_change.rs @@ -0,0 +1,54 @@ +//! Publish a lifecycle fact, its projection and terminal cleanup in the caller's transaction. + +use serde_json::json; + +use crate::runs::_helpers::events::append_run_event; +use crate::runs::_helpers::projection::project_change; +use crate::runs::models::{ChangingRun, RunChange, RunChangeRequest, RunOutcome}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn publish_change( + transaction: ControlTransaction, + request: &RunChangeRequest, + state: &ChangingRun, +) -> Result { + let (transaction, mut cursor) = append_run_event( + transaction, + (request.run_id, request.event_id), + request.change.event_type(), + json!({"change": request}), + ) + .await?; + let mut transaction = project_change(transaction, request, state).await?; + if matches!(request.change, RunChange::CancelRequested { .. }) + && state.worker_instance.is_none() + { + let terminal = RunChangeRequest { + event_id: uuid::Uuid::new_v5(&request.event_id, b"cancelled-before-launch"), + run_id: request.run_id, + claim_id: None, + change: RunChange::Finished { + outcome: RunOutcome::Cancelled, + reason: "cancelled before launch".into(), + exit_code: None, + }, + }; + (transaction, cursor) = append_run_event( + transaction, + (terminal.run_id, terminal.event_id), + terminal.change.event_type(), + json!({"change": terminal}), + ) + .await?; + transaction = project_change(transaction, &terminal, state).await?; + } + let (transaction, _) = query( + transaction, + "UPDATE sqb_runs SET last_event_storage_id=$1 WHERE run_id=$2", + vec![Parameter::Integer(cursor), Parameter::Id(request.run_id)], + ) + .await?; + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/runs/constants.rs b/crates/sqlbuild-orchestration/src/runs/constants.rs index d92a2e052a..801d9c9edb 100644 --- a/crates/sqlbuild-orchestration/src/runs/constants.rs +++ b/crates/sqlbuild-orchestration/src/runs/constants.rs @@ -2,3 +2,6 @@ pub(crate) const NODE_PENDING: &str = "pending"; pub(crate) const NODE_RUNNING: &str = "running"; + +pub(crate) const HEARTBEAT_TIMEOUT_SECONDS: i64 = 45; +pub(crate) const MONITOR_BATCH_SIZE: usize = 100; diff --git a/crates/sqlbuild-orchestration/src/runs/main/change_run.rs b/crates/sqlbuild-orchestration/src/runs/main/change_run.rs index 3bef1033c8..02fe0308c1 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/change_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/change_run.rs @@ -3,9 +3,8 @@ use serde_json::json; use crate::runs::_helpers::changes::{validate_authority, validate_change, validate_retry}; -use crate::runs::_helpers::events::append_run_event; -use crate::runs::_helpers::projection::project_change; -use crate::runs::models::{ChangingRun, RunAuthority, RunChange, RunChangeRequest, RunOutcome}; +use crate::runs::_helpers::publish_change::publish_change; +use crate::runs::models::{ChangingRun, RunAuthority, RunChangeRequest}; use crate::scheduler::main::check_lease::check_lease; use crate::store::errors::ControlError; use crate::store::main::begin::begin; @@ -50,42 +49,7 @@ pub async fn change_run( .ok_or_else(|| ControlError::Invalid("run does not exist".into()))?, )?; validate_change(&state, request)?; - let (transaction, mut cursor) = append_run_event( - transaction, - (request.run_id, request.event_id), - request.change.event_type(), - body, - ) - .await?; - let mut transaction = project_change(transaction, request, &state).await?; - if matches!(request.change, RunChange::CancelRequested { .. }) - && state.worker_instance.is_none() - { - let terminal = RunChangeRequest { - event_id: uuid::Uuid::new_v5(&request.event_id, b"cancelled-before-launch"), - run_id: request.run_id, - claim_id: None, - change: RunChange::Finished { - outcome: RunOutcome::Cancelled, - reason: "cancelled before launch".into(), - exit_code: None, - }, - }; - (transaction, cursor) = append_run_event( - transaction, - (terminal.run_id, terminal.event_id), - terminal.change.event_type(), - json!({"change": terminal}), - ) - .await?; - transaction = project_change(transaction, &terminal, &state).await?; - } - let (mut transaction, _) = query( - transaction, - "UPDATE sqb_runs SET last_event_storage_id=$1 WHERE run_id=$2", - vec![Parameter::Integer(cursor), Parameter::Id(request.run_id)], - ) - .await?; + let mut transaction = publish_change(transaction, request, &state).await?; if let RunAuthority::Scheduler(lease) = authority { transaction = check_lease(transaction, lease).await?; } diff --git a/crates/sqlbuild-orchestration/src/runs/main/mod.rs b/crates/sqlbuild-orchestration/src/runs/main/mod.rs index c19b71681a..ffceba28a4 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/mod.rs @@ -3,6 +3,7 @@ pub mod claim_next_run; pub mod heartbeat_run; pub mod list_run_nodes; pub mod list_runs; +pub mod monitor_runs; pub(crate) mod queue_run; pub mod request_run; pub mod run_execution; diff --git a/crates/sqlbuild-orchestration/src/runs/main/monitor_runs.rs b/crates/sqlbuild-orchestration/src/runs/main/monitor_runs.rs new file mode 100644 index 0000000000..b23d4eb791 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/monitor_runs.rs @@ -0,0 +1,55 @@ +//! Observe heartbeat loss under the scheduler fence without releasing a live worker's claims. + +use chrono::TimeDelta; +use uuid::Uuid; + +use crate::runs::_helpers::publish_change::publish_change; +use crate::runs::constants::{HEARTBEAT_TIMEOUT_SECONDS, MONITOR_BATCH_SIZE}; +use crate::runs::models::{ChangingRun, RunChange, RunChangeRequest}; +use crate::scheduler::main::check_lease::check_lease; +use crate::scheduler::models::SchedulerLease; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Serialized writers make heartbeat inspection and the resulting fact one atomic observation. +pub async fn monitor_runs( + store: &ControlStore, + lease: &SchedulerLease, +) -> Result { + let mut transaction = check_lease(begin(store, TransactionMode::Write).await?, lease).await?; + let cutoff = transaction.now - TimeDelta::seconds(HEARTBEAT_TIMEOUT_SECONDS); + let rows; + (transaction, rows) = query( + transaction, + &format!( + "SELECT r.run_id, r.status, r.cancel_requested_at, r.node_counts_json, \ + r.last_event_storage_id, l.worker_instance FROM sqb_runs r \ + JOIN sqb_run_liveness l ON l.run_id=r.run_id \ + WHERE r.status='running' AND l.last_heartbeat_at<$1 \ + ORDER BY l.last_heartbeat_at, r.run_id LIMIT {MONITOR_BATCH_SIZE}" + ), + vec![Parameter::Time(cutoff)], + ) + .await?; + let observed = rows.len(); + for row in rows { + let run_id: Uuid = serde_json::from_value(row["run_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let previous: i64 = serde_json::from_value(row["last_event_storage_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let state: ChangingRun = decode_row(row)?; + let request = RunChangeRequest { + event_id: Uuid::new_v5(&run_id, format!("unresponsive/{previous}").as_bytes()), + run_id, + claim_id: state.worker_instance, + change: RunChange::Unresponsive, + }; + transaction = publish_change(transaction, &request, &state).await?; + } + commit(check_lease(transaction, lease).await?).await?; + Ok(observed) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs new file mode 100644 index 0000000000..ab6a639022 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs @@ -0,0 +1,37 @@ +//! Drain a bounded due-trigger page, renewing the lease between persisted evaluations. + +use uuid::Uuid; + +use crate::scheduler::main::evaluate_trigger::evaluate_trigger; +use crate::scheduler::main::renew_lease::renew_lease; +use crate::scheduler::models::{SchedulerCycle, SchedulerLease}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub(crate) async fn drain_due_triggers( + store: &ControlStore, + mut lease: SchedulerLease, + mut cycle: SchedulerCycle, +) -> Result<(SchedulerLease, SchedulerCycle), ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "SELECT s.trigger_id FROM sqb_trigger_state s JOIN sqb_triggers t ON t.trigger_id=s.trigger_id \ + JOIN sqb_locations l ON l.location_id=t.location_id \ + WHERE s.status='running' AND s.next_due_at<=$1 AND t.removed_in_snapshot_id IS NULL AND l.archived_at IS NULL \ + ORDER BY s.next_due_at, s.trigger_id LIMIT 100", vec![Parameter::Time(now)]).await?; + commit(transaction).await?; + for row in rows { + let trigger_id: Uuid = serde_json::from_value(row["trigger_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if let Some(plan) = evaluate_trigger(store, &lease, trigger_id).await? { + cycle.evaluated += 1; + cycle.ticks += plan.actions.len(); + } + lease = renew_lease(store, &lease).await?; + } + Ok((lease, cycle)) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/mod.rs index 85bb8cc89b..5102baaac9 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod calendar; +pub(crate) mod drain_due_triggers; pub(crate) mod expansion; pub(crate) mod persistence; pub(crate) mod ticks; diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/scheduler_cycle.rs b/crates/sqlbuild-orchestration/src/scheduler/main/scheduler_cycle.rs index 011d28e303..8db0fea9b5 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/scheduler_cycle.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/scheduler_cycle.rs @@ -3,18 +3,19 @@ use std::time::Duration; use chrono::{DateTime, Utc}; -use uuid::Uuid; +use crate::runs::constants::MONITOR_BATCH_SIZE; +use crate::runs::main::monitor_runs::monitor_runs; +use crate::scheduler::_helpers::drain_due_triggers::drain_due_triggers; use crate::scheduler::constants::RENEW_INTERVAL_SECONDS; use crate::scheduler::main::acquire_lease::acquire_lease; -use crate::scheduler::main::evaluate_trigger::evaluate_trigger; use crate::scheduler::main::renew_lease::renew_lease; use crate::scheduler::models::{SchedulerCycle, SchedulerState}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; use crate::store::main::query::query; -use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::models::{ControlStore, TransactionMode}; pub async fn scheduler_cycle( store: &ControlStore, @@ -29,31 +30,20 @@ pub async fn scheduler_cycle( Err(ControlError::Fenced) => None, Err(error) => return Err(error), }; - let mut cycle = SchedulerCycle { + let cycle = SchedulerCycle { leader: state.lease.is_some(), evaluated: 0, ticks: 0, + unresponsive: 0, wait_for: Duration::from_secs(RENEW_INTERVAL_SECONDS), }; - let Some(mut lease) = state.lease.clone() else { + let Some(lease) = state.lease.clone() else { return Ok((state, cycle)); }; - let transaction = begin(store, TransactionMode::Read).await?; - let now = transaction.now; - let (transaction, rows) = query(transaction, - "SELECT s.trigger_id FROM sqb_trigger_state s JOIN sqb_triggers t ON t.trigger_id=s.trigger_id \ - JOIN sqb_locations l ON l.location_id=t.location_id \ - WHERE s.status='running' AND s.next_due_at<=$1 AND t.removed_in_snapshot_id IS NULL AND l.archived_at IS NULL \ - ORDER BY s.next_due_at, s.trigger_id LIMIT 100", vec![Parameter::Time(now)]).await?; - commit(transaction).await?; - for row in rows { - let trigger_id: Uuid = serde_json::from_value(row["trigger_id"].clone()) - .map_err(|error| ControlError::Invalid(error.to_string()))?; - if let Some(plan) = evaluate_trigger(store, &lease, trigger_id).await? { - cycle.evaluated += 1; - cycle.ticks += plan.actions.len(); - } - lease = renew_lease(store, &lease).await?; + let (lease, mut cycle) = drain_due_triggers(store, lease, cycle).await?; + cycle.unresponsive = monitor_runs(store, &lease).await?; + if cycle.unresponsive == MONITOR_BATCH_SIZE { + cycle.wait_for = Duration::ZERO; } state.lease = Some(lease); let transaction = begin(store, TransactionMode::Read).await?; diff --git a/crates/sqlbuild-orchestration/src/scheduler/models.rs b/crates/sqlbuild-orchestration/src/scheduler/models.rs index a3a0c2ccad..019083278d 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/models.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/models.rs @@ -36,6 +36,7 @@ pub struct SchedulerCycle { pub leader: bool, pub evaluated: usize, pub ticks: usize, + pub unresponsive: usize, pub wait_for: std::time::Duration, } diff --git a/crates/sqlbuild-orchestration/tests/runs.rs b/crates/sqlbuild-orchestration/tests/runs.rs index 5b2f42488c..12d6c66d52 100644 --- a/crates/sqlbuild-orchestration/tests/runs.rs +++ b/crates/sqlbuild-orchestration/tests/runs.rs @@ -6,6 +6,8 @@ mod fault_types; mod helpers; #[path = "store/leases/helpers.rs"] mod lease_helpers; +#[path = "runs/monitor/helpers.rs"] +mod monitor_helpers; #[path = "store/replicas/helpers.rs"] mod replica_helpers; #[path = "store/helpers.rs"] @@ -22,5 +24,7 @@ mod test_coalesce; mod test_faults; #[path = "runs/test_lifecycle.rs"] mod test_lifecycle; +#[path = "runs/test_monitor.rs"] +mod test_monitor; #[path = "runs/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs new file mode 100644 index 0000000000..7f90e2f877 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs @@ -0,0 +1,70 @@ +//! Advance persisted liveness, not the scheduler clock, to exercise real monitoring transactions. + +use std::collections::BTreeMap; + +use chrono::{TimeDelta, Utc}; +use sqlbuild_orchestration::runs::main::change_run::change_run; +use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; +use sqlbuild_orchestration::runs::models::{ + RunAuthority, RunChange, RunChangeRequest, RunClaim, RunRequest, +}; +use sqlbuild_orchestration::scheduler::models::SchedulerLease; +use sqlbuild_orchestration::store::models::ControlStore; +use uuid::Uuid; + +use crate::helpers::queue; + +pub(crate) async fn start_run( + store: &ControlStore, + lease: &SchedulerLease, + input: &RunRequest, +) -> RunClaim { + queue(store, input).await; + let claim = claim_next_run(store, lease, &BTreeMap::new()) + .await + .expect("claim queued run") + .expect("run available"); + assert_eq!(claim.request.run_id, input.run_id); + change_run( + store, + RunAuthority::Worker { + run_id: input.run_id, + claim_id: claim.claim_id, + }, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: input.run_id, + claim_id: Some(claim.claim_id), + change: RunChange::Started, + }, + ) + .await + .expect("start claimed worker"); + claim +} + +pub(crate) async fn age_heartbeat(store: &ControlStore, run_id: Uuid, seconds: i64) { + let timestamp = Utc::now() - TimeDelta::seconds(seconds); + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + rusqlite::Connection::open(path) + .expect("open liveness fixture") + .execute( + "UPDATE sqb_run_liveness SET last_heartbeat_at=?1 WHERE run_id=?2", + rusqlite::params![ + timestamp.to_rfc3339_opts(chrono::SecondsFormat::Micros, true), + run_id.to_string() + ], + ) + .expect("age SQLite heartbeat"); + return; + }; + sqlx_core::query::query("UPDATE sqb_run_liveness SET last_heartbeat_at=$1 WHERE run_id=$2") + .bind(timestamp) + .bind(run_id) + .execute(pool) + .await + .expect("age PostgreSQL heartbeat"); +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs b/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs new file mode 100644 index 0000000000..c4847d7316 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs @@ -0,0 +1,343 @@ +//! Real scheduler cycles detect silent workers without surrendering their physical write claims. + +use std::collections::BTreeMap; +use std::time::Duration; + +use sqlbuild_orchestration::runs::main::change_run::change_run; +use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; +use sqlbuild_orchestration::runs::main::heartbeat_run::heartbeat_run; +use sqlbuild_orchestration::runs::main::list_runs::list_runs; +use sqlbuild_orchestration::runs::main::monitor_runs::monitor_runs; +use sqlbuild_orchestration::runs::models::{ + RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, +}; +use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::scheduler::main::renew_lease::renew_lease; +use sqlbuild_orchestration::scheduler::main::scheduler_cycle::scheduler_cycle; +use sqlbuild_orchestration::scheduler::models::SchedulerState; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::main::create_location::create_location; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; +use uuid::Uuid; + +use crate::fault_helpers::{install_fault, remove_fault}; +use crate::helpers::{queue, request, snapshot}; +use crate::lease_helpers::expire_lease; +use crate::monitor_helpers::{age_heartbeat, start_run}; +use crate::store_helpers::{close_test_store, open_test_store, run}; +use crate::store_test_types::Backend; +use crate::test_types::{MonitorBacklogTestCase, RunMonitorTestCase, TransactionFault}; + +#[test] +fn given_silent_and_live_workers_when_the_scheduler_monitors_then_fenced_atomic_facts_retain_claims_and_allow_recovery() + { + let test_cases = [ + RunMonitorTestCase { + description: "SQLite observes only running workers with expired heartbeats", + backend: Backend::Sqlite, + expected_statuses: [ + RunStatus::Unresponsive, + RunStatus::Running, + RunStatus::Launched, + RunStatus::Failed, + RunStatus::Queued, + ], + }, + RunMonitorTestCase { + description: "PostgreSQL observes only running workers with expired heartbeats", + backend: Backend::Postgres, + expected_statuses: [ + RunStatus::Unresponsive, + RunStatus::Running, + RunStatus::Launched, + RunStatus::Failed, + RunStatus::Queued, + ], + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("snapshot"); + let scope = (location, published.snapshot_id); + let holder = Uuid::now_v7(); + let lease = acquire_lease(store, holder) + .await + .expect("elect") + .expect("leader"); + let silent = start_run(store, &lease, &request(scope, 1, 0, &["orders"])).await; + start_run(store, &lease, &request(scope, 2, 0, &["customers"])).await; + age_heartbeat(store, Uuid::from_u128(2), 40).await; + queue(store, &request(scope, 3, 0, &["inventory"])).await; + let launching = claim_next_run(store, &lease, &BTreeMap::new()) + .await + .expect("claim") + .expect("launching"); + change_run( + store, + RunAuthority::Scheduler(&lease), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: launching.request.run_id, + claim_id: Some(launching.claim_id), + change: RunChange::Launched { + container_id: Some("orders-worker".into()), + }, + }, + ) + .await + .expect("launcher acknowledges"); + let failed = start_run(store, &lease, &request(scope, 4, 0, &["products"])).await; + change_run( + store, + RunAuthority::Worker { + run_id: failed.request.run_id, + claim_id: failed.claim_id, + }, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: failed.request.run_id, + claim_id: Some(failed.claim_id), + change: RunChange::Finished { + outcome: RunOutcome::Failed, + reason: "worker failed".into(), + exit_code: Some(1), + }, + }, + ) + .await + .expect("terminal run"); + queue(store, &request(scope, 5, 0, &["orders"])).await; + age_heartbeat(store, silent.request.run_id, 50).await; + age_heartbeat(store, launching.request.run_id, 50).await; + let before = read_recall_events(store, 0, 100) + .await + .expect("before monitoring"); + let projection = list_runs(store, (location, "development"), 100) + .await + .expect("before monitoring"); + let fault = TransactionFault { + description: "monitor projection failure rolls back the observation", + table: "sqb_runs", + operation: "UPDATE", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected monitor failure')", + postgres_body: "RAISE EXCEPTION 'injected monitor failure'", + }; + install_fault(store, &fault).await; + assert!( + monitor_runs(store, &lease).await.is_err(), + "{}", + test_case.description + ); + remove_fault(store, &fault).await; + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("fact rollback"), + before + ); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("projection rollback"), + projection + ); + expire_lease(store).await; + let (state, observed) = scheduler_cycle(store, SchedulerState::new(holder)) + .await + .expect("monitoring cycle"); + assert_eq!(observed.unresponsive, 1, "{}", test_case.description); + let statuses: BTreeMap<_, _> = list_runs(store, (location, "development"), 100) + .await + .expect("monitored statuses") + .into_iter() + .map(|record| (record.run_id, record.status)) + .collect(); + assert_eq!( + std::array::from_fn(|index| statuses[&Uuid::from_u128(index as u128 + 1)]), + test_case.expected_statuses, + "{}", + test_case.description + ); + let recorded = read_recall_events(store, 0, 100) + .await + .expect("observation"); + assert_eq!(recorded.len(), before.len() + 1); + assert_eq!( + recorded.last().expect("fact")["event_type"], + "run_unresponsive" + ); + let (state, repeated) = scheduler_cycle(store, state) + .await + .expect("repeated monitor"); + assert_eq!(repeated.unresponsive, 0); + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("no repeated facts"), + recorded + ); + let (_, follower) = scheduler_cycle(store, SchedulerState::new(Uuid::now_v7())) + .await + .expect("follower"); + assert!(!follower.leader); + assert_eq!(follower.unresponsive, 0); + assert!( + !heartbeat_run(store, (silent.request.run_id, silent.claim_id)) + .await + .expect("worker recovers") + ); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("recovered projection") + .into_iter() + .find(|record| record.run_id == silent.request.run_id) + .expect("recovered run") + .status, + RunStatus::Running + ); + age_heartbeat(store, silent.request.run_id, 50).await; + let (_, second_loss) = scheduler_cycle(store, state) + .await + .expect("new heartbeat loss"); + assert_eq!(second_loss.unresponsive, 1); + let events = read_recall_events(store, 0, 100) + .await + .expect("successive losses"); + assert_eq!( + events + .iter() + .filter(|event| event["event_type"] == "run_unresponsive") + .count(), + 2 + ); + assert_eq!( + events + .iter() + .filter(|event| event["event_type"] == "run_recovered") + .count(), + 1 + ); + expire_lease(store).await; + assert!(matches!( + monitor_runs(store, &lease).await, + Err(ControlError::Fenced) + )); + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("fenced monitor"), + events + ); + let successor = acquire_lease(store, Uuid::now_v7()) + .await + .expect("takeover") + .expect("successor"); + assert!( + claim_next_run(store, &successor, &BTreeMap::new()) + .await + .expect("write claims remain held after heartbeat loss and takeover") + .is_none() + ); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_a_silent_worker_backlog_when_monitoring_then_bounded_transactions_continue_immediately_until_drained() + { + let test_cases = [ + MonitorBacklogTestCase { + description: "SQLite drains more than one monitor page", + backend: Backend::Sqlite, + run_count: 105, + expected_batches: [100, 5, 0], + }, + MonitorBacklogTestCase { + description: "PostgreSQL drains more than one monitor page", + backend: Backend::Postgres, + run_count: 105, + expected_batches: [100, 5, 0], + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("snapshot"); + let holder = Uuid::now_v7(); + let mut lease = acquire_lease(store, holder) + .await + .expect("elect") + .expect("leader"); + for index in 0..test_case.run_count { + let mut input = request( + (location, published.snapshot_id), + index as u128 + 1, + 0, + &["orders"], + ); + input.write_set.clear(); + start_run(store, &lease, &input).await; + age_heartbeat(store, input.run_id, 50).await; + lease = renew_lease(store, &lease) + .await + .expect("renew during setup"); + } + expire_lease(store).await; + let mut state = SchedulerState::new(holder); + let mut batches = Vec::new(); + for expected in test_case.expected_batches { + let cycle; + (state, cycle) = scheduler_cycle(store, state).await.expect("monitor batch"); + batches.push(cycle.unresponsive); + assert_eq!( + cycle.wait_for, + if expected == 100 { + Duration::ZERO + } else { + Duration::from_secs(5) + } + ); + } + assert_eq!( + batches, test_case.expected_batches, + "{}", + test_case.description + ); + let events = read_recall_events(store, 0, 1000) + .await + .expect("monitor facts"); + assert_eq!( + events + .iter() + .filter(|event| event["event_type"] == "run_unresponsive") + .count(), + test_case.run_count + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_types.rs b/crates/sqlbuild-orchestration/tests/runs/test_types.rs index 81471a4f67..f0112b2903 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_types.rs @@ -42,3 +42,16 @@ pub(crate) struct RunLifecycleTestCase { pub(crate) backend: Backend, pub(crate) expected_status: RunStatus, } + +pub(crate) struct RunMonitorTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_statuses: [RunStatus; 5], +} + +pub(crate) struct MonitorBacklogTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) run_count: usize, + pub(crate) expected_batches: [usize; 3], +} From e4dec0151bc3d95bfe0fd9137edab0aa5e43e1e7 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 01:35:59 +0000 Subject: [PATCH 11/59] feat: keep worker heartbeats independent of execution Own worker heartbeats on a dedicated native runtime and connection pool. Propagate cancellation and revoked credentials without cancelling work on transient API failures; stop and join with the execution owner. Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- crates/sqlbuild-cli/tests/native_commands.rs | 2 + .../tests/native_commands/worker/helpers.rs | 180 ++++++++++++++++++ .../native_commands/worker/test_heartbeat.rs | 63 ++++++ .../native_commands/worker/test_types.rs | 14 ++ .../native_commands/worker/test_worker.rs | 6 + .../src/worker/_helpers/heartbeat.rs | 90 +++++++++ .../src/worker/_helpers/mod.rs | 2 + .../src/worker/_helpers/response.rs | 15 ++ .../src/worker/_helpers/session.rs | 22 +++ .../src/worker/errors.rs | 2 + .../src/worker/main/connect_worker.rs | 15 +- .../src/worker/main/start_worker.rs | 18 +- .../src/worker/models.rs | 15 ++ .../tests/api/test_worker.rs | 12 +- 14 files changed, 431 insertions(+), 25 deletions(-) create mode 100644 crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs create mode 100644 crates/sqlbuild-cli/tests/native_commands/worker/test_heartbeat.rs create mode 100644 crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs create mode 100644 crates/sqlbuild-cli/tests/native_commands/worker/test_worker.rs create mode 100644 crates/sqlbuild-orchestration/src/worker/_helpers/heartbeat.rs create mode 100644 crates/sqlbuild-orchestration/src/worker/_helpers/session.rs diff --git a/crates/sqlbuild-cli/tests/native_commands.rs b/crates/sqlbuild-cli/tests/native_commands.rs index 23a9b5ba97..e93113875b 100644 --- a/crates/sqlbuild-cli/tests/native_commands.rs +++ b/crates/sqlbuild-cli/tests/native_commands.rs @@ -4,3 +4,5 @@ mod helpers; mod test_control; #[path = "native_commands/test_types.rs"] mod test_types; +#[path = "native_commands/worker/test_worker.rs"] +mod worker; diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs new file mode 100644 index 0000000000..2460bfd6ab --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs @@ -0,0 +1,180 @@ +//! Privileged test operators stay separate from the API-only worker session. + +use std::collections::BTreeMap; +use std::time::{Duration, SystemTime}; + +use sqlbuild_orchestration::credentials::main::issue_token::issue_token; +use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; +use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; +use sqlbuild_orchestration::runs::main::change_run::change_run; +use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; +use sqlbuild_orchestration::runs::main::request_run::request_run; +use sqlbuild_orchestration::runs::models::{ + RunAuthority, RunChange, RunChangeRequest, RunKind, RunOrigin, RunRequest, +}; +use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::scheduler::models::Overlap; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; +use sqlbuild_orchestration::store::main::create_location::create_location; +use sqlbuild_orchestration::store::main::open_store::open_store; +use sqlbuild_orchestration::worker::main::connect_worker::connect_worker; +use sqlbuild_orchestration::worker::main::start_worker::start_worker; +use sqlbuild_orchestration::worker::models::WorkerSession; +use uuid::Uuid; + +use crate::test_types::TestControl; +use crate::worker::test_types::WorkerControl; + +pub(crate) async fn start_claimed_worker( + control: &TestControl, + url: &str, +) -> (WorkerSession, WorkerControl) { + let path = control.directory.path().join(".sqlbuild/control.sqlite3"); + let store = open_store(path.to_str().expect("test database path")) + .await + .expect("native control store"); + let location_id = create_location(&store, "orders", "test-admin") + .await + .expect("location"); + let baseline = SnapshotInput { + location_id, + target: "development".into(), + generation: 1, + git_revision: None, + git_dirty: false, + git_author: None, + image_id: None, + environment_hash: "orders-environment".into(), + sqb_version: env!("CARGO_PKG_VERSION").into(), + nodes: BTreeMap::from([( + "orders".into(), + SnapshotNode { + kind: "model".into(), + definition: serde_json::json!({"materialization": "table"}), + raw_sql: Some("select 1 as id".into()), + compiled_sql: Some("select 1 as id".into()), + python_source: None, + }, + )]), + documents: BTreeMap::new(), + }; + let publication = publish_snapshot(&store, &baseline).await.expect("snapshot"); + let run_id = Uuid::now_v7(); + let request = RunRequest { + run_id, + location_id, + target: "development".into(), + snapshot_id: publication.snapshot_id, + image_id: None, + kind: RunKind::Adhoc, + origin: RunOrigin::Manual, + job_key: None, + parent_run_id: None, + root_run_id: run_id, + trigger_id: None, + tick_id: None, + rerun_of_run_id: None, + requested_by: "test-admin".into(), + priority: 0, + selection: ["orders".into()].into(), + write_set: ["orders".into()].into(), + settings: BTreeMap::new(), + concurrency_groups: Default::default(), + overlap: Overlap::Allow, + }; + request_run(&store, &request).await.expect("queue run"); + let lease = acquire_lease(&store, Uuid::now_v7()) + .await + .expect("acquire lease") + .expect("leader"); + let claim = claim_next_run(&store, &lease, &BTreeMap::new()) + .await + .expect("claim") + .expect("queued work"); + let identity = (run_id, claim.claim_id); + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let token = issue_token( + &store, + &context, + CredentialScope::Run { + run_id, + claim_id: identity.1, + }, + (SystemTime::now() + Duration::from_secs(3600)).into(), + ) + .await + .expect("run-scoped token"); + change_run( + &store, + RunAuthority::Scheduler(&lease), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id, + claim_id: Some(identity.1), + change: RunChange::Launched { container_id: None }, + }, + ) + .await + .expect("launched worker"); + let client = connect_worker(url, &token.bearer, identity).expect("worker connection"); + let session = start_worker(&client, &baseline) + .await + .expect("verified worker"); + ( + session, + WorkerControl { + store, + context, + token_id: token.identity.token_id, + identity, + }, + ) +} + +pub(crate) fn cancel_worker(worker: WorkerControl, _control: &mut TestControl) { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("independent operator runtime"); + runtime + .block_on(change_run( + &worker.store, + RunAuthority::Operator("test-admin"), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: worker.identity.0, + claim_id: Some(worker.identity.1), + change: RunChange::CancelRequested { + actor: "test-admin".into(), + }, + }, + )) + .expect("request cancellation while worker runtime is blocked"); +} + +pub(crate) fn revoke_worker(worker: WorkerControl, _control: &mut TestControl) { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("independent operator runtime"); + assert!( + runtime + .block_on(revoke_token( + &worker.store, + &worker.context, + worker.token_id + )) + .expect("revoke worker credential") + ); +} + +pub(crate) fn disconnect_worker(_worker: WorkerControl, control: &mut TestControl) { + control.child.kill().expect("stop API process"); + control.child.wait().expect("reap API process"); +} diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/test_heartbeat.rs b/crates/sqlbuild-cli/tests/native_commands/worker/test_heartbeat.rs new file mode 100644 index 0000000000..011e0e5490 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/worker/test_heartbeat.rs @@ -0,0 +1,63 @@ +//! The standalone server remains runnable while the worker's runtime is deliberately blocked. + +use std::time::{Duration, Instant}; + +use crate::helpers::{api_ready, start_control}; +use crate::worker::helpers::{ + cancel_worker, disconnect_worker, revoke_worker, start_claimed_worker, +}; +use crate::worker::test_types::WorkerHeartbeatTestCase; + +#[test] +fn given_blocking_worker_work_when_control_changes_then_heartbeats_remain_independent() { + let test_cases = [ + WorkerHeartbeatTestCase { + description: "cancellation reaches a blocked worker runtime", + interruption: cancel_worker, + expected_cancelled: true, + }, + WorkerHeartbeatTestCase { + description: "credential revocation reaches a blocked worker runtime", + interruption: revoke_worker, + expected_cancelled: true, + }, + WorkerHeartbeatTestCase { + description: "temporary API unavailability does not cancel warehouse work", + interruption: disconnect_worker, + expected_cancelled: false, + }, + ]; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("worker runtime"); + for test_case in test_cases { + runtime.block_on(async { + let mut control = start_control(&["server", "--port", "0", "--no-scheduler"]); + let url = api_ready(&mut control).await; + let (session, worker) = start_claimed_worker(&control, &url).await; + let operator = std::thread::spawn(move || { + std::thread::sleep(Duration::from_secs(2)); + (test_case.interruption)(worker, &mut control); + control + }); + // No Tokio IO can run on this caller until after the assertion. A heartbeat spawned + // on this runtime, or sharing its connection pool, cannot deliver cancellation. + std::thread::sleep(Duration::from_secs(12)); + assert_eq!( + session.cancel_requested(), + test_case.expected_cancelled, + "{}", + test_case.description + ); + let stopping = Instant::now(); + drop(session); + assert!( + stopping.elapsed() < Duration::from_secs(2), + "{}", + test_case.description + ); + drop(operator.join().expect("operator completed")); + }); + } +} diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs b/crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs new file mode 100644 index 0000000000..73fb9bf997 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs @@ -0,0 +1,14 @@ +use crate::test_types::TestControl; + +pub(crate) struct WorkerHeartbeatTestCase { + pub(crate) description: &'static str, + pub(crate) interruption: fn(WorkerControl, &mut TestControl), + pub(crate) expected_cancelled: bool, +} + +pub(crate) struct WorkerControl { + pub(crate) store: sqlbuild_orchestration::store::models::ControlStore, + pub(crate) context: sqlbuild_orchestration::credentials::models::AuditContext, + pub(crate) token_id: uuid::Uuid, + pub(crate) identity: (uuid::Uuid, uuid::Uuid), +} diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/test_worker.rs b/crates/sqlbuild-cli/tests/native_commands/worker/test_worker.rs new file mode 100644 index 0000000000..13b54de371 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/worker/test_worker.rs @@ -0,0 +1,6 @@ +#[path = "helpers.rs"] +mod helpers; +#[path = "test_heartbeat.rs"] +mod test_heartbeat; +#[path = "test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/heartbeat.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/heartbeat.rs new file mode 100644 index 0000000000..a8afb1fce8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/heartbeat.rs @@ -0,0 +1,90 @@ +//! A dedicated native runtime keeps heartbeat IO independent of warehouse work and Python's GIL. + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use crate::runs::models::RunExecution; +use crate::worker::_helpers::response::http_client; +use crate::worker::errors::WorkerError; +use crate::worker::models::{WorkerClient, WorkerSession}; + +async fn heartbeat_loop( + client: WorkerClient, + cancelled: Arc, + mut stop: tokio::sync::watch::Receiver, +) { + let mut interval = tokio::time::interval(Duration::from_secs(10)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + tokio::select! { + biased; + _ = stop.changed() => return, + _ = interval.tick() => {} + } + let result = tokio::select! { + biased; + _ = stop.changed() => return, + result = client.heartbeat() => result, + }; + match result { + Ok(true) => cancelled.store(true, Ordering::Release), + Err(WorkerError::Rejected(status)) + if matches!(status.as_u16(), 401 | 403 | 404 | 410) => + { + cancelled.store(true, Ordering::Release); + return; + } + Ok(false) | Err(_) => {} + } + } +} + +pub(crate) async fn start_session( + client: WorkerClient, + execution: RunExecution, +) -> Result { + let cancelled = Arc::new(AtomicBool::new(false)); + let signal = Arc::clone(&cancelled); + let (stop, stopping) = tokio::sync::watch::channel(false); + let (ready, started) = tokio::sync::oneshot::channel(); + let thread = std::thread::Builder::new() + .name("sqb-worker-heartbeat".into()) + .spawn(move || { + let runtime = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(error) => { + let _ = ready.send(Err(WorkerError::from(error))); + return; + } + }; + runtime.block_on(async move { + let heartbeat = match http_client(client.authorization.clone()) { + Ok(http) => WorkerClient { + client: http, + ..client + }, + Err(error) => { + let _ = ready.send(Err(error)); + return; + } + }; + let _ = ready.send(Ok(())); + heartbeat_loop(heartbeat, signal, stopping).await; + }); + runtime.shutdown_timeout(Duration::from_secs(1)); + })?; + let session = WorkerSession { + execution, + cancel_requested: cancelled, + stop, + thread: Some(thread), + }; + started + .await + .map_err(|_| WorkerError::Configuration("heartbeat thread stopped during startup"))??; + Ok(session) +} diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs index 57d94ff592..1bbe75bead 100644 --- a/crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs @@ -1,2 +1,4 @@ pub(crate) mod client; +pub(crate) mod heartbeat; pub(crate) mod response; +pub(crate) mod session; diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/response.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/response.rs index e4da90cca5..0790ae0288 100644 --- a/crates/sqlbuild-orchestration/src/worker/_helpers/response.rs +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/response.rs @@ -1,9 +1,24 @@ //! Do not echo response bodies: server diagnostics can contain user configuration. +use std::time::Duration; + +use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderValue}; use serde::de::DeserializeOwned; use crate::worker::errors::WorkerError; +/// Each runtime owns its connection pool; sharing pools could tie heartbeat IO to build work. +pub(crate) fn http_client(authorization: HeaderValue) -> Result { + let mut headers = HeaderMap::new(); + headers.insert(AUTHORIZATION, authorization); + Ok(reqwest::Client::builder() + .default_headers(headers) + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(30)) + .connect_timeout(Duration::from_secs(10)) + .build()?) +} + pub(crate) async fn response( request: reqwest::RequestBuilder, ) -> Result { diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/session.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/session.rs new file mode 100644 index 0000000000..e7bdd3ad08 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/session.rs @@ -0,0 +1,22 @@ +//! The execution owner drains its heartbeat before worker runtime teardown. + +use std::sync::atomic::Ordering; + +use crate::worker::models::WorkerSession; + +impl WorkerSession { + /// Cancellation and credential revocation are monotonic for one pinned worker invocation. + #[must_use] + pub fn cancel_requested(&self) -> bool { + self.cancel_requested.load(Ordering::Acquire) + } +} + +impl Drop for WorkerSession { + fn drop(&mut self) { + let _ = self.stop.send(true); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} diff --git a/crates/sqlbuild-orchestration/src/worker/errors.rs b/crates/sqlbuild-orchestration/src/worker/errors.rs index 30a6a18f8e..e5e62c936a 100644 --- a/crates/sqlbuild-orchestration/src/worker/errors.rs +++ b/crates/sqlbuild-orchestration/src/worker/errors.rs @@ -8,6 +8,8 @@ pub enum WorkerError { Transport(#[from] reqwest::Error), #[error("worker API rejected the request with HTTP {0}")] Rejected(reqwest::StatusCode), + #[error("worker heartbeat could not start: {0}")] + Heartbeat(#[from] std::io::Error), #[error(transparent)] Verification(#[from] crate::store::errors::ControlError), #[error("run cancellation was requested before execution")] diff --git a/crates/sqlbuild-orchestration/src/worker/main/connect_worker.rs b/crates/sqlbuild-orchestration/src/worker/main/connect_worker.rs index f1c543d1dc..cf90ad449f 100644 --- a/crates/sqlbuild-orchestration/src/worker/main/connect_worker.rs +++ b/crates/sqlbuild-orchestration/src/worker/main/connect_worker.rs @@ -1,10 +1,9 @@ //! Bound worker requests have deadlines and must never follow credential-bearing redirects. -use std::time::Duration; - -use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderValue}; +use reqwest::header::HeaderValue; use uuid::Uuid; +use crate::worker::_helpers::response::http_client; use crate::worker::errors::WorkerError; use crate::worker::models::WorkerClient; @@ -37,16 +36,10 @@ pub fn connect_worker( let mut authorization = HeaderValue::from_str(&format!("Bearer {bearer}")) .map_err(|_| WorkerError::Configuration("run bearer is not a valid HTTP credential"))?; authorization.set_sensitive(true); - let mut headers = HeaderMap::new(); - headers.insert(AUTHORIZATION, authorization); - let client = reqwest::Client::builder() - .default_headers(headers) - .redirect(reqwest::redirect::Policy::none()) - .timeout(Duration::from_secs(30)) - .connect_timeout(Duration::from_secs(10)) - .build()?; + let client = http_client(authorization.clone())?; Ok(WorkerClient { client, + authorization, run_url, run_id, claim_id, diff --git a/crates/sqlbuild-orchestration/src/worker/main/start_worker.rs b/crates/sqlbuild-orchestration/src/worker/main/start_worker.rs index c5710f3e31..72569448be 100644 --- a/crates/sqlbuild-orchestration/src/worker/main/start_worker.rs +++ b/crates/sqlbuild-orchestration/src/worker/main/start_worker.rs @@ -2,27 +2,25 @@ use uuid::Uuid; -use crate::runs::models::{RunChange, RunExecution}; +use crate::runs::models::RunChange; use crate::snapshots::main::verify_execution::verify_execution; use crate::snapshots::models::SnapshotInput; +use crate::worker::_helpers::heartbeat::start_session; use crate::worker::errors::WorkerError; -use crate::worker::models::WorkerClient; +use crate::worker::models::{WorkerClient, WorkerSession}; /// Verify the baseline compile before planning, cloning, applying run settings or executing. pub async fn start_worker( client: &WorkerClient, baseline: &SnapshotInput, -) -> Result { +) -> Result { let expected = client.execution().await?; verify_execution(baseline, &expected)?; if client.heartbeat().await? { return Err(WorkerError::Cancelled); } - client - .report( - Uuid::new_v5(&expected.claim.claim_id, b"worker-started"), - RunChange::Started, - ) - .await?; - Ok(expected) + let event_id = Uuid::new_v5(&expected.claim.claim_id, b"worker-started"); + let session = start_session(client.clone(), expected).await?; + client.report(event_id, RunChange::Started).await?; + Ok(session) } diff --git a/crates/sqlbuild-orchestration/src/worker/models.rs b/crates/sqlbuild-orchestration/src/worker/models.rs index 813b4b61e3..02e05c2e89 100644 --- a/crates/sqlbuild-orchestration/src/worker/models.rs +++ b/crates/sqlbuild-orchestration/src/worker/models.rs @@ -1,11 +1,26 @@ //! A worker owns only an API credential, never a control-store connection. +use std::sync::Arc; +use std::sync::atomic::AtomicBool; + use uuid::Uuid; +use crate::runs::models::RunExecution; + #[derive(Clone, Debug)] pub struct WorkerClient { pub(crate) client: reqwest::Client, + pub(crate) authorization: reqwest::header::HeaderValue, pub(crate) run_url: reqwest::Url, pub(crate) run_id: Uuid, pub(crate) claim_id: Uuid, } + +/// Keep this owner alive through execution and terminal reporting, then stop and join its heartbeat. +#[derive(Debug)] +pub struct WorkerSession { + pub execution: RunExecution, + pub(crate) cancel_requested: Arc, + pub(crate) stop: tokio::sync::watch::Sender, + pub(crate) thread: Option>, +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_worker.rs b/crates/sqlbuild-orchestration/tests/api/test_worker.rs index dd61d4724c..5cb27bd833 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_worker.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_worker.rs @@ -137,13 +137,16 @@ fn given_a_claimed_run_when_a_worker_connects_then_verification_precedes_start_a ) .await .expect("record process launch"); - let execution = start_worker(&worker, &baseline) + let session = start_worker(&worker, &baseline) .await .expect("verified start"); - assert_eq!(execution.claim, claim, "{}", test_case.description); + assert_eq!(session.execution.claim, claim, "{}", test_case.description); assert_eq!( - start_worker(&worker, &baseline).await.expect("retry start"), - execution, + start_worker(&worker, &baseline) + .await + .expect("retry start") + .execution, + session.execution, "{}", test_case.description ); @@ -300,6 +303,7 @@ fn given_a_claimed_run_when_a_worker_connects_then_verification_precedes_start_a "{}", test_case.description ); + drop(session); stop_api(api).await; close_test_store(fixture).await; }); From 89dbad91d38b49658e375b75e0602baf0a70863b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 01:41:31 +0000 Subject: [PATCH 12/59] refactor: share authenticated control-plane transport Move bounded client construction, credential redaction, URL validation and JSON response handling to the API owner so worker and location flows can share them. Keep independent heartbeat pools and worker-facing error categories. Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../src/api/_helpers/client.rs | 38 +++++++++++++++++++ .../src/api/_helpers/mod.rs | 1 + .../sqlbuild-orchestration/src/api/errors.rs | 10 +++++ .../src/api/main/client_response.rs | 15 ++++++++ .../src/api/main/connect_client.rs | 31 +++++++++++++++ .../src/api/main/mod.rs | 2 + .../sqlbuild-orchestration/src/api/models.rs | 7 ++++ .../src/worker/_helpers/client.rs | 23 +++++++---- .../src/worker/_helpers/heartbeat.rs | 9 ++--- .../src/worker/_helpers/mod.rs | 1 - .../src/worker/_helpers/response.rs | 30 --------------- .../src/worker/errors.rs | 10 +++++ .../src/worker/main/connect_worker.rs | 32 ++-------------- .../src/worker/models.rs | 4 +- 14 files changed, 140 insertions(+), 73 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/client.rs create mode 100644 crates/sqlbuild-orchestration/src/api/main/client_response.rs create mode 100644 crates/sqlbuild-orchestration/src/api/main/connect_client.rs delete mode 100644 crates/sqlbuild-orchestration/src/worker/_helpers/response.rs diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/client.rs b/crates/sqlbuild-orchestration/src/api/_helpers/client.rs new file mode 100644 index 0000000000..98532b9054 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/client.rs @@ -0,0 +1,38 @@ +//! Each client runtime owns its pool; credentials and server response bodies stay out of diagnostics. + +use std::time::Duration; + +use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderValue}; + +use crate::api::errors::ClientError; +use crate::api::models::ControlClient; + +pub(crate) fn http_client(authorization: HeaderValue) -> Result { + let mut headers = HeaderMap::new(); + headers.insert(AUTHORIZATION, authorization); + Ok(reqwest::Client::builder() + .default_headers(headers) + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(30)) + .connect_timeout(Duration::from_secs(10)) + .build()?) +} + +impl ControlClient { + pub(crate) fn independent(&self) -> Result { + Ok(Self { + client: http_client(self.authorization.clone())?, + authorization: self.authorization.clone(), + base_url: self.base_url.clone(), + }) + } + + pub(crate) fn endpoint(&self, segments: &[&str]) -> Result { + let mut url = self.base_url.clone(); + url.path_segments_mut() + .map_err(|_| ClientError::Configuration("server URL cannot contain an API path"))? + .pop_if_empty() + .extend(segments.iter().copied()); + Ok(url) + } +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs index 41d8c461c5..4f00fe75c5 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod authentication; +pub(crate) mod client; pub(crate) mod runs; pub(crate) mod snapshots; diff --git a/crates/sqlbuild-orchestration/src/api/errors.rs b/crates/sqlbuild-orchestration/src/api/errors.rs index 011ed4974a..ea9da3591d 100644 --- a/crates/sqlbuild-orchestration/src/api/errors.rs +++ b/crates/sqlbuild-orchestration/src/api/errors.rs @@ -7,6 +7,16 @@ use axum::response::{IntoResponse, Response}; use crate::api::models::ApiErrorBody; use crate::store::errors::ControlError; +#[derive(Debug, thiserror::Error)] +pub enum ClientError { + #[error("invalid API connection: {0}")] + Configuration(&'static str), + #[error("API request failed: {0}")] + Transport(#[from] reqwest::Error), + #[error("API rejected the request with HTTP {0}")] + Rejected(reqwest::StatusCode), +} + #[derive(Debug)] pub struct ApiError { pub(crate) status: StatusCode, diff --git a/crates/sqlbuild-orchestration/src/api/main/client_response.rs b/crates/sqlbuild-orchestration/src/api/main/client_response.rs new file mode 100644 index 0000000000..2eb643424f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/main/client_response.rs @@ -0,0 +1,15 @@ +//! Share bounded JSON transport without exposing server bodies or credential-bearing URLs. + +use serde::de::DeserializeOwned; + +use crate::api::errors::ClientError; + +pub(crate) async fn client_response( + request: reqwest::RequestBuilder, +) -> Result { + let response = request.send().await.map_err(reqwest::Error::without_url)?; + if !response.status().is_success() { + return Err(ClientError::Rejected(response.status())); + } + Ok(response.json().await.map_err(reqwest::Error::without_url)?) +} diff --git a/crates/sqlbuild-orchestration/src/api/main/connect_client.rs b/crates/sqlbuild-orchestration/src/api/main/connect_client.rs new file mode 100644 index 0000000000..c0b5258123 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/main/connect_client.rs @@ -0,0 +1,31 @@ +//! Validate the API origin before attaching a sensitive bearer credential. + +use reqwest::header::HeaderValue; + +use crate::api::_helpers::client::http_client; +use crate::api::errors::ClientError; +use crate::api::models::ControlClient; + +pub(crate) fn connect_client(server_url: &str, bearer: &str) -> Result { + let base_url = reqwest::Url::parse(server_url) + .map_err(|_| ClientError::Configuration("server URL must be an absolute HTTP URL"))?; + if !matches!(base_url.scheme(), "http" | "https") + || base_url.host_str().is_none() + || !base_url.username().is_empty() + || base_url.password().is_some() + || base_url.query().is_some() + || base_url.fragment().is_some() + { + return Err(ClientError::Configuration( + "server URL must use HTTP or HTTPS without credentials, query or fragment", + )); + } + let mut authorization = HeaderValue::from_str(&format!("Bearer {bearer}")) + .map_err(|_| ClientError::Configuration("bearer is not a valid HTTP credential"))?; + authorization.set_sensitive(true); + Ok(ControlClient { + client: http_client(authorization.clone())?, + authorization, + base_url, + }) +} diff --git a/crates/sqlbuild-orchestration/src/api/main/mod.rs b/crates/sqlbuild-orchestration/src/api/main/mod.rs index e72ebe13fc..f445706317 100644 --- a/crates/sqlbuild-orchestration/src/api/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/api/main/mod.rs @@ -1,2 +1,4 @@ +pub(crate) mod client_response; +pub(crate) mod connect_client; pub mod openapi_document; pub mod serve_api; diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index e7db400f75..c0ec5c75a9 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -28,3 +28,10 @@ pub struct HeartbeatResponse { pub struct ApiErrorBody { pub error: String, } + +#[derive(Clone, Debug)] +pub(crate) struct ControlClient { + pub(crate) client: reqwest::Client, + pub(crate) authorization: reqwest::header::HeaderValue, + pub(crate) base_url: reqwest::Url, +} diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/client.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/client.rs index cbac454f9b..79bfd1f284 100644 --- a/crates/sqlbuild-orchestration/src/worker/_helpers/client.rs +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/client.rs @@ -2,17 +2,21 @@ use uuid::Uuid; +use crate::api::main::client_response::client_response; use crate::api::models::{HeartbeatResponse, MutationResponse}; use crate::runs::models::{RunChange, RunChangeRequest, RunExecution}; -use crate::worker::_helpers::response::response; use crate::worker::errors::WorkerError; use crate::worker::models::WorkerClient; impl WorkerClient { /// Fetch the launcher's immutable request, not the location's latest deployment. pub async fn execution(&self) -> Result { - let execution: RunExecution = - response(self.client.get(format!("{}execution", self.run_url))).await?; + let execution: RunExecution = client_response( + self.connection + .client + .get(format!("{}execution", self.run_url)), + ) + .await?; if execution.claim.request.run_id != self.run_id || execution.claim.claim_id != self.claim_id { @@ -31,8 +35,9 @@ impl WorkerClient { claim_id: Some(self.claim_id), change, }; - let result: MutationResponse = response( - self.client + let result: MutationResponse = client_response( + self.connection + .client .post(format!("{}events", self.run_url)) .json(&request), ) @@ -42,8 +47,12 @@ impl WorkerClient { /// Run independently from warehouse execution; cancellation is a cooperative request. pub async fn heartbeat(&self) -> Result { - let result: HeartbeatResponse = - response(self.client.post(format!("{}heartbeat", self.run_url))).await?; + let result: HeartbeatResponse = client_response( + self.connection + .client + .post(format!("{}heartbeat", self.run_url)), + ) + .await?; Ok(result.cancel_requested) } } diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/heartbeat.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/heartbeat.rs index a8afb1fce8..25beaf886c 100644 --- a/crates/sqlbuild-orchestration/src/worker/_helpers/heartbeat.rs +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/heartbeat.rs @@ -5,7 +5,6 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; use crate::runs::models::RunExecution; -use crate::worker::_helpers::response::http_client; use crate::worker::errors::WorkerError; use crate::worker::models::{WorkerClient, WorkerSession}; @@ -62,13 +61,13 @@ pub(crate) async fn start_session( } }; runtime.block_on(async move { - let heartbeat = match http_client(client.authorization.clone()) { - Ok(http) => WorkerClient { - client: http, + let heartbeat = match client.connection.independent() { + Ok(connection) => WorkerClient { + connection, ..client }, Err(error) => { - let _ = ready.send(Err(error)); + let _ = ready.send(Err(WorkerError::from(error))); return; } }; diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs index 1bbe75bead..9ec828d671 100644 --- a/crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs @@ -1,4 +1,3 @@ pub(crate) mod client; pub(crate) mod heartbeat; -pub(crate) mod response; pub(crate) mod session; diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/response.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/response.rs deleted file mode 100644 index 0790ae0288..0000000000 --- a/crates/sqlbuild-orchestration/src/worker/_helpers/response.rs +++ /dev/null @@ -1,30 +0,0 @@ -//! Do not echo response bodies: server diagnostics can contain user configuration. - -use std::time::Duration; - -use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderValue}; -use serde::de::DeserializeOwned; - -use crate::worker::errors::WorkerError; - -/// Each runtime owns its connection pool; sharing pools could tie heartbeat IO to build work. -pub(crate) fn http_client(authorization: HeaderValue) -> Result { - let mut headers = HeaderMap::new(); - headers.insert(AUTHORIZATION, authorization); - Ok(reqwest::Client::builder() - .default_headers(headers) - .redirect(reqwest::redirect::Policy::none()) - .timeout(Duration::from_secs(30)) - .connect_timeout(Duration::from_secs(10)) - .build()?) -} - -pub(crate) async fn response( - request: reqwest::RequestBuilder, -) -> Result { - let response = request.send().await.map_err(reqwest::Error::without_url)?; - if !response.status().is_success() { - return Err(WorkerError::Rejected(response.status())); - } - Ok(response.json().await.map_err(reqwest::Error::without_url)?) -} diff --git a/crates/sqlbuild-orchestration/src/worker/errors.rs b/crates/sqlbuild-orchestration/src/worker/errors.rs index e5e62c936a..49240df632 100644 --- a/crates/sqlbuild-orchestration/src/worker/errors.rs +++ b/crates/sqlbuild-orchestration/src/worker/errors.rs @@ -15,3 +15,13 @@ pub enum WorkerError { #[error("run cancellation was requested before execution")] Cancelled, } + +impl From for WorkerError { + fn from(error: crate::api::errors::ClientError) -> Self { + match error { + crate::api::errors::ClientError::Configuration(message) => Self::Configuration(message), + crate::api::errors::ClientError::Transport(error) => Self::Transport(error), + crate::api::errors::ClientError::Rejected(status) => Self::Rejected(status), + } + } +} diff --git a/crates/sqlbuild-orchestration/src/worker/main/connect_worker.rs b/crates/sqlbuild-orchestration/src/worker/main/connect_worker.rs index cf90ad449f..15a7a4d081 100644 --- a/crates/sqlbuild-orchestration/src/worker/main/connect_worker.rs +++ b/crates/sqlbuild-orchestration/src/worker/main/connect_worker.rs @@ -1,45 +1,21 @@ //! Bound worker requests have deadlines and must never follow credential-bearing redirects. -use reqwest::header::HeaderValue; use uuid::Uuid; -use crate::worker::_helpers::response::http_client; +use crate::api::main::connect_client::connect_client; use crate::worker::errors::WorkerError; use crate::worker::models::WorkerClient; -fn worker_url(server_url: &str, run_id: Uuid) -> Result { - let mut url = reqwest::Url::parse(server_url) - .map_err(|_| WorkerError::Configuration("server URL must be an absolute HTTP URL"))?; - if !matches!(url.scheme(), "http" | "https") - || url.host_str().is_none() - || !url.username().is_empty() - || url.password().is_some() - || url.query().is_some() - || url.fragment().is_some() - { - return Err(WorkerError::Configuration( - "server URL must use HTTP or HTTPS without credentials, query or fragment", - )); - } - let prefix = url.path().trim_end_matches('/'); - url.set_path(&format!("{prefix}/api/runs/{run_id}/")); - Ok(url) -} - /// Connect to an API origin or path prefix using the launcher's run-scoped bearer. pub fn connect_worker( server_url: &str, bearer: &str, (run_id, claim_id): (Uuid, Uuid), ) -> Result { - let run_url = worker_url(server_url, run_id)?; - let mut authorization = HeaderValue::from_str(&format!("Bearer {bearer}")) - .map_err(|_| WorkerError::Configuration("run bearer is not a valid HTTP credential"))?; - authorization.set_sensitive(true); - let client = http_client(authorization.clone())?; + let connection = connect_client(server_url, bearer)?; + let run_url = connection.endpoint(&["api", "runs", &run_id.to_string(), ""])?; Ok(WorkerClient { - client, - authorization, + connection, run_url, run_id, claim_id, diff --git a/crates/sqlbuild-orchestration/src/worker/models.rs b/crates/sqlbuild-orchestration/src/worker/models.rs index 02e05c2e89..81799ca75f 100644 --- a/crates/sqlbuild-orchestration/src/worker/models.rs +++ b/crates/sqlbuild-orchestration/src/worker/models.rs @@ -5,12 +5,12 @@ use std::sync::atomic::AtomicBool; use uuid::Uuid; +use crate::api::models::ControlClient; use crate::runs::models::RunExecution; #[derive(Clone, Debug)] pub struct WorkerClient { - pub(crate) client: reqwest::Client, - pub(crate) authorization: reqwest::header::HeaderValue, + pub(crate) connection: ControlClient, pub(crate) run_url: reqwest::Url, pub(crate) run_id: Uuid, pub(crate) claim_id: Uuid, From c122815532235b91894ae8625ca5557a36f0d5f8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 01:51:29 +0000 Subject: [PATCH 13/59] feat: publish retained compiler snapshots through the control API Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- Cargo.lock | 1 + .../src/snapshots/main/mod.rs | 1 + .../snapshots/main/publish_remote_snapshot.rs | 23 +++++ .../tests/api/test_contract.rs | 20 +++- crates/sqlbuild-python/Cargo.toml | 1 + .../_helpers/compiled_project/snapshot.rs | 18 +++- src/sqlbuild/_native.pyi | 1 + .../compiler/compiled_project/helpers.py | 17 +++- .../compiled_project/publication/__init__.py | 1 + .../publication/_test_types.py | 18 ++++ .../compiled_project/publication/helpers.py | 95 ++++++++++++++++++ .../publication/test_publication.py | 97 +++++++++++++++++++ .../test_orchestration_snapshot.py | 16 +-- 13 files changed, 288 insertions(+), 21 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/snapshots/main/publish_remote_snapshot.rs create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/__init__.py create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py diff --git a/Cargo.lock b/Cargo.lock index 92db47c581..060ae30b32 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2647,6 +2647,7 @@ dependencies = [ "sqlbuild-sqltext", "sqlbuild-state", "stacker", + "tokio", ] [[package]] diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs index 88c2bf1eba..8411aaf37d 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs @@ -1,3 +1,4 @@ pub mod current_snapshot; +pub mod publish_remote_snapshot; pub mod publish_snapshot; pub mod verify_execution; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/publish_remote_snapshot.rs b/crates/sqlbuild-orchestration/src/snapshots/main/publish_remote_snapshot.rs new file mode 100644 index 0000000000..681cc0f61e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/publish_remote_snapshot.rs @@ -0,0 +1,23 @@ +//! A code location sends retained compiled data over HTTP and never opens the control store. + +use crate::api::errors::ClientError; +use crate::api::main::client_response::client_response; +use crate::api::main::connect_client::connect_client; +use crate::snapshots::models::{SnapshotInput, SnapshotRecord}; + +pub async fn publish_remote_snapshot( + server_url: &str, + bearer: &str, + input: &SnapshotInput, +) -> Result { + let connection = connect_client(server_url, bearer)?; + let url = connection.endpoint(&[ + "api", + "locations", + &input.location_id.to_string(), + "targets", + &input.target, + "snapshots", + ])?; + client_response(connection.client.post(url).json(input)).await +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_contract.rs b/crates/sqlbuild-orchestration/tests/api/test_contract.rs index 53d9903ae5..bea9439da0 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contract.rs @@ -4,6 +4,7 @@ use std::time::Duration; use chrono::Utc; use reqwest::StatusCode; use serde_json::json; +use sqlbuild_orchestration::api::errors::ClientError; use sqlbuild_orchestration::credentials::main::issue_token::issue_token; use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; @@ -15,6 +16,7 @@ use sqlbuild_orchestration::runs::models::{ RunAuthority, RunChange, RunChangeRequest, RunExecution, RunStatus, }; use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; use sqlbuild_orchestration::snapshots::main::verify_execution::verify_execution; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotRecord}; use sqlbuild_orchestration::store::main::create_location::create_location; @@ -107,16 +109,26 @@ fn given_a_real_http_server_when_locations_and_workers_report_then_scopes_retrie assert!(openapi["components"]["schemas"]["RunChangeRequest"].is_object()); assert!(openapi["components"]["schemas"]["RunExecution"].is_object()); let input = snapshot(location_id); + let published = publish_remote_snapshot(&api.url, &location_token.bearer, &input) + .await + .expect("publish retained native data"); let response = client - .post(&path) + .get(&path) .bearer_auth(&location_token.bearer) - .json(&input) .send() .await - .expect("publish through API"); + .expect("read published snapshot through API"); assert_eq!(response.status(), StatusCode::OK); assert_eq!(response.headers()["cache-control"], "no-store"); - let published: SnapshotRecord = response.json().await.expect("snapshot response"); + drop(response); + assert!(matches!( + publish_remote_snapshot(&api.url, &location_token.bearer, &input).await, + Err(ClientError::Rejected(StatusCode::CONFLICT)) + )); + assert!(matches!( + publish_remote_snapshot(&api.url, &location_token.bearer, &snapshot(other)).await, + Err(ClientError::Rejected(StatusCode::FORBIDDEN)) + )); let response = client .post(&path) .bearer_auth(&location_token.bearer) diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index be0cc75d7c..3b066a9b33 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -47,6 +47,7 @@ sqlbuild-scopes.workspace = true sqlbuild-sqltext.workspace = true sqlbuild-state.workspace = true stacker.workspace = true +tokio.workspace = true [lints] workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs index b0b0d878fe..fa850b69dd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs @@ -3,11 +3,12 @@ use std::collections::BTreeMap; use std::path::PathBuf; -use pyo3::exceptions::PyValueError; +use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult, Python}; use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; use serde_json::Value; use sqlbuild_core::models::DiagnosticSeverity; +use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; use crate::bindings::_helpers::compiled_project::snapshot_models::model_nodes; @@ -29,6 +30,21 @@ impl NativeOrchestrationSnapshot { py.detach(|| serde_json::to_string(&self.input)) .map_err(|error| PyValueError::new_err(error.to_string())) } + + /// Publish retained data without projecting it through Python, releasing the GIL during IO. + fn publish(&self, py: Python<'_>, server_url: &str, bearer: &str) -> PyResult { + py.detach(|| { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; + let result = runtime.block_on(publish_remote_snapshot(server_url, bearer, &self.input)); + runtime.shutdown_timeout(std::time::Duration::from_secs(1)); + let record = result.map_err(|error| PyRuntimeError::new_err(error.to_string()))?; + serde_json::to_string(&record) + .map_err(|error| PyRuntimeError::new_err(error.to_string())) + }) + } } /// Compile catalog nodes from their owners, never from Python model or graph projections. diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 3452f57cab..78a8f577b2 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1742,6 +1742,7 @@ class NativeOrchestrationSnapshot: """A compiled data-only publication retained natively until transport.""" def to_json(self) -> str: ... + def publish(self, server_url: str, bearer: str) -> str: ... def build_orchestration_snapshot( project: CompiledProjectView, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index f351592f4d..024306d0a4 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -8,7 +8,8 @@ from collections.abc import Callable, Iterator from pathlib import Path from types import ModuleType -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any +from uuid import UUID import pytest @@ -47,6 +48,20 @@ if TYPE_CHECKING: from sqlbuild._native import CompileInputsView +METADATA: dict[str, Any] = { + "location_id": str(UUID(int=1)), + "target": "development", + "generation": 1, + "git_revision": None, + "git_dirty": False, + "git_author": None, + "image_id": None, + "environment_hash": "test-environment", + "sqb_version": "not-the-installed-version", + "nodes": {}, + "documents": {}, +} + TYPE_PROOF_FILES: dict[str, str] = { "sqlbuild_project.toml": ( 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRCONTRACT105"]\n\n' diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/__init__.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/__init__.py new file mode 100644 index 0000000000..75debee041 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/__init__.py @@ -0,0 +1 @@ +"""Retained compiler snapshots published through the real native control API.""" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py new file mode 100644 index 0000000000..8c7301d86d --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -0,0 +1,18 @@ +"""Publication cases and isolated server credentials.""" + +from dataclasses import dataclass +from typing import NamedTuple + + +class SnapshotApi(NamedTuple): + url: str + location_id: str + bearer: str + + +@dataclass(frozen=True) +class SnapshotPublicationTestCase: + description: str + target: str + expected_generation: int + expected_nodes: frozenset[str] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py new file mode 100644 index 0000000000..cbc2554451 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -0,0 +1,95 @@ +"""A standalone native API and real SQLite store, with test-only credential provisioning.""" + +from __future__ import annotations + +import hashlib +import json +import os +import secrets +import sqlite3 +import subprocess +import sys +import time +from collections.abc import Iterator +from contextlib import contextmanager +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any +from urllib.parse import quote +from urllib.request import Request, urlopen +from uuid import uuid4 + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + SnapshotApi, +) + + +def api_address(process: subprocess.Popen[bytes], log: Path) -> str: + """Wait for bounded native startup, never publishing an unready address.""" + deadline: float = time.monotonic() + 10 + while "Control API ready: " not in log.read_text(): + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, "Native API did not become ready" + time.sleep(0.02) + return "http://" + log.read_text().partition("Control API ready: ")[2].splitlines()[0] + + +@contextmanager +def snapshot_api(directory: Path) -> Iterator[SnapshotApi]: + """Only this fixture seeds the control database; the code location uses HTTP exclusively.""" + directory.mkdir() + log: Path = directory / "server.log" + environment: dict[str, str] = os.environ.copy() + environment.pop("SQLBUILD_CONTROL_DATABASE_URL", None) + with log.open("wb") as stderr: + process: subprocess.Popen[bytes] = subprocess.Popen( + [str(Path(sys.executable).with_name("sqb")), "server", "--port", "0", "--no-scheduler"], + cwd=directory, + env=environment, + stdout=subprocess.DEVNULL, + stderr=stderr, + ) + try: + url: str = api_address(process, log) + location_id: str = str(uuid4()) + token_id: str = str(uuid4()) + bearer: str = f"{token_id}.{secrets.token_hex(32)}" + now: datetime = datetime.now(UTC) + with sqlite3.connect(directory / ".sqlbuild/control.sqlite3") as connection: + _ = connection.execute( + "INSERT INTO sqb_locations (location_id,name,created_by,created_at) " + "VALUES (?, 'orders', 'test-admin', ?)", + (location_id, now.isoformat()), + ) + _ = connection.execute( + "INSERT INTO sqb_access_tokens " + "(token_id,secret_hash,location_id,scope_json,created_at,expires_at) " + "VALUES (?,?,?,?,?,?)", + ( + token_id, + hashlib.sha256(bearer.encode()).hexdigest(), + location_id, + json.dumps({"kind": "location", "location_id": location_id}), + now.isoformat(), + (now + timedelta(hours=1)).isoformat(), + ), + ) + yield SnapshotApi(url=url, location_id=location_id, bearer=bearer) + finally: + process.terminate() + try: + _ = process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + _ = process.wait(timeout=5) + + +def current_snapshot(api: SnapshotApi, target: str) -> dict[str, Any]: + """Observe the public API instead of inspecting stored snapshot implementation details.""" + request: Request = Request( + f"{api.url}/api/locations/{api.location_id}/targets/{quote(target, safe='')}/snapshots", + headers={"Authorization": f"Bearer {api.bearer}"}, + ) + with urlopen(request, timeout=5) as response: + result: dict[str, Any] = json.load(response) + return result diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py new file mode 100644 index 0000000000..ce5ca06abd --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py @@ -0,0 +1,97 @@ +"""Real compile, native transport, authenticated server and durable publication.""" + +import json +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild._native import ( + NativeOrchestrationSnapshot, + build_compiled_project, + build_orchestration_snapshot, + discover_project_inputs, +) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.types import CompiledProjectView +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( + METADATA, + TYPE_PROOF_FILES, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + SnapshotPublicationTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + current_snapshot, + snapshot_api, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotPublicationTestCase( + description="ordinary target", + target="development", + expected_generation=2, + expected_nodes=frozenset({"model:orders", "model:stg_orders"}), + ), + SnapshotPublicationTestCase( + description="target requiring URL path encoding", + target="preview #1", + expected_generation=2, + expected_nodes=frozenset({"model:orders", "model:stg_orders"}), + ), + ], + ids=lambda case: case.description, +) +def test_given_retained_compiler_data_when_publishing_then_http_updates_atomically_and_rejections_keep_last_good( + test_case: SnapshotPublicationTestCase, tmp_path: Path +) -> None: + project_dir: Path = tmp_path / "project" + write_files(project_dir=project_dir, files=TYPE_PROOF_FILES) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=project_dir), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + assert not any(item.severity == "error" for item in project.diagnostics) + with snapshot_api(tmp_path / "control") as api: + metadata: dict[str, Any] = METADATA | { + "location_id": api.location_id, + "target": test_case.target, + } + first: NativeOrchestrationSnapshot = build_orchestration_snapshot( + project, json.dumps(metadata), project_dir + ) + record: dict[str, Any] = json.loads(first.publish(api.url, api.bearer)) + assert record["generation"] == 1 + assert record["location_id"] == api.location_id + initial: dict[str, Any] = current_snapshot(api, test_case.target) + assert frozenset(initial["nodes"]) == test_case.expected_nodes + assert initial["nodes"]["model:orders"]["definition"]["deps"] == ["model:stg_orders"] + edited: dict[str, str] = TYPE_PROOF_FILES | { + "models/stg_orders.sql": TYPE_PROOF_FILES["models/stg_orders.sql"].replace( + "CAST(5 AS INTEGER)", "CAST(9 AS INTEGER)" + ) + } + write_files(project_dir=project_dir, files=edited) + changed: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=project_dir), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + next_snapshot: NativeOrchestrationSnapshot = build_orchestration_snapshot( + changed, json.dumps(metadata | {"generation": 2}), project_dir + ) + updated: dict[str, Any] = json.loads(next_snapshot.publish(api.url, api.bearer)) + assert updated["generation"] == test_case.expected_generation + assert updated["snapshot_id"] != record["snapshot_id"] + with pytest.raises(RuntimeError, match="HTTP 409"): + _ = first.publish(api.url, api.bearer) + with pytest.raises(RuntimeError, match="HTTP 401"): + _ = next_snapshot.publish(api.url, "invalid-credential") + current: dict[str, Any] = current_snapshot(api, test_case.target) + assert current["record"] == updated + assert "CAST(9 AS INTEGER)" in current["nodes"]["model:stg_orders"]["raw_sql"] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index e53269d8a3..f32d0b2601 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -5,7 +5,6 @@ import json from pathlib import Path from typing import Any -from uuid import UUID import pytest @@ -24,25 +23,12 @@ ) from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( FUNCTION_PROJECT_FILES, + METADATA, PARITY_FILES, TYPE_PROOF_FILES, ) from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files -METADATA: dict[str, Any] = { - "location_id": str(UUID(int=1)), - "target": "development", - "generation": 1, - "git_revision": None, - "git_dirty": False, - "git_author": None, - "image_id": None, - "environment_hash": "test-environment", - "sqb_version": "not-the-installed-version", - "nodes": {}, - "documents": {}, -} - @pytest.mark.parametrize( "test_case", From 0692195d12b2557cc0945cc555893d7201f715cf Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 02:08:00 +0000 Subject: [PATCH 14/59] feat: ingest scoped worker output through the control API Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- Cargo.lock | 2 + .../src/api/_helpers/runs.rs | 21 ++- .../src/api/main/serve_api.rs | 1 + .../sqlbuild-orchestration/src/api/models.rs | 3 +- .../src/runs/_helpers/mod.rs | 1 + .../src/runs/_helpers/output.rs | 80 ++++++++ .../src/runs/constants.rs | 3 + .../src/runs/main/mod.rs | 1 + .../src/runs/main/publish_run_output.rs | 36 ++++ .../sqlbuild-orchestration/src/runs/models.rs | 14 ++ .../src/worker/_helpers/client.rs | 17 +- crates/sqlbuild-orchestration/tests/api.rs | 2 + .../tests/api/output/helpers.rs | 178 ++++++++++++++++++ .../tests/api/test_contract.rs | 33 ++++ .../tests/api/test_worker.rs | 12 ++ crates/sqlbuild-recall/Cargo.toml | 2 + crates/sqlbuild-recall/src/records/models.rs | 21 ++- 17 files changed, 421 insertions(+), 6 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/output.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/publish_run_output.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/output/helpers.rs diff --git a/Cargo.lock b/Cargo.lock index 060ae30b32..2246634aaa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2658,11 +2658,13 @@ dependencies = [ "chrono", "rusqlite", "rustix", + "serde", "serde_json", "sha2", "sqlbuild-config", "sqlbuild-core", "tempfile", + "utoipa", ] [[package]] diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs b/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs index ef93a4a15a..a0f39f3324 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs @@ -3,6 +3,7 @@ use axum::extract::{Path, State}; use axum::http::StatusCode; use axum::{Extension, Json}; +use sqlbuild_recall::records::models::OutputChunk; use uuid::Uuid; use crate::api::errors::ApiError; @@ -10,8 +11,9 @@ use crate::api::models::{HeartbeatResponse, MutationResponse}; use crate::credentials::models::{CredentialScope, TokenIdentity}; use crate::runs::main::change_run::change_run; use crate::runs::main::heartbeat_run::heartbeat_run; +use crate::runs::main::publish_run_output::publish_run_output; use crate::runs::main::run_execution::run_execution; -use crate::runs::models::{RunAuthority, RunChangeRequest, RunExecution}; +use crate::runs::models::{OutputPublication, RunAuthority, RunChangeRequest, RunExecution}; use crate::store::models::ControlStore; #[utoipa::path(get, path = "/api/runs/{run_id}/execution", params(("run_id" = Uuid, Path)), @@ -68,6 +70,23 @@ pub(crate) async fn heartbeat( })) } +#[utoipa::path(post, path = "/api/runs/{run_id}/output", params(("run_id" = Uuid, Path)), + request_body = Vec, + responses((status = 200, body = OutputPublication), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Run not authorized")), + security(("bearer" = [])))] +pub(crate) async fn output( + State(store): State, + Extension(identity): Extension, + Path(run_id): Path, + Json(chunks): Json>, +) -> Result, ApiError> { + let claim_id = require_worker(&identity, run_id)?; + Ok(Json( + publish_run_output(&store, (run_id, claim_id), &chunks).await?, + )) +} + fn require_worker(identity: &TokenIdentity, run_id: Uuid) -> Result { if let CredentialScope::Run { run_id: permitted, diff --git a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs index 99927066e8..2dd444bef6 100644 --- a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs +++ b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs @@ -25,6 +25,7 @@ pub async fn serve_api( .route("/api/runs/{run_id}/execution", get(runs::execution)) .route("/api/runs/{run_id}/events", post(runs::report)) .route("/api/runs/{run_id}/heartbeat", post(runs::heartbeat)) + .route("/api/runs/{run_id}/output", post(runs::output)) .layer(DefaultBodyLimit::max(64 * 1024 * 1024)) .route_layer(middleware::from_fn_with_state(store.clone(), authenticate)); let application = protected diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index c0ec5c75a9..6120833396 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -10,7 +10,8 @@ use crate::api::_helpers::{runs, snapshots}; snapshots::current, runs::execution, runs::report, - runs::heartbeat + runs::heartbeat, + runs::output ))] pub(crate) struct ControlApi; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs index 75a1590516..5bfdb31c61 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs @@ -2,5 +2,6 @@ pub(crate) mod changes; pub(crate) mod claims; pub(crate) mod events; pub(crate) mod nodes; +pub(crate) mod output; pub(crate) mod projection; pub(crate) mod publish_change; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/output.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/output.rs new file mode 100644 index 0000000000..d6daa4602a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/output.rs @@ -0,0 +1,80 @@ +//! Resolve output ownership from the physical claim and normalize Recall input before append. + +use chrono::{DateTime, SecondsFormat}; +use sqlbuild_recall::records::main::output_row::output_row; +use sqlbuild_recall::records::models::{OutputChunk, RecallRow, RowEnvelope}; +use uuid::Uuid; + +use crate::runs::constants::{OUTPUT_BATCH_BYTES, OUTPUT_BATCH_CHUNKS}; +use crate::runs::models::OutputMetadata; +use crate::store::errors::ControlError; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn output_envelope( + transaction: ControlTransaction, + (run_id, claim_id): (Uuid, Uuid), +) -> Result<(ControlTransaction, RowEnvelope), ControlError> { + let (transaction, rows) = query( + transaction, + "SELECT l.claimed_at, s.sqb_version FROM sqb_run_liveness l \ + JOIN sqb_runs r ON r.run_id=l.run_id JOIN sqb_snapshots s ON s.snapshot_id=r.snapshot_id \ + WHERE l.run_id=$1 AND l.worker_instance=$2", + vec![Parameter::Id(run_id), Parameter::Id(claim_id)], + ) + .await?; + let metadata: OutputMetadata = decode_row(rows.into_iter().next().ok_or_else(|| { + ControlError::Invalid("output does not belong to an active run claim".into()) + })?)?; + Ok(( + transaction, + RowEnvelope { + invocation_id: format!("worker/{claim_id}"), + invocation_started_at: metadata + .claimed_at + .to_rfc3339_opts(SecondsFormat::Micros, true), + producer_version: metadata.sqb_version, + }, + )) +} + +/// Keep MAX(sequence) + 1 representable for Recall completeness accounting. +pub(crate) fn output_rows( + envelope: &RowEnvelope, + chunks: &[OutputChunk], +) -> Result, ControlError> { + if chunks.len() > OUTPUT_BATCH_CHUNKS + || chunks.iter().map(|chunk| chunk.data.len()).sum::() > OUTPUT_BATCH_BYTES + { + return Err(ControlError::Invalid( + "output batch exceeds its limit".into(), + )); + } + let mut rows = Vec::with_capacity(chunks.len()); + for chunk in chunks { + let mut chunk = chunk.clone(); + let mut integers = vec![chunk.sequence, chunk.byte_offset, chunk.dropped_before]; + if let Some(omitted) = chunk.omitted { + integers.extend([omitted.bytes, omitted.sequences]); + if !chunk.data.is_empty() || omitted.sequences == 0 { + return Err(ControlError::Invalid( + "invalid output omission marker".into(), + )); + } + } + if chunk.sequence >= i64::MAX as u64 + || integers.iter().any(|value| i64::try_from(*value).is_err()) + { + return Err(ControlError::Invalid( + "output counter is out of range".into(), + )); + } + chunk.occurred_at = DateTime::parse_from_rfc3339(&chunk.occurred_at) + .map_err(|_| ControlError::Invalid("invalid output timestamp".into()))? + .to_utc() + .to_rfc3339_opts(SecondsFormat::Micros, true); + rows.push(output_row(envelope, &chunk)); + } + Ok(rows) +} diff --git a/crates/sqlbuild-orchestration/src/runs/constants.rs b/crates/sqlbuild-orchestration/src/runs/constants.rs index 801d9c9edb..f53d82c1b5 100644 --- a/crates/sqlbuild-orchestration/src/runs/constants.rs +++ b/crates/sqlbuild-orchestration/src/runs/constants.rs @@ -5,3 +5,6 @@ pub(crate) const NODE_RUNNING: &str = "running"; pub(crate) const HEARTBEAT_TIMEOUT_SECONDS: i64 = 45; pub(crate) const MONITOR_BATCH_SIZE: usize = 100; + +pub(crate) const OUTPUT_BATCH_CHUNKS: usize = 256; +pub(crate) const OUTPUT_BATCH_BYTES: usize = 1_048_576; diff --git a/crates/sqlbuild-orchestration/src/runs/main/mod.rs b/crates/sqlbuild-orchestration/src/runs/main/mod.rs index ffceba28a4..46615363df 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/mod.rs @@ -4,6 +4,7 @@ pub mod heartbeat_run; pub mod list_run_nodes; pub mod list_runs; pub mod monitor_runs; +pub mod publish_run_output; pub(crate) mod queue_run; pub mod request_run; pub mod run_execution; diff --git a/crates/sqlbuild-orchestration/src/runs/main/publish_run_output.rs b/crates/sqlbuild-orchestration/src/runs/main/publish_run_output.rs new file mode 100644 index 0000000000..35633a3a0e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/publish_run_output.rs @@ -0,0 +1,36 @@ +//! Exact-byte Recall output is namespaced by the authenticated physical claim, never a body ID. + +use sqlbuild_recall::records::models::OutputChunk; +use uuid::Uuid; + +use crate::runs::_helpers::output::{output_envelope, output_rows}; +use crate::runs::models::OutputPublication; +use crate::store::errors::ControlError; +use crate::store::main::append_recall::append_recall; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{ControlStore, TransactionMode}; + +/// The active-claim check and the complete batch commit share a serialized write transaction. +pub async fn publish_run_output( + store: &ControlStore, + (run_id, claim_id): (Uuid, Uuid), + chunks: &[OutputChunk], +) -> Result { + let transaction = begin(store, TransactionMode::Write).await?; + let (mut transaction, envelope) = output_envelope(transaction, (run_id, claim_id)).await?; + let mut published = OutputPublication::default(); + for row in output_rows(&envelope, chunks)? { + let outcome; + (transaction, outcome) = append_recall(transaction, &row).await?; + if outcome.inserted { + published.inserted += 1; + } else if outcome.conflict.is_some() { + published.conflicts += 1; + } else { + published.duplicates += 1; + } + } + commit(transaction).await?; + Ok(published) +} diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index 636dc6891a..bf98fc240b 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -264,3 +264,17 @@ pub(crate) struct ChangingRun { pub(crate) cancel_requested_at: Option>, pub(crate) node_counts_json: BTreeMap, } + +/// Differing retries retain conflict evidence while preserving the first output bytes. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct OutputPublication { + pub inserted: u32, + pub duplicates: u32, + pub conflicts: u32, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct OutputMetadata { + pub(crate) claimed_at: DateTime, + pub(crate) sqb_version: String, +} diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/client.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/client.rs index 79bfd1f284..813c99c3ac 100644 --- a/crates/sqlbuild-orchestration/src/worker/_helpers/client.rs +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/client.rs @@ -1,10 +1,11 @@ //! A worker's authenticated HTTP requests carry only its own run and claim. +use sqlbuild_recall::records::models::OutputChunk; use uuid::Uuid; use crate::api::main::client_response::client_response; use crate::api::models::{HeartbeatResponse, MutationResponse}; -use crate::runs::models::{RunChange, RunChangeRequest, RunExecution}; +use crate::runs::models::{OutputPublication, RunChange, RunChangeRequest, RunExecution}; use crate::worker::errors::WorkerError; use crate::worker::models::WorkerClient; @@ -45,6 +46,20 @@ impl WorkerClient { Ok(result.applied) } + /// Retry the same chunks after an uncertain response; differing retries never overwrite bytes. + pub async fn publish_output( + &self, + chunks: &[OutputChunk], + ) -> Result { + Ok(client_response( + self.connection + .client + .post(format!("{}output", self.run_url)) + .json(chunks), + ) + .await?) + } + /// Run independently from warehouse execution; cancellation is a cooperative request. pub async fn heartbeat(&self) -> Result { let result: HeartbeatResponse = client_response( diff --git a/crates/sqlbuild-orchestration/tests/api.rs b/crates/sqlbuild-orchestration/tests/api.rs index 14ce5705f3..9c411b15ae 100644 --- a/crates/sqlbuild-orchestration/tests/api.rs +++ b/crates/sqlbuild-orchestration/tests/api.rs @@ -1,5 +1,7 @@ #[path = "api/helpers.rs"] mod helpers; +#[path = "api/output/helpers.rs"] +mod output_helpers; #[path = "runs/helpers.rs"] mod run_helpers; #[path = "store/helpers.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/output/helpers.rs b/crates/sqlbuild-orchestration/tests/api/output/helpers.rs new file mode 100644 index 0000000000..6c628f05de --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/output/helpers.rs @@ -0,0 +1,178 @@ +//! Worker output contract helpers inspect durable bytes on both databases. + +use serde_json::json; +use sqlbuild_orchestration::runs::models::OutputPublication; +use sqlbuild_orchestration::store::main::read_recall_view::read_recall_view; +use sqlbuild_orchestration::store::models::ControlStore; +use sqlbuild_orchestration::worker::errors::WorkerError; +use sqlbuild_orchestration::worker::models::WorkerClient; +use sqlbuild_recall::records::models::{OmittedOutput, OutputChunk, OutputStream}; +use uuid::Uuid; + +pub(crate) fn chunk(sequence: u64, stream: OutputStream, data: &[u8]) -> OutputChunk { + OutputChunk { + sequence, + stream, + occurred_at: "2026-01-02T03:04:05Z".into(), + data: data.to_vec(), + byte_offset: 0, + dropped_before: 0, + omitted: None, + } +} + +async fn stored_bytes(store: &ControlStore, invocation: &str) -> Vec<(i64, Vec)> { + let sql = "SELECT sequence, data FROM recall_output WHERE invocation_id=$1 ORDER BY sequence"; + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!("SQLite store") + }; + let connection = rusqlite::Connection::open(path).expect("inspect SQLite bytes"); + let mut statement = connection.prepare(sql).expect("output query"); + return statement + .query_map([invocation], |row| Ok((row.get(0)?, row.get(1)?))) + .expect("stored rows") + .collect::>() + .expect("decode bytes"); + }; + sqlx_core::query_as::query_as(sql) + .bind(invocation) + .fetch_all(pool) + .await + .expect("inspect PostgreSQL bytes") +} + +pub(crate) async fn output_contract(store: &ControlStore, worker: &WorkerClient, claim_id: Uuid) { + let invocation = format!("worker/{claim_id}"); + let first = chunk(0, OutputStream::Stdout, b"orders\0\xff\n"); + let warning = chunk(1, OutputStream::Stderr, b"warning\xc3\x28"); + let omission = OutputChunk { + omitted: Some(OmittedOutput { + bytes: 43, + sequences: 2, + }), + byte_offset: first.data.len() as u64, + dropped_before: 1, + ..chunk(3, OutputStream::Stdout, b"") + }; + let chunks = [first.clone(), warning.clone(), omission]; + assert_eq!( + worker + .publish_output(&chunks) + .await + .expect("publish exact bytes"), + OutputPublication { + inserted: 3, + duplicates: 0, + conflicts: 0 + } + ); + assert_eq!( + worker + .publish_output(&chunks) + .await + .expect("retry accepted output"), + OutputPublication { + inserted: 0, + duplicates: 3, + conflicts: 0 + } + ); + let mut equivalent = first.clone(); + equivalent.occurred_at = "2026-01-02T04:04:05+01:00".into(); + assert_eq!( + worker + .publish_output(&[equivalent]) + .await + .expect("equivalent timestamp retry") + .duplicates, + 1 + ); + let mut conflict = first.clone(); + conflict.data = b"different".to_vec(); + assert_eq!( + worker + .publish_output(&[conflict]) + .await + .expect("record retry conflict") + .conflicts, + 1 + ); + assert_eq!( + stored_bytes(store, &invocation).await, + vec![ + (0, first.data.clone()), + (1, warning.data.clone()), + (3, vec![]) + ] + ); + let conflicts = read_recall_view(store, ("conflicts", &invocation), 10) + .await + .expect("durable conflicts"); + assert_eq!(conflicts.len(), 1); + let counts = read_recall_view(store, ("output_invocations", &invocation), 10) + .await + .expect("output accounting"); + assert_eq!(counts[0]["stdout_bytes"], json!(first.data.len())); + assert_eq!(counts[0]["stderr_bytes"], json!(warning.data.len())); + assert_eq!(counts[0]["truncated_bytes"], json!(43)); + + let valid = chunk(4, OutputStream::Stdout, b"must roll back"); + let mut invalid = chunk(5, OutputStream::Stderr, b"invalid timestamp"); + invalid.occurred_at = "not-a-time".into(); + for rejected in [ + vec![valid.clone(), invalid], + vec![OutputChunk { + sequence: u64::MAX, + ..valid.clone() + }], + vec![OutputChunk { + sequence: i64::MAX as u64, + ..valid.clone() + }], + vec![OutputChunk { + byte_offset: u64::MAX, + ..valid.clone() + }], + vec![OutputChunk { + dropped_before: u64::MAX, + ..valid.clone() + }], + vec![OutputChunk { + omitted: Some(OmittedOutput { + bytes: 1, + sequences: 0, + }), + ..valid.clone() + }], + vec![valid.clone(); 257], + vec![chunk(4, OutputStream::Stdout, &vec![0; 1_048_577])], + ] { + assert!(matches!( + worker.publish_output(&rejected).await, + Err(WorkerError::Rejected(reqwest::StatusCode::BAD_REQUEST)) + )); + } + assert_eq!( + stored_bytes(store, &invocation).await.len(), + 3, + "invalid batches are atomic" + ); + assert_eq!( + worker + .publish_output(&vec![first; 256]) + .await + .expect("chunk count boundary") + .duplicates, + 256 + ); + let boundary = chunk(4, OutputStream::Stdout, &vec![0; 1_048_576]); + assert_eq!( + worker + .publish_output(&[boundary]) + .await + .expect("byte count boundary") + .inserted, + 1 + ); +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_contract.rs b/crates/sqlbuild-orchestration/tests/api/test_contract.rs index bea9439da0..31668c3a0b 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contract.rs @@ -374,6 +374,39 @@ fn given_a_real_http_server_when_locations_and_workers_report_then_scopes_retrie .status(), StatusCode::FORBIDDEN ); + for (output_run, credential) in [ + (Uuid::now_v7(), &worker.bearer), + (run_id, &location_token.bearer), + ] { + assert_eq!( + client + .post(format!("{}/api/runs/{output_run}/output", api.url)) + .bearer_auth(credential) + .json(&json!([])) + .send() + .await + .expect("output scope denial") + .status(), + StatusCode::FORBIDDEN + ); + } + let mut forged = json!(crate::output_helpers::chunk( + 0, + sqlbuild_recall::records::models::OutputStream::Stdout, + b"forged" + )); + forged["invocation_id"] = json!("scheduler/another-run"); + assert_eq!( + client + .post(format!("{}/api/runs/{run_id}/output", api.url)) + .bearer_auth(&worker.bearer) + .json(&json!([forged])) + .send() + .await + .expect("body cannot choose its output namespace") + .status(), + StatusCode::UNPROCESSABLE_ENTITY + ); let cancellation = RunChangeRequest { event_id: Uuid::now_v7(), change: RunChange::CancelRequested { diff --git a/crates/sqlbuild-orchestration/tests/api/test_worker.rs b/crates/sqlbuild-orchestration/tests/api/test_worker.rs index 5cb27bd833..83fa332103 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_worker.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_worker.rs @@ -23,6 +23,7 @@ use sqlbuild_orchestration::worker::main::start_worker::start_worker; use uuid::Uuid; use crate::helpers::{start_api, stop_api}; +use crate::output_helpers::{chunk, output_contract}; use crate::run_helpers::{queue, request, snapshot}; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; @@ -140,6 +141,7 @@ fn given_a_claimed_run_when_a_worker_connects_then_verification_precedes_start_a let session = start_worker(&worker, &baseline) .await .expect("verified start"); + output_contract(store, &worker, identity.1).await; assert_eq!(session.execution.claim, claim, "{}", test_case.description); assert_eq!( start_worker(&worker, &baseline) @@ -206,6 +208,16 @@ fn given_a_claimed_run_when_a_worker_connects_then_verification_precedes_start_a ) .await .expect("finish"); + assert!(matches!( + worker + .publish_output(&[chunk( + 5, + sqlbuild_recall::records::models::OutputStream::Stdout, + b"late" + )]) + .await, + Err(WorkerError::Rejected(reqwest::StatusCode::UNAUTHORIZED)) + )); let runs = list_runs(store, (location, "development"), 10) .await .expect("completed"); diff --git a/crates/sqlbuild-recall/Cargo.toml b/crates/sqlbuild-recall/Cargo.toml index cfabda69e4..8b54b4542e 100644 --- a/crates/sqlbuild-recall/Cargo.toml +++ b/crates/sqlbuild-recall/Cargo.toml @@ -11,10 +11,12 @@ rust-version.workspace = true blake3.workspace = true chrono.workspace = true rusqlite.workspace = true +serde.workspace = true serde_json.workspace = true sha2.workspace = true sqlbuild-config.workspace = true sqlbuild-core.workspace = true +utoipa.workspace = true [target.'cfg(unix)'.dependencies] rustix.workspace = true diff --git a/crates/sqlbuild-recall/src/records/models.rs b/crates/sqlbuild-recall/src/records/models.rs index e99a1cf280..a7798e3e5c 100644 --- a/crates/sqlbuild-recall/src/records/models.rs +++ b/crates/sqlbuild-recall/src/records/models.rs @@ -62,7 +62,10 @@ pub struct RowEnvelope { } /// A process output stream. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, utoipa::ToSchema, +)] +#[serde(rename_all = "snake_case")] pub enum OutputStream { Stdout, Stderr, @@ -89,7 +92,8 @@ impl OutputStream { } /// One exact-byte output chunk. -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] pub struct OutputChunk { /// Per invocation, from 0, shared by both streams. pub sequence: u64, @@ -105,7 +109,18 @@ pub struct OutputChunk { } /// Output of one stream that was not recorded because it exceeded `max_output_bytes`. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[derive( + Clone, + Copy, + Debug, + Default, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + utoipa::ToSchema, +)] +#[serde(deny_unknown_fields)] pub struct OmittedOutput { pub bytes: u64, /// Sequences this gap accounts for, its own included. From d6e441a9ec54c4da4450e239b321f1ea978aee99 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 02:23:59 +0000 Subject: [PATCH 15/59] feat: bootstrap audited locations and credentials natively Add privileged native administration with JSON results, bounded location credentials and atomic security audit. Detect lost result delivery without claiming committed mutations rolled back. Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- Cargo.lock | 1 + crates/sqlbuild-cli/Cargo.toml | 1 + .../sqlbuild-cli/src/argv/command_table.json | 199 ++++++++++++++++++ .../src/native_commands/_helpers/decision.rs | 36 +++- .../src/native_commands/_helpers/execution.rs | 21 +- .../src/native_commands/constants.rs | 1 + .../src/native_commands/main/_run_admin.rs | 45 ++++ .../src/native_commands/main/_run_control.rs | 15 +- .../src/native_commands/main/mod.rs | 1 + .../main/run_control_invocation.rs | 57 +++-- .../src/native_commands/models.rs | 10 + crates/sqlbuild-cli/tests/native_commands.rs | 4 + .../tests/native_commands/admin/helpers.rs | 55 +++++ .../tests/native_commands/admin/test_admin.rs | 182 ++++++++++++++++ .../tests/native_commands/admin/test_types.rs | 14 ++ .../tests/native_commands/test_control.rs | 47 +++++ .../tests/native_commands/worker/helpers.rs | 16 +- .../main/create_location.rs | 28 ++- .../src/credentials/main/mod.rs | 1 + .../src/store/main/mod.rs | 1 - crates/sqlbuild-orchestration/tests/api.rs | 2 + .../tests/api/test_contract.rs | 2 +- .../tests/api/test_worker.rs | 2 +- .../tests/credentials/helpers.rs | 18 ++ .../tests/credentials/locations/helpers.rs | 24 +++ .../tests/credentials/test_contract.rs | 9 +- .../sqlbuild-orchestration/tests/platform.rs | 2 + .../tests/platform/test_watchdog.rs | 2 +- crates/sqlbuild-orchestration/tests/runs.rs | 2 + .../tests/runs/test_claims.rs | 2 +- .../tests/runs/test_coalesce.rs | 2 +- .../tests/runs/test_faults.rs | 2 +- .../tests/runs/test_lifecycle.rs | 2 +- .../tests/runs/test_monitor.rs | 2 +- .../sqlbuild-orchestration/tests/scheduler.rs | 2 + .../tests/scheduler/ticks/helpers.rs | 2 +- .../tests/scheduler/ticks/test_loop.rs | 2 +- .../tests/scheduler/ticks/test_triggers.rs | 2 +- .../sqlbuild-orchestration/tests/snapshots.rs | 2 + .../tests/snapshots/helpers.rs | 2 +- 40 files changed, 760 insertions(+), 60 deletions(-) create mode 100644 crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs create mode 100644 crates/sqlbuild-cli/tests/native_commands/admin/helpers.rs create mode 100644 crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs create mode 100644 crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs rename crates/sqlbuild-orchestration/src/{store => credentials}/main/create_location.rs (57%) create mode 100644 crates/sqlbuild-orchestration/tests/credentials/locations/helpers.rs diff --git a/Cargo.lock b/Cargo.lock index 2246634aaa..7f825f1ccc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2490,6 +2490,7 @@ name = "sqlbuild-cli" version = "0.186.2" dependencies = [ "blake3", + "chrono", "ctrlc", "getrandom 0.4.3", "reqwest", diff --git a/crates/sqlbuild-cli/Cargo.toml b/crates/sqlbuild-cli/Cargo.toml index 686080a599..fd9d3be80e 100644 --- a/crates/sqlbuild-cli/Cargo.toml +++ b/crates/sqlbuild-cli/Cargo.toml @@ -17,6 +17,7 @@ path = "src/bin/sqlbuild.rs" [dependencies] blake3.workspace = true +chrono.workspace = true getrandom.workspace = true serde.workspace = true serde_json = { workspace = true, features = ["arbitrary_precision"] } diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index ec58647f34..c5dad55e42 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -78,6 +78,200 @@ "nargs": "A...", "group": 0, "parsers": [ + { + "name": "admin", + "parser": { + "prog": "sqb admin", + "description": "Privileged local administration using direct control database access. Results are JSON; issued bearer credentials appear only once on stdout.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--database-env" + ], + "dest": "control_database_env", + "default": "SQLBUILD_CONTROL_DATABASE_URL", + "help": "environment variable holding the control database URL (default: SQLBUILD_CONTROL_DATABASE_URL; SQLite when unset)", + "group": 1 + }, + { + "kind": "parsers", + "dest": "admin_command", + "nargs": "A...", + "required": true, + "group": 0, + "parsers": [ + { + "name": "create-location", + "parser": { + "prog": "sqb admin create-location", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "NAME", + "required": true, + "help": "location label; identity is assigned by the control store", + "group": 0 + } + ] + } + }, + { + "name": "issue-location-token", + "parser": { + "prog": "sqb admin issue-location-token", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "LOCATION_ID", + "required": true, + "help": "location UUID", + "group": 0 + }, + { + "kind": "store", + "options": [ + "--expires-at" + ], + "dest": "admin_expires_at", + "required": true, + "help": "future expiry as an RFC3339 timestamp with timezone", + "group": 1 + } + ] + } + }, + { + "name": "revoke-token", + "parser": { + "prog": "sqb admin revoke-token", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "TOKEN_ID", + "required": true, + "help": "credential UUID, never its bearer secret", + "group": 0 + } + ] + } + } + ], + "choices_help": [ + { + "dest": "create-location", + "metavar": "create-location", + "help": "register a location with an atomic security audit record" + }, + { + "dest": "issue-location-token", + "metavar": "issue-location-token", + "help": "issue a bounded location credential, returned once as JSON" + }, + { + "dest": "revoke-token", + "metavar": "revoke-token", + "help": "revoke a credential by UUID" + } + ] + } + ] + } + }, { "name": "server", "parser": { @@ -7719,6 +7913,11 @@ } ], "choices_help": [ + { + "dest": "admin", + "metavar": "admin", + "help": "administer locations and credentials through privileged database access" + }, { "dest": "server", "metavar": "server", diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index 21c6342413..8ccde9b2a8 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -3,7 +3,9 @@ use std::net::{IpAddr, SocketAddr}; use std::path::PathBuf; +use chrono::{DateTime, Utc}; use sqlbuild_orchestration::store::errors::ControlError; +use uuid::Uuid; use crate::argv::constants::{JSON_OPTION, REVIEW_PROG}; use crate::argv::models::{ArgvOutcome, Namespace, Value}; @@ -15,7 +17,8 @@ use crate::native_commands::constants::{ }; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::models::{ - CostOptions, NativeCommand, NativeInvocation, RulesAction, RulesOptions, SkillsOptions, + ControlAdmin, CostOptions, NativeCommand, NativeInvocation, RulesAction, RulesOptions, + SkillsOptions, }; use sqlbuild_core::cost_command::constants::DESCENDING_ORDER; @@ -39,6 +42,37 @@ pub(crate) fn flag(namespace: &Namespace, name: &str) -> bool { namespace.get(name) == Some(&Value::Bool(true)) } +pub(crate) fn control_admin(namespace: &Namespace) -> Result { + let value = text(namespace, "admin_subject").unwrap_or_default(); + if text(namespace, "admin_command").as_deref() == Some("create-location") { + if value.trim().is_empty() { + return Err(ControlError::Invalid("location name is required".into())); + } + return Ok(ControlAdmin::CreateLocation(value)); + } + let id = Uuid::parse_str(&value) + .map_err(|_| ControlError::Invalid("a valid location or token UUID is required".into()))?; + match text(namespace, "admin_command").as_deref() { + Some("issue-location-token") => { + let expiry = DateTime::parse_from_rfc3339( + &text(namespace, "admin_expires_at").unwrap_or_default(), + ) + .map_err(|_| ControlError::Invalid("--expires-at must be an RFC3339 timestamp".into()))? + .with_timezone(&Utc); + if expiry <= Utc::now() { + return Err(ControlError::Invalid( + "token expiry must be in the future".into(), + )); + } + Ok(ControlAdmin::IssueLocationToken(id, expiry)) + } + Some("revoke-token") => Ok(ControlAdmin::RevokeToken(id)), + _ => Err(ControlError::Invalid( + "an administration command is required".into(), + )), + } +} + pub(crate) fn control_address( namespace: &Namespace, standalone: bool, diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index ac2f27678e..0a84181be4 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -5,16 +5,20 @@ use std::path::{Path, PathBuf}; use std::time::SystemTime; use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use sqlbuild_orchestration::store::main::open_store::open_store; +use sqlbuild_orchestration::store::models::ControlStore; use tokio::sync::watch; use tokio::task::JoinSet; -use crate::argv::models::Stream; +use crate::argv::models::{Namespace, Stream}; use crate::compute_logs::constants::DEFAULT_RETENTION_COUNT; use crate::compute_logs::main::abandon_capture::abandon_capture; use crate::compute_logs::main::finish_capture::finish_capture; use crate::compute_logs::main::open_log_root::open_log_root; use crate::compute_logs::main::start_capture::start_capture; use crate::compute_logs::models::{CaptureEnd, CaptureStart, OpenCapture}; +use crate::native_commands::_helpers::decision::control_database; use crate::native_commands::_helpers::init::run_init; use crate::native_commands::_helpers::package::installed_package; use crate::native_commands::_helpers::paths::project_location; @@ -29,6 +33,21 @@ use crate::native_commands::models::{ CommandOutput, InstalledPackage, LostStdout, NativeCommand, NativeInvocation, ProjectLocation, }; +/// Open and migrate the privileged store, reporting progress without contaminating JSON stdout. +pub(crate) async fn open_control_store( + namespace: &Namespace, +) -> Result { + let url = control_database(namespace)?; + drop(write_output(&[( + Stream::Stderr, + "Opening control store...\n", + )])); + let store = open_store(&url).await?; + migrate_store(&store).await?; + drop(write_output(&[(Stream::Stderr, "Control store ready\n")])); + Ok(store) +} + async fn shutdown_signal() -> Result<(), ControlError> { #[cfg(unix)] { diff --git a/crates/sqlbuild-cli/src/native_commands/constants.rs b/crates/sqlbuild-cli/src/native_commands/constants.rs index 7f993a2320..74c0358e56 100644 --- a/crates/sqlbuild-cli/src/native_commands/constants.rs +++ b/crates/sqlbuild-cli/src/native_commands/constants.rs @@ -43,6 +43,7 @@ pub(crate) const INIT_COMMAND: &str = "init"; pub(crate) const SKILLS_COMMAND: &str = "skills"; pub(crate) const SERVER_COMMAND: &str = "server"; pub(crate) const SCHEDULER_COMMAND: &str = "scheduler"; +pub(crate) const ADMIN_COMMAND: &str = "admin"; pub(crate) const RULES_COMMAND: &str = "rules"; pub(crate) const RULES_LIST_ACTION: &str = "list"; pub(crate) const RULES_SHOW_ACTION: &str = "show"; diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs new file mode 100644 index 0000000000..06b21d9aea --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs @@ -0,0 +1,45 @@ +//! Direct database ownership authorizes bootstrap, not an untrusted HTTP request or supplied role. + +use serde_json::{Value, json}; +use sqlbuild_orchestration::credentials::main::create_location::create_location; +use sqlbuild_orchestration::credentials::main::issue_token::issue_token; +use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; +use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; +use sqlbuild_orchestration::store::errors::ControlError; +use uuid::Uuid; + +use crate::argv::models::Namespace; +use crate::native_commands::_helpers::decision::control_admin; +use crate::native_commands::_helpers::execution::open_control_store; +use crate::native_commands::models::ControlAdmin; + +pub(crate) async fn run_admin(namespace: &Namespace) -> Result { + let action = control_admin(namespace)?; + let store = open_control_store(namespace).await?; + let context = AuditContext { + actor: "local-control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + match action { + ControlAdmin::CreateLocation(name) => { + let id = create_location(&store, &name, &context).await?; + Ok(json!({"location_id": id, "name": name})) + } + ControlAdmin::IssueLocationToken(location_id, expires_at) => { + let token = issue_token( + &store, + &context, + CredentialScope::Location { location_id }, + expires_at, + ) + .await?; + Ok(json!(token)) + } + ControlAdmin::RevokeToken(token_id) => { + let revoked = revoke_token(&store, &context, token_id).await?; + Ok(json!({"token_id": token_id, "revoked": revoked})) + } + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs index 06f74ae5df..1f39c05c8b 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs @@ -5,14 +5,12 @@ use sqlbuild_orchestration::platform::main::run_watchdog::run_watchdog; use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler; use sqlbuild_orchestration::scheduler::models::SchedulerState; use sqlbuild_orchestration::store::errors::ControlError; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; -use sqlbuild_orchestration::store::main::open_store::open_store; use tokio::sync::watch; use uuid::Uuid; use crate::argv::models::{Namespace, Stream}; -use crate::native_commands::_helpers::decision::{control_address, control_database, flag}; -use crate::native_commands::_helpers::execution::supervise_control_roles; +use crate::native_commands::_helpers::decision::{control_address, flag}; +use crate::native_commands::_helpers::execution::{open_control_store, supervise_control_roles}; use crate::native_commands::_helpers::terminal::write_output; async fn stopped(mut shutdown: watch::Receiver) { @@ -28,14 +26,7 @@ pub(crate) async fn run_control( standalone: bool, ) -> Result<(), ControlError> { let address = control_address(namespace, standalone)?; - let url = control_database(namespace)?; - drop(write_output(&[( - Stream::Stderr, - "Opening control store...\n", - )])); - let store = open_store(&url).await?; - migrate_store(&store).await?; - drop(write_output(&[(Stream::Stderr, "Control store ready\n")])); + let store = open_control_store(namespace).await?; let (stop, shutdown) = watch::channel(false); let (_changes, changes) = watch::channel(()); let mut tasks = tokio::task::JoinSet::new(); diff --git a/crates/sqlbuild-cli/src/native_commands/main/mod.rs b/crates/sqlbuild-cli/src/native_commands/main/mod.rs index c52a4a29a2..e93969bdc4 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/mod.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/mod.rs @@ -1,4 +1,5 @@ pub(super) mod _cost_report_options; +pub(super) mod _run_admin; pub(super) mod _run_control; pub(super) mod _run_cost; pub(crate) mod parse_native_invocation; diff --git a/crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs index 699407935b..6ff78bf47b 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs @@ -5,12 +5,16 @@ use sqlbuild_orchestration::api::main::openapi_document::openapi_document; use crate::argv::models::{Namespace, Stream}; use crate::native_commands::_helpers::decision::{flag, text}; use crate::native_commands::_helpers::terminal::{finish_lost_stdout, write_output}; -use crate::native_commands::constants::{SCHEDULER_COMMAND, SERVER_COMMAND}; +use crate::native_commands::constants::{ADMIN_COMMAND, SCHEDULER_COMMAND, SERVER_COMMAND}; +use crate::native_commands::main::_run_admin::run_admin; use crate::native_commands::main::_run_control::run_control; pub(crate) fn run_control_invocation(namespace: &Namespace) -> Option { let command = text(namespace, "command")?; - if !matches!(command.as_str(), SERVER_COMMAND | SCHEDULER_COMMAND) { + if !matches!( + command.as_str(), + SERVER_COMMAND | SCHEDULER_COMMAND | ADMIN_COMMAND + ) { return None; } if flag(namespace, "control_openapi") { @@ -42,22 +46,45 @@ pub(crate) fn run_control_invocation(namespace: &Namespace) -> Option { return Some(1); } }; - Some( - match runtime.block_on(run_control(namespace, command == SCHEDULER_COMMAND)) { - Ok(()) => { + let admin = command == ADMIN_COMMAND; + let role = if admin { "administration" } else { "service" }; + let result = runtime.block_on(async { + if admin { + run_admin(namespace).await.map(Some) + } else { + run_control(namespace, command == SCHEDULER_COMMAND) + .await + .map(|()| None) + } + }); + Some(match result { + Ok(output) => { + let lost = + output.and_then(|value| write_output(&[(Stream::Stdout, &format!("{value}\n"))])); + if admin && lost.is_some() { drop(write_output(&[( Stream::Stderr, - "Control service stopped successfully\n", + "Control administration JSON result delivery failed; changes may already be committed\n", )])); - 0 + return Some(1); } - Err(error) => { - drop(write_output(&[( - Stream::Stderr, - &format!("Control service failed: {error}\n"), - )])); - 1 + let exit_code = finish_lost_stdout(lost.as_ref(), 0); + if exit_code == 0 { + let message = if admin { + "Control administration completed successfully\n" + } else { + "Control service stopped successfully\n" + }; + drop(write_output(&[(Stream::Stderr, message)])); } - }, - ) + exit_code + } + Err(error) => { + drop(write_output(&[( + Stream::Stderr, + &format!("Control {role} failed: {error}\n"), + )])); + 1 + } + }) } diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index f328de4e67..9ae73c2995 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -2,8 +2,18 @@ use std::path::PathBuf; +use chrono::{DateTime, Utc}; +use uuid::Uuid; + use crate::argv::models::Namespace; +/// Privileged local operations require direct ownership of the control database. +pub(crate) enum ControlAdmin { + CreateLocation(String), + IssueLocationToken(Uuid, DateTime), + RevokeToken(Uuid), +} + /// Parsing can finish an invocation before package discovery or any Python handoff. #[derive(Debug)] pub(crate) enum NativeParse { diff --git a/crates/sqlbuild-cli/tests/native_commands.rs b/crates/sqlbuild-cli/tests/native_commands.rs index e93113875b..208f02150b 100644 --- a/crates/sqlbuild-cli/tests/native_commands.rs +++ b/crates/sqlbuild-cli/tests/native_commands.rs @@ -1,5 +1,9 @@ +#[path = "native_commands/admin/helpers.rs"] +mod admin_helpers; #[path = "native_commands/helpers.rs"] mod helpers; +#[path = "native_commands/admin/test_admin.rs"] +mod test_admin; #[path = "native_commands/test_control.rs"] mod test_control; #[path = "native_commands/test_types.rs"] diff --git a/crates/sqlbuild-cli/tests/native_commands/admin/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/admin/helpers.rs new file mode 100644 index 0000000000..ae7e64d9c4 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/admin/helpers.rs @@ -0,0 +1,55 @@ +//! Run the isolated native executable; never include returned bearer bytes in assertion messages. + +use std::process::Command; + +use serde_json::Value; + +use crate::test_types::TestControl; + +pub(crate) fn administer(control: &TestControl, arguments: &[&str]) -> Value { + let output = Command::new( + control + .directory + .path() + .join(format!("sqb{}", std::env::consts::EXE_SUFFIX)), + ) + .arg("admin") + .args(arguments) + .current_dir(control.directory.path()) + .env("PATH", "") + .env_remove("SQLBUILD_CONTROL_DATABASE_URL") + .output() + .expect("native administration process"); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + String::from_utf8_lossy(&output.stderr), + "Opening control store...\nControl store ready\nControl administration completed successfully\n" + ); + serde_json::from_slice(&output.stdout).expect("JSON-only admin stdout") +} + +pub(crate) fn audit_actions(control: &TestControl, bearer: &str) -> Vec { + let connection = + rusqlite::Connection::open(control.directory.path().join(".sqlbuild/control.sqlite3")) + .expect("inspect control audit"); + let mut query = connection.prepare("SELECT action, actor, granting_role, scope_json || before_json || after_json FROM sqb_security_audit ORDER BY occurred_at, event_id") + .expect("audit records"); + query + .query_map([], |row| { + let action: String = row.get(0)?; + let actor: String = row.get(1)?; + let role: String = row.get(2)?; + let payload: String = row.get(3)?; + assert_eq!(actor, "local-control-store-owner"); + assert_eq!(role, "control-store-owner"); + assert!(!payload.contains(bearer)); + Ok(action) + }) + .expect("read audit") + .collect::>() + .expect("decode audit") +} diff --git a/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs b/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs new file mode 100644 index 0000000000..70e03d2970 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs @@ -0,0 +1,182 @@ +//! Native bootstrap credentials authorize real publication and stop working after revocation. + +#[path = "test_types.rs"] +mod test_types; + +use std::collections::BTreeMap; + +use chrono::{Duration, Utc}; +use sqlbuild_orchestration::api::errors::ClientError; +use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; +use sqlbuild_orchestration::snapshots::models::SnapshotInput; +use uuid::Uuid; + +use crate::admin_helpers::{administer, audit_actions}; +use crate::helpers::{api_ready, start_control}; +#[cfg(unix)] +use crate::test_admin::test_types::AdminDeliveryTestCase; +use crate::test_admin::test_types::AdminTestCase; + +#[test] +fn given_a_standalone_control_server_when_bootstrapped_by_an_operator_then_scoped_publication_and_revocation_are_audited() + { + let test_cases = [AdminTestCase { + description: "native operator bootstrap preserves labels and scopes credentials", + location_name: "orders / café", + expected_name: "orders / café", + expected_actions: [ + "location.created", + "location.created", + "token.issued", + "token.revoked", + ], + }]; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("test runtime"); + for test_case in test_cases { + runtime.block_on(async { + let mut control = start_control(&["server", "--port", "0", "--no-scheduler"]); + let url = api_ready(&mut control).await; + let first = administer(&control, &["create-location", test_case.location_name]); + let second = administer(&control, &["create-location", "customers"]); + let location_id = + Uuid::parse_str(first["location_id"].as_str().expect("location UUID")) + .expect("UUID"); + let other_id = Uuid::parse_str(second["location_id"].as_str().expect("location UUID")) + .expect("UUID"); + assert_ne!(location_id, other_id); + assert_eq!( + first["name"], test_case.expected_name, + "{}", + test_case.description + ); + let expiry = (Utc::now() + Duration::hours(1)).to_rfc3339(); + let credential = administer( + &control, + &[ + "issue-location-token", + &location_id.to_string(), + "--expires-at", + &expiry, + ], + ); + let bearer = credential["bearer"].as_str().expect("one-time bearer"); + assert_eq!(credential["identity"]["scope"]["kind"], "location"); + assert_eq!( + credential["identity"]["scope"]["location_id"], + location_id.to_string() + ); + let mut snapshot = SnapshotInput { + location_id, + target: "development".into(), + generation: 1, + git_revision: None, + git_dirty: false, + git_author: None, + image_id: None, + environment_hash: "orders-environment".into(), + sqb_version: env!("CARGO_PKG_VERSION").into(), + nodes: BTreeMap::new(), + documents: BTreeMap::new(), + }; + let published = publish_remote_snapshot(&url, bearer, &snapshot) + .await + .expect("operator-issued credential publishes"); + assert_eq!(published.location_id, location_id); + snapshot.location_id = other_id; + assert!(matches!( + publish_remote_snapshot(&url, bearer, &snapshot).await, + Err(ClientError::Rejected(reqwest::StatusCode::FORBIDDEN)) + )); + snapshot.location_id = location_id; + snapshot.generation = 2; + let token_id = credential["identity"]["token_id"] + .as_str() + .expect("credential ID"); + for expected in [true, false] { + let revoked = administer(&control, &["revoke-token", token_id]); + assert_eq!(revoked["revoked"], expected); + assert_eq!(revoked["token_id"], token_id); + } + assert!(matches!( + publish_remote_snapshot(&url, bearer, &snapshot).await, + Err(ClientError::Rejected(reqwest::StatusCode::UNAUTHORIZED)) + )); + let mut actions = audit_actions(&control, bearer); + actions.sort(); + assert_eq!( + actions, test_case.expected_actions, + "{}", + test_case.description + ); + #[cfg(unix)] + crate::helpers::terminate(&mut control).await; + }); + } +} + +#[cfg(unix)] +#[test] +fn given_a_committed_admin_operation_when_json_output_is_lost_then_all_response_sizes_fail_without_claiming_rollback() + { + let test_cases = [ + AdminDeliveryTestCase { + description: "small response", + name_bytes: 8, + expected_exit: 1, + expected_rows: (1, 1), + }, + AdminDeliveryTestCase { + description: "Python compatibility drop window", + name_bytes: 5000, + expected_exit: 1, + expected_rows: (1, 1), + }, + AdminDeliveryTestCase { + description: "large response", + name_bytes: 9000, + expected_exit: 1, + expected_rows: (1, 1), + }, + ]; + for test_case in test_cases { + let directory = tempfile::tempdir().expect("isolated administration directory"); + let binary = crate::helpers::isolated_binary(directory.path()); + let output = std::process::Command::new(binary) + .args([ + "admin", + "create-location", + &"x".repeat(test_case.name_bytes), + ]) + .current_dir(directory.path()) + .env("PATH", "") + .env_remove("SQLBUILD_CONTROL_DATABASE_URL") + .stdout( + std::fs::OpenOptions::new() + .write(true) + .open("/dev/full") + .expect("failing output device"), + ) + .output() + .expect("native administration"); + assert_eq!( + output.status.code(), + Some(test_case.expected_exit), + "{}", + test_case.description + ); + assert_eq!( + String::from_utf8_lossy(&output.stderr), + "Opening control store...\nControl store ready\nControl administration JSON result delivery failed; changes may already be committed\n", + "{}", + test_case.description + ); + let connection = + rusqlite::Connection::open(directory.path().join(".sqlbuild/control.sqlite3")) + .expect("inspect committed operation"); + let rows: (i64, i64) = connection.query_row("SELECT (SELECT COUNT(*) FROM sqb_locations), (SELECT COUNT(*) FROM sqb_security_audit)", [], |row| Ok((row.get(0)?, row.get(1)?))).expect("location and audit remain committed"); + assert_eq!(rows, test_case.expected_rows, "{}", test_case.description); + } +} diff --git a/crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs b/crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs new file mode 100644 index 0000000000..62aac53bee --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs @@ -0,0 +1,14 @@ +pub(crate) struct AdminTestCase { + pub(crate) description: &'static str, + pub(crate) location_name: &'static str, + pub(crate) expected_name: &'static str, + pub(crate) expected_actions: [&'static str; 4], +} + +#[cfg(unix)] +pub(crate) struct AdminDeliveryTestCase { + pub(crate) description: &'static str, + pub(crate) name_bytes: usize, + pub(crate) expected_exit: i32, + pub(crate) expected_rows: (i64, i64), +} diff --git a/crates/sqlbuild-cli/tests/native_commands/test_control.rs b/crates/sqlbuild-cli/tests/native_commands/test_control.rs index 85a2e4e44c..4f4ffc1f3c 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_control.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_control.rs @@ -21,6 +21,53 @@ fn given_a_standalone_binary_when_printing_help_or_rejecting_invalid_input_then_ expected_stdout_prefix: "usage: sqb ", expected_stderr: "", }, + CommandTestCase { + description: "native privileged administration help", + arguments: &["admin", "--help"], + expected_success: true, + expected_stdout_prefix: "usage: sqb admin ", + expected_stderr: "", + }, + CommandTestCase { + description: "blank location rejected before store initialization", + arguments: &["admin", "create-location", " "], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: location name is required\n", + }, + CommandTestCase { + description: "invalid credential ID rejected before store initialization", + arguments: &["admin", "revoke-token", "not-a-uuid"], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: a valid location or token UUID is required\n", + }, + CommandTestCase { + description: "invalid expiry rejected before store initialization", + arguments: &[ + "admin", + "issue-location-token", + "00000000-0000-0000-0000-000000000001", + "--expires-at", + "tomorrow", + ], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: --expires-at must be an RFC3339 timestamp\n", + }, + CommandTestCase { + description: "expired credential rejected before store initialization", + arguments: &[ + "admin", + "issue-location-token", + "00000000-0000-0000-0000-000000000001", + "--expires-at", + "2000-01-01T00:00:00Z", + ], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: token expiry must be in the future\n", + }, CommandTestCase { description: "invalid port is rejected before store initialization", arguments: &["server", "--port", "65536"], diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs index 2460bfd6ab..c951d675f2 100644 --- a/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs +++ b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs @@ -3,6 +3,7 @@ use std::collections::BTreeMap; use std::time::{Duration, SystemTime}; +use sqlbuild_orchestration::credentials::main::create_location::create_location; use sqlbuild_orchestration::credentials::main::issue_token::issue_token; use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; @@ -16,7 +17,6 @@ use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; use sqlbuild_orchestration::scheduler::models::Overlap; use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; -use sqlbuild_orchestration::store::main::create_location::create_location; use sqlbuild_orchestration::store::main::open_store::open_store; use sqlbuild_orchestration::worker::main::connect_worker::connect_worker; use sqlbuild_orchestration::worker::main::start_worker::start_worker; @@ -34,7 +34,13 @@ pub(crate) async fn start_claimed_worker( let store = open_store(path.to_str().expect("test database path")) .await .expect("native control store"); - let location_id = create_location(&store, "orders", "test-admin") + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let location_id = create_location(&store, "orders", &context) .await .expect("location"); let baseline = SnapshotInput { @@ -93,12 +99,6 @@ pub(crate) async fn start_claimed_worker( .expect("claim") .expect("queued work"); let identity = (run_id, claim.claim_id); - let context = AuditContext { - actor: "test-admin".into(), - granting_role: "admin".into(), - request_id: Uuid::now_v7(), - ip_address: None, - }; let token = issue_token( &store, &context, diff --git a/crates/sqlbuild-orchestration/src/store/main/create_location.rs b/crates/sqlbuild-orchestration/src/credentials/main/create_location.rs similarity index 57% rename from crates/sqlbuild-orchestration/src/store/main/create_location.rs rename to crates/sqlbuild-orchestration/src/credentials/main/create_location.rs index 075de8a140..88d4d8d148 100644 --- a/crates/sqlbuild-orchestration/src/store/main/create_location.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/create_location.rs @@ -1,7 +1,10 @@ //! Create a server-owned location identity independent of its mutable label. +use serde_json::json; use uuid::Uuid; +use crate::credentials::_helpers::audit::audit; +use crate::credentials::models::AuditContext; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; @@ -12,19 +15,30 @@ use crate::store::models::{ControlStore, Parameter, TransactionMode}; pub async fn create_location( store: &ControlStore, name: &str, - actor: &str, + context: &AuditContext, ) -> Result { - if name.trim().is_empty() || actor.trim().is_empty() { - return Err(ControlError::Invalid( - "location name and actor are required".into(), - )); + if name.trim().is_empty() { + return Err(ControlError::Invalid("location name is required".into())); } let mut transaction = begin(store, TransactionMode::Write).await?; let id = Uuid::now_v7(); let now = transaction.now; (transaction, _) = query(transaction, "INSERT INTO sqb_locations (location_id, name, created_by, created_at) VALUES ($1, $2, $3, $4)", - vec![Parameter::Id(id), Parameter::Text(name.into()), Parameter::Text(actor.into()), Parameter::Time(now)]).await?; - commit(transaction).await?; + vec![Parameter::Id(id), Parameter::Text(name.into()), Parameter::Text(context.actor.clone()), Parameter::Time(now)]).await?; + commit( + audit( + transaction, + context, + (Uuid::new_v5(&id, b"created"), "location.created"), + ( + json!({"location_id": id}), + json!(null), + json!({"name": name}), + ), + ) + .await?, + ) + .await?; Ok(id) } diff --git a/crates/sqlbuild-orchestration/src/credentials/main/mod.rs b/crates/sqlbuild-orchestration/src/credentials/main/mod.rs index 6b63069d7b..49e61796cf 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/mod.rs @@ -1,4 +1,5 @@ pub mod authenticate_token; +pub mod create_location; pub mod create_principal; pub mod issue_token; pub mod revoke_token; diff --git a/crates/sqlbuild-orchestration/src/store/main/mod.rs b/crates/sqlbuild-orchestration/src/store/main/mod.rs index d22e12f725..de522badbc 100644 --- a/crates/sqlbuild-orchestration/src/store/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/store/main/mod.rs @@ -2,7 +2,6 @@ pub(crate) mod append_control_event; pub(crate) mod append_recall; pub(crate) mod begin; pub(crate) mod commit; -pub mod create_location; pub(crate) mod decode_row; pub mod migrate_store; pub mod open_store; diff --git a/crates/sqlbuild-orchestration/tests/api.rs b/crates/sqlbuild-orchestration/tests/api.rs index 9c411b15ae..543523edd6 100644 --- a/crates/sqlbuild-orchestration/tests/api.rs +++ b/crates/sqlbuild-orchestration/tests/api.rs @@ -1,5 +1,7 @@ #[path = "api/helpers.rs"] mod helpers; +#[path = "credentials/locations/helpers.rs"] +mod location_helpers; #[path = "api/output/helpers.rs"] mod output_helpers; #[path = "runs/helpers.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/test_contract.rs b/crates/sqlbuild-orchestration/tests/api/test_contract.rs index 31668c3a0b..72ea9f093b 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contract.rs @@ -19,11 +19,11 @@ use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; use sqlbuild_orchestration::snapshots::main::verify_execution::verify_execution; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotRecord}; -use sqlbuild_orchestration::store::main::create_location::create_location; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use uuid::Uuid; use crate::helpers::{start_api, stop_api}; +use crate::location_helpers::create_location; use crate::run_helpers::{queue, request, snapshot}; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; diff --git a/crates/sqlbuild-orchestration/tests/api/test_worker.rs b/crates/sqlbuild-orchestration/tests/api/test_worker.rs index 83fa332103..e488f61f11 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_worker.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_worker.rs @@ -15,7 +15,6 @@ use sqlbuild_orchestration::runs::models::{ use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; use sqlbuild_orchestration::store::errors::ControlError; -use sqlbuild_orchestration::store::main::create_location::create_location; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::worker::errors::WorkerError; use sqlbuild_orchestration::worker::main::connect_worker::connect_worker; @@ -23,6 +22,7 @@ use sqlbuild_orchestration::worker::main::start_worker::start_worker; use uuid::Uuid; use crate::helpers::{start_api, stop_api}; +use crate::location_helpers::create_location; use crate::output_helpers::{chunk, output_contract}; use crate::run_helpers::{queue, request, snapshot}; use crate::store_helpers::{close_test_store, open_test_store, run}; diff --git a/crates/sqlbuild-orchestration/tests/credentials/helpers.rs b/crates/sqlbuild-orchestration/tests/credentials/helpers.rs index e01295e856..f86c76460c 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/helpers.rs @@ -1,5 +1,23 @@ use sqlbuild_orchestration::store::models::ControlStore; +pub(crate) async fn location_state(store: &ControlStore) -> (i64, i64) { + let sql = "SELECT (SELECT COUNT(*) FROM sqb_locations), \ + (SELECT COUNT(*) FROM sqb_security_audit WHERE action='location.created')"; + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + return rusqlite::Connection::open(path) + .expect("inspect locations") + .query_row(sql, [], |row| Ok((row.get(0)?, row.get(1)?))) + .expect("location and audit counts"); + }; + sqlx_core::query_as::query_as(sql) + .fetch_one(pool) + .await + .expect("location and audit counts") +} + pub(crate) async fn stored_hashes(store: &ControlStore) -> Vec { let ControlStore::Postgres(pool) = store else { let ControlStore::Sqlite(path) = store else { diff --git a/crates/sqlbuild-orchestration/tests/credentials/locations/helpers.rs b/crates/sqlbuild-orchestration/tests/credentials/locations/helpers.rs new file mode 100644 index 0000000000..c6edcc2874 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/locations/helpers.rs @@ -0,0 +1,24 @@ +//! Shared location fixtures use the same audited identity operation as operator bootstrap. + +use sqlbuild_orchestration::credentials::models::AuditContext; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::models::ControlStore; +use uuid::Uuid; + +pub(crate) async fn create_location( + store: &ControlStore, + name: &str, + actor: &str, +) -> Result { + sqlbuild_orchestration::credentials::main::create_location::create_location( + store, + name, + &AuditContext { + actor: actor.into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }, + ) + .await +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs index 4c39180fad..3c3c62f087 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs @@ -2,6 +2,7 @@ use std::collections::BTreeMap; use chrono::{Duration, Utc}; use sqlbuild_orchestration::credentials::main::authenticate_token::authenticate_token; +use sqlbuild_orchestration::credentials::main::create_location::create_location; use sqlbuild_orchestration::credentials::main::create_principal::create_principal; use sqlbuild_orchestration::credentials::main::issue_token::issue_token; use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; @@ -9,12 +10,11 @@ use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope, use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::create_location::create_location; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use uuid::Uuid; use crate::fault_helpers::{install_fault, remove_fault}; -use crate::helpers::stored_hashes; +use crate::helpers::{location_state, stored_hashes}; use crate::replica_helpers::second_store; use crate::run_helpers::{queue, request, snapshot}; use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; @@ -69,6 +69,8 @@ fn given_scoped_tokens_when_revoked_or_the_run_ends_then_all_replicas_reject_the .await .is_err() ); + assert!(create_location(store, "orders", &context).await.is_err()); + assert_eq!(location_state(&replica).await, (0, 0)); remove_fault(store, &fault).await; let principal_id = create_principal( store, @@ -77,9 +79,10 @@ fn given_scoped_tokens_when_revoked_or_the_run_ends_then_all_replicas_reject_the ) .await .expect("principal creation and audit rolled back together"); - let location_id = create_location(store, "orders", "test-admin") + let location_id = create_location(store, "orders", &context) .await .expect("location"); + assert_eq!(location_state(&replica).await, (1, 1)); let scopes = [ CredentialScope::Principal { principal_id }, CredentialScope::Location { location_id }, diff --git a/crates/sqlbuild-orchestration/tests/platform.rs b/crates/sqlbuild-orchestration/tests/platform.rs index 7908cbf127..3243ae357c 100644 --- a/crates/sqlbuild-orchestration/tests/platform.rs +++ b/crates/sqlbuild-orchestration/tests/platform.rs @@ -1,5 +1,7 @@ #[path = "platform/helpers.rs"] mod helpers; +#[path = "credentials/locations/helpers.rs"] +mod location_helpers; #[path = "store/replicas/helpers.rs"] mod replica_helpers; #[path = "store/helpers.rs"] diff --git a/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs b/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs index 6954c6bfe0..e119342ea7 100644 --- a/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs +++ b/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs @@ -5,11 +5,11 @@ use sqlbuild_orchestration::platform::main::run_watchdog::run_watchdog; use sqlbuild_orchestration::platform::models::PlatformEvent; use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::create_location::create_location; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use uuid::Uuid; use crate::helpers::{snapshot, wait_for_scans}; +use crate::location_helpers::create_location; use crate::replica_helpers::second_store; use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; use crate::store_test_types::Backend; diff --git a/crates/sqlbuild-orchestration/tests/runs.rs b/crates/sqlbuild-orchestration/tests/runs.rs index 12d6c66d52..2ab841868c 100644 --- a/crates/sqlbuild-orchestration/tests/runs.rs +++ b/crates/sqlbuild-orchestration/tests/runs.rs @@ -6,6 +6,8 @@ mod fault_types; mod helpers; #[path = "store/leases/helpers.rs"] mod lease_helpers; +#[path = "credentials/locations/helpers.rs"] +mod location_helpers; #[path = "runs/monitor/helpers.rs"] mod monitor_helpers; #[path = "store/replicas/helpers.rs"] diff --git a/crates/sqlbuild-orchestration/tests/runs/test_claims.rs b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs index 509232d7f2..fdf5e34d4e 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_claims.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs @@ -6,12 +6,12 @@ use sqlbuild_orchestration::runs::main::list_runs::list_runs; use sqlbuild_orchestration::runs::models::RunStatus; use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::create_location::create_location; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; use uuid::Uuid; use crate::helpers::{queue, request, snapshot}; +use crate::location_helpers::create_location; use crate::replica_helpers::second_store; use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; use crate::store_test_types::Backend; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs b/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs index 8c0ea844dd..b38db00540 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs @@ -8,11 +8,11 @@ use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; use sqlbuild_orchestration::scheduler::main::evaluate_trigger::evaluate_trigger; use sqlbuild_orchestration::scheduler::models::TickDecision; use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::create_location::create_location; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use uuid::Uuid; use crate::helpers::snapshot; +use crate::location_helpers::create_location; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; use crate::test_types::CoalescedRunTestCase; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_faults.rs b/crates/sqlbuild-orchestration/tests/runs/test_faults.rs index 6af9167fbb..d32ffd0472 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_faults.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_faults.rs @@ -9,7 +9,6 @@ use sqlbuild_orchestration::runs::models::{RunAuthority, RunChange, RunChangeReq use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; use sqlbuild_orchestration::store::errors::ControlError; -use sqlbuild_orchestration::store::main::create_location::create_location; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; use uuid::Uuid; @@ -17,6 +16,7 @@ use uuid::Uuid; use crate::fault_helpers::{install_fault, remove_fault}; use crate::helpers::{queue, request, snapshot}; use crate::lease_helpers::expire_lease; +use crate::location_helpers::create_location; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; use crate::test_types::{ClaimFailureTestCase, TransactionFault}; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs index 0f02f31093..d06a397736 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs @@ -10,13 +10,13 @@ use sqlbuild_orchestration::runs::models::{ }; use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::create_location::create_location; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; use uuid::Uuid; use crate::fault_helpers::{install_fault, remove_fault}; use crate::helpers::{queue, request, snapshot}; +use crate::location_helpers::create_location; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; use crate::test_types::{RunLifecycleTestCase, TransactionFault}; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs b/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs index c4847d7316..e567d9d25b 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs @@ -17,7 +17,6 @@ use sqlbuild_orchestration::scheduler::main::scheduler_cycle::scheduler_cycle; use sqlbuild_orchestration::scheduler::models::SchedulerState; use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; use sqlbuild_orchestration::store::errors::ControlError; -use sqlbuild_orchestration::store::main::create_location::create_location; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; use uuid::Uuid; @@ -25,6 +24,7 @@ use uuid::Uuid; use crate::fault_helpers::{install_fault, remove_fault}; use crate::helpers::{queue, request, snapshot}; use crate::lease_helpers::expire_lease; +use crate::location_helpers::create_location; use crate::monitor_helpers::{age_heartbeat, start_run}; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; diff --git a/crates/sqlbuild-orchestration/tests/scheduler.rs b/crates/sqlbuild-orchestration/tests/scheduler.rs index 7142cff07b..774474ceef 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler.rs @@ -6,6 +6,8 @@ mod fault_types; mod helpers; #[path = "store/leases/helpers.rs"] mod lease_helpers; +#[path = "credentials/locations/helpers.rs"] +mod location_helpers; #[path = "store/replicas/helpers.rs"] mod replica_helpers; #[path = "store/helpers.rs"] diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs index fa51f828f5..1cbed7a105 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs @@ -12,13 +12,13 @@ use sqlbuild_orchestration::scheduler::models::TickOutcome; use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; use sqlbuild_orchestration::store::errors::ControlError; -use sqlbuild_orchestration::store::main::create_location::create_location; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; use uuid::Uuid; use crate::fault_helpers::{install_fault, remove_fault}; use crate::lease_helpers::expire_lease; +use crate::location_helpers::create_location; use crate::replica_helpers::second_store; use crate::store_helpers::{close_store, close_test_store, open_test_store}; use crate::store_test_types::Backend; diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs index f9341cf027..fbe1a63283 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs @@ -6,11 +6,11 @@ use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler; use sqlbuild_orchestration::scheduler::main::scheduler_cycle::scheduler_cycle; use sqlbuild_orchestration::scheduler::models::SchedulerState; use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::create_location::create_location; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use uuid::Uuid; use crate::lease_helpers::expire_lease; +use crate::location_helpers::create_location; use crate::replica_helpers::second_store; use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; use crate::store_test_types::Backend; diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs index 4df7178b12..cf2cfb68ac 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs @@ -9,11 +9,11 @@ use sqlbuild_orchestration::declarations::models::{ use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; use sqlbuild_orchestration::scheduler::main::evaluate_trigger::evaluate_trigger; use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::create_location::create_location; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use uuid::Uuid; use crate::fault_helpers::{install_fault, remove_fault}; +use crate::location_helpers::create_location; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; use crate::ticks::helpers::snapshot; diff --git a/crates/sqlbuild-orchestration/tests/snapshots.rs b/crates/sqlbuild-orchestration/tests/snapshots.rs index 58d1347746..13be1a8abc 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots.rs @@ -2,6 +2,8 @@ mod contract; #[path = "snapshots/helpers.rs"] mod helpers; +#[path = "credentials/locations/helpers.rs"] +mod location_helpers; #[path = "store/replicas/helpers.rs"] mod replica_helpers; #[path = "store/helpers.rs"] diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index b1533caba3..9030c6f077 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -7,12 +7,12 @@ use sqlbuild_orchestration::snapshots::main::current_snapshot::current_snapshot; use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; use sqlbuild_orchestration::store::errors::ControlError; -use sqlbuild_orchestration::store::main::create_location::create_location; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::open_store::open_store; use sqlbuild_orchestration::store::models::ControlStore; use uuid::Uuid; +use crate::location_helpers::create_location; use crate::replica_helpers::second_store; use crate::store_helpers::{close_store, close_test_store, open_test_store}; use crate::store_test_types::Backend; From c005a3765bece65699566f97b70b8104aa4acf24 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 02:59:45 +0000 Subject: [PATCH 16/59] fix: use a scalar expression in the DuckShop function fixture Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../project/functions/sql/edge_cases/edge_next_quantity.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/duckshop/project/functions/sql/edge_cases/edge_next_quantity.sql b/benchmarks/duckshop/project/functions/sql/edge_cases/edge_next_quantity.sql index c4fa936dea..d49cb02a03 100644 --- a/benchmarks/duckshop/project/functions/sql/edge_cases/edge_next_quantity.sql +++ b/benchmarks/duckshop/project/functions/sql/edge_cases/edge_next_quantity.sql @@ -5,4 +5,4 @@ FUNCTION ( returns INTEGER, ); -SELECT quantity + 1 +quantity + 1 From 133bc3e53f1e6c838734a73a1c9d7ab963fe054e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 04:47:59 +0000 Subject: [PATCH 17/59] feat: compile authored jobs and cron schedules into snapshots Keep the public Python API limited to authoring records and native delegation. Resolve selectors and validate schedule data in Rust before publication. Preserve stable shorthand trigger names across cron edits. Add the explicitly approved narrow Python facade boundary entries without increasing movable Python or native crossing budgets. Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../bindings/_helpers/compiled_project/mod.rs | 1 + .../_helpers/compiled_project/snapshot.rs | 21 ++- .../compiled_project/snapshot_declarations.rs | 121 ++++++++++++++++ .../src/bindings/_helpers/native_host/mod.rs | 1 + .../native_host/orchestration_declarations.rs | 19 +++ python-boundary.toml | 15 ++ src/sqlbuild/_native.pyi | 2 + src/sqlbuild/orchestration.py | 8 ++ .../main/compile_snapshot.py | 23 +++ .../orchestration_declarations/models.py | 31 +++++ .../compiler/compiled_project/_test_types.py | 18 +++ .../test_orchestration_snapshot.py | 131 ++++++++++++++++++ 12 files changed, 389 insertions(+), 2 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs create mode 100644 src/sqlbuild/orchestration.py create mode 100644 src/sqlbuild/orchestration_declarations/main/compile_snapshot.py create mode 100644 src/sqlbuild/orchestration_declarations/models.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs index 1fb46a0eb0..67cb07e670 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs @@ -5,6 +5,7 @@ pub(crate) mod rows; pub(crate) mod schema_facts; pub(crate) mod snapshot; pub(crate) mod snapshot_boundary; +pub(crate) mod snapshot_declarations; pub(crate) mod snapshot_models; pub(crate) mod snapshot_resources; pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs index fa850b69dd..7665d7a2d6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs @@ -4,16 +4,21 @@ use std::collections::BTreeMap; use std::path::PathBuf; use pyo3::exceptions::{PyRuntimeError, PyValueError}; -use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult, Python}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult, Python, +}; use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; use serde_json::Value; use sqlbuild_core::models::DiagnosticSeverity; use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; +use crate::bindings::_helpers::compiled_project::snapshot_declarations::compile_declarations; use crate::bindings::_helpers::compiled_project::snapshot_models::model_nodes; use crate::bindings::_helpers::compiled_project::snapshot_resources::resource_nodes; +use crate::bindings::_helpers::compiled_project::values::plain_value; use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; +use crate::bindings::_helpers::native_host::orchestration_declarations::validate_jobs; use crate::bindings::_helpers::native_host::orchestration_environment::environment_hash; use crate::bindings::_helpers::records::fields::field; @@ -49,12 +54,14 @@ impl NativeOrchestrationSnapshot { /// Compile catalog nodes from their owners, never from Python model or graph projections. #[pyfunction] +#[pyo3(signature = (project, metadata_json, project_dir, *, declarations=None))] pub(crate) fn build_orchestration_snapshot( - py: Python<'_>, project: PyRef<'_, NativeProjectView>, metadata_json: &str, project_dir: PathBuf, + declarations: Option>>, ) -> PyResult { + let py = project.py(); let mut input: SnapshotInput = serde_json::from_str(metadata_json) .map_err(|error| PyValueError::new_err(error.to_string()))?; if !input.nodes.is_empty() { @@ -83,6 +90,16 @@ pub(crate) fn build_orchestration_snapshot( let owner = project.owner().bind(py).borrow(); input.nodes = model_nodes(py, &project, &owner, input.nodes)?; input.nodes = resource_nodes(py, &project, &owner, input.nodes)?; + drop(owner); + if let Some(declarations) = declarations { + validate_jobs(py, &declarations)?; + let authored = declarations + .iter() + .map(plain_value) + .collect::>>()?; + let graph = project.lineage_graph(py)?; + input = compile_declarations(input, graph.bind(py).get().graph(), authored)?; + } Ok(NativeOrchestrationSnapshot { input }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs new file mode 100644 index 0000000000..bc2b0b3a9b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -0,0 +1,121 @@ +//! Compile authoring records against retained graph data before publication. + +use std::collections::{BTreeMap, BTreeSet}; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::PyResult; +use serde::Deserialize; +use serde_json::Value; +use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; +use sqlbuild_analysis::graph::models::ProjectGraph; +use sqlbuild_orchestration::declarations::main::snapshot_definitions::snapshot_definitions; +use sqlbuild_orchestration::declarations::models::{ + JobDefinition, OnFailure, Schedule, TriggerDefinition, TriggerStatus, +}; +use sqlbuild_orchestration::scheduler::models::{CatchUp, Overlap}; +use sqlbuild_orchestration::snapshots::models::SnapshotInput; + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct AuthoredSchedule { + cron: String, + timezone: String, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct AuthoredJob { + name: String, + description: String, + select: String, + exclude: String, + overlap: Overlap, + on_failure: OnFailure, + concurrency_groups: Vec, + schedule: Option, + targets: Vec, +} + +pub(crate) fn compile_declarations( + mut input: SnapshotInput, + graph: &ProjectGraph, + declarations: Vec, +) -> PyResult { + if declarations.is_empty() { + return Ok(input); + } + if input.documents.contains_key("jobs") || input.documents.contains_key("triggers") { + return Err(PyValueError::new_err( + "authored declarations cannot overwrite snapshot job or trigger documents", + )); + } + let mut jobs: BTreeMap = BTreeMap::new(); + let mut triggers: BTreeMap = BTreeMap::new(); + let mut names: BTreeSet = BTreeSet::new(); + for declaration in declarations { + let job: AuthoredJob = serde_json::from_value(declaration) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + if !names.insert(job.name.clone()) { + return Err(PyValueError::new_err(format!("duplicate job {}", job.name))); + } + if !job.targets.is_empty() && !job.targets.contains(&input.target) { + continue; + } + let tokens = |value: &str| { + value + .split_whitespace() + .map(str::to_owned) + .collect::>() + }; + if job.select.trim().is_empty() { + return Err(PyValueError::new_err(format!( + "job {} needs a selection", + job.name + ))); + } + let selection: BTreeSet = + resolve_selectors(graph, &tokens(&job.select), &tokens(&job.exclude)) + .map_err(|error| PyValueError::new_err(error.message))? + .into_iter() + .map(|(kind, name)| format!("{kind}:{name}")) + .collect(); + if let Some(schedule) = job.schedule { + triggers.insert( + job.name.clone(), + TriggerDefinition { + description: job.description.clone(), + job_key: job.name.clone(), + schedule: Schedule::Cron { + expression: schedule.cron, + timezone: schedule.timezone, + }, + starts_at: None, + default_status: TriggerStatus::Running, + catch_up: CatchUp::Latest, + }, + ); + } + jobs.insert( + job.name, + JobDefinition { + description: job.description, + selection, + settings: BTreeMap::new(), + overlap: job.overlap, + on_failure: job.on_failure, + concurrency_groups: job.concurrency_groups.into_iter().collect(), + }, + ); + } + for (key, document) in [ + ("jobs", serde_json::to_value(jobs)), + ("triggers", serde_json::to_value(triggers)), + ] { + input.documents.insert( + key.into(), + document.map_err(|error| PyValueError::new_err(error.to_string()))?, + ); + } + snapshot_definitions(&input).map_err(|error| PyValueError::new_err(error.to_string()))?; + Ok(input) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index 637d9ef489..25adcc06b4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -4,6 +4,7 @@ pub(crate) mod connections; pub(crate) mod header_projection; pub(crate) mod host; pub(crate) mod model_lookup; +pub(crate) mod orchestration_declarations; pub(crate) mod orchestration_environment; pub(crate) mod registration; pub(crate) mod results; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs new file mode 100644 index 0000000000..c42491171b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs @@ -0,0 +1,19 @@ +//! The public authoring records enter native compilation here, never scheduler execution. + +use pyo3::exceptions::PyTypeError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, Python}; + +pub(crate) fn validate_jobs( + py: Python<'_>, + declarations: &[Bound<'_, PyAny>], +) -> Result<(), PyErr> { + let job = py.import("sqlbuild.orchestration")?.getattr("Job")?; + for declaration in declarations { + if !declaration.is_instance(&job)? { + return Err(PyTypeError::new_err( + "snapshot declarations must be sqlbuild.orchestration.Job objects", + )); + } + } + Ok(()) +} diff --git a/python-boundary.toml b/python-boundary.toml index ee2346f85d..b02d91a88d 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -12,6 +12,21 @@ # data instead of projecting those public containers and immediately reading them back. remaining_lines = 0 +[[stays]] +path = "orchestration.py" +rule = 3 +note = "Public orchestration facade: re-exports only." + +[[stays]] +path = "orchestration_declarations/models.py" +rule = 3 +note = "Public job and schedule authoring records only; selection resolution, validation and scheduling stay native." + +[[stays]] +path = "orchestration_declarations/main/compile_snapshot.py" +rule = 3 +note = "Public declaration compiler: one-line delegation to build_orchestration_snapshot; no Python discovery, validation or snapshot logic." + [[stays]] path = "executor" rule = 1 diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 78a8f577b2..6a4f400584 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1748,6 +1748,8 @@ def build_orchestration_snapshot( project: CompiledProjectView, metadata_json: str, project_dir: Path, + *, + declarations: Sequence[object] | None = None, ) -> NativeOrchestrationSnapshot: ... class NativeProjectView: diff --git a/src/sqlbuild/orchestration.py b/src/sqlbuild/orchestration.py new file mode 100644 index 0000000000..94bb6c3ace --- /dev/null +++ b/src/sqlbuild/orchestration.py @@ -0,0 +1,8 @@ +"""Public authoring and snapshot compilation API for orchestration declarations.""" + +from sqlbuild.orchestration_declarations.main.compile_snapshot import compile_snapshot +from sqlbuild.orchestration_declarations.models import Job, Schedule +from sqlbuild.orchestration_declarations.models import Job as job +from sqlbuild.orchestration_declarations.models import Schedule as schedule + +__all__ = ("Job", "Schedule", "compile_snapshot", "job", "schedule") diff --git a/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py b/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py new file mode 100644 index 0000000000..2be661c7c1 --- /dev/null +++ b/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py @@ -0,0 +1,23 @@ +"""Delegate authoring-record compilation to the native retained snapshot owner.""" + +from __future__ import annotations + +from collections.abc import Sequence +from pathlib import Path + +from sqlbuild._native import NativeOrchestrationSnapshot, build_orchestration_snapshot +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.orchestration_declarations.models import Job + + +def compile_snapshot( + *, + project: CompiledProjectView, + metadata_json: str, + project_dir: Path, + declarations: Sequence[Job] = (), +) -> NativeOrchestrationSnapshot: + """Compile authoring records and retained project facts into a data-only snapshot.""" + return build_orchestration_snapshot( + project, metadata_json, project_dir, declarations=declarations + ) diff --git a/src/sqlbuild/orchestration_declarations/models.py b/src/sqlbuild/orchestration_declarations/models.py new file mode 100644 index 0000000000..278aa55335 --- /dev/null +++ b/src/sqlbuild/orchestration_declarations/models.py @@ -0,0 +1,31 @@ +"""Authoring records only; selection, validation and scheduling belong to Rust.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Literal + + +@dataclass(frozen=True, kw_only=True) +class Schedule: + """A wall-clock cron schedule attached to a job.""" + + cron: str + timezone: str + + +@dataclass(frozen=True, eq=False) +class Job: + """A named selection to execute, optionally on a wall-clock schedule.""" + + name: str + description: str = field(kw_only=True) + select: str = field(kw_only=True) + exclude: str = field(default="", kw_only=True) + overlap: Literal["coalesce", "queue", "allow"] = field(default="coalesce", kw_only=True) + on_failure: Literal["skip_dependents", "skip_downstream_jobs", "stop_group"] = field( + default="skip_dependents", kw_only=True + ) + concurrency_groups: tuple[str, ...] = field(default=(), kw_only=True) + schedule: Schedule | None = field(default=None, kw_only=True) + targets: tuple[str, ...] = field(default=(), kw_only=True) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index 6f5b1c5053..eba1f44a39 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -214,3 +214,21 @@ class SnapshotEnvironmentTestCase: first_lockfile: str second_lockfile: str expected_environment_count: int + + +@dataclass(frozen=True) +class SnapshotScheduleTestCase: + description: str + timezone: str + expected_selection: tuple[str, ...] + + +@dataclass(frozen=True) +class SnapshotDeclarationRejectionTestCase: + description: str + job_description: str = "Orders." + select: str = "orders" + cron: str = "0 2 * * *" + timezone: str = "UTC" + copies: int = 1 + expected_message: str = "" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index f32d0b2601..e5787f636d 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +from dataclasses import replace from pathlib import Path from typing import Any @@ -16,10 +17,13 @@ ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.orchestration import Job, compile_snapshot, job, schedule from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( SnapshotCatalogTestCase, + SnapshotDeclarationRejectionTestCase, SnapshotEnvironmentTestCase, SnapshotRejectionTestCase, + SnapshotScheduleTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( FUNCTION_PROJECT_FILES, @@ -266,3 +270,130 @@ def test_given_a_dependency_lock_when_it_changes_then_environment_identity_chang assert changed == json.loads( build_orchestration_snapshot(project, json.dumps(METADATA), tmp_path).to_json() ) + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotScheduleTestCase( + description="daylight saving zone", + timezone="Europe/London", + expected_selection=("model:orders",), + ), + SnapshotScheduleTestCase( + description="fixed offset zone", timezone="UTC", expected_selection=("model:orders",) + ), + ], + ids=lambda case: case.description, +) +def test_given_authored_jobs_when_compiling_then_native_snapshots_resolve_selection_and_cron( + tmp_path: Path, test_case: SnapshotScheduleTestCase +) -> None: + write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + nightly: Job = job( + "nightly_orders", + description="Refresh order reporting.", + select="+orders", + exclude="stg_orders", + overlap="queue", + concurrency_groups=("order_reporting",), + schedule=schedule(cron="15 2 * * *", timezone=test_case.timezone), + targets=("development",), + ) + excluded: Job = replace(nightly, name="other_target", targets=("production",)) + first: dict[str, Any] = json.loads( + compile_snapshot( + project=project, + metadata_json=json.dumps(METADATA), + project_dir=tmp_path, + declarations=[nightly, excluded], + ).to_json() + ) + assert first["documents"]["jobs"] == { + "nightly_orders": { + "description": "Refresh order reporting.", + "selection": list(test_case.expected_selection), + "settings": {}, + "overlap": "queue", + "on_failure": "skip_dependents", + "concurrency_groups": ["order_reporting"], + } + } + trigger: dict[str, Any] = first["documents"]["triggers"]["nightly_orders"] + assert trigger["job_key"] == "nightly_orders" + assert trigger["schedule"] == { + "kind": "cron", + "expression": "15 2 * * *", + "timezone": test_case.timezone, + } + edited: Job = replace(nightly, schedule=schedule(cron="30 3 * * *", timezone="UTC")) + second: dict[str, Any] = json.loads( + compile_snapshot( + project=project, + metadata_json=json.dumps(METADATA), + project_dir=tmp_path, + declarations=[edited], + ).to_json() + ) + assert second["documents"]["triggers"].keys() == first["documents"]["triggers"].keys() + assert second["documents"]["triggers"]["nightly_orders"]["schedule"]["expression"] == ( + "30 3 * * *" + ) + assert second["nodes"] == first["nodes"] + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotDeclarationRejectionTestCase( + description="blank description", + job_description=" ", + expected_message="needs a description", + ), + SnapshotDeclarationRejectionTestCase( + description="blank selection", select="", expected_message="needs a selection" + ), + SnapshotDeclarationRejectionTestCase( + description="missing model", select="missing", expected_message="missing" + ), + SnapshotDeclarationRejectionTestCase( + description="invalid cron", cron="not a cron", expected_message="cron" + ), + SnapshotDeclarationRejectionTestCase( + description="invalid timezone", timezone="Unknown/Zone", expected_message="time zone" + ), + SnapshotDeclarationRejectionTestCase( + description="duplicate names", copies=2, expected_message="duplicate job orders" + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_declarations_when_compiling_then_native_validation_rejects_them( + tmp_path: Path, test_case: SnapshotDeclarationRejectionTestCase +) -> None: + write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + declarations: list[Job] = [ + job( + "orders", + description=test_case.job_description, + select=test_case.select, + schedule=schedule(cron=test_case.cron, timezone=test_case.timezone), + ) + ] * test_case.copies + with pytest.raises(ValueError, match=test_case.expected_message): + compile_snapshot( + project=project, + metadata_json=json.dumps(METADATA), + project_dir=tmp_path, + declarations=declarations, + ) From de751491d87ee4a68923c1c7b6b5a55303f3a6c6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 05:01:02 +0000 Subject: [PATCH 18/59] feat: discover orchestration declarations in code locations Collect exported jobs and nested factory containers natively, deduplicating aliases and terminating cyclic containers. Explicit empty declarations skip user module execution. Remove failed imports from the module cache so publication retries cannot reuse partial definitions. Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../edge_cases/orchestration/failed_import.py | 3 + .../orchestration/edge_cases/definitions.py | 13 ++ crates/sqlbuild-discovery/src/constants.rs | 2 + .../src/python_paths/constants.rs | 5 +- .../tests/test_unsupported_python_paths.rs | 2 + .../src/entries/_helpers/user_code.rs | 5 + crates/sqlbuild-host/src/entries/models.rs | 1 + .../_helpers/compiled_project/snapshot.rs | 15 ++- .../compiled_project/snapshot_declarations.rs | 64 +++++++--- .../native_host/orchestration_declarations.rs | 37 +++++- fensu.toml | 2 +- src/sqlbuild/_native_host.py | 4 + src/sqlbuild/_native_host.pyi | 3 + .../_helpers/filesystem/project_modules.py | 13 ++ .../main/compile_snapshot.py | 2 +- .../compiler/compiled_project/_test_types.py | 16 +++ .../test_orchestration_snapshot.py | 116 ++++++++++++++++++ 17 files changed, 279 insertions(+), 24 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/failed_import.py create mode 100644 benchmarks/duckshop/project/orchestration/edge_cases/definitions.py diff --git a/benchmarks/duckshop/edge_cases/orchestration/failed_import.py b/benchmarks/duckshop/edge_cases/orchestration/failed_import.py new file mode 100644 index 0000000000..6483f0b2ab --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/failed_import.py @@ -0,0 +1,3 @@ +"""Negative fixture: a failed declaration import must fail every publication retry.""" + +raise RuntimeError("broken declaration") diff --git a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py new file mode 100644 index 0000000000..7d6e411ee8 --- /dev/null +++ b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py @@ -0,0 +1,13 @@ +"""Exercise alias deduplication and cyclic factory containers during native discovery.""" + +from sqlbuild.orchestration import job, schedule + +edge_orders = job( + "edge_snapshot_orders", + description="Refresh only the synthetic edge-case models.", + select="tag:edge_case", + schedule=schedule(cron="15 2 * * *", timezone="UTC"), +) +edge_alias = edge_orders +edge_factory_results = [[edge_orders]] +edge_factory_results.append(edge_factory_results) diff --git a/crates/sqlbuild-discovery/src/constants.rs b/crates/sqlbuild-discovery/src/constants.rs index 096580ebab..86a7a4dcd5 100644 --- a/crates/sqlbuild-discovery/src/constants.rs +++ b/crates/sqlbuild-discovery/src/constants.rs @@ -2,6 +2,8 @@ /// Root of SQL model files. pub const MODELS_DIRECTORY: &str = "models"; +/// Root of code-location orchestration declarations, never imported by the scheduler. +pub const ORCHESTRATION_DIRECTORY: &str = "orchestration"; /// Suffix of SQL files. pub const SQL_FILE_SUFFIX: &str = ".sql"; /// The folder below a resource tree that holds grouped scoped declarations. diff --git a/crates/sqlbuild-discovery/src/python_paths/constants.rs b/crates/sqlbuild-discovery/src/python_paths/constants.rs index eacdb349f9..4483a9f780 100644 --- a/crates/sqlbuild-discovery/src/python_paths/constants.rs +++ b/crates/sqlbuild-discovery/src/python_paths/constants.rs @@ -1,7 +1,9 @@ //! The documented roots project Python may live under, and the directories the walk skips. +use crate::constants::ORCHESTRATION_DIRECTORY; + /// Top-level directories whose Python files are always supported. -pub(crate) const DIRECT_PYTHON_ROOTS: [&str; 10] = [ +pub(crate) const DIRECT_PYTHON_ROOTS: [&str; 11] = [ "adapters", "materializations", "providers", @@ -12,6 +14,7 @@ pub(crate) const DIRECT_PYTHON_ROOTS: [&str; 10] = [ "enums", "constants", "event_exporters", + ORCHESTRATION_DIRECTORY, ]; /// `(root, language)` directory pairs whose Python files are supported. pub(crate) const LANGUAGE_PYTHON_ROOTS: [(&str, &str); 2] = diff --git a/crates/sqlbuild-discovery/src/python_paths/tests/test_unsupported_python_paths.rs b/crates/sqlbuild-discovery/src/python_paths/tests/test_unsupported_python_paths.rs index 64ecd45aa6..9a4843ec44 100644 --- a/crates/sqlbuild-discovery/src/python_paths/tests/test_unsupported_python_paths.rs +++ b/crates/sqlbuild-discovery/src/python_paths/tests/test_unsupported_python_paths.rs @@ -11,6 +11,8 @@ fn given_project_python_when_listing_unsupported_paths_then_only_undocumented_ro files: &[ "adapter.py", "python/orders.py", + "orchestration/jobs.py", + "orchestration/nested/calendar.py", "functions/python/f.py", "macros/m.py", "models/_macros/m.py", diff --git a/crates/sqlbuild-host/src/entries/_helpers/user_code.rs b/crates/sqlbuild-host/src/entries/_helpers/user_code.rs index 7deafb99c8..51d567c4e1 100644 --- a/crates/sqlbuild-host/src/entries/_helpers/user_code.rs +++ b/crates/sqlbuild-host/src/entries/_helpers/user_code.rs @@ -64,6 +64,11 @@ pub(crate) const ENTRIES: &[HostEntry] = &[ kind: HostKind::Function, edge: HostEdge::UserCode, }, + HostEntry { + name: "load_orchestration_module", + kind: HostKind::Function, + edge: HostEdge::UserCode, + }, HostEntry { name: "module_functions", kind: HostKind::Function, diff --git a/crates/sqlbuild-host/src/entries/models.rs b/crates/sqlbuild-host/src/entries/models.rs index 5c8482411a..057a1d4203 100644 --- a/crates/sqlbuild-host/src/entries/models.rs +++ b/crates/sqlbuild-host/src/entries/models.rs @@ -140,6 +140,7 @@ pub enum UserCodeHost { LoadProviderModule, LoadPythonNodeModule, LoadSinkModule, + LoadOrchestrationModule, ModuleFunctions, ModuleProviderClasses, ParameterAnnotations, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs index 7665d7a2d6..cee2c105f2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs @@ -13,10 +13,11 @@ use sqlbuild_core::models::DiagnosticSeverity; use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; -use crate::bindings::_helpers::compiled_project::snapshot_declarations::compile_declarations; +use crate::bindings::_helpers::compiled_project::snapshot_declarations::{ + AuthoredJob, compile_declarations, discover_declarations, +}; use crate::bindings::_helpers::compiled_project::snapshot_models::model_nodes; use crate::bindings::_helpers::compiled_project::snapshot_resources::resource_nodes; -use crate::bindings::_helpers::compiled_project::values::plain_value; use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; use crate::bindings::_helpers::native_host::orchestration_declarations::validate_jobs; use crate::bindings::_helpers::native_host::orchestration_environment::environment_hash; @@ -91,12 +92,16 @@ pub(crate) fn build_orchestration_snapshot( input.nodes = model_nodes(py, &project, &owner, input.nodes)?; input.nodes = resource_nodes(py, &project, &owner, input.nodes)?; drop(owner); - if let Some(declarations) = declarations { + let declarations = match declarations { + Some(declarations) => declarations, + None => discover_declarations(py, &project_dir)?, + }; + if !declarations.is_empty() { validate_jobs(py, &declarations)?; let authored = declarations .iter() - .map(plain_value) - .collect::>>()?; + .map(|declaration| declaration.extract::()) + .collect::>>()?; let graph = project.lineage_graph(py)?; input = compile_declarations(input, graph.bind(py).get().graph(), authored)?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index bc2b0b3a9b..af7683934d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -1,13 +1,18 @@ //! Compile authoring records against retained graph data before publication. use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; +use pyo3::FromPyObject; use pyo3::exceptions::PyValueError; -use pyo3::prelude::PyResult; -use serde::Deserialize; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use serde_json::Value; use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; use sqlbuild_analysis::graph::models::ProjectGraph; +use sqlbuild_core::constants::PYTHON_INIT_MODULE_STEM; +use sqlbuild_discovery::constants::{ORCHESTRATION_DIRECTORY, PYTHON_FILE_SUFFIX}; +use sqlbuild_discovery::tree::main::rglob::rglob; +use sqlbuild_discovery::tree::models::ProjectTree; use sqlbuild_orchestration::declarations::main::snapshot_definitions::snapshot_definitions; use sqlbuild_orchestration::declarations::models::{ JobDefinition, OnFailure, Schedule, TriggerDefinition, TriggerStatus, @@ -15,22 +20,51 @@ use sqlbuild_orchestration::declarations::models::{ use sqlbuild_orchestration::scheduler::models::{CatchUp, Overlap}; use sqlbuild_orchestration::snapshots::models::SnapshotInput; -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] +use crate::bindings::_helpers::native_host::orchestration_declarations::module_jobs; + +/// Dedicated declaration modules are imported only in the code-location interpreter. +pub(crate) fn discover_declarations<'py>( + py: Python<'py>, + project_dir: &Path, +) -> PyResult>> { + let tree = ProjectTree::new(project_dir); + let files = py + .detach(|| { + rglob(&tree, ORCHESTRATION_DIRECTORY, |entry| { + !entry.is_dir + && entry.name.ends_with(PYTHON_FILE_SUFFIX) + && entry.name.strip_suffix(PYTHON_FILE_SUFFIX) != Some(PYTHON_INIT_MODULE_STEM) + }) + }) + .map_err(|error| { + PyValueError::new_err(format!("orchestration discovery failed: {error:?}")) + })?; + let mut declarations: Vec> = Vec::new(); + let mut seen: BTreeSet = BTreeSet::new(); + for file in files { + for job in module_jobs(py, &tree.absolute(&file), project_dir)? { + if seen.insert(job.as_ptr() as usize) { + declarations.push(job); + } + } + } + Ok(declarations) +} + +#[derive(FromPyObject)] struct AuthoredSchedule { cron: String, timezone: String, } -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -struct AuthoredJob { +#[derive(FromPyObject)] +pub(crate) struct AuthoredJob { name: String, description: String, select: String, exclude: String, - overlap: Overlap, - on_failure: OnFailure, + overlap: String, + on_failure: String, concurrency_groups: Vec, schedule: Option, targets: Vec, @@ -39,7 +73,7 @@ struct AuthoredJob { pub(crate) fn compile_declarations( mut input: SnapshotInput, graph: &ProjectGraph, - declarations: Vec, + declarations: Vec, ) -> PyResult { if declarations.is_empty() { return Ok(input); @@ -52,9 +86,7 @@ pub(crate) fn compile_declarations( let mut jobs: BTreeMap = BTreeMap::new(); let mut triggers: BTreeMap = BTreeMap::new(); let mut names: BTreeSet = BTreeSet::new(); - for declaration in declarations { - let job: AuthoredJob = serde_json::from_value(declaration) - .map_err(|error| PyValueError::new_err(error.to_string()))?; + for job in declarations { if !names.insert(job.name.clone()) { return Err(PyValueError::new_err(format!("duplicate job {}", job.name))); } @@ -101,8 +133,10 @@ pub(crate) fn compile_declarations( description: job.description, selection, settings: BTreeMap::new(), - overlap: job.overlap, - on_failure: job.on_failure, + overlap: serde_json::from_value::(Value::String(job.overlap)) + .map_err(|error| PyValueError::new_err(error.to_string()))?, + on_failure: serde_json::from_value::(Value::String(job.on_failure)) + .map_err(|error| PyValueError::new_err(error.to_string()))?, concurrency_groups: job.concurrency_groups.into_iter().collect(), }, ); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs index c42491171b..6c3d086654 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs @@ -1,7 +1,42 @@ //! The public authoring records enter native compilation here, never scheduler execution. +use std::collections::HashSet; +use std::path::Path; + use pyo3::exceptions::PyTypeError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; +use sqlbuild_host::entries::models::UserCodeHost; + +use crate::bindings::_helpers::native_host::host::entry; + +/// Only module execution crosses into Python; discovery and collection stay native. +pub(crate) fn module_jobs<'py>( + py: Python<'py>, + file_path: &Path, + project_dir: &Path, +) -> PyResult>> { + let options = PyDict::new(py); + options.set_item("file_path", file_path)?; + options.set_item("project_dir", project_dir)?; + let module = entry(py, UserCodeHost::LoadOrchestrationModule)?.call((), Some(&options))?; + let job_class = py.import("sqlbuild.orchestration")?.getattr("Job")?; + let attributes = module.getattr("__dict__")?.cast_into::()?; + let mut pending: Vec> = attributes.values().iter().collect(); + let mut seen: HashSet = HashSet::new(); + let mut jobs: Vec> = Vec::new(); + while let Some(value) = pending.pop() { + if !seen.insert(value.as_ptr() as usize) { + continue; + } + if value.is_instance(&job_class)? { + jobs.push(value); + } else if value.is_instance_of::() || value.is_instance_of::() { + pending.extend(value.try_iter()?.collect::>>()?); + } + } + Ok(jobs) +} pub(crate) fn validate_jobs( py: Python<'_>, diff --git a/fensu.toml b/fensu.toml index 2338c3e56a..c06a29ffdd 100644 --- a/fensu.toml +++ b/fensu.toml @@ -229,7 +229,7 @@ reason = "Native code resolves these through the sqlbuild._native_host name tabl [[targets.python.dead_code.roots]] modules = ["sqlbuild.compiler.discovery._helpers.filesystem.project_modules"] -symbols = ["import_python_root_module", "install_python_root_package", "load_materialize_function", "load_provider_module", "load_python_node_module", "load_sink_module", "module_functions", "module_provider_classes", "parameter_annotations", "provider_usages", "requires_arguments", "restore_python_root_modules", "source_line", "validate_event_exporter_signature"] +symbols = ["import_python_root_module", "install_python_root_package", "load_materialize_function", "load_orchestration_module", "load_provider_module", "load_python_node_module", "load_sink_module", "module_functions", "module_provider_classes", "parameter_annotations", "provider_usages", "requires_arguments", "restore_python_root_modules", "source_line", "validate_event_exporter_signature"] reason = "Native code resolves these through the sqlbuild._native_host name table." [[targets.python.dead_code.roots]] diff --git a/src/sqlbuild/_native_host.py b/src/sqlbuild/_native_host.py index 9dca6ce419..fde2109986 100644 --- a/src/sqlbuild/_native_host.py +++ b/src/sqlbuild/_native_host.py @@ -167,6 +167,10 @@ "sqlbuild.compiler.discovery._helpers.filesystem.project_modules", "load_sink_module", ), + "load_orchestration_module": ( + "sqlbuild.compiler.discovery._helpers.filesystem.project_modules", + "load_orchestration_module", + ), "module_functions": ( "sqlbuild.compiler.discovery._helpers.filesystem.project_modules", "module_functions", diff --git a/src/sqlbuild/_native_host.pyi b/src/sqlbuild/_native_host.pyi index c68e33d507..4ca6064432 100644 --- a/src/sqlbuild/_native_host.pyi +++ b/src/sqlbuild/_native_host.pyi @@ -130,6 +130,9 @@ from sqlbuild.compiler.discovery._helpers.filesystem.project_modules import ( from sqlbuild.compiler.discovery._helpers.filesystem.project_modules import ( load_materialize_function as load_materialize_function, ) +from sqlbuild.compiler.discovery._helpers.filesystem.project_modules import ( + load_orchestration_module as load_orchestration_module, +) from sqlbuild.compiler.discovery._helpers.filesystem.project_modules import ( load_provider_module as load_provider_module, ) diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py index f28340d71b..1eef8c4987 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py @@ -244,6 +244,17 @@ def load_sink_module(*, file_path: Path, project_dir: Path) -> ModuleType: ) +def load_orchestration_module(*, file_path: Path, project_dir: Path) -> ModuleType: + """Execute an orchestration module with the project's normal package imports.""" + + return _load_project_package_module( + file_path=file_path, + project_dir=project_dir, + file_label="orchestration", + error_type=PythonNodeDiscoveryError, + ) + + def load_materialize_function( *, file_path: Path, project_dir: Path ) -> Callable[..., object] | None: @@ -338,6 +349,8 @@ def _exec_project_module( sys.modules[module_name] = module spec.loader.exec_module(module) except Exception as error: + if sys.modules.get(module_name) is module: + sys.modules.pop(module_name, None) raise error_type( f"Failed to import {file_label} file {file_path.relative_to(project_dir)}: {error}" ) from error diff --git a/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py b/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py index 2be661c7c1..3cba43a00c 100644 --- a/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py +++ b/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py @@ -15,7 +15,7 @@ def compile_snapshot( project: CompiledProjectView, metadata_json: str, project_dir: Path, - declarations: Sequence[Job] = (), + declarations: Sequence[Job] | None = None, ) -> NativeOrchestrationSnapshot: """Compile authoring records and retained project facts into a data-only snapshot.""" return build_orchestration_snapshot( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index eba1f44a39..6fd1933af6 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -232,3 +232,19 @@ class SnapshotDeclarationRejectionTestCase: timezone: str = "UTC" copies: int = 1 expected_message: str = "" + + +@dataclass(frozen=True) +class SnapshotDiscoveryTestCase: + description: str + files: dict[str, str] + expected_jobs: tuple[str, ...] + declarations: tuple[()] | None = None + + +@dataclass(frozen=True) +class SnapshotDiscoveryRejectionTestCase: + description: str + files: dict[str, str] + expected_error: type[Exception] + expected_message: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index e5787f636d..f9e8c85502 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -11,6 +11,7 @@ from sqlbuild._native import ( NativeOrchestrationSnapshot, + PythonNodeDiscoveryError, build_compiled_project, build_orchestration_snapshot, discover_project_inputs, @@ -21,6 +22,8 @@ from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( SnapshotCatalogTestCase, SnapshotDeclarationRejectionTestCase, + SnapshotDiscoveryRejectionTestCase, + SnapshotDiscoveryTestCase, SnapshotEnvironmentTestCase, SnapshotRejectionTestCase, SnapshotScheduleTestCase, @@ -397,3 +400,116 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject project_dir=tmp_path, declarations=declarations, ) + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotDiscoveryTestCase( + description="relative imports aliases and factory containers", + files={ + "orchestration/shared.py": ( + "from sqlbuild.orchestration import job\n" + "staging = job('staging', description='Stage orders.', select='stg_orders')\n" + ), + "orchestration/jobs.py": ( + "from sqlbuild.orchestration import job\n" + "from .shared import staging\n" + "alias = staging\n" + "def factory():\n" + " return [job('daily', description='Daily orders.', select='orders')]\n" + "jobs = [factory(), staging]\n" + "jobs.append(jobs)\n" + ), + }, + expected_jobs=("daily", "staging"), + ), + SnapshotDiscoveryTestCase( + description="explicit empty declarations do not execute project files", + files={"orchestration/jobs.py": "raise RuntimeError('must not import')\n"}, + expected_jobs=(), + declarations=(), + ), + SnapshotDiscoveryTestCase( + description="DuckShop edge-case declaration containers", + files={ + "orchestration/edge_cases/definitions.py": Path( + "benchmarks/duckshop/project/orchestration/edge_cases/definitions.py" + ).read_text(), + "models/orders.sql": TYPE_PROOF_FILES["models/orders.sql"].replace( + "MODEL (", "MODEL (tags ['edge_case'],", 1 + ), + }, + expected_jobs=("edge_snapshot_orders",), + ), + SnapshotDiscoveryTestCase( + description="a project without declarations has no job document", + files={}, + expected_jobs=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_declaration_modules_when_compiling_then_native_discovery_collects_jobs( + tmp_path: Path, test_case: SnapshotDiscoveryTestCase +) -> None: + write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES | test_case.files) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + publication: dict[str, Any] = json.loads( + compile_snapshot( + project=project, + metadata_json=json.dumps(METADATA), + project_dir=tmp_path, + declarations=test_case.declarations, + ).to_json() + ) + assert tuple(publication["documents"].get("jobs", {})) == test_case.expected_jobs + for definition in publication["documents"].get("jobs", {}).values(): + assert set(definition["selection"]) <= publication["nodes"].keys() + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotDiscoveryRejectionTestCase( + description="distinct objects cannot share a job name", + files={ + "orchestration/jobs.py": ( + "from sqlbuild.orchestration import job\n" + "jobs = [job('daily', description='Orders.', select='orders') for _ in range(2)]\n" + ), + }, + expected_error=ValueError, + expected_message="duplicate job daily", + ), + SnapshotDiscoveryRejectionTestCase( + description="broken imports fail on every retry", + files={ + "orchestration/jobs.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/failed_import.py" + ).read_text() + }, + expected_error=PythonNodeDiscoveryError, + expected_message="Failed to import orchestration file orchestration/jobs.py", + ), + ], + ids=lambda case: case.description, +) +def test_given_broken_declaration_modules_when_compiling_then_no_partial_snapshot_is_returned( + tmp_path: Path, test_case: SnapshotDiscoveryRejectionTestCase +) -> None: + write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES | test_case.files) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + for _ in range(2): + with pytest.raises(test_case.expected_error, match=test_case.expected_message): + compile_snapshot( + project=project, metadata_json=json.dumps(METADATA), project_dir=tmp_path + ) From c0d9cc1a5b5eca0422ac1778e94a35313bb27ac2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 05:47:23 +0000 Subject: [PATCH 19/59] feat: validate authored orchestration job settings Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../orchestration/conflicting_settings.py | 10 ++ .../orchestration/invalid_settings.py | 10 ++ .../orchestration/edge_cases/definitions.py | 2 + .../src/argv/_helpers/job_settings.rs | 128 ++++++++++++++++++ crates/sqlbuild-cli/src/argv/_helpers/mod.rs | 1 + .../src/argv/_helpers/normalization.rs | 40 +++--- crates/sqlbuild-cli/src/argv/constants.rs | 2 + .../src/argv/main/job_settings_arguments.rs | 32 +++++ crates/sqlbuild-cli/src/argv/main/mod.rs | 1 + .../argv/tests/test_parse_cli_arguments.rs | 98 +++++++++++++- .../sqlbuild-cli/src/argv/tests/test_types.rs | 18 +++ .../compiled_project/snapshot_declarations.rs | 20 ++- .../native_host/orchestration_declarations.rs | 12 +- .../orchestration_declarations/models.py | 2 + .../compiler/compiled_project/_test_types.py | 2 + .../test_orchestration_snapshot.py | 82 ++++++++++- 16 files changed, 437 insertions(+), 23 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/conflicting_settings.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/invalid_settings.py create mode 100644 crates/sqlbuild-cli/src/argv/_helpers/job_settings.rs create mode 100644 crates/sqlbuild-cli/src/argv/main/job_settings_arguments.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/conflicting_settings.py b/benchmarks/duckshop/edge_cases/orchestration/conflicting_settings.py new file mode 100644 index 0000000000..ae5a4c9dec --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/conflicting_settings.py @@ -0,0 +1,10 @@ +"""Negative fixture: mutually exclusive build flags must fail before publication.""" + +from sqlbuild.orchestration import job + +orders = job( + "orders", + description="Refresh orders.", + select="orders", + settings={"load_sources": False, "reload": True}, +) diff --git a/benchmarks/duckshop/edge_cases/orchestration/invalid_settings.py b/benchmarks/duckshop/edge_cases/orchestration/invalid_settings.py new file mode 100644 index 0000000000..248b1b4ce1 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/invalid_settings.py @@ -0,0 +1,10 @@ +"""Negative fixture: job settings must not override the frozen selection.""" + +from sqlbuild.orchestration import job + +orders = job( + "orders", + description="Refresh orders.", + select="orders", + settings={"select": "customers"}, +) diff --git a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py index 7d6e411ee8..db9dff756a 100644 --- a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py +++ b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py @@ -6,6 +6,8 @@ "edge_snapshot_orders", description="Refresh only the synthetic edge-case models.", select="tag:edge_case", + settings={"run_tests": False}, + concurrency=2, schedule=schedule(cron="15 2 * * *", timezone="UTC"), ) edge_alias = edge_orders diff --git a/crates/sqlbuild-cli/src/argv/_helpers/job_settings.rs b/crates/sqlbuild-cli/src/argv/_helpers/job_settings.rs new file mode 100644 index 0000000000..19bb1563e3 --- /dev/null +++ b/crates/sqlbuild-cli/src/argv/_helpers/job_settings.rs @@ -0,0 +1,128 @@ +//! Render one authored setting and check emitted options against the build parser. + +use std::collections::BTreeMap; + +use serde_json::Value as Json; + +use crate::argv::_helpers::table::root_parser; +use crate::argv::constants::CONCURRENCY_DEST; +use crate::argv::models::{ActionKind, ActionSpec, Default, ParserSpec, Value}; + +pub(crate) fn setting_argument( + parser: &ParserSpec, + name: &str, + value: &Json, +) -> Result, String> { + if matches!( + name, + "help" + | "target" + | "project_dir" + | "select" + | "exclude" + | "select_file" + | "positional_select" + | "json" + | "json_output" + | "event_output" + | "manifest" + | "dbt_project_dir" + | "dbt_profiles_dir" + | "dbt_target" + | "dbt_target_path" + ) { + return Err(format!("job setting {name} is owned by the run request")); + } + let actions: Vec<&ActionSpec> = parser + .actions + .iter() + .filter(|action| action.dest.as_deref() == Some(name)) + .collect(); + let Some(action) = actions.first() else { + return Err(format!("unknown build setting {name}")); + }; + if matches!(action.kind, ActionKind::StoreTrue | ActionKind::StoreFalse) { + return boolean_argument(name, value, &actions); + } + let option = &action.option_strings[0]; + let text = setting_text(name, value, action)?; + if action + .choices + .as_ref() + .is_some_and(|choices| !choices.contains(&text)) + { + return Err(format!("job setting {name} is not an accepted choice")); + } + Ok(Some(format!("{option}={text}"))) +} + +fn boolean_argument( + name: &str, + value: &Json, + actions: &[&ActionSpec], +) -> Result, String> { + let action = actions[0]; + let flag = value + .as_bool() + .ok_or_else(|| format!("job setting {name} must be boolean"))?; + if let Some(chosen) = actions + .iter() + .find(|action| action.constant == Value::Bool(flag)) + { + return Ok(Some(chosen.option_strings[0].clone())); + } + if action.default != Default::Value(Value::Bool(flag)) + && !(action.default == Default::Suppress + && root_parser().actions.iter().any(|global| { + global.dest.as_deref() == Some(name) + && global.default == Default::Value(Value::Bool(flag)) + })) + { + return Err(format!("job setting {name} cannot be set to {flag}")); + } + Ok(None) +} + +fn setting_text(name: &str, value: &Json, action: &ActionSpec) -> Result { + match action.converter.as_deref() { + Some("int" | "_positive_integer") => { + let integer = value + .as_i64() + .ok_or_else(|| format!("job setting {name} must be an integer"))?; + if integer < 1 + && (name == CONCURRENCY_DEST + || action.converter.as_deref() == Some("_positive_integer")) + { + return Err(format!("job setting {name} must be positive")); + } + Ok(integer.to_string()) + } + Some("parse_cli_vars") => { + if !value.is_object() { + return Err(format!("job setting {name} must be a JSON object")); + } + Ok(value.to_string()) + } + _ => value + .as_str() + .ok_or_else(|| format!("job setting {name} must be text")) + .map(str::to_owned), + } +} + +pub(crate) fn validate_mutex(parser: &ParserSpec, arguments: &[String]) -> Result<(), String> { + let mut mutex_groups: BTreeMap = BTreeMap::new(); + for argument in arguments { + let option = argument + .split_once('=') + .map_or(argument.as_str(), |(option, _)| option); + if let Some(group) = parser.actions[parser.option_index[option]].mutex_group + && let Some(previous) = mutex_groups.insert(group, option) + { + return Err(format!( + "job settings {previous} and {option} are mutually exclusive" + )); + } + } + Ok(()) +} diff --git a/crates/sqlbuild-cli/src/argv/_helpers/mod.rs b/crates/sqlbuild-cli/src/argv/_helpers/mod.rs index 22763ff51e..1d2d932e20 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/mod.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod dbt_execution; pub(crate) mod formatting; +pub(crate) mod job_settings; pub(crate) mod namespace; pub(crate) mod normalization; pub(crate) mod parsing; diff --git a/crates/sqlbuild-cli/src/argv/_helpers/normalization.rs b/crates/sqlbuild-cli/src/argv/_helpers/normalization.rs index 311d9ef353..b53bf9bf04 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/normalization.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/normalization.rs @@ -77,24 +77,10 @@ impl Run<'_> { .map_err(|message| self.error(parser, &message))?; namespace.set("concurrency", concurrency); } - if CURSOR_COMMANDS.contains(&command) { - let untyped: bool = ["start_cursor", "end_cursor"] - .iter() - .any(|name| namespace.value(name) != Value::None); - let typed: bool = [ - "start_cursor_ts", - "end_cursor_ts", - "start_cursor_int", - "end_cursor_int", - ] - .iter() - .any(|name| namespace.value(name) != Value::None); - if untyped && typed { - return Err(self.error( - parser, - "--start-cursor/--end-cursor cannot be combined with the -ts or -int cursor flags", - )); - } + if CURSOR_COMMANDS.contains(&command) + && let Some(message) = cursor_error(|name| namespace.value(name) != Value::None) + { + return Err(self.error(parser, message)); } if command == RENAME_COMMAND && let Some(message) = rename_error(&namespace) @@ -156,6 +142,24 @@ impl Run<'_> { } } +/// The command line and authored job settings share the same cursor-family restriction. +pub(crate) fn cursor_error(present: impl Fn(&str) -> bool) -> Option<&'static str> { + let untyped = ["start_cursor", "end_cursor"] + .iter() + .any(|name| present(name)); + let typed = [ + "start_cursor_ts", + "end_cursor_ts", + "start_cursor_int", + "end_cursor_int", + ] + .iter() + .any(|name| present(name)); + (untyped && typed).then_some( + "--start-cursor/--end-cursor cannot be combined with the -ts or -int cursor flags", + ) +} + /// The first rule `--since` and `--explain` break, on the commands that accept them. fn since_error(namespace: &Namespace) -> Option<&'static str> { match namespace.value("since") { diff --git a/crates/sqlbuild-cli/src/argv/constants.rs b/crates/sqlbuild-cli/src/argv/constants.rs index ef1b833fbb..3c079c4086 100644 --- a/crates/sqlbuild-cli/src/argv/constants.rs +++ b/crates/sqlbuild-cli/src/argv/constants.rs @@ -36,6 +36,8 @@ pub(crate) const MAXIMUM_BLANK_RUN: usize = 2; /// `CliNamespace` attributes the post-parse rules read and write. pub(crate) const COMMAND_DEST: &str = "command"; +pub(crate) const BUILD_COMMAND: &str = "build"; +pub(crate) const CONCURRENCY_DEST: &str = "concurrency"; pub(crate) const DBT_COMMAND: &str = "dbt"; pub(crate) const DBT_DEBUG_COMMAND: &str = "debug"; pub(crate) const DBT_PASSTHROUGH_COMMANDS: [&str; 4] = ["plan", "run", "build", "debug"]; diff --git a/crates/sqlbuild-cli/src/argv/main/job_settings_arguments.rs b/crates/sqlbuild-cli/src/argv/main/job_settings_arguments.rs new file mode 100644 index 0000000000..46d7b8ba46 --- /dev/null +++ b/crates/sqlbuild-cli/src/argv/main/job_settings_arguments.rs @@ -0,0 +1,32 @@ +//! Validate frozen build settings against the same table as the command-line parser. + +use std::collections::BTreeMap; + +use serde_json::Value as Json; + +use crate::argv::_helpers::job_settings::{setting_argument, validate_mutex}; +use crate::argv::_helpers::normalization::cursor_error; +use crate::argv::_helpers::table::root_parser; +use crate::argv::constants::BUILD_COMMAND; + +/// Produce argument tokens, never shell text; execution identity is owned by the run request. +pub fn job_settings_arguments(settings: &BTreeMap) -> Result, String> { + let parser = root_parser() + .actions + .iter() + .flat_map(|action| &action.parsers) + .find(|(name, _)| name == BUILD_COMMAND) + .map(|(_, parser)| parser) + .ok_or_else(|| "build command is unavailable".to_owned())?; + let mut arguments: Vec = Vec::new(); + for (name, value) in settings { + if let Some(argument) = setting_argument(parser, name, value)? { + arguments.push(argument); + } + } + validate_mutex(parser, &arguments)?; + if let Some(message) = cursor_error(|name| settings.contains_key(name)) { + return Err(format!("job settings: {message}")); + } + Ok(arguments) +} diff --git a/crates/sqlbuild-cli/src/argv/main/mod.rs b/crates/sqlbuild-cli/src/argv/main/mod.rs index 0dabda8f2b..d456903240 100644 --- a/crates/sqlbuild-cli/src/argv/main/mod.rs +++ b/crates/sqlbuild-cli/src/argv/main/mod.rs @@ -1,3 +1,4 @@ pub mod cli_namespace_defaults; +pub mod job_settings_arguments; pub mod parse_cli_arguments; pub mod parse_dbt_execution_arguments; diff --git a/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs b/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs index d47a6702c5..ab5d022216 100644 --- a/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs +++ b/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs @@ -1,6 +1,14 @@ +use std::collections::BTreeMap; + +use serde_json::json; + +use crate::argv::main::job_settings_arguments::job_settings_arguments; use crate::argv::models::{ArgvOutcome, Value}; use crate::argv::tests::helpers::{parse, strings}; -use crate::argv::tests::test_types::{ExitTestCase, HelpTextTestCase, ParsedValueTestCase}; +use crate::argv::tests::test_types::{ + ExitTestCase, HelpTextTestCase, JobSettingsRejectionTestCase, JobSettingsTestCase, + ParsedValueTestCase, +}; #[test] fn given_accepted_arguments_when_parsing_then_attributes_hold_the_parsed_values() { @@ -172,3 +180,91 @@ fn given_command_help_when_parsing_then_hidden_options_are_left_out() { assert_eq!(absent, test_case.expected_absent, "{help}"); } } + +#[test] +fn given_job_settings_when_validating_then_values_survive_the_build_parser() { + let test_cases = [ + JobSettingsTestCase { + description: "inverse flags and option-like values survive the build parser", + settings: BTreeMap::from([ + ("concurrency".into(), json!(3)), + ("load_sources".into(), json!(false)), + ("reload".into(), json!(false)), + ("run_tests".into(), json!(false)), + ("full_refresh".into(), json!(true)), + ("warehouse".into(), json!("--full-refresh")), + ]), + expected_values: vec![ + ("concurrency", Value::Int("3".into())), + ("load_sources", Value::Bool(false)), + ("reload", Value::Bool(false)), + ("run_tests", Value::Bool(false)), + ("full_refresh", Value::Bool(true)), + ("warehouse", Value::Str("--full-refresh".into())), + ], + expected_empty: false, + }, + JobSettingsTestCase { + description: "default-valued flags do not become explicit options", + settings: BTreeMap::from([ + ("run_tests".into(), json!(true)), + ("full_refresh".into(), json!(false)), + ("debug".into(), json!(false)), + ]), + expected_values: vec![], + expected_empty: true, + }, + ]; + for test_case in test_cases { + let settings = job_settings_arguments(&test_case.settings).expect(test_case.description); + assert_eq!( + settings.is_empty(), + test_case.expected_empty, + "{}", + test_case.description + ); + let mut arguments: Vec = vec!["build".to_owned()]; + arguments.extend(settings); + let arguments: Vec<&str> = arguments.iter().map(String::as_str).collect(); + let parsed = parse(&arguments).parsed.expect(test_case.description); + for (name, expected_value) in test_case.expected_values { + assert_eq!( + parsed.get(name), + Some(&expected_value), + "{}", + test_case.description + ); + } + } +} + +#[test] +fn given_conflicting_job_settings_when_validating_then_options_are_rejected() { + let test_cases = [ + JobSettingsRejectionTestCase { + description: "no-load conflicts with reload", + settings: BTreeMap::from([ + ("load_sources".into(), json!(false)), + ("reload".into(), json!(true)), + ]), + expected_error: "mutually exclusive", + }, + JobSettingsRejectionTestCase { + description: "load conflicts with reload", + settings: BTreeMap::from([ + ("load_sources".into(), json!(true)), + ("reload".into(), json!(true)), + ]), + expected_error: "mutually exclusive", + }, + ]; + for test_case in test_cases { + assert!( + job_settings_arguments(&test_case.settings) + .expect_err(test_case.description) + .contains(test_case.expected_error), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/argv/tests/test_types.rs b/crates/sqlbuild-cli/src/argv/tests/test_types.rs index 4452d1689e..793e4a87a3 100644 --- a/crates/sqlbuild-cli/src/argv/tests/test_types.rs +++ b/crates/sqlbuild-cli/src/argv/tests/test_types.rs @@ -1,3 +1,6 @@ +use std::collections::BTreeMap; + +use serde_json::Value as Json; use sqlbuild_core::text::main::python_str_repr::python_str_repr; use sqlbuild_core::dbt_cli::models::DbtParsedFlags; @@ -63,3 +66,18 @@ pub(super) struct DbtExecutionTestCase { pub(super) args: &'static [&'static str], pub(super) expected_parsed: Result, } + +/// Authored settings and the build namespace or validation failure they produce. +pub(super) struct JobSettingsTestCase { + pub(super) description: &'static str, + pub(super) settings: BTreeMap, + pub(super) expected_values: Vec<(&'static str, Value)>, + pub(super) expected_empty: bool, +} + +/// Mutually exclusive authored options rejected before a snapshot is published. +pub(super) struct JobSettingsRejectionTestCase { + pub(super) description: &'static str, + pub(super) settings: BTreeMap, + pub(super) expected_error: &'static str, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index af7683934d..4e2fce5b69 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -9,6 +9,7 @@ use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use serde_json::Value; use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; use sqlbuild_analysis::graph::models::ProjectGraph; +use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; use sqlbuild_core::constants::PYTHON_INIT_MODULE_STEM; use sqlbuild_discovery::constants::{ORCHESTRATION_DIRECTORY, PYTHON_FILE_SUFFIX}; use sqlbuild_discovery::tree::main::rglob::rglob; @@ -20,7 +21,9 @@ use sqlbuild_orchestration::declarations::models::{ use sqlbuild_orchestration::scheduler::models::{CatchUp, Overlap}; use sqlbuild_orchestration::snapshots::models::SnapshotInput; -use crate::bindings::_helpers::native_host::orchestration_declarations::module_jobs; +use crate::bindings::_helpers::native_host::orchestration_declarations::{ + authored_setting, module_jobs, +}; /// Dedicated declaration modules are imported only in the code-location interpreter. pub(crate) fn discover_declarations<'py>( @@ -65,6 +68,10 @@ pub(crate) struct AuthoredJob { exclude: String, overlap: String, on_failure: String, + #[pyo3(from_py_with = authored_setting)] + settings: Value, + #[pyo3(from_py_with = authored_setting)] + concurrency: Value, concurrency_groups: Vec, schedule: Option, targets: Vec, @@ -93,6 +100,15 @@ pub(crate) fn compile_declarations( if !job.targets.is_empty() && !job.targets.contains(&input.target) { continue; } + let mut settings: BTreeMap = serde_json::from_value(job.settings) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + if !job.concurrency.is_null() { + if settings.contains_key("concurrency") { + return Err(PyValueError::new_err("job concurrency is declared twice")); + } + settings.insert("concurrency".into(), job.concurrency); + } + job_settings_arguments(&settings).map_err(PyValueError::new_err)?; let tokens = |value: &str| { value .split_whitespace() @@ -132,7 +148,7 @@ pub(crate) fn compile_declarations( JobDefinition { description: job.description, selection, - settings: BTreeMap::new(), + settings, overlap: serde_json::from_value::(Value::String(job.overlap)) .map_err(|error| PyValueError::new_err(error.to_string()))?, on_failure: serde_json::from_value::(Value::String(job.on_failure)) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs index 6c3d086654..76ee58e258 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs @@ -3,13 +3,23 @@ use std::collections::HashSet; use std::path::Path; -use pyo3::exceptions::PyTypeError; +use pyo3::exceptions::{PyTypeError, PyValueError}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; use sqlbuild_host::entries::models::UserCodeHost; use crate::bindings::_helpers::native_host::host::entry; +/// JSON authoring values cross once; the interpreter rejects circular user containers safely. +pub(crate) fn authored_setting(value: &Bound<'_, PyAny>) -> PyResult { + let text: String = value + .py() + .import("json")? + .call_method1("dumps", (value,))? + .extract()?; + serde_json::from_str(&text).map_err(|error| PyValueError::new_err(error.to_string())) +} + /// Only module execution crosses into Python; discovery and collection stay native. pub(crate) fn module_jobs<'py>( py: Python<'py>, diff --git a/src/sqlbuild/orchestration_declarations/models.py b/src/sqlbuild/orchestration_declarations/models.py index 278aa55335..e3496353b1 100644 --- a/src/sqlbuild/orchestration_declarations/models.py +++ b/src/sqlbuild/orchestration_declarations/models.py @@ -26,6 +26,8 @@ class Job: on_failure: Literal["skip_dependents", "skip_downstream_jobs", "stop_group"] = field( default="skip_dependents", kw_only=True ) + settings: dict[str, object] = field(default_factory=dict, kw_only=True) + concurrency: int | None = field(default=None, kw_only=True) concurrency_groups: tuple[str, ...] = field(default=(), kw_only=True) schedule: Schedule | None = field(default=None, kw_only=True) targets: tuple[str, ...] = field(default=(), kw_only=True) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index 6fd1933af6..1744939e31 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -230,6 +230,8 @@ class SnapshotDeclarationRejectionTestCase: select: str = "orders" cron: str = "0 2 * * *" timezone: str = "UTC" + settings: dict[str, object] = field(default_factory=dict) + concurrency: int | None = None copies: int = 1 expected_message: str = "" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index f9e8c85502..58c60b9f7a 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -304,6 +304,8 @@ def test_given_authored_jobs_when_compiling_then_native_snapshots_resolve_select select="+orders", exclude="stg_orders", overlap="queue", + settings={"run_tests": False, "vars": {"batch": 3}, "warehouse": "reporting"}, + concurrency=4, concurrency_groups=("order_reporting",), schedule=schedule(cron="15 2 * * *", timezone=test_case.timezone), targets=("development",), @@ -321,7 +323,12 @@ def test_given_authored_jobs_when_compiling_then_native_snapshots_resolve_select "nightly_orders": { "description": "Refresh order reporting.", "selection": list(test_case.expected_selection), - "settings": {}, + "settings": { + "concurrency": 4, + "run_tests": False, + "vars": {"batch": 3}, + "warehouse": "reporting", + }, "overlap": "queue", "on_failure": "skip_dependents", "concurrency_groups": ["order_reporting"], @@ -373,6 +380,57 @@ def test_given_authored_jobs_when_compiling_then_native_snapshots_resolve_select SnapshotDeclarationRejectionTestCase( description="duplicate names", copies=2, expected_message="duplicate job orders" ), + SnapshotDeclarationRejectionTestCase( + description="unknown setting", + settings={"concurreny": 4}, + expected_message="unknown build setting concurreny", + ), + SnapshotDeclarationRejectionTestCase( + description="selection cannot escape its pinned run", + settings={"select": "customers"}, + expected_message="owned by the run request", + ), + SnapshotDeclarationRejectionTestCase( + description="zero concurrency", + concurrency=0, + expected_message="must be positive", + ), + SnapshotDeclarationRejectionTestCase( + description="boolean is not concurrency", + concurrency=True, + expected_message="concurrency must be an integer", + ), + SnapshotDeclarationRejectionTestCase( + description="integer is not a boolean flag", + settings={"full_refresh": 1}, + expected_message="must be boolean", + ), + SnapshotDeclarationRejectionTestCase( + description="integer strings are not typed settings", + settings={"concurrency": "3"}, + expected_message="must be an integer", + ), + SnapshotDeclarationRejectionTestCase( + description="variables need an object", + settings={"vars": ["orders"]}, + expected_message="must be a JSON object", + ), + SnapshotDeclarationRejectionTestCase( + description="mutually exclusive load settings", + settings={"load_sources": False, "reload": True}, + expected_message="mutually exclusive", + ), + SnapshotDeclarationRejectionTestCase( + description="conflicting cursor families", + settings={"start_cursor": "2", "end_cursor_int": "4"}, + expected_message="cannot be combined", + ), + SnapshotDeclarationRejectionTestCase( + description="duplicate concurrency declarations", + concurrency=3, + settings={"concurrency": 4}, + expected_message="declared twice", + ), ], ids=lambda case: case.description, ) @@ -390,6 +448,8 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject "orders", description=test_case.job_description, select=test_case.select, + settings=test_case.settings, + concurrency=test_case.concurrency, schedule=schedule(cron=test_case.cron, timezone=test_case.timezone), ) ] * test_case.copies @@ -496,6 +556,26 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects expected_error=PythonNodeDiscoveryError, expected_message="Failed to import orchestration file orchestration/jobs.py", ), + SnapshotDiscoveryRejectionTestCase( + description="DuckShop settings cannot replace the selection", + files={ + "orchestration/jobs.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/invalid_settings.py" + ).read_text() + }, + expected_error=ValueError, + expected_message="owned by the run request", + ), + SnapshotDiscoveryRejectionTestCase( + description="DuckShop mutually exclusive settings fail on every retry", + files={ + "orchestration/jobs.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/conflicting_settings.py" + ).read_text() + }, + expected_error=ValueError, + expected_message="mutually exclusive", + ), ], ids=lambda case: case.description, ) From b4462ad5def88896bc5e69a201dea2155ae8f1ac Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 05:53:57 +0000 Subject: [PATCH 20/59] feat: administer principal credentials from native CLI Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../sqlbuild-cli/src/argv/command_table.json | 112 ++++++++++++++++ .../src/native_commands/_helpers/decision.rs | 24 +++- .../src/native_commands/main/_run_admin.rs | 17 ++- .../src/native_commands/models.rs | 4 +- .../tests/native_commands/admin/test_admin.rs | 120 +++++++++++++++++- .../tests/native_commands/admin/test_types.rs | 7 + .../tests/native_commands/test_control.rs | 7 + 7 files changed, 276 insertions(+), 15 deletions(-) diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index c5dad55e42..ec7cee1ac8 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -162,6 +162,59 @@ ] } }, + { + "name": "create-principal", + "parser": { + "prog": "sqb admin create-principal", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "NAME", + "required": true, + "help": "principal label; creation grants no permissions", + "group": 0 + }, + { + "kind": "store", + "options": [ + "--kind" + ], + "dest": "admin_principal_kind", + "choices": [ + "person", + "service_account" + ], + "default": "person", + "help": "identity kind (default: person)", + "group": 1 + } + ] + } + }, { "name": "issue-location-token", "parser": { @@ -211,6 +264,55 @@ ] } }, + { + "name": "issue-principal-token", + "parser": { + "prog": "sqb admin issue-principal-token", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "PRINCIPAL_ID", + "required": true, + "help": "principal UUID", + "group": 0 + }, + { + "kind": "store", + "options": [ + "--expires-at" + ], + "dest": "admin_expires_at", + "required": true, + "help": "future expiry as an RFC3339 timestamp with timezone", + "group": 1 + } + ] + } + }, { "name": "revoke-token", "parser": { @@ -257,11 +359,21 @@ "metavar": "create-location", "help": "register a location with an atomic security audit record" }, + { + "dest": "create-principal", + "metavar": "create-principal", + "help": "create a person or service account without granting permissions" + }, { "dest": "issue-location-token", "metavar": "issue-location-token", "help": "issue a bounded location credential, returned once as JSON" }, + { + "dest": "issue-principal-token", + "metavar": "issue-principal-token", + "help": "issue a bounded principal credential, returned once as JSON" + }, { "dest": "revoke-token", "metavar": "revoke-token", diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index 8ccde9b2a8..4055146ffc 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -4,6 +4,7 @@ use std::net::{IpAddr, SocketAddr}; use std::path::PathBuf; use chrono::{DateTime, Utc}; +use sqlbuild_orchestration::credentials::models::{CredentialScope, PrincipalKind}; use sqlbuild_orchestration::store::errors::ControlError; use uuid::Uuid; @@ -44,16 +45,27 @@ pub(crate) fn flag(namespace: &Namespace, name: &str) -> bool { pub(crate) fn control_admin(namespace: &Namespace) -> Result { let value = text(namespace, "admin_subject").unwrap_or_default(); - if text(namespace, "admin_command").as_deref() == Some("create-location") { + let action = text(namespace, "admin_command"); + if action.as_deref() == Some("create-location") { if value.trim().is_empty() { return Err(ControlError::Invalid("location name is required".into())); } return Ok(ControlAdmin::CreateLocation(value)); } + if action.as_deref() == Some("create-principal") { + if value.trim().is_empty() { + return Err(ControlError::Invalid("principal name is required".into())); + } + let kind = match text(namespace, "admin_principal_kind").as_deref() { + Some("service_account") => PrincipalKind::ServiceAccount, + _ => PrincipalKind::Person, + }; + return Ok(ControlAdmin::CreatePrincipal(value, kind)); + } let id = Uuid::parse_str(&value) .map_err(|_| ControlError::Invalid("a valid location or token UUID is required".into()))?; - match text(namespace, "admin_command").as_deref() { - Some("issue-location-token") => { + match action.as_deref() { + Some("issue-location-token" | "issue-principal-token") => { let expiry = DateTime::parse_from_rfc3339( &text(namespace, "admin_expires_at").unwrap_or_default(), ) @@ -64,7 +76,11 @@ pub(crate) fn control_admin(namespace: &Namespace) -> Result CredentialScope::Principal { principal_id: id }, + _ => CredentialScope::Location { location_id: id }, + }; + Ok(ControlAdmin::IssueToken(scope, expiry)) } Some("revoke-token") => Ok(ControlAdmin::RevokeToken(id)), _ => Err(ControlError::Invalid( diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs index 06b21d9aea..8929365faf 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs @@ -2,9 +2,10 @@ use serde_json::{Value, json}; use sqlbuild_orchestration::credentials::main::create_location::create_location; +use sqlbuild_orchestration::credentials::main::create_principal::create_principal; use sqlbuild_orchestration::credentials::main::issue_token::issue_token; use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; -use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; +use sqlbuild_orchestration::credentials::models::AuditContext; use sqlbuild_orchestration::store::errors::ControlError; use uuid::Uuid; @@ -27,14 +28,12 @@ pub(crate) async fn run_admin(namespace: &Namespace) -> Result { - let token = issue_token( - &store, - &context, - CredentialScope::Location { location_id }, - expires_at, - ) - .await?; + ControlAdmin::CreatePrincipal(name, kind) => { + let id = create_principal(&store, &context, (&name, kind)).await?; + Ok(json!({"principal_id": id, "name": name, "kind": kind})) + } + ControlAdmin::IssueToken(scope, expires_at) => { + let token = issue_token(&store, &context, scope, expires_at).await?; Ok(json!(token)) } ControlAdmin::RevokeToken(token_id) => { diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 9ae73c2995..df6d08a534 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -3,6 +3,7 @@ use std::path::PathBuf; use chrono::{DateTime, Utc}; +use sqlbuild_orchestration::credentials::models::{CredentialScope, PrincipalKind}; use uuid::Uuid; use crate::argv::models::Namespace; @@ -10,7 +11,8 @@ use crate::argv::models::Namespace; /// Privileged local operations require direct ownership of the control database. pub(crate) enum ControlAdmin { CreateLocation(String), - IssueLocationToken(Uuid, DateTime), + CreatePrincipal(String, PrincipalKind), + IssueToken(CredentialScope, DateTime), RevokeToken(Uuid), } diff --git a/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs b/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs index 70e03d2970..389afeb33b 100644 --- a/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs +++ b/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs @@ -15,7 +15,7 @@ use crate::admin_helpers::{administer, audit_actions}; use crate::helpers::{api_ready, start_control}; #[cfg(unix)] use crate::test_admin::test_types::AdminDeliveryTestCase; -use crate::test_admin::test_types::AdminTestCase; +use crate::test_admin::test_types::{AdminTestCase, PrincipalAdminTestCase}; #[test] fn given_a_standalone_control_server_when_bootstrapped_by_an_operator_then_scoped_publication_and_revocation_are_audited() @@ -117,6 +117,124 @@ fn given_a_standalone_control_server_when_bootstrapped_by_an_operator_then_scope } } +#[test] +fn given_native_principal_administration_when_issuing_credentials_then_identity_is_audited_without_granting_location_permissions() + { + let test_cases = [ + PrincipalAdminTestCase { + description: "a person credential has no implicit location authority", + kind: "person", + expected_name: "orders operator / café", + expected_actions: ["principal.created", "token.issued", "token.revoked"], + }, + PrincipalAdminTestCase { + description: "a service account credential has no implicit location authority", + kind: "service_account", + expected_name: "orders automation", + expected_actions: ["principal.created", "token.issued", "token.revoked"], + }, + ]; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("test runtime"); + for test_case in test_cases { + runtime.block_on(async { + let mut control = start_control(&["server", "--port", "0", "--no-scheduler"]); + let url = api_ready(&mut control).await; + let principal = administer( + &control, + &[ + "create-principal", + test_case.expected_name, + "--kind", + test_case.kind, + ], + ); + let principal_id = Uuid::parse_str( + principal["principal_id"] + .as_str() + .expect(test_case.description), + ) + .expect(test_case.description); + assert_eq!( + principal["name"], test_case.expected_name, + "{}", + test_case.description + ); + assert_eq!( + principal["kind"], test_case.kind, + "{}", + test_case.description + ); + let expiry = (Utc::now() + Duration::hours(1)).to_rfc3339(); + let credential = administer( + &control, + &[ + "issue-principal-token", + &principal_id.to_string(), + "--expires-at", + &expiry, + ], + ); + assert_eq!( + credential["identity"]["scope"]["kind"], "principal", + "{}", + test_case.description + ); + assert_eq!( + credential["identity"]["scope"]["principal_id"], + principal_id.to_string(), + "{}", + test_case.description + ); + let bearer = credential["bearer"].as_str().expect(test_case.description); + let snapshot = SnapshotInput { + location_id: Uuid::now_v7(), + target: "development".into(), + generation: 1, + git_revision: None, + git_dirty: false, + git_author: None, + image_id: None, + environment_hash: "orders-environment".into(), + sqb_version: env!("CARGO_PKG_VERSION").into(), + nodes: BTreeMap::new(), + documents: BTreeMap::new(), + }; + assert!( + matches!( + publish_remote_snapshot(&url, bearer, &snapshot).await, + Err(ClientError::Rejected(reqwest::StatusCode::FORBIDDEN)) + ), + "{}", + test_case.description + ); + let token_id = credential["identity"]["token_id"] + .as_str() + .expect(test_case.description); + let revoked = administer(&control, &["revoke-token", token_id]); + assert_eq!(revoked["revoked"], true, "{}", test_case.description); + assert!( + matches!( + publish_remote_snapshot(&url, bearer, &snapshot).await, + Err(ClientError::Rejected(reqwest::StatusCode::UNAUTHORIZED)) + ), + "{}", + test_case.description + ); + assert_eq!( + audit_actions(&control, bearer), + test_case.expected_actions, + "{}", + test_case.description + ); + #[cfg(unix)] + crate::helpers::terminate(&mut control).await; + }); + } +} + #[cfg(unix)] #[test] fn given_a_committed_admin_operation_when_json_output_is_lost_then_all_response_sizes_fail_without_claiming_rollback() diff --git a/crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs b/crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs index 62aac53bee..8999c65a6b 100644 --- a/crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs +++ b/crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs @@ -5,6 +5,13 @@ pub(crate) struct AdminTestCase { pub(crate) expected_actions: [&'static str; 4], } +pub(crate) struct PrincipalAdminTestCase { + pub(crate) description: &'static str, + pub(crate) kind: &'static str, + pub(crate) expected_name: &'static str, + pub(crate) expected_actions: [&'static str; 3], +} + #[cfg(unix)] pub(crate) struct AdminDeliveryTestCase { pub(crate) description: &'static str, diff --git a/crates/sqlbuild-cli/tests/native_commands/test_control.rs b/crates/sqlbuild-cli/tests/native_commands/test_control.rs index 4f4ffc1f3c..4da4952d73 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_control.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_control.rs @@ -35,6 +35,13 @@ fn given_a_standalone_binary_when_printing_help_or_rejecting_invalid_input_then_ expected_stdout_prefix: "", expected_stderr: "Control administration failed: invalid control-plane data: location name is required\n", }, + CommandTestCase { + description: "blank principal rejected before store initialization", + arguments: &["admin", "create-principal", " "], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: principal name is required\n", + }, CommandTestCase { description: "invalid credential ID rejected before store initialization", arguments: &["admin", "revoke-token", "not-a-uuid"], From 7547ab065daa5883c0a869af6d6dfd17ae49a3c1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 06:04:37 +0000 Subject: [PATCH 21/59] feat: expose location-scoped orchestration reads Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- Cargo.lock | 2 + Cargo.toml | 2 +- .../src/api/_helpers/authentication.rs | 15 + .../src/api/_helpers/mod.rs | 2 + .../src/api/_helpers/run_reads.rs | 67 ++++ .../src/api/_helpers/snapshots.rs | 13 +- .../src/api/_helpers/trigger_reads.rs | 96 +++++ .../src/api/main/serve_api.rs | 22 +- .../sqlbuild-orchestration/src/api/models.rs | 15 +- .../src/declarations/models.rs | 14 +- .../sqlbuild-orchestration/src/runs/models.rs | 6 +- .../src/scheduler/models.rs | 11 +- crates/sqlbuild-orchestration/tests/api.rs | 2 + .../tests/api/helpers.rs | 15 + .../tests/api/test_reads.rs | 365 ++++++++++++++++++ .../tests/api/test_types.rs | 7 + 16 files changed, 625 insertions(+), 29 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/run_reads.rs create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/trigger_reads.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/test_reads.rs diff --git a/Cargo.lock b/Cargo.lock index 7f825f1ccc..c468288a19 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -112,6 +112,7 @@ checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" dependencies = [ "axum-core", "bytes", + "form_urlencoded", "futures-util", "http", "http-body", @@ -127,6 +128,7 @@ dependencies = [ "serde_core", "serde_json", "serde_path_to_error", + "serde_urlencoded", "sync_wrapper", "tokio", "tower", diff --git a/Cargo.toml b/Cargo.toml index ea73843c9b..b28edb4dda 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,7 +64,7 @@ publish = false rust-version = "1.88" [workspace.dependencies] -axum = { version = "0.8", default-features = false, features = ["http1", "json", "tokio"] } +axum = { version = "0.8", default-features = false, features = ["http1", "json", "query", "tokio"] } fensu-policy = "=0.22.0" # `sqb` names compute log captures with random ids, as `uuid.uuid4().hex`. getrandom = "0.4" diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication.rs index 2920fd6ea0..c7f6dd2e06 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication.rs @@ -4,9 +4,11 @@ use axum::extract::{Request, State}; use axum::http::{HeaderValue, StatusCode, header}; use axum::middleware::Next; use axum::response::Response; +use uuid::Uuid; use crate::api::errors::ApiError; use crate::credentials::main::authenticate_token::authenticate_token; +use crate::credentials::models::{CredentialScope, TokenIdentity}; use crate::store::models::ControlStore; pub(crate) async fn authenticate( @@ -36,3 +38,16 @@ pub(crate) async fn authenticate( .insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); Ok(response) } + +pub(crate) fn require_location( + identity: &TokenIdentity, + location_id: Uuid, +) -> Result<(), ApiError> { + if identity.scope != (CredentialScope::Location { location_id }) { + return Err(ApiError { + status: StatusCode::FORBIDDEN, + message: "credential does not authorize this location".into(), + }); + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs index 4f00fe75c5..ed6ed88be8 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs @@ -1,4 +1,6 @@ pub(crate) mod authentication; pub(crate) mod client; +pub(crate) mod run_reads; pub(crate) mod runs; pub(crate) mod snapshots; +pub(crate) mod trigger_reads; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/run_reads.rs b/crates/sqlbuild-orchestration/src/api/_helpers/run_reads.rs new file mode 100644 index 0000000000..c8c002e3d0 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/run_reads.rs @@ -0,0 +1,67 @@ +//! Location credentials can observe only their own runs; they cannot request or mutate work. + +use axum::extract::{Path, Query, State}; +use axum::http::StatusCode; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::_helpers::authentication::require_location; +use crate::api::errors::ApiError; +use crate::api::models::ReadQuery; +use crate::credentials::models::TokenIdentity; +use crate::runs::main::list_run_nodes::list_run_nodes; +use crate::runs::main::list_runs::list_runs; +use crate::runs::models::{RunNodeRecord, RunRecord}; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/runs", + params(("location_id" = Uuid, Path), ("target" = String, Path), + ("limit" = Option, Query, description = "Maximum recent runs; defaults to 100, capped at 1000")), + responses((status = 200, body = Vec), (status = 401, description = "Invalid credential"), + (status = 403, description = "Location not authorized")), security(("bearer" = [])))] +pub(crate) async fn recent( + State(store): State, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, + Query(options): Query, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + Ok(Json( + list_runs(&store, (location_id, &target), options.limit.unwrap_or(100)).await?, + )) +} + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/runs/{run_id}/nodes", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("run_id" = Uuid, Path)), + responses((status = 200, body = Vec), (status = 401, description = "Invalid credential"), + (status = 403, description = "Location not authorized"), (status = 404, description = "Run not found")), + security(("bearer" = [])))] +pub(crate) async fn nodes( + State(store): State, + Extension(identity): Extension, + Path((location_id, target, run_id)): Path<(Uuid, String, Uuid)>, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + let transaction = begin(&store, TransactionMode::Read).await?; + let (transaction, rows) = query( + transaction, + "SELECT run_id FROM sqb_runs WHERE run_id=$1 AND location_id=$2 AND target=$3", + vec![ + Parameter::Id(run_id), + Parameter::Id(location_id), + Parameter::Text(target), + ], + ) + .await?; + commit(transaction).await?; + if rows.is_empty() { + return Err(ApiError { + status: StatusCode::NOT_FOUND, + message: "run not found".into(), + }); + } + Ok(Json(list_run_nodes(&store, run_id).await?)) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/snapshots.rs b/crates/sqlbuild-orchestration/src/api/_helpers/snapshots.rs index 198925e8fc..e5719f11ce 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/snapshots.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/snapshots.rs @@ -5,8 +5,9 @@ use axum::http::StatusCode; use axum::{Extension, Json}; use uuid::Uuid; +use crate::api::_helpers::authentication::require_location; use crate::api::errors::ApiError; -use crate::credentials::models::{CredentialScope, TokenIdentity}; +use crate::credentials::models::TokenIdentity; use crate::snapshots::main::current_snapshot::current_snapshot; use crate::snapshots::main::publish_snapshot::publish_snapshot; use crate::snapshots::models::{CurrentSnapshot, SnapshotInput, SnapshotRecord}; @@ -53,13 +54,3 @@ pub(crate) async fn current( })?; Ok(Json(current)) } - -fn require_location(identity: &TokenIdentity, location_id: Uuid) -> Result<(), ApiError> { - if identity.scope != (CredentialScope::Location { location_id }) { - return Err(ApiError { - status: StatusCode::FORBIDDEN, - message: "credential does not authorize this location".into(), - }); - } - Ok(()) -} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/trigger_reads.rs b/crates/sqlbuild-orchestration/src/api/_helpers/trigger_reads.rs new file mode 100644 index 0000000000..ec5dfa88eb --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/trigger_reads.rs @@ -0,0 +1,96 @@ +//! Trigger definitions and immutable history are visible only within the authenticated location. + +use axum::extract::{Path, Query, State}; +use axum::http::StatusCode; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::_helpers::authentication::require_location; +use crate::api::errors::ApiError; +use crate::api::models::ReadQuery; +use crate::credentials::models::TokenIdentity; +use crate::declarations::main::list_trigger_events::list_trigger_events; +use crate::declarations::main::list_triggers::list_triggers; +use crate::declarations::models::{TriggerChangeRecord, TriggerRecord}; +use crate::scheduler::main::list_ticks::list_ticks; +use crate::scheduler::models::TickRecord; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/triggers", + params(("location_id" = Uuid, Path), ("target" = String, Path)), + responses((status = 200, body = Vec), (status = 401, description = "Invalid credential"), + (status = 403, description = "Location not authorized")), security(("bearer" = [])))] +pub(crate) async fn current( + State(store): State, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + Ok(Json(list_triggers(&store, location_id, &target).await?)) +} + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/triggers/{trigger_id}/events", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("trigger_id" = Uuid, Path), + ("limit" = Option, Query, description = "Maximum events; defaults to 100, capped at 1000")), + responses((status = 200, body = Vec), (status = 401, description = "Invalid credential"), + (status = 403, description = "Location not authorized"), (status = 404, description = "Trigger not found")), + security(("bearer" = [])))] +pub(crate) async fn events( + State(store): State, + Extension(identity): Extension, + Path((location_id, target, trigger_id)): Path<(Uuid, String, Uuid)>, + Query(options): Query, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + require_trigger(&store, (location_id, &target, trigger_id)).await?; + Ok(Json( + list_trigger_events(&store, trigger_id, options.limit.unwrap_or(100)).await?, + )) +} + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/triggers/{trigger_id}/ticks", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("trigger_id" = Uuid, Path), + ("limit" = Option, Query, description = "Maximum recent ticks; defaults to 100, capped at 1000")), + responses((status = 200, body = Vec), (status = 401, description = "Invalid credential"), + (status = 403, description = "Location not authorized"), (status = 404, description = "Trigger not found")), + security(("bearer" = [])))] +pub(crate) async fn ticks( + State(store): State, + Extension(identity): Extension, + Path((location_id, target, trigger_id)): Path<(Uuid, String, Uuid)>, + Query(options): Query, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + require_trigger(&store, (location_id, &target, trigger_id)).await?; + Ok(Json( + list_ticks(&store, trigger_id, options.limit.unwrap_or(100)).await?, + )) +} + +async fn require_trigger( + store: &ControlStore, + (location_id, target, trigger_id): (Uuid, &str, Uuid), +) -> Result<(), ApiError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query( + transaction, + "SELECT trigger_id FROM sqb_triggers WHERE trigger_id=$1 AND location_id=$2 AND target=$3", + vec![ + Parameter::Id(trigger_id), + Parameter::Id(location_id), + Parameter::Text(target.into()), + ], + ) + .await?; + commit(transaction).await?; + if rows.is_empty() { + return Err(ApiError { + status: StatusCode::NOT_FOUND, + message: "trigger not found".into(), + }); + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs index 2dd444bef6..b6ea25ac48 100644 --- a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs +++ b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs @@ -5,7 +5,7 @@ use axum::routing::{get, post}; use axum::{Json, Router, middleware}; use crate::api::_helpers::authentication::authenticate; -use crate::api::_helpers::{runs, snapshots}; +use crate::api::_helpers::{run_reads, runs, snapshots, trigger_reads}; use crate::api::main::openapi_document::openapi_document; use crate::store::errors::ControlError; use crate::store::main::migrate_store::migrate_store; @@ -22,6 +22,26 @@ pub async fn serve_api( "/api/locations/{location_id}/targets/{target}/snapshots", get(snapshots::current).post(snapshots::publish), ) + .route( + "/api/locations/{location_id}/targets/{target}/runs", + get(run_reads::recent), + ) + .route( + "/api/locations/{location_id}/targets/{target}/runs/{run_id}/nodes", + get(run_reads::nodes), + ) + .route( + "/api/locations/{location_id}/targets/{target}/triggers", + get(trigger_reads::current), + ) + .route( + "/api/locations/{location_id}/targets/{target}/triggers/{trigger_id}/events", + get(trigger_reads::events), + ) + .route( + "/api/locations/{location_id}/targets/{target}/triggers/{trigger_id}/ticks", + get(trigger_reads::ticks), + ) .route("/api/runs/{run_id}/execution", get(runs::execution)) .route("/api/runs/{run_id}/events", post(runs::report)) .route("/api/runs/{run_id}/heartbeat", post(runs::heartbeat)) diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index 6120833396..722d7fad9c 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -2,7 +2,7 @@ use serde::{Deserialize, Serialize}; -use crate::api::_helpers::{runs, snapshots}; +use crate::api::_helpers::{run_reads, runs, snapshots, trigger_reads}; #[derive(Debug, utoipa::OpenApi)] #[openapi(paths( @@ -11,10 +11,21 @@ use crate::api::_helpers::{runs, snapshots}; runs::execution, runs::report, runs::heartbeat, - runs::output + runs::output, + run_reads::recent, + run_reads::nodes, + trigger_reads::current, + trigger_reads::events, + trigger_reads::ticks ))] pub(crate) struct ControlApi; +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ReadQuery { + pub(crate) limit: Option, +} + #[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] pub struct MutationResponse { pub applied: bool, diff --git a/crates/sqlbuild-orchestration/src/declarations/models.rs b/crates/sqlbuild-orchestration/src/declarations/models.rs index 9561c50ca1..4120b8c9b6 100644 --- a/crates/sqlbuild-orchestration/src/declarations/models.rs +++ b/crates/sqlbuild-orchestration/src/declarations/models.rs @@ -16,7 +16,7 @@ pub enum OnFailure { StopGroup, } -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(rename_all = "snake_case")] pub enum TriggerStatus { #[default] @@ -50,7 +50,7 @@ pub struct JobDefinition { pub concurrency_groups: BTreeSet, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum Schedule { Cron { @@ -63,7 +63,7 @@ pub enum Schedule { }, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(deny_unknown_fields)] pub struct TriggerDefinition { pub description: String, @@ -83,7 +83,7 @@ pub struct Definitions { pub triggers: BTreeMap, } -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] pub struct TriggerRecord { pub trigger_id: uuid::Uuid, pub name: String, @@ -95,7 +95,7 @@ pub struct TriggerRecord { } /// Operator changes are immutable facts; clearing an override restores the published code default. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum TriggerChange { Enable, @@ -105,7 +105,7 @@ pub enum TriggerChange { ResetCursor, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(deny_unknown_fields)] pub struct TriggerChangeRequest { pub event_id: uuid::Uuid, @@ -115,7 +115,7 @@ pub struct TriggerChangeRequest { pub change: TriggerChange, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] pub struct TriggerChangeRecord { pub request_json: TriggerChangeRequest, pub occurred_at: DateTime, diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index bf98fc240b..b6e379e372 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -48,7 +48,7 @@ impl RunOrigin { } } -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(rename_all = "snake_case")] pub enum RunStatus { Queued, @@ -88,7 +88,7 @@ pub struct RunRequest { pub overlap: crate::scheduler::models::Overlap, } -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] pub struct RunRecord { pub run_id: Uuid, pub location_id: Uuid, @@ -128,7 +128,7 @@ impl NodeOutcome { } } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] pub struct RunNodeRecord { pub node_key: String, pub attempt: i64, diff --git a/crates/sqlbuild-orchestration/src/scheduler/models.rs b/crates/sqlbuild-orchestration/src/scheduler/models.rs index 019083278d..9f4d421736 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/models.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/models.rs @@ -47,13 +47,16 @@ pub struct CronSchedule { pub(crate) timezone: chrono_tz::Tz, } -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(rename_all = "snake_case")] pub enum CatchUp { #[default] Latest, /// Process the oldest bounded batch; later due times remain due for the next transaction. - All { max_runs: std::num::NonZeroUsize }, + All { + #[schema(value_type = usize, minimum = 1)] + max_runs: std::num::NonZeroUsize, + }, } #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] @@ -127,7 +130,7 @@ pub(crate) struct DueTrigger { pub(crate) status: TriggerStatus, } -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(rename_all = "snake_case")] pub enum TickOutcome { Requested, @@ -137,7 +140,7 @@ pub enum TickOutcome { Deferred, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] pub struct TickRecord { pub tick_id: Uuid, pub trigger_id: Uuid, diff --git a/crates/sqlbuild-orchestration/tests/api.rs b/crates/sqlbuild-orchestration/tests/api.rs index 543523edd6..81c5ac5beb 100644 --- a/crates/sqlbuild-orchestration/tests/api.rs +++ b/crates/sqlbuild-orchestration/tests/api.rs @@ -12,6 +12,8 @@ mod store_helpers; mod store_test_types; #[path = "api/test_contract.rs"] mod test_contract; +#[path = "api/test_reads.rs"] +mod test_reads; #[path = "api/test_types.rs"] mod test_types; #[path = "api/test_worker.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/helpers.rs b/crates/sqlbuild-orchestration/tests/api/helpers.rs index 4364fba3a3..bb65bdcf07 100644 --- a/crates/sqlbuild-orchestration/tests/api/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/helpers.rs @@ -53,3 +53,18 @@ pub(crate) async fn stop_api(api: TestApi) { .expect("join API server") .expect("API server stopped cleanly"); } + +pub(crate) async fn read_json( + client: &reqwest::Client, + (url, bearer): (&str, &str), +) -> serde_json::Value { + let response = client + .get(url) + .bearer_auth(bearer) + .send() + .await + .expect("HTTP read"); + assert_eq!(response.status(), reqwest::StatusCode::OK); + assert_eq!(response.headers()["cache-control"], "no-store"); + response.json().await.expect("typed JSON response") +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_reads.rs b/crates/sqlbuild-orchestration/tests/api/test_reads.rs new file mode 100644 index 0000000000..b2df105562 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_reads.rs @@ -0,0 +1,365 @@ +//! Real HTTP reads keep both location and target scope, including guessed nested resource IDs. + +use std::collections::BTreeMap; +use std::time::Duration; + +use chrono::Utc; +use reqwest::StatusCode; +use serde_json::json; +use sqlbuild_orchestration::credentials::main::create_principal::create_principal; +use sqlbuild_orchestration::credentials::main::issue_token::issue_token; +use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; +use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope, PrincipalKind}; +use sqlbuild_orchestration::declarations::main::change_trigger::change_trigger; +use sqlbuild_orchestration::declarations::main::list_triggers::list_triggers; +use sqlbuild_orchestration::declarations::models::{TriggerChange, TriggerChangeRequest}; +use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; +use sqlbuild_orchestration::runs::main::request_run::request_run; +use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::scheduler::main::evaluate_trigger::evaluate_trigger; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use uuid::Uuid; + +use crate::helpers::{read_json, start_api, stop_api}; +use crate::location_helpers::create_location; +use crate::run_helpers::{request, snapshot}; +use crate::store_helpers::{close_test_store, open_test_store, run}; +use crate::store_test_types::Backend; +use crate::test_types::ApiReadTestCase; + +#[test] +fn given_published_locations_and_scheduled_work_when_read_through_http_then_resources_and_histories_remain_scoped() + { + let test_cases = [ + ApiReadTestCase { + description: "SQLite scoped HTTP reads", + backend: Backend::Sqlite, + expected_runs: 3, + expected_ticks: 3, + }, + ApiReadTestCase { + description: "PostgreSQL scoped HTTP reads", + backend: Backend::Postgres, + expected_runs: 3, + expected_ticks: 3, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location_id = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "inventory", "test-admin") + .await + .expect("other location"); + let mut input = snapshot(location_id); + input.documents.insert( + "jobs".into(), + json!({"orders": {"description": "Build orders", "selection": ["orders"]}}), + ); + input.documents.insert("triggers".into(), json!({"daily_orders": {"description": "Refresh orders", "job_key": "orders", "starts_at": "2020-01-01T00:00:00Z", + "schedule": {"kind": "calendar", "times": ["2020-01-01T00:00:00Z", "2020-01-02T00:00:00Z", "2020-01-03T00:00:00Z", "2030-01-01T00:00:00Z"], "refresh_at": "2029-01-01T00:00:00Z"}}})); + let published = publish_snapshot(store, &input) + .await + .expect("publish declarations"); + let trigger_id = list_triggers(store, location_id, "development") + .await + .expect("trigger")[0] + .trigger_id; + let mut foreign = input.clone(); + foreign.location_id = other; + let foreign_snapshot = publish_snapshot(store, &foreign) + .await + .expect("publish other location"); + let foreign_trigger = list_triggers(store, other, "development") + .await + .expect("foreign trigger")[0] + .trigger_id; + let foreign_run = request( + (other, foreign_snapshot.snapshot_id), + 1001, + 0, + &["inventory"], + ); + request_run(store, &foreign_run) + .await + .expect("queue foreign run"); + let mut another_target = input.clone(); + another_target.target = "validation".into(); + publish_snapshot(store, &another_target) + .await + .expect("publish second target"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + evaluate_trigger(store, &lease, trigger_id) + .await + .expect("evaluate due schedule"); + let mut manual_ids = Vec::new(); + for (id, node) in [(100, "customers"), (101, "products")] { + let run = request((location_id, published.snapshot_id), id, 0, &[node]); + manual_ids.push(run.run_id); + request_run(store, &run).await.expect("queue manual run"); + } + let change = TriggerChangeRequest { + event_id: Uuid::now_v7(), + trigger_id, + actor: "test-admin".into(), + reason: "Pause orders refresh".into(), + change: TriggerChange::Pause, + }; + change_trigger(store, &change).await.expect("pause trigger"); + let audit = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let expiry = Utc::now() + chrono::Duration::hours(1); + let token = issue_token( + store, + &audit, + CredentialScope::Location { location_id }, + expiry, + ) + .await + .expect("location token"); + let principal_id = + create_principal(store, &audit, ("orders operator", PrincipalKind::Person)) + .await + .expect("unprivileged person"); + let principal = issue_token( + store, + &audit, + CredentialScope::Principal { principal_id }, + expiry, + ) + .await + .expect("principal token"); + let claim = claim_next_run(store, &lease, &BTreeMap::new()) + .await + .expect("claim") + .expect("work"); + let worker = issue_token( + store, + &audit, + CredentialScope::Run { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }, + expiry, + ) + .await + .expect("worker token"); + let api = start_api(store).await; + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(5)) + .build() + .expect("HTTP client"); + let prefix = format!( + "{}/api/locations/{location_id}/targets/development", + api.url + ); + let run_path = format!("{prefix}/runs"); + let node_path = format!("{prefix}/runs/{}/nodes", manual_ids[0]); + let trigger_path = format!("{prefix}/triggers"); + let event_path = format!("{trigger_path}/{trigger_id}/events"); + let tick_path = format!("{trigger_path}/{trigger_id}/ticks"); + let paths = [ + &run_path, + &node_path, + &trigger_path, + &event_path, + &tick_path, + ]; + let runs = read_json(&client, (&run_path, &token.bearer)).await; + assert_eq!( + runs.as_array().expect("runs").len(), + test_case.expected_runs, + "{}", + test_case.description + ); + assert!( + runs.as_array() + .expect("runs") + .iter() + .all(|run| run["location_id"] == location_id.to_string() + && run["target"] == "development") + ); + let nodes = read_json(&client, (&node_path, &token.bearer)).await; + assert_eq!(nodes.as_array().expect("nodes").len(), 1); + assert_eq!(nodes[0]["node_key"], "customers"); + assert_eq!(nodes[0]["status"], "pending"); + let triggers = read_json(&client, (&trigger_path, &token.bearer)).await; + assert_eq!(triggers.as_array().expect("triggers").len(), 1); + assert_eq!(triggers[0]["trigger_id"], trigger_id.to_string()); + assert_eq!(triggers[0]["status"], "paused"); + let events = read_json(&client, (&event_path, &token.bearer)).await; + assert_eq!(events.as_array().expect("events").len(), 1); + assert_eq!(events[0]["request_json"], json!(change)); + let ticks = read_json(&client, (&tick_path, &token.bearer)).await; + assert_eq!( + ticks.as_array().expect("ticks").len(), + test_case.expected_ticks + ); + assert_eq!(ticks[0]["outcome"], "requested"); + assert_eq!(ticks[0]["scheduled_for"], "2020-01-03T00:00:00Z"); + assert_eq!(ticks[1]["outcome"], "covered"); + for path in [&run_path, &event_path, &tick_path] { + assert_eq!( + read_json(&client, (&format!("{path}?limit=1"), &token.bearer)) + .await + .as_array() + .expect("bounded history") + .len(), + 1 + ); + assert_eq!( + read_json(&client, (&format!("{path}?limit=0"), &token.bearer)).await, + json!([]) + ); + assert_eq!( + client + .get(format!("{path}?limit=invalid")) + .bearer_auth(&token.bearer) + .send() + .await + .expect("invalid limit") + .status(), + StatusCode::BAD_REQUEST + ); + } + for path in paths { + assert_eq!( + client + .get(path) + .send() + .await + .expect("unauthenticated read") + .status(), + StatusCode::UNAUTHORIZED + ); + for bearer in [&principal.bearer, &worker.bearer] { + assert_eq!( + client + .get(path) + .bearer_auth(bearer) + .send() + .await + .expect("wrong credential scope") + .status(), + StatusCode::FORBIDDEN + ); + } + assert_eq!( + client + .get(path.replace(&location_id.to_string(), &other.to_string())) + .bearer_auth(&token.bearer) + .send() + .await + .expect("cross-location read") + .status(), + StatusCode::FORBIDDEN + ); + } + for path in [ + node_path.replace(&manual_ids[0].to_string(), &foreign_run.run_id.to_string()), + event_path.replace(&trigger_id.to_string(), &foreign_trigger.to_string()), + tick_path.replace(&trigger_id.to_string(), &foreign_trigger.to_string()), + node_path.replace("/development/", "/validation/"), + event_path.replace("/development/", "/validation/"), + tick_path.replace("/development/", "/validation/"), + ] { + assert_eq!( + client + .get(path) + .bearer_auth(&token.bearer) + .send() + .await + .expect("nested ID cannot bypass scope") + .status(), + StatusCode::NOT_FOUND + ); + } + assert_eq!( + read_json( + &client, + ( + &run_path.replace("/development/", "/validation/"), + &token.bearer + ) + ) + .await, + json!([]) + ); + let validation = read_json( + &client, + ( + &trigger_path.replace("/development/", "/validation/"), + &token.bearer, + ), + ) + .await; + assert_eq!(validation.as_array().expect("other target").len(), 1); + assert_ne!(validation[0]["trigger_id"], trigger_id.to_string()); + let openapi: serde_json::Value = client + .get(format!("{}/api/openapi.json", api.url)) + .send() + .await + .expect("OpenAPI") + .json() + .await + .expect("OpenAPI JSON"); + for path in [ + "/api/locations/{location_id}/targets/{target}/runs", + "/api/locations/{location_id}/targets/{target}/runs/{run_id}/nodes", + "/api/locations/{location_id}/targets/{target}/triggers", + "/api/locations/{location_id}/targets/{target}/triggers/{trigger_id}/events", + "/api/locations/{location_id}/targets/{target}/triggers/{trigger_id}/ticks", + ] { + assert_eq!( + openapi["paths"][path]["get"]["security"], + json!([{"bearer": []}]) + ); + } + input.generation += 1; + input.documents.insert("triggers".into(), json!({})); + publish_snapshot(store, &input) + .await + .expect("remove declaration"); + assert_eq!( + read_json(&client, (&trigger_path, &token.bearer)).await, + json!([]) + ); + assert_eq!( + read_json(&client, (&event_path, &token.bearer)).await, + events + ); + assert_eq!(read_json(&client, (&tick_path, &token.bearer)).await, ticks); + revoke_token(store, &audit, token.identity.token_id) + .await + .expect("revoke location"); + for path in paths { + assert_eq!( + client + .get(path) + .bearer_auth(&token.bearer) + .send() + .await + .expect("revoked read") + .status(), + StatusCode::UNAUTHORIZED + ); + } + stop_api(api).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index 5ebd106605..2c63a4f4cd 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -14,6 +14,13 @@ pub(crate) struct WorkerRedirectTestCase { pub(crate) expected_status: reqwest::StatusCode, } +pub(crate) struct ApiReadTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_runs: usize, + pub(crate) expected_ticks: usize, +} + pub(crate) struct TestApi { pub(crate) url: String, pub(crate) shutdown: tokio::sync::oneshot::Sender<()>, From 5903c7135822ed9ca208993f407c83601fa0fc7d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 06:20:30 +0000 Subject: [PATCH 22/59] feat: compile authored schedule runtime controls Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../orchestration/edge_cases/definitions.py | 9 ++- .../compiled_project/snapshot_declarations.rs | 28 +++++-- .../orchestration_declarations/models.py | 6 +- .../compiler/compiled_project/_test_types.py | 12 ++- .../publication/test_publication.py | 29 ++++++- .../test_orchestration_snapshot.py | 80 +++++++++++++++++-- 6 files changed, 149 insertions(+), 15 deletions(-) diff --git a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py index db9dff756a..3fcca7b42e 100644 --- a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py +++ b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py @@ -8,7 +8,14 @@ select="tag:edge_case", settings={"run_tests": False}, concurrency=2, - schedule=schedule(cron="15 2 * * *", timezone="UTC"), + schedule=schedule( + cron="15 2 * * *", + timezone="UTC", + starts_at="2026-07-03T01:17:23.456789+05:30", + default_status="paused", + catch_up="all", + max_catch_up=3, + ), ) edge_alias = edge_orders edge_factory_results = [[edge_orders]] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index 4e2fce5b69..54ad885b83 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -16,9 +16,9 @@ use sqlbuild_discovery::tree::main::rglob::rglob; use sqlbuild_discovery::tree::models::ProjectTree; use sqlbuild_orchestration::declarations::main::snapshot_definitions::snapshot_definitions; use sqlbuild_orchestration::declarations::models::{ - JobDefinition, OnFailure, Schedule, TriggerDefinition, TriggerStatus, + JobDefinition, OnFailure, Schedule, TriggerDefinition, }; -use sqlbuild_orchestration::scheduler::models::{CatchUp, Overlap}; +use sqlbuild_orchestration::scheduler::models::Overlap; use sqlbuild_orchestration::snapshots::models::SnapshotInput; use crate::bindings::_helpers::native_host::orchestration_declarations::{ @@ -58,6 +58,12 @@ pub(crate) fn discover_declarations<'py>( struct AuthoredSchedule { cron: String, timezone: String, + #[pyo3(from_py_with = authored_setting)] + starts_at: Value, + default_status: String, + catch_up: String, + #[pyo3(from_py_with = authored_setting)] + max_catch_up: Value, } #[derive(FromPyObject)] @@ -128,6 +134,11 @@ pub(crate) fn compile_declarations( .map(|(kind, name)| format!("{kind}:{name}")) .collect(); if let Some(schedule) = job.schedule { + let catch_up = if schedule.max_catch_up.is_null() { + Value::String(schedule.catch_up) + } else { + serde_json::json!({schedule.catch_up: {"max_runs": schedule.max_catch_up}}) + }; triggers.insert( job.name.clone(), TriggerDefinition { @@ -137,9 +148,16 @@ pub(crate) fn compile_declarations( expression: schedule.cron, timezone: schedule.timezone, }, - starts_at: None, - default_status: TriggerStatus::Running, - catch_up: CatchUp::Latest, + starts_at: serde_json::from_value(schedule.starts_at) + .map_err(|error| PyValueError::new_err(format!("starts_at: {error}")))?, + default_status: serde_json::from_value(Value::String(schedule.default_status)) + .map_err(|error| { + PyValueError::new_err(format!("default_status: {error}")) + })?, + catch_up: serde_json::from_value(catch_up) + .map_err(|error| PyValueError::new_err(format!( + "catch_up: {error}; use 'latest' without a bound or 'all' with a positive max_catch_up" + )))?, }, ); } diff --git a/src/sqlbuild/orchestration_declarations/models.py b/src/sqlbuild/orchestration_declarations/models.py index e3496353b1..fa8ff9efc1 100644 --- a/src/sqlbuild/orchestration_declarations/models.py +++ b/src/sqlbuild/orchestration_declarations/models.py @@ -8,10 +8,14 @@ @dataclass(frozen=True, kw_only=True) class Schedule: - """A wall-clock cron schedule attached to a job.""" + """Cron with an RFC 3339 start; catch_up='all' requires a positive max_catch_up.""" cron: str timezone: str + starts_at: str | None = None + default_status: Literal["running", "paused"] = "running" + catch_up: Literal["latest", "all"] = "latest" + max_catch_up: int | None = None @dataclass(frozen=True, eq=False) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index 1744939e31..53f479aa3c 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -4,6 +4,8 @@ from dataclasses import dataclass, field +from sqlbuild.orchestration import Schedule + @dataclass(frozen=True) class HeaderProjectionTestCase: @@ -219,8 +221,15 @@ class SnapshotEnvironmentTestCase: @dataclass(frozen=True) class SnapshotScheduleTestCase: description: str - timezone: str + schedule: Schedule expected_selection: tuple[str, ...] + expected_controls: dict[str, object] = field( + default_factory=lambda: { + "starts_at": None, + "default_status": "running", + "catch_up": "latest", + } + ) @dataclass(frozen=True) @@ -234,6 +243,7 @@ class SnapshotDeclarationRejectionTestCase: concurrency: int | None = None copies: int = 1 expected_message: str = "" + schedule: Schedule | None = None @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py index ce5ca06abd..90d620f024 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py @@ -3,6 +3,8 @@ import json from pathlib import Path from typing import Any +from urllib.parse import quote +from urllib.request import Request, urlopen import pytest @@ -50,7 +52,15 @@ def test_given_retained_compiler_data_when_publishing_then_http_updates_atomical test_case: SnapshotPublicationTestCase, tmp_path: Path ) -> None: project_dir: Path = tmp_path / "project" - write_files(project_dir=project_dir, files=TYPE_PROOF_FILES) + files: dict[str, str] = TYPE_PROOF_FILES | { + "orchestration/edge_cases/definitions.py": Path( + "benchmarks/duckshop/project/orchestration/edge_cases/definitions.py" + ).read_text(), + "models/orders.sql": TYPE_PROOF_FILES["models/orders.sql"].replace( + "MODEL (", "MODEL (tags ['edge_case'],", 1 + ), + } + write_files(project_dir=project_dir, files=files) project: CompiledProjectView = build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=project_dir), adapter=DuckDbAdapter(), @@ -71,7 +81,10 @@ def test_given_retained_compiler_data_when_publishing_then_http_updates_atomical initial: dict[str, Any] = current_snapshot(api, test_case.target) assert frozenset(initial["nodes"]) == test_case.expected_nodes assert initial["nodes"]["model:orders"]["definition"]["deps"] == ["model:stg_orders"] - edited: dict[str, str] = TYPE_PROOF_FILES | { + assert initial["documents"]["triggers"]["edge_snapshot_orders"]["catch_up"] == { + "all": {"max_runs": 3} + } + edited: dict[str, str] = files | { "models/stg_orders.sql": TYPE_PROOF_FILES["models/stg_orders.sql"].replace( "CAST(5 AS INTEGER)", "CAST(9 AS INTEGER)" ) @@ -95,3 +108,15 @@ def test_given_retained_compiler_data_when_publishing_then_http_updates_atomical current: dict[str, Any] = current_snapshot(api, test_case.target) assert current["record"] == updated assert "CAST(9 AS INTEGER)" in current["nodes"]["model:stg_orders"]["raw_sql"] + request: Request = Request( + f"{api.url}/api/locations/{api.location_id}/targets/" + f"{quote(test_case.target, safe='')}/triggers", + headers={"Authorization": f"Bearer {api.bearer}"}, + ) + with urlopen(request, timeout=5) as response: + triggers: list[dict[str, Any]] = json.load(response) + assert len(triggers) == 1 + assert triggers[0]["status"] == "paused" + assert triggers[0]["definition_json"]["starts_at"] == "2026-07-02T19:47:23.456789Z" + assert triggers[0]["definition_json"]["default_status"] == "paused" + assert triggers[0]["definition_json"]["catch_up"] == {"all": {"max_runs": 3}} diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 58c60b9f7a..03b75e763a 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -280,11 +280,42 @@ def test_given_a_dependency_lock_when_it_changes_then_environment_identity_chang [ SnapshotScheduleTestCase( description="daylight saving zone", - timezone="Europe/London", + schedule=schedule(cron="15 2 * * *", timezone="Europe/London"), expected_selection=("model:orders",), ), SnapshotScheduleTestCase( - description="fixed offset zone", timezone="UTC", expected_selection=("model:orders",) + description="fixed offset zone", + schedule=schedule(cron="15 2 * * *", timezone="UTC"), + expected_selection=("model:orders",), + ), + SnapshotScheduleTestCase( + description="paused bounded catch-up with a zoned start", + schedule=schedule( + cron="15 2 * * *", + timezone="Europe/London", + starts_at="2026-07-03T01:17:23.456789+05:30", + default_status="paused", + catch_up="all", + max_catch_up=3, + ), + expected_selection=("model:orders",), + expected_controls={ + "starts_at": "2026-07-02T19:47:23.456789Z", + "default_status": "paused", + "catch_up": {"all": {"max_runs": 3}}, + }, + ), + SnapshotScheduleTestCase( + description="largest supported catch-up batch", + schedule=schedule( + cron="15 2 * * *", timezone="UTC", catch_up="all", max_catch_up=10_000 + ), + expected_selection=("model:orders",), + expected_controls={ + "starts_at": None, + "default_status": "running", + "catch_up": {"all": {"max_runs": 10_000}}, + }, ), ], ids=lambda case: case.description, @@ -307,7 +338,7 @@ def test_given_authored_jobs_when_compiling_then_native_snapshots_resolve_select settings={"run_tests": False, "vars": {"batch": 3}, "warehouse": "reporting"}, concurrency=4, concurrency_groups=("order_reporting",), - schedule=schedule(cron="15 2 * * *", timezone=test_case.timezone), + schedule=test_case.schedule, targets=("development",), ) excluded: Job = replace(nightly, name="other_target", targets=("production",)) @@ -339,8 +370,9 @@ def test_given_authored_jobs_when_compiling_then_native_snapshots_resolve_select assert trigger["schedule"] == { "kind": "cron", "expression": "15 2 * * *", - "timezone": test_case.timezone, + "timezone": test_case.schedule.timezone, } + assert {key: trigger[key] for key in test_case.expected_controls} == test_case.expected_controls edited: Job = replace(nightly, schedule=schedule(cron="30 3 * * *", timezone="UTC")) second: dict[str, Any] = json.loads( compile_snapshot( @@ -377,6 +409,43 @@ def test_given_authored_jobs_when_compiling_then_native_snapshots_resolve_select SnapshotDeclarationRejectionTestCase( description="invalid timezone", timezone="Unknown/Zone", expected_message="time zone" ), + SnapshotDeclarationRejectionTestCase( + description="zero catch-up bound", + schedule=schedule(cron="0 2 * * *", timezone="UTC", catch_up="all", max_catch_up=0), + expected_message="catch_up", + ), + SnapshotDeclarationRejectionTestCase( + description="negative catch-up bound", + schedule=schedule(cron="0 2 * * *", timezone="UTC", catch_up="all", max_catch_up=-1), + expected_message="catch_up", + ), + SnapshotDeclarationRejectionTestCase( + description="boolean is not a catch-up bound", + schedule=schedule(cron="0 2 * * *", timezone="UTC", catch_up="all", max_catch_up=True), + expected_message="catch_up", + ), + SnapshotDeclarationRejectionTestCase( + description="unbounded all catch-up", + schedule=schedule(cron="0 2 * * *", timezone="UTC", catch_up="all"), + expected_message="catch_up", + ), + SnapshotDeclarationRejectionTestCase( + description="catch-up exceeds transaction bound", + schedule=schedule( + cron="0 2 * * *", timezone="UTC", catch_up="all", max_catch_up=10_001 + ), + expected_message="catch-up exceeds 10000", + ), + SnapshotDeclarationRejectionTestCase( + description="latest does not silently ignore a bound", + schedule=schedule(cron="0 2 * * *", timezone="UTC", max_catch_up=3), + expected_message="latest.*without a bound", + ), + SnapshotDeclarationRejectionTestCase( + description="start needs an explicit offset", + schedule=schedule(cron="0 2 * * *", timezone="UTC", starts_at="2026-07-03T01:17:23"), + expected_message="starts_at", + ), SnapshotDeclarationRejectionTestCase( description="duplicate names", copies=2, expected_message="duplicate job orders" ), @@ -450,7 +519,8 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject select=test_case.select, settings=test_case.settings, concurrency=test_case.concurrency, - schedule=schedule(cron=test_case.cron, timezone=test_case.timezone), + schedule=test_case.schedule + or schedule(cron=test_case.cron, timezone=test_case.timezone), ) ] * test_case.copies with pytest.raises(ValueError, match=test_case.expected_message): From 0f4abcdef23b33b25ef91177a5d349c662ff9516 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 06:40:38 +0000 Subject: [PATCH 23/59] feat: add atomic privileged trigger controls Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../sqlbuild-cli/src/argv/command_table.json | 90 ++++++++++++++ .../src/native_commands/_helpers/decision.rs | 32 ++++- .../src/native_commands/main/_run_admin.rs | 22 ++++ .../src/native_commands/models.rs | 7 ++ .../tests/native_commands/admin/helpers.rs | 64 ++++++++-- .../tests/native_commands/admin/test_admin.rs | 54 ++++++++- .../tests/native_commands/admin/test_types.rs | 10 +- .../tests/native_commands/test_control.rs | 112 +++++++++++++++++- .../tests/native_commands/test_types.rs | 6 + .../src/credentials/_helpers/mod.rs | 1 - .../audit.rs => main/append_audit.rs} | 2 +- .../src/credentials/main/create_location.rs | 4 +- .../src/credentials/main/create_principal.rs | 4 +- .../src/credentials/main/issue_token.rs | 4 +- .../src/credentials/main/mod.rs | 1 + .../src/credentials/main/revoke_token.rs | 4 +- .../src/declarations/main/change_trigger.rs | 32 ++++- .../tests/api/test_reads.rs | 4 +- .../tests/scheduler/ticks/helpers.rs | 48 +++++++- .../tests/scheduler/ticks/test_triggers.rs | 100 ++++++++++++---- 20 files changed, 540 insertions(+), 61 deletions(-) rename crates/sqlbuild-orchestration/src/credentials/{_helpers/audit.rs => main/append_audit.rs} (97%) diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index ec7cee1ac8..cd0b2193e9 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -351,6 +351,91 @@ } ] } + }, + { + "name": "change-trigger", + "parser": { + "prog": "sqb admin change-trigger", + "description": "Change runtime trigger state using privileged database access; the fact, projection and security audit commit together.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "TRIGGER_ID", + "required": true, + "help": "current trigger UUID", + "group": 0 + }, + { + "kind": "store", + "options": [], + "dest": "admin_trigger_change", + "metavar": "CHANGE", + "choices": [ + "enable", + "pause", + "reset-default", + "set-cursor", + "reset-cursor" + ], + "required": true, + "help": "runtime change; reset-default restores the published status default", + "group": 0 + }, + { + "kind": "store", + "options": [ + "--event-id" + ], + "dest": "admin_event_id", + "required": true, + "help": "request UUID; reuse it with identical arguments after an uncertain result", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--reason" + ], + "dest": "admin_reason", + "required": true, + "help": "nonblank reason recorded with the operator fact and security audit", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--cursor" + ], + "dest": "admin_cursor", + "help": "RFC3339 timestamp with offset; required only for set-cursor", + "group": 1 + } + ] + } } ], "choices_help": [ @@ -378,6 +463,11 @@ "dest": "revoke-token", "metavar": "revoke-token", "help": "revoke a credential by UUID" + }, + { + "dest": "change-trigger", + "metavar": "change-trigger", + "help": "change trigger status or cursor with an atomic audit and retry identity" } ] } diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index 4055146ffc..0673c8aa18 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -4,7 +4,9 @@ use std::net::{IpAddr, SocketAddr}; use std::path::PathBuf; use chrono::{DateTime, Utc}; +use serde_json::json; use sqlbuild_orchestration::credentials::models::{CredentialScope, PrincipalKind}; +use sqlbuild_orchestration::declarations::models::TriggerChange; use sqlbuild_orchestration::store::errors::ControlError; use uuid::Uuid; @@ -63,8 +65,36 @@ pub(crate) fn control_admin(namespace: &Namespace) -> Result { + let event_id = Uuid::parse_str(&text(namespace, "admin_event_id").unwrap_or_default()) + .map_err(|_| ControlError::Invalid("--event-id must be a UUID".into()))?; + let reason = text(namespace, "admin_reason").unwrap_or_default(); + if reason.trim().is_empty() { + return Err(ControlError::Invalid("--reason must not be blank".into())); + } + let mut change = json!({ + "kind": text(namespace, "admin_trigger_change").unwrap_or_default().replace('-', "_"), + }); + let cursor = text(namespace, "admin_cursor"); + if let Some(cursor) = &cursor { + change["time"] = json!(cursor); + } + let change: TriggerChange = serde_json::from_value(change) + .map_err(|error| ControlError::Invalid(format!("trigger change: {error}")))?; + if cursor.is_some() && !matches!(change, TriggerChange::SetCursor { .. }) { + return Err(ControlError::Invalid( + "--cursor is only valid for set-cursor".into(), + )); + } + Ok(ControlAdmin::ChangeTrigger { + event_id, + trigger_id: id, + change, + reason, + }) + } Some("issue-location-token" | "issue-principal-token") => { let expiry = DateTime::parse_from_rfc3339( &text(namespace, "admin_expires_at").unwrap_or_default(), diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs index 8929365faf..90bc31d618 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs @@ -6,6 +6,8 @@ use sqlbuild_orchestration::credentials::main::create_principal::create_principa use sqlbuild_orchestration::credentials::main::issue_token::issue_token; use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; use sqlbuild_orchestration::credentials::models::AuditContext; +use sqlbuild_orchestration::declarations::main::change_trigger::change_trigger; +use sqlbuild_orchestration::declarations::models::TriggerChangeRequest; use sqlbuild_orchestration::store::errors::ControlError; use uuid::Uuid; @@ -40,5 +42,25 @@ pub(crate) async fn run_admin(namespace: &Namespace) -> Result { + let applied = change_trigger( + &store, + &context, + &TriggerChangeRequest { + event_id, + trigger_id, + actor: context.actor.clone(), + reason, + change, + }, + ) + .await?; + Ok(json!({"event_id": event_id, "trigger_id": trigger_id, "applied": applied})) + } } } diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index df6d08a534..b7b6f00b4b 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -4,6 +4,7 @@ use std::path::PathBuf; use chrono::{DateTime, Utc}; use sqlbuild_orchestration::credentials::models::{CredentialScope, PrincipalKind}; +use sqlbuild_orchestration::declarations::models::TriggerChange; use uuid::Uuid; use crate::argv::models::Namespace; @@ -14,6 +15,12 @@ pub(crate) enum ControlAdmin { CreatePrincipal(String, PrincipalKind), IssueToken(CredentialScope, DateTime), RevokeToken(Uuid), + ChangeTrigger { + event_id: Uuid, + trigger_id: Uuid, + change: TriggerChange, + reason: String, + }, } /// Parsing can finish an invocation before package discovery or any Python handoff. diff --git a/crates/sqlbuild-cli/tests/native_commands/admin/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/admin/helpers.rs index ae7e64d9c4..d43fbfa744 100644 --- a/crates/sqlbuild-cli/tests/native_commands/admin/helpers.rs +++ b/crates/sqlbuild-cli/tests/native_commands/admin/helpers.rs @@ -1,13 +1,29 @@ //! Run the isolated native executable; never include returned bearer bytes in assertion messages. -use std::process::Command; +use std::process::{Command, Output}; use serde_json::Value; +use sqlbuild_orchestration::declarations::models::TriggerRecord; +use uuid::Uuid; use crate::test_types::TestControl; pub(crate) fn administer(control: &TestControl, arguments: &[&str]) -> Value { - let output = Command::new( + let output = invoke_admin(control, arguments); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + String::from_utf8_lossy(&output.stderr), + "Opening control store...\nControl store ready\nControl administration completed successfully\n" + ); + serde_json::from_slice(&output.stdout).expect("JSON-only admin stdout") +} + +pub(crate) fn invoke_admin(control: &TestControl, arguments: &[&str]) -> Output { + Command::new( control .directory .path() @@ -19,17 +35,39 @@ pub(crate) fn administer(control: &TestControl, arguments: &[&str]) -> Value { .env("PATH", "") .env_remove("SQLBUILD_CONTROL_DATABASE_URL") .output() - .expect("native administration process"); - assert!( - output.status.success(), - "{}", - String::from_utf8_lossy(&output.stderr) - ); - assert_eq!( - String::from_utf8_lossy(&output.stderr), - "Opening control store...\nControl store ready\nControl administration completed successfully\n" - ); - serde_json::from_slice(&output.stdout).expect("JSON-only admin stdout") + .expect("native administration process") +} + +pub(crate) async fn read_triggers( + url: &str, + bearer: &str, + (location, target): (Uuid, &str), +) -> Vec { + let mut endpoint = reqwest::Url::parse(url).expect("API address"); + endpoint + .path_segments_mut() + .expect("API path") + .pop_if_empty() + .extend([ + "api", + "locations", + &location.to_string(), + "targets", + target, + "triggers", + ]); + reqwest::Client::new() + .get(endpoint) + .bearer_auth(bearer) + .timeout(std::time::Duration::from_secs(5)) + .send() + .await + .expect("trigger response") + .error_for_status() + .expect("authorized trigger read") + .json() + .await + .expect("trigger records") } pub(crate) fn audit_actions(control: &TestControl, bearer: &str) -> Vec { diff --git a/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs b/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs index 389afeb33b..f2ab692398 100644 --- a/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs +++ b/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs @@ -8,14 +8,14 @@ use std::collections::BTreeMap; use chrono::{Duration, Utc}; use sqlbuild_orchestration::api::errors::ClientError; use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; -use sqlbuild_orchestration::snapshots::models::SnapshotInput; +use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; use uuid::Uuid; -use crate::admin_helpers::{administer, audit_actions}; +use crate::admin_helpers::{administer, audit_actions, invoke_admin, read_triggers}; use crate::helpers::{api_ready, start_control}; #[cfg(unix)] use crate::test_admin::test_types::AdminDeliveryTestCase; -use crate::test_admin::test_types::{AdminTestCase, PrincipalAdminTestCase}; +use crate::test_admin::test_types::{AdminTestCase, PrincipalAdminTestCase, TriggerAdminTestCase}; #[test] fn given_a_standalone_control_server_when_bootstrapped_by_an_operator_then_scoped_publication_and_revocation_are_audited() @@ -29,6 +29,11 @@ fn given_a_standalone_control_server_when_bootstrapped_by_an_operator_then_scope "location.created", "token.issued", "token.revoked", + "trigger.changed", + "trigger.changed", + "trigger.changed", + "trigger.changed", + "trigger.changed", ], }]; let runtime = tokio::runtime::Builder::new_current_thread() @@ -78,13 +83,52 @@ fn given_a_standalone_control_server_when_bootstrapped_by_an_operator_then_scope image_id: None, environment_hash: "orders-environment".into(), sqb_version: env!("CARGO_PKG_VERSION").into(), - nodes: BTreeMap::new(), - documents: BTreeMap::new(), + nodes: BTreeMap::from([("model:orders".into(), SnapshotNode { + kind: "model".into(), + definition: serde_json::json!({"materialization": "table"}), + raw_sql: Some("select 1 as order_id".into()), + compiled_sql: Some("select 1 as order_id".into()), + python_source: None, + })]), + documents: BTreeMap::from([ + ("jobs".into(), serde_json::json!({"orders": {"description": "Refresh orders.", "selection": ["model:orders"]}})), + ("triggers".into(), serde_json::json!({"daily_orders": { + "description": "Daily orders.", "job_key": "orders", "default_status": "paused", + "schedule": {"kind": "cron", "expression": "0 2 * * *", "timezone": "UTC"}, + }})), + ]), }; let published = publish_remote_snapshot(&url, bearer, &snapshot) .await .expect("operator-issued credential publishes"); assert_eq!(published.location_id, location_id); + let trigger_id = read_triggers(&url, bearer, (location_id, "development")).await[0].trigger_id.to_string(); + for case in [ + TriggerAdminTestCase { description: "enable overrides the paused code default", change: "enable", cursor_arguments: &[], expected_status: "running", expected_cursor: None }, + TriggerAdminTestCase { description: "pause overrides an enabled trigger", change: "pause", cursor_arguments: &[], expected_status: "paused", expected_cursor: None }, + TriggerAdminTestCase { description: "reset-default restores the paused code default", change: "reset-default", cursor_arguments: &[], expected_status: "paused", expected_cursor: None }, + TriggerAdminTestCase { description: "set-cursor normalizes the timestamp offset", change: "set-cursor", cursor_arguments: &["--cursor", "2026-07-03T01:17:23+05:30"], expected_status: "paused", expected_cursor: Some("2026-07-02T19:47:23+00:00") }, + TriggerAdminTestCase { description: "reset-cursor removes the durable cursor", change: "reset-cursor", cursor_arguments: &[], expected_status: "paused", expected_cursor: None }, + ] { + let event_id = Uuid::now_v7().to_string(); + let mut arguments = vec!["change-trigger", &trigger_id, case.change, "--event-id", &event_id, "--reason", "Order refresh maintenance"]; + arguments.extend_from_slice(case.cursor_arguments); + for expected in [true, false] { + let result = administer(&control, &arguments); + assert_eq!(result["applied"], expected, "{}", case.description); + assert_eq!(result["event_id"], event_id); + } + let mut conflict = arguments.clone(); + conflict[6] = "Different request content"; + assert!(!invoke_admin(&control, &conflict).status.success(), "{}", case.description); + let triggers = read_triggers(&url, bearer, (location_id, "development")).await; + assert_eq!(triggers[0].status.as_str(), case.expected_status, "{}", case.description); + assert_eq!(triggers[0].last_tick_for.map(|value| value.to_rfc3339()).as_deref(), case.expected_cursor, "{}", case.description); + } + snapshot.generation = 2; + snapshot.documents.get_mut("triggers").expect("trigger document")["daily_orders"]["default_status"] = serde_json::json!("running"); + publish_remote_snapshot(&url, bearer, &snapshot).await.expect("publish a new default"); + assert_eq!(read_triggers(&url, bearer, (location_id, "development")).await[0].status.as_str(), "running", "reset-default removed the operator override"); snapshot.location_id = other_id; assert!(matches!( publish_remote_snapshot(&url, bearer, &snapshot).await, diff --git a/crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs b/crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs index 8999c65a6b..3adc9b5e24 100644 --- a/crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs +++ b/crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs @@ -2,7 +2,7 @@ pub(crate) struct AdminTestCase { pub(crate) description: &'static str, pub(crate) location_name: &'static str, pub(crate) expected_name: &'static str, - pub(crate) expected_actions: [&'static str; 4], + pub(crate) expected_actions: [&'static str; 9], } pub(crate) struct PrincipalAdminTestCase { @@ -12,6 +12,14 @@ pub(crate) struct PrincipalAdminTestCase { pub(crate) expected_actions: [&'static str; 3], } +pub(crate) struct TriggerAdminTestCase { + pub(crate) description: &'static str, + pub(crate) change: &'static str, + pub(crate) cursor_arguments: &'static [&'static str], + pub(crate) expected_status: &'static str, + pub(crate) expected_cursor: Option<&'static str>, +} + #[cfg(unix)] pub(crate) struct AdminDeliveryTestCase { pub(crate) description: &'static str, diff --git a/crates/sqlbuild-cli/tests/native_commands/test_control.rs b/crates/sqlbuild-cli/tests/native_commands/test_control.rs index 4da4952d73..74eaf195d3 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_control.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_control.rs @@ -1,7 +1,7 @@ use std::process::Command; use crate::helpers::{api_ready, isolated_binary, start_control, wait_for_health}; -use crate::test_types::{CommandTestCase, OpenApiTestCase, ServiceTestCase}; +use crate::test_types::{CommandTestCase, OpenApiTestCase, ServiceTestCase, TriggerCursorTestCase}; #[test] fn given_a_standalone_binary_when_printing_help_or_rejecting_invalid_input_then_python_and_database_access_are_not_required() @@ -28,6 +28,45 @@ fn given_a_standalone_binary_when_printing_help_or_rejecting_invalid_input_then_ expected_stdout_prefix: "usage: sqb admin ", expected_stderr: "", }, + CommandTestCase { + description: "native trigger administration help", + arguments: &["admin", "change-trigger", "--help"], + expected_success: true, + expected_stdout_prefix: "usage: sqb admin change-trigger ", + expected_stderr: "", + }, + CommandTestCase { + description: "blank trigger reason rejected before store initialization", + arguments: &[ + "admin", + "change-trigger", + "00000000-0000-0000-0000-000000000001", + "pause", + "--event-id", + "00000000-0000-0000-0000-000000000002", + "--reason", + " ", + ], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: --reason must not be blank\n", + }, + CommandTestCase { + description: "set-cursor requires a timestamp before store initialization", + arguments: &[ + "admin", + "change-trigger", + "00000000-0000-0000-0000-000000000001", + "set-cursor", + "--event-id", + "00000000-0000-0000-0000-000000000002", + "--reason", + "Refresh orders", + ], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: trigger change: missing field `time`\n", + }, CommandTestCase { description: "blank location rejected before store initialization", arguments: &["admin", "create-location", " "], @@ -47,7 +86,7 @@ fn given_a_standalone_binary_when_printing_help_or_rejecting_invalid_input_then_ arguments: &["admin", "revoke-token", "not-a-uuid"], expected_success: false, expected_stdout_prefix: "", - expected_stderr: "Control administration failed: invalid control-plane data: a valid location or token UUID is required\n", + expected_stderr: "Control administration failed: invalid control-plane data: a valid subject UUID is required\n", }, CommandTestCase { description: "invalid expiry rejected before store initialization", @@ -118,6 +157,75 @@ fn given_a_standalone_binary_when_printing_help_or_rejecting_invalid_input_then_ } } +#[test] +fn given_a_cursor_free_trigger_change_when_a_cursor_is_supplied_then_it_fails_before_opening_the_store() + { + let test_cases = [ + TriggerCursorTestCase { + description: "enable rejects an extra cursor", + change: "enable", + expected_success: false, + }, + TriggerCursorTestCase { + description: "pause rejects an extra cursor", + change: "pause", + expected_success: false, + }, + TriggerCursorTestCase { + description: "reset-default rejects an extra cursor", + change: "reset-default", + expected_success: false, + }, + TriggerCursorTestCase { + description: "reset-cursor rejects an extra cursor", + change: "reset-cursor", + expected_success: false, + }, + ]; + let directory = tempfile::tempdir().expect("isolated CLI directory"); + let binary = isolated_binary(directory.path()); + for test_case in test_cases { + for cursor in ["2026-01-01T00:00:00Z", "nonsense"] { + let output = Command::new(&binary) + .args([ + "admin", + "change-trigger", + "00000000-0000-0000-0000-000000000001", + test_case.change, + "--event-id", + "00000000-0000-0000-0000-000000000002", + "--reason", + "Refresh orders", + "--cursor", + cursor, + ]) + .current_dir(directory.path()) + .env("PATH", "") + .env("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://not-opened") + .output() + .expect("standalone native command"); + assert_eq!( + output.status.success(), + test_case.expected_success, + "{}: {cursor}", + test_case.description + ); + assert_eq!( + String::from_utf8_lossy(&output.stderr), + "Control administration failed: invalid control-plane data: --cursor is only valid for set-cursor\n", + "{}: {cursor}", + test_case.description + ); + assert!( + output.stdout.is_empty(), + "{}: {cursor}", + test_case.description + ); + assert!(!directory.path().join(".sqlbuild").exists()); + } + } +} + #[test] fn given_a_standalone_server_when_exporting_openapi_then_the_worker_contract_is_machine_readable_without_python_or_a_database() { diff --git a/crates/sqlbuild-cli/tests/native_commands/test_types.rs b/crates/sqlbuild-cli/tests/native_commands/test_types.rs index 15974f0788..4796bc1277 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_types.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_types.rs @@ -9,6 +9,12 @@ pub(crate) struct CommandTestCase { pub(crate) expected_stderr: &'static str, } +pub(crate) struct TriggerCursorTestCase { + pub(crate) description: &'static str, + pub(crate) change: &'static str, + pub(crate) expected_success: bool, +} + pub(crate) struct OpenApiTestCase { pub(crate) description: &'static str, pub(crate) expected_path: &'static str, diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs index 7093af372d..ae89a5d233 100644 --- a/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs @@ -1,2 +1 @@ -pub(crate) mod audit; pub(crate) mod scopes; diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/audit.rs b/crates/sqlbuild-orchestration/src/credentials/main/append_audit.rs similarity index 97% rename from crates/sqlbuild-orchestration/src/credentials/_helpers/audit.rs rename to crates/sqlbuild-orchestration/src/credentials/main/append_audit.rs index 7a78c89f96..30b2c7362d 100644 --- a/crates/sqlbuild-orchestration/src/credentials/_helpers/audit.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/append_audit.rs @@ -8,7 +8,7 @@ use crate::store::errors::ControlError; use crate::store::main::query::query; use crate::store::models::{ControlTransaction, Parameter}; -pub(crate) async fn audit( +pub(crate) async fn append_audit( transaction: ControlTransaction, context: &AuditContext, (event_id, action): (Uuid, &str), diff --git a/crates/sqlbuild-orchestration/src/credentials/main/create_location.rs b/crates/sqlbuild-orchestration/src/credentials/main/create_location.rs index 88d4d8d148..a046c16405 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/create_location.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/create_location.rs @@ -3,7 +3,7 @@ use serde_json::json; use uuid::Uuid; -use crate::credentials::_helpers::audit::audit; +use crate::credentials::main::append_audit::append_audit; use crate::credentials::models::AuditContext; use crate::store::errors::ControlError; use crate::store::main::begin::begin; @@ -27,7 +27,7 @@ pub async fn create_location( "INSERT INTO sqb_locations (location_id, name, created_by, created_at) VALUES ($1, $2, $3, $4)", vec![Parameter::Id(id), Parameter::Text(name.into()), Parameter::Text(context.actor.clone()), Parameter::Time(now)]).await?; commit( - audit( + append_audit( transaction, context, (Uuid::new_v5(&id, b"created"), "location.created"), diff --git a/crates/sqlbuild-orchestration/src/credentials/main/create_principal.rs b/crates/sqlbuild-orchestration/src/credentials/main/create_principal.rs index b85b43d51e..4b85936350 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/create_principal.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/create_principal.rs @@ -3,7 +3,7 @@ use serde_json::json; use uuid::Uuid; -use crate::credentials::_helpers::audit::audit; +use crate::credentials::main::append_audit::append_audit; use crate::credentials::models::{AuditContext, PrincipalKind}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; @@ -38,7 +38,7 @@ pub async fn create_principal( ) .await?; commit( - audit( + append_audit( transaction, context, (Uuid::new_v5(&id, b"created"), "principal.created"), diff --git a/crates/sqlbuild-orchestration/src/credentials/main/issue_token.rs b/crates/sqlbuild-orchestration/src/credentials/main/issue_token.rs index 1ccb56aba1..090bf85c66 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/issue_token.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/issue_token.rs @@ -7,9 +7,9 @@ use serde_json::json; use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; -use crate::credentials::_helpers::audit::audit; use crate::credentials::_helpers::scopes::{scope_is_active, scope_parameters}; use crate::credentials::constants::TOKEN_SECRET_BYTES; +use crate::credentials::main::append_audit::append_audit; use crate::credentials::models::{AuditContext, CredentialScope, IssuedToken, TokenIdentity}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; @@ -65,7 +65,7 @@ pub async fn issue_token( expires_at, }; commit( - audit( + append_audit( transaction, context, (Uuid::new_v5(&token_id, b"issued"), "token.issued"), diff --git a/crates/sqlbuild-orchestration/src/credentials/main/mod.rs b/crates/sqlbuild-orchestration/src/credentials/main/mod.rs index 49e61796cf..bde23d947a 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod append_audit; pub mod authenticate_token; pub mod create_location; pub mod create_principal; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/revoke_token.rs b/crates/sqlbuild-orchestration/src/credentials/main/revoke_token.rs index 6be1bb1e78..a7366783ae 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/revoke_token.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/revoke_token.rs @@ -3,7 +3,7 @@ use serde_json::json; use uuid::Uuid; -use crate::credentials::_helpers::audit::audit; +use crate::credentials::main::append_audit::append_audit; use crate::credentials::models::AuditContext; use crate::store::errors::ControlError; use crate::store::main::begin::begin; @@ -31,7 +31,7 @@ pub async fn revoke_token( .cloned() .ok_or_else(|| ControlError::Invalid("stored token scope is missing".into()))?; commit( - audit( + append_audit( transaction, context, (Uuid::new_v5(&token_id, b"revoked"), "token.revoked"), diff --git a/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs b/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs index 4ccd6765e0..0977da705b 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs @@ -1,5 +1,9 @@ //! Apply an operator fact and its projection in one transaction, with retry-safe request identity. +use serde_json::json; + +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::AuditContext; use crate::declarations::_helpers::trigger_state::trigger_timing; use crate::declarations::models::{ EditableTriggerState, TriggerChange, TriggerChangeRequest, TriggerStatus, @@ -14,11 +18,12 @@ use crate::store::models::{ControlStore, Parameter, TransactionMode}; /// Authorization belongs to the API boundary; `actor` is the authenticated principal, not client input. pub async fn change_trigger( store: &ControlStore, + context: &AuditContext, request: &TriggerChangeRequest, ) -> Result { - if request.actor.trim().is_empty() { + if request.actor.trim().is_empty() || request.actor != context.actor { return Err(ControlError::Invalid( - "trigger changes require an actor".into(), + "trigger changes require the authenticated audit actor".into(), )); } let body = @@ -51,6 +56,10 @@ pub async fn change_trigger( decode_row(rows.into_iter().next().ok_or_else(|| { ControlError::Invalid("trigger does not exist in the current snapshot".into()) })?)?; + let before = json!({ + "status_override": state.status_override, + "last_tick_for": state.last_tick_for, + }); match request.change { TriggerChange::Enable => state.status_override = Some(TriggerStatus::Running), TriggerChange::Pause => state.status_override = Some(TriggerStatus::Paused), @@ -72,6 +81,23 @@ pub async fn change_trigger( "UPDATE sqb_trigger_state SET status=$1, status_override=$2, last_tick_for=$3, next_due_at=$4, updated_at=$5 WHERE trigger_id=$6", vec![Parameter::Text(status.as_str().into()), Parameter::OptionalText(state.status_override.map(|value| value.as_str().into())), Parameter::OptionalTime(state.last_tick_for), Parameter::OptionalTime(next), Parameter::Time(now), Parameter::Id(request.trigger_id)]).await?; - commit(transaction).await?; + commit( + append_audit( + transaction, + context, + (request.event_id, "trigger.changed"), + ( + json!({"trigger_id": request.trigger_id, "snapshot_id": state.snapshot_id}), + before, + json!({ + "status_override": state.status_override, + "last_tick_for": state.last_tick_for, + "reason": request.reason, + }), + ), + ) + .await?, + ) + .await?; Ok(true) } diff --git a/crates/sqlbuild-orchestration/tests/api/test_reads.rs b/crates/sqlbuild-orchestration/tests/api/test_reads.rs index b2df105562..6849aae676 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_reads.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_reads.rs @@ -115,13 +115,15 @@ fn given_published_locations_and_scheduled_work_when_read_through_http_then_reso reason: "Pause orders refresh".into(), change: TriggerChange::Pause, }; - change_trigger(store, &change).await.expect("pause trigger"); let audit = AuditContext { actor: "test-admin".into(), granting_role: "admin".into(), request_id: Uuid::now_v7(), ip_address: None, }; + change_trigger(store, &audit, &change) + .await + .expect("pause trigger"); let expiry = Utc::now() + chrono::Duration::hours(1); let token = issue_token( store, diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs index 1cbed7a105..7d4bac4e82 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs @@ -2,7 +2,7 @@ use std::collections::BTreeMap; -use serde_json::json; +use serde_json::{Value, json}; use sqlbuild_orchestration::declarations::main::list_triggers::list_triggers; use sqlbuild_orchestration::runs::main::list_runs::list_runs; use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; @@ -14,6 +14,7 @@ use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; use sqlbuild_orchestration::store::errors::ControlError; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; +use sqlbuild_orchestration::store::models::ControlStore; use uuid::Uuid; use crate::fault_helpers::{install_fault, remove_fault}; @@ -59,6 +60,51 @@ pub(crate) fn snapshot(location_id: Uuid) -> SnapshotInput { } } +async fn trigger_audit_rows(store: &ControlStore) -> Vec<(String, String, String, String, String)> { + let sql = "SELECT CAST(event_id AS TEXT), actor, granting_role, CAST(before_json AS TEXT), CAST(after_json AS TEXT) FROM sqb_security_audit WHERE action='trigger.changed'"; + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + return rusqlite::Connection::open(path) + .expect("inspect trigger audit") + .prepare(sql) + .expect("audit query") + .query_map([], |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + )) + }) + .expect("read trigger audits") + .collect::>() + .expect("audit rows"); + }; + sqlx_core::query_as::query_as(sql) + .fetch_all(pool) + .await + .expect("trigger audits") +} + +pub(crate) async fn trigger_audits(store: &ControlStore) -> Vec { + trigger_audit_rows(store) + .await + .into_iter() + .map(|(event_id, actor, granting_role, before, after)| { + json!({ + "event_id": event_id, + "actor": actor, + "granting_role": granting_role, + "before_json": serde_json::from_str::(&before).expect("before state"), + "after_json": serde_json::from_str::(&after).expect("after state"), + }) + }) + .collect() +} + pub(crate) async fn transaction_contract(backend: Backend) -> Option<(usize, usize)> { let fixture = open_test_store(backend).await?; let store = &fixture.store; diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs index cf2cfb68ac..34e9c4b198 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs @@ -1,5 +1,6 @@ //! Operator trigger facts remain consistent across publication and retry boundaries. +use sqlbuild_orchestration::credentials::models::AuditContext; use sqlbuild_orchestration::declarations::main::change_trigger::change_trigger; use sqlbuild_orchestration::declarations::main::list_trigger_events::list_trigger_events; use sqlbuild_orchestration::declarations::main::list_triggers::list_triggers; @@ -16,7 +17,7 @@ use crate::fault_helpers::{install_fault, remove_fault}; use crate::location_helpers::create_location; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; -use crate::ticks::helpers::snapshot; +use crate::ticks::helpers::{snapshot, trigger_audits}; use crate::ticks::test_types::{TransactionFault, TriggerControlsTestCase}; #[test] @@ -59,7 +60,13 @@ fn given_operator_changes_when_retried_and_republished_then_facts_and_effective_ reason: "maintenance".into(), change: TriggerChange::Pause, }; - assert!(change_trigger(store, &pause).await.expect("pause")); + let audit = AuditContext { + actor: "orders-operator".into(), + granting_role: "operator".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + assert!(change_trigger(store, &audit, &pause).await.expect("pause")); let paused = list_triggers(store, location, "development") .await .expect("paused trigger") @@ -86,27 +93,49 @@ fn given_operator_changes_when_retried_and_republished_then_facts_and_effective_ change: TriggerChange::Enable, ..pause.clone() }; - let fault = TransactionFault { - description: "operator event and projection", - table: "sqb_trigger_state", - operation: "UPDATE", - condition: "", - sqlite_body: "SELECT RAISE(ABORT,'injected state failure')", - postgres_body: "RAISE EXCEPTION 'injected state failure'", - }; - install_fault(store, &fault).await; - assert!(change_trigger(store, &enable).await.is_err()); - remove_fault(store, &fault).await; - assert_eq!( - list_trigger_events(store, trigger_id, 100) + for fault in [ + TransactionFault { + description: "operator event and projection", + table: "sqb_trigger_state", + operation: "UPDATE", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected state failure')", + postgres_body: "RAISE EXCEPTION 'injected state failure'", + }, + TransactionFault { + description: "operator security audit", + table: "sqb_security_audit", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected audit failure')", + postgres_body: "RAISE EXCEPTION 'injected audit failure'", + }, + ] { + install_fault(store, &fault).await; + assert!(change_trigger(store, &audit, &enable).await.is_err()); + remove_fault(store, &fault).await; + assert_eq!( + list_trigger_events(store, trigger_id, 100) + .await + .expect("rolled-back fact") + .len(), + 1 + ); + assert_eq!( + list_triggers(store, location, "development") + .await + .expect("rolled back projection")[0] + .status, + TriggerStatus::Paused + ); + } + assert!( + change_trigger(store, &audit, &enable) .await - .expect("rolled-back fact") - .len(), - 1 + .expect("retry enable") ); - assert!(change_trigger(store, &enable).await.expect("retry enable")); assert!( - !change_trigger(store, &pause) + !change_trigger(store, &audit, &pause) .await .expect("retry old pause is inert") ); @@ -115,7 +144,7 @@ fn given_operator_changes_when_retried_and_republished_then_facts_and_effective_ ..pause.clone() }; assert!( - change_trigger(store, &conflict).await.is_err(), + change_trigger(store, &audit, &conflict).await.is_err(), "same event ID cannot change content" ); input.generation = 2; @@ -139,7 +168,7 @@ fn given_operator_changes_when_retried_and_republished_then_facts_and_effective_ change: TriggerChange::ResetDefault, ..pause.clone() }; - change_trigger(store, &reset) + change_trigger(store, &audit, &reset) .await .expect("restore new code default"); assert_eq!( @@ -157,7 +186,7 @@ fn given_operator_changes_when_retried_and_republished_then_facts_and_effective_ change: TriggerChange::SetCursor { time }, ..pause.clone() }; - change_trigger(store, &cursor) + change_trigger(store, &audit, &cursor) .await .expect("set durable cursor"); assert_eq!( @@ -172,7 +201,7 @@ fn given_operator_changes_when_retried_and_republished_then_facts_and_effective_ change: TriggerChange::ResetCursor, ..pause.clone() }; - change_trigger(store, &reset_cursor) + change_trigger(store, &audit, &reset_cursor) .await .expect("reset cursor"); assert!( @@ -191,6 +220,29 @@ fn given_operator_changes_when_retried_and_republished_then_facts_and_effective_ .iter() .all(|event| event.request_json.actor == "orders-operator") ); + let other_actor = AuditContext { + actor: "customers-operator".into(), + ..audit.clone() + }; + assert!(change_trigger(store, &other_actor, &pause).await.is_err()); + let audits = trigger_audits(store).await; + assert_eq!(audits.len(), test_case.expected_events); + assert!( + audits + .iter() + .all(|row| row["actor"] == "orders-operator" + && row["granting_role"] == "operator") + ); + let cleared = audits + .iter() + .find(|row| row["event_id"] == reset_cursor.event_id.to_string()) + .expect("cursor reset audit"); + assert_eq!( + cleared["before_json"]["last_tick_for"], + "2020-01-02T00:00:00Z" + ); + assert!(cleared["after_json"]["last_tick_for"].is_null()); + assert_eq!(cleared["after_json"]["reason"], "maintenance"); close_test_store(fixture).await; }); } From 0273d2e2eb203febd716087eaa747b7b2aac4fbd Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 07:08:11 +0000 Subject: [PATCH 24/59] feat: deliver claim-scoped Recall through native workers Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../src/api/_helpers/runs.rs | 23 +- .../src/api/constants.rs | 3 + .../src/api/main/serve_api.rs | 4 +- crates/sqlbuild-orchestration/src/api/mod.rs | 1 + .../sqlbuild-orchestration/src/api/models.rs | 1 + .../src/runs/_helpers/mod.rs | 1 + .../src/runs/_helpers/output.rs | 8 +- .../src/runs/_helpers/recall.rs | 103 +++++ .../src/runs/constants.rs | 2 + .../src/runs/main/mod.rs | 1 + .../src/runs/main/publish_run_output.rs | 4 +- .../src/runs/main/publish_run_recall.rs | 39 ++ .../sqlbuild-orchestration/src/runs/models.rs | 17 +- .../src/worker/_helpers/mod.rs | 1 + .../src/worker/_helpers/recall.rs | 215 +++++++++ crates/sqlbuild-orchestration/tests/api.rs | 2 + .../tests/api/output/helpers.rs | 2 +- .../tests/api/recall/helpers.rs | 411 ++++++++++++++++++ .../tests/api/test_contract.rs | 30 +- .../tests/api/test_types.rs | 7 + .../tests/api/test_worker.rs | 102 ++++- .../src/records/_helpers/mod.rs | 1 + .../src/records/_helpers/record_row.rs | 61 +++ crates/sqlbuild-recall/src/records/models.rs | 12 +- crates/sqlbuild-recall/src/schema/models.rs | 15 +- crates/sqlbuild-recall/src/store/models.rs | 6 +- 26 files changed, 1046 insertions(+), 26 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/api/constants.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/recall.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/publish_run_recall.rs create mode 100644 crates/sqlbuild-orchestration/src/worker/_helpers/recall.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/recall/helpers.rs create mode 100644 crates/sqlbuild-recall/src/records/_helpers/record_row.rs diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs b/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs index a0f39f3324..3762a428bf 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs @@ -4,6 +4,7 @@ use axum::extract::{Path, State}; use axum::http::StatusCode; use axum::{Extension, Json}; use sqlbuild_recall::records::models::OutputChunk; +use sqlbuild_recall::store::models::InsertOutcome; use uuid::Uuid; use crate::api::errors::ApiError; @@ -12,8 +13,11 @@ use crate::credentials::models::{CredentialScope, TokenIdentity}; use crate::runs::main::change_run::change_run; use crate::runs::main::heartbeat_run::heartbeat_run; use crate::runs::main::publish_run_output::publish_run_output; +use crate::runs::main::publish_run_recall::publish_run_recall; use crate::runs::main::run_execution::run_execution; -use crate::runs::models::{OutputPublication, RunAuthority, RunChangeRequest, RunExecution}; +use crate::runs::models::{ + OutputPublication, RecallBatch, RunAuthority, RunChangeRequest, RunExecution, +}; use crate::store::models::ControlStore; #[utoipa::path(get, path = "/api/runs/{run_id}/execution", params(("run_id" = Uuid, Path)), @@ -87,6 +91,23 @@ pub(crate) async fn output( )) } +#[utoipa::path(post, path = "/api/runs/{run_id}/recall", params(("run_id" = Uuid, Path)), + request_body = RecallBatch, + responses((status = 200, body = InsertOutcome), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Run not authorized")), + security(("bearer" = [])))] +pub(crate) async fn recall( + State(store): State, + Extension(identity): Extension, + Path(run_id): Path, + Json(batch): Json, +) -> Result, ApiError> { + let claim_id = require_worker(&identity, run_id)?; + Ok(Json( + publish_run_recall(&store, (run_id, claim_id), &batch).await?, + )) +} + fn require_worker(identity: &TokenIdentity, run_id: Uuid) -> Result { if let CredentialScope::Run { run_id: permitted, diff --git a/crates/sqlbuild-orchestration/src/api/constants.rs b/crates/sqlbuild-orchestration/src/api/constants.rs new file mode 100644 index 0000000000..d3d0c93f8d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/constants.rs @@ -0,0 +1,3 @@ +//! Shared HTTP transport bounds for the server and native clients. + +pub(crate) const BODY_BYTES: usize = 64 * 1024 * 1024; diff --git a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs index b6ea25ac48..52c2efba75 100644 --- a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs +++ b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs @@ -6,6 +6,7 @@ use axum::{Json, Router, middleware}; use crate::api::_helpers::authentication::authenticate; use crate::api::_helpers::{run_reads, runs, snapshots, trigger_reads}; +use crate::api::constants::BODY_BYTES; use crate::api::main::openapi_document::openapi_document; use crate::store::errors::ControlError; use crate::store::main::migrate_store::migrate_store; @@ -46,7 +47,8 @@ pub async fn serve_api( .route("/api/runs/{run_id}/events", post(runs::report)) .route("/api/runs/{run_id}/heartbeat", post(runs::heartbeat)) .route("/api/runs/{run_id}/output", post(runs::output)) - .layer(DefaultBodyLimit::max(64 * 1024 * 1024)) + .route("/api/runs/{run_id}/recall", post(runs::recall)) + .layer(DefaultBodyLimit::max(BODY_BYTES)) .route_layer(middleware::from_fn_with_state(store.clone(), authenticate)); let application = protected .route( diff --git a/crates/sqlbuild-orchestration/src/api/mod.rs b/crates/sqlbuild-orchestration/src/api/mod.rs index 8fa5a583a4..951756c642 100644 --- a/crates/sqlbuild-orchestration/src/api/mod.rs +++ b/crates/sqlbuild-orchestration/src/api/mod.rs @@ -1,6 +1,7 @@ //! Python-free HTTP endpoints; locations and workers have no direct database access. pub(crate) mod _helpers; +pub(crate) mod constants; pub mod errors; pub mod main; pub mod models; diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index 722d7fad9c..c2604cc5c5 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -12,6 +12,7 @@ use crate::api::_helpers::{run_reads, runs, snapshots, trigger_reads}; runs::report, runs::heartbeat, runs::output, + runs::recall, run_reads::recent, run_reads::nodes, trigger_reads::current, diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs index 5bfdb31c61..a3adbd20a7 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs @@ -5,3 +5,4 @@ pub(crate) mod nodes; pub(crate) mod output; pub(crate) mod projection; pub(crate) mod publish_change; +pub(crate) mod recall; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/output.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/output.rs index d6daa4602a..beb7a601ef 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/output.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/output.rs @@ -6,13 +6,13 @@ use sqlbuild_recall::records::models::{OutputChunk, RecallRow, RowEnvelope}; use uuid::Uuid; use crate::runs::constants::{OUTPUT_BATCH_BYTES, OUTPUT_BATCH_CHUNKS}; -use crate::runs::models::OutputMetadata; +use crate::runs::models::ClaimMetadata; use crate::store::errors::ControlError; use crate::store::main::decode_row::decode_row; use crate::store::main::query::query; use crate::store::models::{ControlTransaction, Parameter}; -pub(crate) async fn output_envelope( +pub(crate) async fn claim_envelope( transaction: ControlTransaction, (run_id, claim_id): (Uuid, Uuid), ) -> Result<(ControlTransaction, RowEnvelope), ControlError> { @@ -24,8 +24,8 @@ pub(crate) async fn output_envelope( vec![Parameter::Id(run_id), Parameter::Id(claim_id)], ) .await?; - let metadata: OutputMetadata = decode_row(rows.into_iter().next().ok_or_else(|| { - ControlError::Invalid("output does not belong to an active run claim".into()) + let metadata: ClaimMetadata = decode_row(rows.into_iter().next().ok_or_else(|| { + ControlError::Invalid("publication does not belong to an active run claim".into()) })?)?; Ok(( transaction, diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/recall.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/recall.rs new file mode 100644 index 0000000000..442b0ba521 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/recall.rs @@ -0,0 +1,103 @@ +//! Global Recall keys cannot escape a worker's claim namespace. + +use std::collections::BTreeMap; + +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use sqlbuild_recall::records::models::{RecallRow, RecallValue, RowEnvelope}; +use sqlbuild_recall::schema::models::Table; +use sqlbuild_recall::store::models::InsertOutcome; + +use crate::runs::models::RecallRecord; +use crate::store::errors::ControlError; +use crate::store::main::append_recall::append_recall; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +/// Recall statement projections join correlations globally, not only by invocation. +pub(crate) fn claim_record( + envelope: &RowEnvelope, + record: &RecallRecord, +) -> Result { + let mut values = record.values.clone(); + for name in [ + "event_id", + "run_id", + "resource_id", + "resource_attempt_id", + "operation_id", + "statement_id", + ] { + if let Some(RecallValue::Text(id)) = values.get_mut(name) { + *id = serde_json::json!([envelope.invocation_id, id]).to_string(); + } + } + match record.table { + Table::Meta | Table::Conflicts => { + return Err(ControlError::Invalid( + "Recall store metadata is server-owned".into(), + )); + } + Table::Artifacts => { + let Some(RecallValue::Blob(content)) = values.get("content") else { + return Err(ControlError::Invalid( + "Recall artifact content must be bytes".into(), + )); + }; + if values.get("digest") + != Some(&RecallValue::Text(format!( + "sha256:{}", + sha256_hex(content) + ))) + || values.get("size") != Some(&RecallValue::Integer(content.len() as i64)) + { + return Err(ControlError::Invalid( + "Recall artifact digest or size differs from its bytes".into(), + )); + } + } + _ => {} + } + RecallRow::from_values(envelope, record.table, &values) + .map_err(|error| ControlError::Invalid(error.to_string())) +} + +pub(crate) async fn append_records( + mut transaction: ControlTransaction, + envelope: &RowEnvelope, + rows: &[RecallRow], +) -> Result<(ControlTransaction, InsertOutcome), ControlError> { + let mut result = InsertOutcome::default(); + let mut delivered: BTreeMap = BTreeMap::new(); + for row in rows { + if row.table == Table::ArtifactRefs + && !rows.iter().any(|candidate| { + candidate.table == Table::Artifacts + && candidate.value("digest") == row.value("digest") + }) + { + let Some(RecallValue::Text(digest)) = row.value("digest") else { + return Err(ControlError::Invalid( + "Recall artifact digest must be text".into(), + )); + }; + let references; + (transaction, references) = query(transaction, + "SELECT 1 AS present FROM recall_artifact_refs WHERE invocation_id=$1 AND digest=$2 LIMIT 1", + vec![Parameter::Text(envelope.invocation_id.clone()), Parameter::Text(digest.clone())]).await?; + if references.is_empty() { + return Err(ControlError::Invalid( + "Recall artifact reference has no invocation-owned content".into(), + )); + } + } + let outcome; + (transaction, outcome) = append_recall(transaction, row).await?; + if let Some(conflict) = outcome.conflict { + result.conflicts.push(conflict); + } else { + *delivered.entry(row.table).or_insert(0) += 1; + } + } + result.delivered = delivered.into_iter().collect(); + Ok((transaction, result)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/constants.rs b/crates/sqlbuild-orchestration/src/runs/constants.rs index f53d82c1b5..661f08ec4f 100644 --- a/crates/sqlbuild-orchestration/src/runs/constants.rs +++ b/crates/sqlbuild-orchestration/src/runs/constants.rs @@ -8,3 +8,5 @@ pub(crate) const MONITOR_BATCH_SIZE: usize = 100; pub(crate) const OUTPUT_BATCH_CHUNKS: usize = 256; pub(crate) const OUTPUT_BATCH_BYTES: usize = 1_048_576; +pub(crate) const RECALL_BATCH_ROWS: usize = 500; +pub(crate) const RECALL_INVOCATION_ID_BYTES: usize = 256; diff --git a/crates/sqlbuild-orchestration/src/runs/main/mod.rs b/crates/sqlbuild-orchestration/src/runs/main/mod.rs index 46615363df..da3eca538f 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/mod.rs @@ -5,6 +5,7 @@ pub mod list_run_nodes; pub mod list_runs; pub mod monitor_runs; pub mod publish_run_output; +pub mod publish_run_recall; pub(crate) mod queue_run; pub mod request_run; pub mod run_execution; diff --git a/crates/sqlbuild-orchestration/src/runs/main/publish_run_output.rs b/crates/sqlbuild-orchestration/src/runs/main/publish_run_output.rs index 35633a3a0e..8390019aab 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/publish_run_output.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/publish_run_output.rs @@ -3,7 +3,7 @@ use sqlbuild_recall::records::models::OutputChunk; use uuid::Uuid; -use crate::runs::_helpers::output::{output_envelope, output_rows}; +use crate::runs::_helpers::output::{claim_envelope, output_rows}; use crate::runs::models::OutputPublication; use crate::store::errors::ControlError; use crate::store::main::append_recall::append_recall; @@ -18,7 +18,7 @@ pub async fn publish_run_output( chunks: &[OutputChunk], ) -> Result { let transaction = begin(store, TransactionMode::Write).await?; - let (mut transaction, envelope) = output_envelope(transaction, (run_id, claim_id)).await?; + let (mut transaction, envelope) = claim_envelope(transaction, (run_id, claim_id)).await?; let mut published = OutputPublication::default(); for row in output_rows(&envelope, chunks)? { let outcome; diff --git a/crates/sqlbuild-orchestration/src/runs/main/publish_run_recall.rs b/crates/sqlbuild-orchestration/src/runs/main/publish_run_recall.rs new file mode 100644 index 0000000000..9a70513ebd --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/publish_run_recall.rs @@ -0,0 +1,39 @@ +//! A worker's complete Recall batch and active-claim check share one transaction. + +use sqlbuild_recall::store::models::InsertOutcome; +use uuid::Uuid; + +use crate::runs::_helpers::output::claim_envelope; +use crate::runs::_helpers::recall::{append_records, claim_record}; +use crate::runs::constants::{RECALL_BATCH_ROWS, RECALL_INVOCATION_ID_BYTES}; +use crate::runs::models::RecallBatch; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn publish_run_recall( + store: &ControlStore, + identity: (Uuid, Uuid), + batch: &RecallBatch, +) -> Result { + if batch.invocation_id.trim().is_empty() + || batch.invocation_id.len() > RECALL_INVOCATION_ID_BYTES + || batch.records.len() > RECALL_BATCH_ROWS + { + return Err(ControlError::Invalid( + "invalid Recall invocation or batch size".into(), + )); + } + let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, mut envelope) = claim_envelope(transaction, identity).await?; + envelope.invocation_id = format!("{}/{}", envelope.invocation_id, batch.invocation_id); + let rows = batch + .records + .iter() + .map(|record| claim_record(&envelope, record)) + .collect::, _>>()?; + let (transaction, result) = append_records(transaction, &envelope, &rows).await?; + commit(transaction).await?; + Ok(result) +} diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index b6e379e372..c0496bd605 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -273,8 +273,23 @@ pub struct OutputPublication { pub conflicts: u32, } +/// One invocation's records; the server owns its claim namespace, start time and version. +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct RecallBatch { + pub invocation_id: String, + pub records: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct RecallRecord { + pub table: sqlbuild_recall::schema::models::Table, + pub values: BTreeMap, +} + #[derive(Debug, Deserialize)] -pub(crate) struct OutputMetadata { +pub(crate) struct ClaimMetadata { pub(crate) claimed_at: DateTime, pub(crate) sqb_version: String, } diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs index 9ec828d671..e3fde3c92a 100644 --- a/crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod client; pub(crate) mod heartbeat; +pub(crate) mod recall; pub(crate) mod session; diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/recall.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/recall.rs new file mode 100644 index 0000000000..b259195962 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/recall.rs @@ -0,0 +1,215 @@ +//! Reuse the native delivery thread and accounting over claim-scoped HTTP, without Python. + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use sqlbuild_recall::records::models::{RecallRow, RowEnvelope}; +use sqlbuild_recall::schema::main::table_spec::table_spec; +use sqlbuild_recall::schema::models::Table; +use sqlbuild_recall::store::errors::{StoreError, StoreErrorKind}; +use sqlbuild_recall::store::models::InsertOutcome; +use sqlbuild_recall::store::types::{RecallStore, RecallStoreOpener}; +use sqlbuild_recall::writer::constants::{BATCH_ROWS, BATCH_WINDOW_MS}; +use sqlbuild_recall::writer::models::SessionStore; + +use crate::api::constants::BODY_BYTES; +use crate::api::main::client_response::client_response; +use crate::runs::constants::RECALL_BATCH_ROWS; +use crate::runs::models::{RecallBatch, RecallRecord}; +use crate::worker::models::WorkerClient; + +#[derive(Debug)] +struct WorkerRecallOpener(WorkerClient); + +struct WorkerRecallStore { + runtime: tokio::runtime::Runtime, + client: WorkerClient, + invocation_id: String, +} + +fn failure(message: impl Into, retryable: bool) -> StoreError { + StoreError { + kind: if retryable { + StoreErrorKind::Busy + } else { + StoreErrorKind::Other + }, + message: message.into(), + } +} + +fn transport_record(row: &RecallRow) -> RecallRecord { + RecallRecord { + table: row.table, + values: table_spec(row.table) + .columns + .iter() + .filter_map(|column| { + let value = row.value(column.name)?; + Some((column.name.to_owned(), value.clone())) + }) + .collect(), + } +} + +impl WorkerClient { + /// Pass this to the native Recall session; all delivery finishes before terminal reporting. + #[must_use] + pub fn recall_store(&self) -> SessionStore { + SessionStore::Hosted(Arc::new(WorkerRecallOpener(self.clone()))) + } + + /// The server owns common envelope columns and recomputes each row's content hash. + pub async fn publish_recall( + &self, + batch: &RecallBatch, + ) -> Result { + Ok(client_response( + self.connection + .client + .post(format!("{}recall", self.run_url)) + .json(batch), + ) + .await?) + } +} + +impl RecallStoreOpener for WorkerRecallOpener { + fn open( + &self, + envelope: &RowEnvelope, + give_up: Instant, + ) -> Result, StoreError> { + if Instant::now() >= give_up { + return Err(failure("Recall delivery deadline expired", true)); + } + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|_| failure("Recall delivery runtime could not start", false))?; + let client = { + let _entered = runtime.enter(); + WorkerClient { + connection: self + .0 + .connection + .independent() + .map_err(|_| failure("Recall HTTP client could not start", false))?, + ..self.0.clone() + } + }; + Ok(Box::new(WorkerRecallStore { + runtime, + client, + invocation_id: envelope.invocation_id.clone(), + })) + } + + fn batch_rows(&self) -> usize { + BATCH_ROWS + } + + fn batch_window(&self) -> Duration { + Duration::from_millis(BATCH_WINDOW_MS) + } +} + +/// Native materialization emits content/reference pairs, whose proof stays in one HTTP request. +impl RecallStore for WorkerRecallStore { + fn insert(&self, rows: &[RecallRow], give_up: Instant) -> Result { + if Instant::now() >= give_up { + return Err(failure("Recall delivery deadline expired", true)); + } + let mut batch = RecallBatch { + invocation_id: self.invocation_id.clone(), + records: Vec::new(), + }; + let base_bytes = serde_json::to_vec(&batch) + .map_err(|_| failure("Recall batch could not be encoded", false))? + .len(); + self.runtime.block_on(async { + let mut result = InsertOutcome::default(); + let mut bytes = base_bytes; + let mut offset = 0; + while offset < rows.len() { + if Instant::now() >= give_up { + return Err(failure("Recall delivery deadline expired", true)); + } + let paired = rows[offset].table == Table::Artifacts + && rows.get(offset + 1).is_some_and(|next| { + next.table == Table::ArtifactRefs + && next.value("digest") == rows[offset].value("digest") + }); + let end = offset + if paired { 2 } else { 1 }; + let records = rows[offset..end] + .iter() + .map(transport_record) + .collect::>(); + let added = serde_json::to_vec(&records) + .map_err(|_| failure("Recall records could not be encoded", false))? + .len() + - 2; + if base_bytes + added > BODY_BYTES { + return Err(failure("Recall record exceeds the HTTP body limit", false)); + } + if !batch.records.is_empty() + && (batch.records.len() + records.len() > RECALL_BATCH_ROWS + || bytes + added + 1 > BODY_BYTES) + { + (batch, result) = self.flush(batch, result, give_up).await?; + bytes = base_bytes; + } + bytes += added + usize::from(!batch.records.is_empty()); + batch.records.extend(records); + offset = end; + } + if !batch.records.is_empty() || rows.is_empty() { + (_, result) = self.flush(batch, result, give_up).await?; + } + result.delivered.sort_by_key(|(table, _)| *table); + Ok(result) + }) + } +} + +impl WorkerRecallStore { + async fn flush( + &self, + mut batch: RecallBatch, + mut result: InsertOutcome, + give_up: Instant, + ) -> Result<(RecallBatch, InsertOutcome), StoreError> { + let outcome = tokio::time::timeout_at( + tokio::time::Instant::from_std(give_up), + self.client.publish_recall(&batch), + ) + .await + .map_err(|_| failure("Recall delivery deadline expired", true))? + .map_err(|error| { + let retryable = match &error { + crate::worker::errors::WorkerError::Transport(_) => true, + crate::worker::errors::WorkerError::Rejected(status) => { + status.is_server_error() || *status == reqwest::StatusCode::TOO_MANY_REQUESTS + } + _ => false, + }; + failure(error.to_string(), retryable) + })?; + for (table, count) in outcome.delivered { + if let Some((_, previous)) = result + .delivered + .iter_mut() + .find(|(found, _)| *found == table) + { + *previous += count; + } else { + result.delivered.push((table, count)); + } + } + result.conflicts.extend(outcome.conflicts); + result.failed.extend(outcome.failed); + result.failure = result.failure.take().or(outcome.failure); + batch.records.clear(); + Ok((batch, result)) + } +} diff --git a/crates/sqlbuild-orchestration/tests/api.rs b/crates/sqlbuild-orchestration/tests/api.rs index 81c5ac5beb..f72b32d6fb 100644 --- a/crates/sqlbuild-orchestration/tests/api.rs +++ b/crates/sqlbuild-orchestration/tests/api.rs @@ -4,6 +4,8 @@ mod helpers; mod location_helpers; #[path = "api/output/helpers.rs"] mod output_helpers; +#[path = "api/recall/helpers.rs"] +mod recall_helpers; #[path = "runs/helpers.rs"] mod run_helpers; #[path = "store/helpers.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/output/helpers.rs b/crates/sqlbuild-orchestration/tests/api/output/helpers.rs index 6c628f05de..230312a314 100644 --- a/crates/sqlbuild-orchestration/tests/api/output/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/output/helpers.rs @@ -21,7 +21,7 @@ pub(crate) fn chunk(sequence: u64, stream: OutputStream, data: &[u8]) -> OutputC } } -async fn stored_bytes(store: &ControlStore, invocation: &str) -> Vec<(i64, Vec)> { +pub(crate) async fn stored_bytes(store: &ControlStore, invocation: &str) -> Vec<(i64, Vec)> { let sql = "SELECT sequence, data FROM recall_output WHERE invocation_id=$1 ORDER BY sequence"; let ControlStore::Postgres(pool) = store else { let ControlStore::Sqlite(path) = store else { diff --git a/crates/sqlbuild-orchestration/tests/api/recall/helpers.rs b/crates/sqlbuild-orchestration/tests/api/recall/helpers.rs new file mode 100644 index 0000000000..edc555b357 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/recall/helpers.rs @@ -0,0 +1,411 @@ +//! Exercise native writer delivery and authenticated retries against both physical stores. + +use std::time::Duration; + +use serde_json::json; +use sqlbuild_orchestration::runs::models::{RecallBatch, RecallRecord}; +use sqlbuild_orchestration::store::main::read_recall_view::read_recall_view; +use sqlbuild_orchestration::store::models::ControlStore; +use sqlbuild_orchestration::worker::errors::WorkerError; +use sqlbuild_orchestration::worker::models::WorkerClient; +use sqlbuild_recall::records::main::artifact_rows::artifact_rows; +use sqlbuild_recall::records::main::event_object_row::event_object_row; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use sqlbuild_recall::records::models::{ + ArtifactInput, InvocationContext, OutputStream, RecallRow, RecallValue, RowEnvelope, +}; +use sqlbuild_recall::schema::main::table_spec::table_spec; +use sqlbuild_recall::schema::models::Table; +use sqlbuild_recall::writer::main::start_recall_session::start_recall_session; +use sqlbuild_recall::writer::models::SessionConfig; +use uuid::Uuid; + +use crate::output_helpers::stored_bytes; + +fn record(row: RecallRow) -> RecallRecord { + RecallRecord { + table: row.table, + values: table_spec(row.table) + .columns + .iter() + .filter_map(|column| { + row.value(column.name) + .map(|value| (column.name.to_owned(), value.clone())) + }) + .collect(), + } +} + +pub(crate) async fn recall_contract( + store: &ControlStore, + worker: &WorkerClient, + claim_id: Uuid, +) -> RecallBatch { + let envelope = RowEnvelope { + invocation_id: "execution".into(), + invocation_started_at: "2026-01-02T03:04:05Z".into(), + producer_version: "untrusted-version".into(), + }; + let invocation = format!("worker/{claim_id}/execution"); + let event = json!({ + "event_id": "same-event-id", "occurred_at": "2026-01-02T03:04:05Z", + "event_type": "statement_completed", "schema_version": 1, "producer": "executor", + "statement_id": "shared-producer-id", + "payload": {"duration_ms": 12.75, "row_count": 9, "outcome": "success"} + }); + let mut batch = RecallBatch { + invocation_id: envelope.invocation_id.clone(), + records: vec![record( + event_object_row(&envelope, event.as_object().expect("event object")) + .expect("encode event"), + )], + }; + for _ in 0..2 { + let outcome = worker + .publish_recall(&batch) + .await + .expect("retry event batch"); + assert_eq!(outcome.delivered, [(Table::Events, 1)]); + assert!(outcome.conflicts.is_empty()); + } + let events = read_recall_view(store, ("events", &invocation), 10) + .await + .expect("read stored events"); + assert_eq!(events.len(), 1); + assert_eq!(events[0]["duration_ms"], json!(12.75)); + assert_eq!(events[0]["row_count"], json!(9)); + assert_ne!(events[0]["event_id"], json!("same-event-id")); + assert_ne!(events[0]["producer_version"], json!("untrusted-version")); + batch.records[0] + .values + .insert("duration_ms".into(), RecallValue::Real(13.25)); + assert_eq!( + worker + .publish_recall(&batch) + .await + .expect("publish conflicting event") + .conflicts + .len(), + 1 + ); + assert_eq!( + read_recall_view(store, ("events", &invocation), 10) + .await + .expect("read original event")[0]["duration_ms"], + json!(12.75) + ); + assert_eq!( + read_recall_view(store, ("conflicts", &invocation), 10) + .await + .expect("read conflict evidence") + .len(), + 1 + ); + + let mut separate = batch.clone(); + separate.invocation_id = "another-invocation".into(); + assert_eq!( + worker + .publish_recall(&separate) + .await + .expect("publish another invocation") + .delivered, + [(Table::Events, 1)] + ); + assert_eq!( + read_recall_view(store, ("statements", &invocation), 10) + .await + .expect("read first invocation statement")[0]["duration_ms"], + json!(12.75) + ); + assert_eq!( + read_recall_view( + store, + ( + "statements", + &format!("worker/{claim_id}/another-invocation") + ), + 10 + ) + .await + .expect("read second invocation statement")[0]["duration_ms"], + json!(13.25) + ); + let artifact = ArtifactInput { + digest: format!("sha256:{}", sha256_hex(b"orders\0\xff")), + kind: "evidence".into(), + media_type: "application/octet-stream".into(), + content: b"orders\0\xff".to_vec(), + seen_at: "2026-01-02T03:04:05Z".into(), + ref_id: "first".into(), + role: "evidence".into(), + statement_id: None, + resource_id: None, + }; + let [content, reference] = artifact_rows(&envelope, &artifact).map(record); + let artifacts = RecallBatch { + invocation_id: "execution".into(), + records: vec![content.clone(), reference.clone()], + }; + assert_eq!( + worker + .publish_recall(&artifacts) + .await + .expect("publish artifact with proof") + .delivered + .len(), + 2 + ); + let mut later_ref = reference.clone(); + later_ref + .values + .insert("ref_id".into(), RecallValue::Text("second".into())); + assert_eq!( + worker + .publish_recall(&RecallBatch { + invocation_id: "execution".into(), + records: vec![later_ref] + }) + .await + .expect("publish later artifact reference") + .delivered, + [(Table::ArtifactRefs, 1)] + ); + assert_eq!( + read_recall_view(store, ("artifacts", &invocation), 10) + .await + .expect("read both artifact references") + .len(), + 2 + ); + + let mut forged = content; + forged + .values + .insert("content".into(), RecallValue::Blob(b"different".to_vec())); + let mut event_next = batch.records[0].clone(); + event_next.values.insert( + "event_id".into(), + RecallValue::Text("must-roll-back".into()), + ); + let mut forged_envelope = event_next.clone(); + forged_envelope.values.insert( + "invocation_id".into(), + RecallValue::Text("scheduler/other".into()), + ); + let mut wrong_type = event_next.clone(); + wrong_type + .values + .insert("row_count".into(), RecallValue::Text("nine".into())); + let mut missing = event_next.clone(); + missing.values.remove("event_id"); + let mut bad_json = event_next.clone(); + bad_json + .values + .insert("payload".into(), RecallValue::Text("not-json".into())); + let mut bad_timestamp = event_next.clone(); + bad_timestamp + .values + .insert("occurred_at".into(), RecallValue::Text("not-a-time".into())); + for records in [ + vec![event_next.clone(), forged], + vec![event_next.clone(), forged_envelope], + vec![event_next.clone(), wrong_type], + vec![event_next.clone(), missing], + vec![event_next.clone(), bad_json], + vec![event_next.clone(), bad_timestamp], + vec![ + event_next.clone(), + RecallRecord { + table: Table::Meta, + values: Default::default(), + }, + ], + vec![ + event_next.clone(), + RecallRecord { + table: Table::Conflicts, + values: Default::default(), + }, + ], + vec![event_next; 501], + ] { + assert!(matches!( + worker + .publish_recall(&RecallBatch { + invocation_id: "execution".into(), + records + }) + .await, + Err(WorkerError::Rejected(reqwest::StatusCode::BAD_REQUEST)) + )); + } + // A later authorization failure rolls back the preceding valid event, not just validation errors. + assert!(matches!( + worker + .publish_recall(&RecallBatch { + invocation_id: "foreign".into(), + records: vec![batch.records[0].clone(), reference] + }) + .await, + Err(WorkerError::Rejected(reqwest::StatusCode::BAD_REQUEST)) + )); + assert!( + read_recall_view(store, ("events", &format!("worker/{claim_id}/foreign")), 10) + .await + .expect("read rolled-back invocation") + .is_empty() + ); + assert_eq!( + read_recall_view(store, ("events", &invocation), 10) + .await + .expect("read unchanged invocation") + .len(), + 1 + ); + + native_session(store, worker, claim_id).await; + large_batches(store, worker, claim_id).await; + batch +} + +async fn native_session(store: &ControlStore, worker: &WorkerClient, claim_id: Uuid) { + let hosted = worker.recall_store(); + let summary = tokio::task::spawn_blocking(move || { + let session = start_recall_session(SessionConfig { + store: hosted, + envelope: RowEnvelope { invocation_id: "native-writer".into(), invocation_started_at: "2026-01-02T03:04:05Z".into(), producer_version: "test-version".into() }, + context: InvocationContext { command: Some("build".into()), git_dirty: Some(true), ..Default::default() }, + retention_days: 30, shutdown_timeout: Duration::from_secs(10), diagnostics_min_severity: "debug".into(), + max_output_bytes: None, heartbeat_interval: Duration::from_secs(60), setup_budget: Duration::from_secs(2), + maintenance: false, defer_context: false, maintenance_owner: "test-worker".into(), audit_evidence: Default::default(), + }); + session.append_output(OutputStream::Stdout, b"customers\0\xff\n"); + session.append_output(OutputStream::Stderr, b"warning\xc3\x28\n"); + session.publish_sql("select 7 as order_id", Some("statement-1"), Some("orders")); + session.publish_diagnostic(&json!({"occurred_at": "2026-01-02T03:04:05Z", "severity": "warning", "logger": "sqlbuild", "source": "test", "message": "orders warning", "fields": {"count": 3}}).to_string()).expect("queue diagnostic"); + session.publish_event(&json!({"event_id": "native-event", "occurred_at": "2026-01-02T03:04:05Z", "event_type": "statement_completed", "producer": "executor", "schema_version": 1, "payload": {"duration_ms": 2.5}}).to_string()).expect("queue event"); + session.close() + }).await.expect("native writer thread"); + assert!(summary.flush_complete, "{summary:?}"); + assert_eq!(summary.total().accepted, summary.total().delivered); + let invocation = format!("worker/{claim_id}/native-writer"); + let output = stored_bytes(store, &invocation).await; + assert_eq!(output.len(), 2); + assert!( + output + .iter() + .any(|(_, bytes)| bytes == b"customers\0\xff\n") + ); + assert!( + output + .iter() + .any(|(_, bytes)| bytes == b"warning\xc3\x28\n") + ); + assert_eq!( + read_recall_view(store, ("diagnostics", &invocation), 10) + .await + .expect("read diagnostic")[0]["message"], + json!("orders warning") + ); + assert_eq!( + read_recall_view(store, ("events", &invocation), 10) + .await + .expect("read native events") + .len(), + 1 + ); + assert_eq!( + read_recall_view(store, ("artifacts", &invocation), 10) + .await + .expect("read captured SQL")[0]["text"], + json!("select 7 as order_id") + ); + let delivered = read_recall_view(store, ("output_invocations", &invocation), 10) + .await + .expect("read delivery accounting"); + assert_eq!(delivered.len(), 1); + assert!( + delivered[0]["flush_complete"] == json!(1) || delivered[0]["flush_complete"] == json!(true) + ); +} + +async fn large_batches(store: &ControlStore, worker: &WorkerClient, claim_id: Uuid) { + use sqlbuild_recall::writer::models::SessionStore; + use std::time::Instant; + + let SessionStore::Hosted(opener) = worker.recall_store() else { + panic!("hosted store") + }; + tokio::task::spawn_blocking(move || { + let envelope = RowEnvelope { + invocation_id: "backlog".into(), + invocation_started_at: "2026-01-02T03:04:05Z".into(), + producer_version: "test".into(), + }; + let store = opener + .open(&envelope, Instant::now() + Duration::from_secs(2)) + .expect("open native HTTP store"); + // A replacement can expand one pending lifecycle event beyond the queue's batch count. + let events = (0..501) + .map(|index| { + event_object_row(&envelope, json!({ + "event_id": format!("backlog-{index}"), "occurred_at": "2026-01-02T03:04:05Z", + "event_type": "statement_completed", "producer": "executor", "schema_version": 1, + "payload": {"row_count": index} + }).as_object().expect("event object")).expect("encode backlog event") + }) + .collect::>(); + let expanded = store.insert(&events, Instant::now() + Duration::from_secs(30)); + // Six maximum-sized native artifacts fit the 32 MiB queue, but their JSON blobs exceed 64 MiB. + let artifacts = (0..6) + .flat_map(|index| { + let content = vec![b'a' + index; 4 * 1024 * 1024]; + artifact_rows( + &envelope, + &ArtifactInput { + digest: format!("sha256:{}", sha256_hex(&content)), + content, + kind: "evidence".into(), + media_type: "application/octet-stream".into(), + seen_at: "2026-01-02T03:04:05Z".into(), + ref_id: format!("backlog-{index}"), + role: "evidence".into(), + statement_id: None, + resource_id: None, + }, + ) + }) + .collect::>(); + let bytes = store.insert(&artifacts, Instant::now() + Duration::from_secs(30)); + assert!( + expanded.is_ok() && bytes.is_ok(), + "materialized rows: {expanded:?}; wire bytes: {bytes:?}" + ); + assert_eq!( + expanded.expect("publish expanded event batch").delivered, + [(Table::Events, 501)] + ); + assert_eq!( + bytes.expect("publish large artifact batch").delivered, + [(Table::Artifacts, 6), (Table::ArtifactRefs, 6)] + ); + }) + .await + .expect("join backlog writer"); + let invocation = format!("worker/{claim_id}/backlog"); + assert_eq!( + read_recall_view(store, ("events", &invocation), 1000) + .await + .expect("read full event backlog") + .len(), + 501 + ); + assert_eq!( + read_recall_view(store, ("artifacts", &invocation), 10) + .await + .expect("read large artifacts") + .len(), + 6 + ); +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_contract.rs b/crates/sqlbuild-orchestration/tests/api/test_contract.rs index 72ea9f093b..df890b2024 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contract.rs @@ -378,17 +378,25 @@ fn given_a_real_http_server_when_locations_and_workers_report_then_scopes_retrie (Uuid::now_v7(), &worker.bearer), (run_id, &location_token.bearer), ] { - assert_eq!( - client - .post(format!("{}/api/runs/{output_run}/output", api.url)) - .bearer_auth(credential) - .json(&json!([])) - .send() - .await - .expect("output scope denial") - .status(), - StatusCode::FORBIDDEN - ); + for (endpoint, body) in [ + ("output", json!([])), + ( + "recall", + json!({"invocation_id": "execution", "records": []}), + ), + ] { + assert_eq!( + client + .post(format!("{}/api/runs/{output_run}/{endpoint}", api.url)) + .bearer_auth(credential) + .json(&body) + .send() + .await + .expect("publication scope denial") + .status(), + StatusCode::FORBIDDEN + ); + } } let mut forged = json!(crate::output_helpers::chunk( 0, diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index 2c63a4f4cd..550618fdb8 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -14,6 +14,13 @@ pub(crate) struct WorkerRedirectTestCase { pub(crate) expected_status: reqwest::StatusCode, } +pub(crate) struct WorkerDeadlineTestCase { + pub(crate) description: &'static str, + pub(crate) budget: std::time::Duration, + pub(crate) expected_kind: sqlbuild_recall::store::errors::StoreErrorKind, + pub(crate) expected_arrival: bool, +} + pub(crate) struct ApiReadTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, diff --git a/crates/sqlbuild-orchestration/tests/api/test_worker.rs b/crates/sqlbuild-orchestration/tests/api/test_worker.rs index e488f61f11..07a1beb282 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_worker.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_worker.rs @@ -24,10 +24,13 @@ use uuid::Uuid; use crate::helpers::{start_api, stop_api}; use crate::location_helpers::create_location; use crate::output_helpers::{chunk, output_contract}; +use crate::recall_helpers::recall_contract; use crate::run_helpers::{queue, request, snapshot}; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; -use crate::test_types::{ApiContractTestCase, TestApi, WorkerRedirectTestCase}; +use crate::test_types::{ + ApiContractTestCase, TestApi, WorkerDeadlineTestCase, WorkerRedirectTestCase, +}; #[test] fn given_a_claimed_run_when_a_worker_connects_then_verification_precedes_start_and_reports_are_retryable() @@ -142,6 +145,7 @@ fn given_a_claimed_run_when_a_worker_connects_then_verification_precedes_start_a .await .expect("verified start"); output_contract(store, &worker, identity.1).await; + let recall_batch = recall_contract(store, &worker, identity.1).await; assert_eq!(session.execution.claim, claim, "{}", test_case.description); assert_eq!( start_worker(&worker, &baseline) @@ -208,6 +212,10 @@ fn given_a_claimed_run_when_a_worker_connects_then_verification_precedes_start_a ) .await .expect("finish"); + assert!(matches!( + worker.publish_recall(&recall_batch).await, + Err(WorkerError::Rejected(reqwest::StatusCode::UNAUTHORIZED)) + )); assert!(matches!( worker .publish_output(&[chunk( @@ -370,3 +378,95 @@ fn given_a_credentialed_worker_when_the_server_redirects_then_the_redirect_is_no }); } } + +#[test] +fn given_a_slow_recall_server_when_the_delivery_deadline_expires_then_the_native_store_stops_waiting() + { + use sqlbuild_recall::records::models::RowEnvelope; + use sqlbuild_recall::store::errors::StoreErrorKind; + use sqlbuild_recall::writer::models::SessionStore; + use std::time::{Duration, Instant}; + + let test_cases = [ + WorkerDeadlineTestCase { + description: "HTTP wait expires within the close budget", + budget: Duration::from_millis(50), + expected_kind: StoreErrorKind::Busy, + expected_arrival: true, + }, + WorkerDeadlineTestCase { + description: "expired budget sends no request", + budget: Duration::ZERO, + expected_kind: StoreErrorKind::Busy, + expected_arrival: false, + }, + ]; + for test_case in test_cases { + run(async { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind slow server"); + let url = format!( + "http://{}", + listener.local_addr().expect("slow server address") + ); + let arrived = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let observed = arrived.clone(); + let application = Router::new().route( + "/api/runs/{run_id}/recall", + axum::routing::post(move || { + let arrived = arrived.clone(); + async move { + arrived.store(true, std::sync::atomic::Ordering::SeqCst); + tokio::time::sleep(Duration::from_secs(2)).await; + axum::Json(sqlbuild_recall::store::models::InsertOutcome::default()) + } + }), + ); + let task = tokio::spawn(async move { axum::serve(listener, application).await }); + let client = connect_worker( + &url, + "synthetic-worker-bearer", + (Uuid::now_v7(), Uuid::now_v7()), + ) + .expect("connect worker to slow server"); + let SessionStore::Hosted(opener) = client.recall_store() else { + panic!("hosted store") + }; + tokio::task::spawn_blocking(move || { + let envelope = RowEnvelope { + invocation_id: "deadline".into(), + invocation_started_at: "2026-01-02T03:04:05Z".into(), + producer_version: "test".into(), + }; + let store = opener + .open(&envelope, Instant::now() + Duration::from_secs(1)) + .expect("open HTTP store"); + let started = Instant::now(); + let error = store + .insert(&[], started + test_case.budget) + .expect_err("deadline rejects delivery"); + assert_eq!( + error.kind, test_case.expected_kind, + "{}", + test_case.description + ); + assert!( + started.elapsed() < Duration::from_secs(1), + "{}", + test_case.description + ); + }) + .await + .expect("join slow-server publication"); + assert_eq!( + observed.load(std::sync::atomic::Ordering::SeqCst), + test_case.expected_arrival, + "{}", + test_case.description + ); + task.abort(); + let _ = task.await; + }); + } +} diff --git a/crates/sqlbuild-recall/src/records/_helpers/mod.rs b/crates/sqlbuild-recall/src/records/_helpers/mod.rs index eaf86fcc66..e04c007fa0 100644 --- a/crates/sqlbuild-recall/src/records/_helpers/mod.rs +++ b/crates/sqlbuild-recall/src/records/_helpers/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod events; pub(crate) mod ids; +pub(crate) mod record_row; pub(crate) mod rows; pub(crate) mod time; diff --git a/crates/sqlbuild-recall/src/records/_helpers/record_row.rs b/crates/sqlbuild-recall/src/records/_helpers/record_row.rs new file mode 100644 index 0000000000..3a99f89e08 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/_helpers/record_row.rs @@ -0,0 +1,61 @@ +//! Decode transported record values against the physical schema, then compute their digest. + +use std::collections::BTreeMap; + +use crate::records::_helpers::rows::finish_row; +use crate::records::errors::RecordError; +use crate::records::models::{RecallRow, RecallValue, RowEnvelope}; +use crate::schema::main::table_spec::table_spec; +use crate::schema::models::{ColumnType, Table}; + +/// Envelope and digest columns belong to the receiver, never the transported values. +pub(crate) fn record_row( + envelope: &RowEnvelope, + table: Table, + values: &BTreeMap, +) -> Result { + let columns = table_spec(table).columns; + let mut decoded = Vec::with_capacity(values.len()); + for (name, value) in values { + let column = columns + .iter() + .find(|column| column.name == name) + .ok_or_else(|| RecordError::new(format!("unknown Recall record column {name}")))?; + let valid = match (value, column.column_type) { + (RecallValue::Null, _) => column.nullable, + (RecallValue::Integer(0 | 1), ColumnType::Boolean) => true, + (RecallValue::Integer(_), ColumnType::Integer) => true, + (RecallValue::Real(value), ColumnType::Real) => value.is_finite(), + (RecallValue::Blob(_), ColumnType::Blob) => true, + (RecallValue::Text(_), ColumnType::Text) => true, + (RecallValue::Text(value), ColumnType::Json) => { + serde_json::from_str::(value).map_err(|error| { + RecordError::new(format!("invalid Recall JSON column {name}: {error}")) + })?; + true + } + (RecallValue::Text(value), ColumnType::Timestamp) => { + chrono::DateTime::parse_from_rfc3339(value).map_err(|error| { + RecordError::new(format!("invalid Recall timestamp column {name}: {error}")) + })?; + true + } + _ => false, + }; + if !valid { + return Err(RecordError::new(format!( + "invalid Recall record column {name}" + ))); + } + decoded.push((column.name, value.clone())); + } + for column in columns.iter().filter(|column| !column.nullable) { + if !values.contains_key(column.name) { + return Err(RecordError::new(format!( + "missing Recall record column {}", + column.name + ))); + } + } + Ok(finish_row(table, envelope, decoded)) +} diff --git a/crates/sqlbuild-recall/src/records/models.rs b/crates/sqlbuild-recall/src/records/models.rs index a7798e3e5c..eaab7e2c60 100644 --- a/crates/sqlbuild-recall/src/records/models.rs +++ b/crates/sqlbuild-recall/src/records/models.rs @@ -3,7 +3,8 @@ use crate::schema::models::Table; /// One stored value. -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +#[serde(tag = "kind", content = "value", rename_all = "snake_case")] pub enum RecallValue { Null, Integer(i64), @@ -34,6 +35,15 @@ pub struct RecallRow { } impl RecallRow { + /// Validate schema-typed values and rebuild the receiver-owned envelope and content digest. + pub fn from_values( + envelope: &RowEnvelope, + table: Table, + values: &std::collections::BTreeMap, + ) -> Result { + crate::records::_helpers::record_row::record_row(envelope, table, values) + } + /// The value of `column`, if the row has it. #[must_use] pub fn value(&self, column: &str) -> Option<&RecallValue> { diff --git a/crates/sqlbuild-recall/src/schema/models.rs b/crates/sqlbuild-recall/src/schema/models.rs index 5da839a462..e622f273fe 100644 --- a/crates/sqlbuild-recall/src/schema/models.rs +++ b/crates/sqlbuild-recall/src/schema/models.rs @@ -91,7 +91,20 @@ pub struct TableSpec { } /// The tables Recall writes. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[derive( + Clone, + Copy, + Debug, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, + serde::Serialize, + serde::Deserialize, + utoipa::ToSchema, +)] +#[serde(rename_all = "snake_case")] pub enum Table { Invocations, Output, diff --git a/crates/sqlbuild-recall/src/store/models.rs b/crates/sqlbuild-recall/src/store/models.rs index 61b2f27614..fd53a8b5c7 100644 --- a/crates/sqlbuild-recall/src/store/models.rs +++ b/crates/sqlbuild-recall/src/store/models.rs @@ -54,7 +54,7 @@ pub enum QuarantineOutcome { } /// A row whose key was already stored with different content; the first row is kept. -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize, utoipa::ToSchema)] pub struct ConflictEvidence { pub table: Table, pub record_key: String, @@ -63,7 +63,9 @@ pub struct ConflictEvidence { } /// What one insert batch did, per table. -#[derive(Clone, Debug, Default, PartialEq, Eq)] +#[derive( + Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize, utoipa::ToSchema, +)] pub struct InsertOutcome { /// Rows written or already present with equal content, by table. pub delivered: Vec<(Table, u64)>, From ecd1f673a82eac07267ae0e2ea6550c14ea857bf Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 07:25:37 +0000 Subject: [PATCH 25/59] feat: retain native worker ownership in the project interpreter Verify retained compiler baselines before worker startup and join native heartbeat work on interpreter-side close. Preserve claim identity and report retry keys without treating resource teardown as physical warehouse completion. Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- Cargo.lock | 1 + crates/sqlbuild-python/Cargo.toml | 1 + .../bindings/_helpers/compiled_project/mod.rs | 1 + .../_helpers/compiled_project/snapshot.rs | 13 ++ .../compiled_project/snapshot_worker.rs | 157 ++++++++++++++++++ fensu.toml | 6 + src/sqlbuild/_native.pyi | 18 ++ .../publication/_test_types.py | 15 ++ .../compiled_project/publication/helpers.py | 85 ++++++++++ .../publication/test_worker.py | 148 +++++++++++++++++ 10 files changed, 445 insertions(+) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_worker.rs create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py diff --git a/Cargo.lock b/Cargo.lock index c468288a19..ad63bc0ef9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2651,6 +2651,7 @@ dependencies = [ "sqlbuild-state", "stacker", "tokio", + "uuid", ] [[package]] diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index 3b066a9b33..270745994d 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -48,6 +48,7 @@ sqlbuild-sqltext.workspace = true sqlbuild-state.workspace = true stacker.workspace = true tokio.workspace = true +uuid.workspace = true [lints] workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs index 67cb07e670..7ae7f4d521 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs @@ -8,4 +8,5 @@ pub(crate) mod snapshot_boundary; pub(crate) mod snapshot_declarations; pub(crate) mod snapshot_models; pub(crate) mod snapshot_resources; +pub(crate) mod snapshot_worker; pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs index cee2c105f2..018f69fb18 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs @@ -18,6 +18,7 @@ use crate::bindings::_helpers::compiled_project::snapshot_declarations::{ }; use crate::bindings::_helpers::compiled_project::snapshot_models::model_nodes; use crate::bindings::_helpers::compiled_project::snapshot_resources::resource_nodes; +use crate::bindings::_helpers::compiled_project::snapshot_worker::NativeOrchestrationWorker; use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; use crate::bindings::_helpers::native_host::orchestration_declarations::validate_jobs; use crate::bindings::_helpers::native_host::orchestration_environment::environment_hash; @@ -51,6 +52,17 @@ impl NativeOrchestrationSnapshot { .map_err(|error| PyRuntimeError::new_err(error.to_string())) }) } + + /// Retain the verified claim and its independent native heartbeat in the worker interpreter. + fn start_worker( + &self, + py: Python<'_>, + server_url: &str, + bearer: &str, + identity: (String, String), + ) -> PyResult { + NativeOrchestrationWorker::start(self, py, (server_url, bearer), identity) + } } /// Compile catalog nodes from their owners, never from Python model or graph projections. @@ -139,5 +151,6 @@ pub(crate) fn destination(value: &sqlbuild_project::models::DestinationFact) -> pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; + module.add_class::()?; module.add_function(wrap_pyfunction!(build_orchestration_snapshot, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_worker.rs new file mode 100644 index 0000000000..966c3709e2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_worker.rs @@ -0,0 +1,157 @@ +//! A worker interpreter retains the native claim owner; it never opens the control database. + +use std::sync::Mutex; + +use pyo3::exceptions::{PyRuntimeError, PyValueError}; +use pyo3::prelude::{PyRef, PyResult, Python}; +use pyo3::{pyclass, pymethods}; +use sqlbuild_orchestration::runs::models::RunChange; +use sqlbuild_orchestration::worker::main::connect_worker::connect_worker; +use sqlbuild_orchestration::worker::main::start_worker::start_worker; +use sqlbuild_orchestration::worker::models::{WorkerClient, WorkerSession}; +use uuid::Uuid; + +use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSnapshot; + +struct WorkerOwner { + runtime: tokio::runtime::Runtime, + client: WorkerClient, + session: WorkerSession, +} + +/// Closing this owner stops and joins heartbeat IO but does not claim warehouse work has stopped. +#[pyclass(frozen, module = "sqlbuild._native")] +pub(crate) struct NativeOrchestrationWorker { + owner: Mutex>, +} + +impl NativeOrchestrationWorker { + /// Verify this retained baseline before the caller applies run settings or opens a warehouse. + pub(crate) fn start( + snapshot: &NativeOrchestrationSnapshot, + py: Python<'_>, + (server_url, bearer): (&str, &str), + (run_id, claim_id): (String, String), + ) -> PyResult { + let identity = ( + Uuid::parse_str(&run_id).map_err(|error| PyValueError::new_err(error.to_string()))?, + Uuid::parse_str(&claim_id).map_err(|error| PyValueError::new_err(error.to_string()))?, + ); + py.detach(|| { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; + let connected = runtime.block_on(async { + let client = connect_worker(server_url, bearer, identity)?; + let session = start_worker(&client, &snapshot.input).await?; + Ok::<_, sqlbuild_orchestration::worker::errors::WorkerError>((client, session)) + }); + match connected { + Ok((client, session)) => Ok(NativeOrchestrationWorker { + owner: Mutex::new(Some(WorkerOwner { + runtime, + client, + session, + })), + }), + Err(error) => { + runtime.shutdown_timeout(std::time::Duration::from_secs(1)); + Err(PyRuntimeError::new_err(error.to_string())) + } + } + }) + } +} + +#[pymethods] +impl NativeOrchestrationWorker { + /// The frozen request supplies selection and settings only after baseline verification. + fn execution_json(&self, py: Python<'_>) -> PyResult { + py.detach(|| { + let owner = self.owner.lock().map_err(|_| unavailable())?; + let owner = owner.as_ref().ok_or_else(unavailable)?; + serde_json::to_string(&owner.session.execution) + .map_err(|error| PyRuntimeError::new_err(error.to_string())) + }) + } + + /// Cancellation remains sticky; a closed worker cannot authorize further work. + fn cancel_requested(&self, py: Python<'_>) -> bool { + py.detach(|| { + self.owner.lock().map_or(true, |owner| { + owner + .as_ref() + .is_none_or(|owner| owner.session.cancel_requested()) + }) + }) + } + + /// Retry an uncertain report with the identical event ID and change, never a new identity. + fn report(&self, py: Python<'_>, event_id: &str, change_json: &str) -> PyResult { + let event_id = + Uuid::parse_str(event_id).map_err(|error| PyValueError::new_err(error.to_string()))?; + let change: RunChange = serde_json::from_str(change_json) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + py.detach(|| { + let owner = self.owner.lock().map_err(|_| unavailable())?; + let owner = owner.as_ref().ok_or_else(unavailable)?; + owner + .runtime + .block_on(owner.client.report(event_id, change)) + .map_err(|error| PyRuntimeError::new_err(error.to_string())) + }) + } + + fn __enter__(slf: PyRef<'_, Self>) -> PyResult> { + if slf.owner.lock().map_err(|_| unavailable())?.is_none() { + return Err(unavailable()); + } + Ok(slf) + } + + fn __exit__( + &self, + py: Python<'_>, + _kind: &pyo3::Bound<'_, pyo3::PyAny>, + _value: &pyo3::Bound<'_, pyo3::PyAny>, + _traceback: &pyo3::Bound<'_, pyo3::PyAny>, + ) -> PyResult<()> { + self.close(py) + } + + /// Join the heartbeat without publishing a terminal outcome or releasing physical claims. + fn close(&self, py: Python<'_>) -> PyResult<()> { + py.detach(|| { + let mut owner = self.owner.lock().map_err(|_| unavailable())?; + if let Some(active) = owner.take() { + active.close(); + } + Ok(()) + }) + } +} + +fn unavailable() -> pyo3::PyErr { + PyRuntimeError::new_err("native worker session is closed or unavailable") +} + +impl WorkerOwner { + fn close(self) { + drop(self.session); + drop(self.client); + self.runtime + .shutdown_timeout(std::time::Duration::from_secs(1)); + } +} + +impl Drop for NativeOrchestrationWorker { + /// GC can hold the GIL; heartbeat teardown joins native IO and never needs Python. + fn drop(&mut self) { + if let Ok(owner) = self.owner.get_mut() + && let Some(owner) = owner.take() + { + owner.close(); + } + } +} diff --git a/fensu.toml b/fensu.toml index c06a29ffdd..191fa4912d 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1576,6 +1576,12 @@ path = "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.r symbols = ["NativeOrchestrationSnapshot"] reason = "The pyclass module attribute names sqlbuild._native but executes no Python call. The struct retains only SnapshotInput; binding functions remain checked." +[[targets.rust.rule_exceptions]] +rule = "XSB072" +path = "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_worker.rs" +symbols = ["NativeOrchestrationWorker"] +reason = "The pyclass module attribute names sqlbuild._native but executes no Python call. The struct retains only native worker ownership; binding functions remain checked." + [[targets.rust.rule_exceptions]] rule = "FPRSS201" path = "crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 6a4f400584..27199c0f53 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1743,6 +1743,24 @@ class NativeOrchestrationSnapshot: def to_json(self) -> str: ... def publish(self, server_url: str, bearer: str) -> str: ... + def start_worker( + self, server_url: str, bearer: str, identity: tuple[str, str] + ) -> NativeOrchestrationWorker: ... + +class NativeOrchestrationWorker: + """A verified claim owner with joined native heartbeat teardown.""" + + def execution_json(self) -> str: ... + def cancel_requested(self) -> bool: ... + def report(self, event_id: str, change_json: str) -> bool: ... + def close(self) -> None: ... + def __enter__(self) -> Self: ... + def __exit__( + self, + kind: type[BaseException] | None, + value: BaseException | None, + traceback: TracebackType | None, + ) -> None: ... def build_orchestration_snapshot( project: CompiledProjectView, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 8c7301d86d..63a996d791 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -10,9 +10,24 @@ class SnapshotApi(NamedTuple): bearer: str +class ClaimedWorker(NamedTuple): + run_id: str + claim_id: str + bearer: str + + @dataclass(frozen=True) class SnapshotPublicationTestCase: description: str target: str expected_generation: int expected_nodes: frozenset[str] + + +@dataclass(frozen=True) +class SnapshotWorkerTestCase: + description: str + model_file: str + original_sql: str + replacement_sql: str + expected_error: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index cbc2554451..27b44181af 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -20,6 +20,7 @@ from uuid import uuid4 from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + ClaimedWorker, SnapshotApi, ) @@ -93,3 +94,87 @@ def current_snapshot(api: SnapshotApi, target: str) -> dict[str, Any]: with urlopen(request, timeout=5) as response: result: dict[str, Any] = json.load(response) return result + + +def claimed_worker(directory: Path, api: SnapshotApi, publication: dict[str, Any]) -> ClaimedWorker: + """Provision only the launch boundary; verification and lifecycle reporting use real HTTP.""" + run_id: str = str(uuid4()) + claim_id: str = str(uuid4()) + token_id: str = str(uuid4()) + bearer: str = f"{token_id}.{secrets.token_hex(32)}" + now: datetime = datetime.now(UTC) + selection: list[str] = ["model:orders"] + request: dict[str, Any] = { + "run_id": run_id, + "location_id": api.location_id, + "target": publication["target"], + "snapshot_id": publication["snapshot_id"], + "image_id": None, + "kind": "job", + "origin": "manual", + "job_key": None, + "parent_run_id": None, + "root_run_id": run_id, + "trigger_id": None, + "tick_id": None, + "rerun_of_run_id": None, + "requested_by": "test-admin", + "priority": 0, + "selection": selection, + "write_set": selection, + "settings": {"full_refresh": True}, + "concurrency_groups": [], + "overlap": "allow", + } + with sqlite3.connect(directory / ".sqlbuild/control.sqlite3") as connection: + _ = connection.execute( + "INSERT INTO sqb_runs (run_id,location_id,target,snapshot_id,kind,origin,root_run_id," + "requested_by,selection_json,request_json,write_set_digest,status,queued_at," + "node_counts_json,last_event_storage_id) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + ( + run_id, + api.location_id, + publication["target"], + publication["snapshot_id"], + "job", + "manual", + run_id, + "test-admin", + json.dumps(selection), + json.dumps(request), + hashlib.sha256(json.dumps(selection).encode()).hexdigest(), + "queued", + now.isoformat(), + json.dumps({"pending": 1}), + 0, + ), + ) + _ = connection.execute( + "INSERT INTO sqb_run_nodes (run_id,node_key,attempt,status,writes) " + "VALUES (?,'model:orders',1,'pending',TRUE)", + (run_id,), + ) + _ = connection.execute( + "INSERT INTO sqb_write_claims (target,node_key,run_id,claimed_at) VALUES (?,?,?,?)", + (publication["target"], "model:orders", run_id, now.isoformat()), + ) + _ = connection.execute( + "INSERT INTO sqb_run_liveness (run_id,worker_instance,claimed_at,last_heartbeat_at) " + "VALUES (?,?,?,?)", + (run_id, claim_id, now.isoformat(), now.isoformat()), + ) + _ = connection.execute( + "INSERT INTO sqb_access_tokens " + "(token_id,secret_hash,run_id,claim_id,scope_json,created_at,expires_at) " + "VALUES (?,?,?,?,?,?,?)", + ( + token_id, + hashlib.sha256(bearer.encode()).hexdigest(), + run_id, + claim_id, + json.dumps({"kind": "run", "run_id": run_id, "claim_id": claim_id}), + now.isoformat(), + (now + timedelta(hours=1)).isoformat(), + ), + ) + return ClaimedWorker(run_id=run_id, claim_id=claim_id, bearer=bearer) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py new file mode 100644 index 0000000000..b357d3504a --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -0,0 +1,148 @@ +"""Retained compiler baselines cross the native worker boundary before lifecycle writes.""" + +import json +from pathlib import Path +from typing import Any +from urllib.request import Request, urlopen +from uuid import uuid4 + +import pytest + +from sqlbuild._native import ( + NativeOrchestrationSnapshot, + build_compiled_project, + build_orchestration_snapshot, + discover_project_inputs, +) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.types import CompiledProjectView +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( + METADATA, + TYPE_PROOF_FILES, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + ClaimedWorker, + SnapshotWorkerTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + claimed_worker, + snapshot_api, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotWorkerTestCase( + description="unselected dependency drift", + model_file="models/stg_orders.sql", + original_sql="CAST(5 AS INTEGER)", + replacement_sql="CAST(17 AS INTEGER)", + expected_error="node content differs", + ), + SnapshotWorkerTestCase( + description="selected model drift", + model_file="models/orders.sql", + original_sql="SELECT order_id, amount", + replacement_sql="SELECT order_id, amount * CAST(2 AS BIGINT) AS amount", + expected_error="node content differs", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_retained_baseline_when_starting_a_worker_then_verification_and_claim_lifetime_are_native( + test_case: SnapshotWorkerTestCase, + tmp_path: Path, +) -> None: + project_dir: Path = tmp_path / "project" + write_files(project_dir=project_dir, files=TYPE_PROOF_FILES) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=project_dir), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + with snapshot_api(tmp_path / "control") as api: + metadata: dict[str, Any] = METADATA | {"location_id": api.location_id} + baseline: NativeOrchestrationSnapshot = build_orchestration_snapshot( + project, json.dumps(metadata), project_dir + ) + publication: dict[str, Any] = json.loads(baseline.publish(api.url, api.bearer)) + claim: ClaimedWorker = claimed_worker(tmp_path / "control", api, publication) + identity: tuple[str, str] = (claim.run_id, claim.claim_id) + changed: dict[str, str] = TYPE_PROOF_FILES | { + test_case.model_file: TYPE_PROOF_FILES[test_case.model_file].replace( + test_case.original_sql, test_case.replacement_sql + ) + } + write_files(project_dir=project_dir, files=changed) + drifted: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=project_dir), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + wrong: NativeOrchestrationSnapshot = build_orchestration_snapshot( + drifted, json.dumps(metadata), project_dir + ) + with pytest.raises(RuntimeError, match=test_case.expected_error): + _ = wrong.start_worker(api.url, claim.bearer, identity) + runs_url: str = f"{api.url}/api/locations/{api.location_id}/targets/development/runs" + location_headers: dict[str, str] = {"Authorization": f"Bearer {api.bearer}"} + with urlopen(Request(runs_url, headers=location_headers), timeout=5) as response: + assert json.load(response)[0]["status"] == "queued" + with pytest.raises(RuntimeError, match="does not match"): + _ = baseline.start_worker(api.url, claim.bearer, (claim.run_id, str(uuid4()))) + with baseline.start_worker(api.url, claim.bearer, identity) as worker: + execution: dict[str, Any] = json.loads(worker.execution_json()) + assert execution["claim"]["request"]["selection"] == ["model:orders"] + assert execution["claim"]["request"]["settings"] == {"full_refresh": True} + assert frozenset(execution["node_hashes"]) == { + "model:orders", + "model:stg_orders", + } + assert not worker.cancel_requested() + event_id: str = str(uuid4()) + change: str = json.dumps({"kind": "node_started", "node_key": "model:orders"}) + assert worker.report(event_id, change) + assert not worker.report(event_id, change) + assert claim.bearer not in repr(worker) + worker.close() + assert worker.cancel_requested() + with pytest.raises(RuntimeError, match="closed"): + _ = worker.execution_json() + with pytest.raises(RuntimeError, match="closed"): + _ = worker.report(str(uuid4()), change) + with pytest.raises(RuntimeError, match="closed"), worker: + pytest.fail("closed worker cannot be entered") + # Closing an interpreter-side owner is not proof of physical warehouse termination. + with urlopen(Request(runs_url, headers=location_headers), timeout=5) as response: + assert json.load(response)[0]["status"] == "running" + with baseline.start_worker(api.url, claim.bearer, identity) as resumed: + assert resumed.report( + str(uuid4()), + json.dumps( + { + "kind": "node_finished", + "node_key": "model:orders", + "outcome": "success", + "rows_affected": 7, + "error_code": None, + "skipped_because_of": None, + } + ), + ) + assert resumed.report( + str(uuid4()), + json.dumps( + { + "kind": "finished", + "outcome": "success", + "reason": "worker execution completed", + "exit_code": 0, + } + ), + ) + with urlopen(Request(runs_url, headers=location_headers), timeout=5) as response: + result: dict[str, Any] = json.load(response)[0] + assert result["status"] == "success" + assert result["node_counts_json"] == {"success": 1} From 472e1a374903b5363a2feac4509060f313ac134d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 07:31:43 +0000 Subject: [PATCH 26/59] refactor: keep scheduler planning entries within their domain Restrict forward and reverse cron expansion and pure tick planning to the scheduler. Preserve scheduling bodies and all existing fault, DST and simulation coverage. Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../src/scheduler/_helpers/expansion.rs | 4 ++-- .../main/{next_cron_time.rs => _next_cron_time.rs} | 2 +- .../src/scheduler/main/{plan_ticks.rs => _plan_ticks.rs} | 5 ++++- .../main/{previous_cron_time.rs => _previous_cron_time.rs} | 2 +- .../src/scheduler/main/evaluate_trigger.rs | 2 +- crates/sqlbuild-orchestration/src/scheduler/main/mod.rs | 6 +++--- .../src/scheduler/main/next_schedule_time.rs | 2 +- .../sqlbuild-orchestration/src/scheduler/tests/helpers.rs | 2 +- .../src/scheduler/tests/test_backlog.rs | 2 +- .../sqlbuild-orchestration/src/scheduler/tests/test_cron.rs | 4 ++-- .../src/scheduler/tests/test_ticks.rs | 2 +- 11 files changed, 18 insertions(+), 15 deletions(-) rename crates/sqlbuild-orchestration/src/scheduler/main/{next_cron_time.rs => _next_cron_time.rs} (95%) rename crates/sqlbuild-orchestration/src/scheduler/main/{plan_ticks.rs => _plan_ticks.rs} (93%) rename crates/sqlbuild-orchestration/src/scheduler/main/{previous_cron_time.rs => _previous_cron_time.rs} (95%) diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs index 8ba7d7e422..75337f66d1 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs @@ -6,9 +6,9 @@ use chrono::{DateTime, Utc}; use crate::declarations::models::{Schedule, TriggerDefinition}; use crate::scheduler::constants::MAX_TICKS_PER_BATCH; -use crate::scheduler::main::next_cron_time::next_cron_time; +use crate::scheduler::main::_next_cron_time::next_cron_time; +use crate::scheduler::main::_previous_cron_time::previous_cron_time; use crate::scheduler::main::parse_cron::parse_cron; -use crate::scheduler::main::previous_cron_time::previous_cron_time; use crate::scheduler::models::CatchUp; use crate::scheduler::types::ScheduleExpansion; use crate::store::errors::ControlError; diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/next_cron_time.rs b/crates/sqlbuild-orchestration/src/scheduler/main/_next_cron_time.rs similarity index 95% rename from crates/sqlbuild-orchestration/src/scheduler/main/next_cron_time.rs rename to crates/sqlbuild-orchestration/src/scheduler/main/_next_cron_time.rs index 8964653860..90c51039cc 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/next_cron_time.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/_next_cron_time.rs @@ -7,7 +7,7 @@ use crate::scheduler::models::CronSchedule; use crate::store::errors::ControlError; /// Ambiguities select the later fold; missing times move to the end of the actual IANA gap. -pub fn next_cron_time( +pub(in crate::scheduler) fn next_cron_time( schedule: &CronSchedule, after: DateTime, ) -> Result, ControlError> { diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/plan_ticks.rs b/crates/sqlbuild-orchestration/src/scheduler/main/_plan_ticks.rs similarity index 93% rename from crates/sqlbuild-orchestration/src/scheduler/main/plan_ticks.rs rename to crates/sqlbuild-orchestration/src/scheduler/main/_plan_ticks.rs index 0e8d05b971..842e9308eb 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/plan_ticks.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/_plan_ticks.rs @@ -10,7 +10,10 @@ use crate::scheduler::models::{TickPlan, TickState}; use crate::store::errors::ControlError; /// Return facts to append and the cursor to commit atomically with them. -pub fn plan_ticks(state: &TickState, now: DateTime) -> Result { +pub(in crate::scheduler) fn plan_ticks( + state: &TickState, + now: DateTime, +) -> Result { if !state.enabled { return Ok(TickPlan { actions: vec![], diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/previous_cron_time.rs b/crates/sqlbuild-orchestration/src/scheduler/main/_previous_cron_time.rs similarity index 95% rename from crates/sqlbuild-orchestration/src/scheduler/main/previous_cron_time.rs rename to crates/sqlbuild-orchestration/src/scheduler/main/_previous_cron_time.rs index de74a44c70..3297e839ba 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/previous_cron_time.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/_previous_cron_time.rs @@ -7,7 +7,7 @@ use crate::scheduler::models::CronSchedule; use crate::store::errors::ControlError; /// Fold and gap mapping is identical to forward expansion, including the later ambiguous instant. -pub fn previous_cron_time( +pub(in crate::scheduler) fn previous_cron_time( schedule: &CronSchedule, at: DateTime, ) -> Result, ControlError> { diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/evaluate_trigger.rs b/crates/sqlbuild-orchestration/src/scheduler/main/evaluate_trigger.rs index 0c079450a9..1a2e98d3b7 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/evaluate_trigger.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/evaluate_trigger.rs @@ -6,8 +6,8 @@ use uuid::Uuid; use crate::declarations::models::TriggerStatus; use crate::scheduler::_helpers::expansion::expand_schedule; use crate::scheduler::_helpers::persistence::persist_ticks; +use crate::scheduler::main::_plan_ticks::plan_ticks; use crate::scheduler::main::check_lease::check_lease; -use crate::scheduler::main::plan_ticks::plan_ticks; use crate::scheduler::models::{DueTrigger, SchedulerLease, TickPlan, TickState}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/mod.rs b/crates/sqlbuild-orchestration/src/scheduler/main/mod.rs index 0e3a7afb49..60225c3875 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/mod.rs @@ -1,12 +1,12 @@ +pub(in crate::scheduler) mod _next_cron_time; +pub(in crate::scheduler) mod _plan_ticks; +pub(in crate::scheduler) mod _previous_cron_time; pub mod acquire_lease; pub(crate) mod check_lease; pub mod evaluate_trigger; pub mod list_ticks; -pub mod next_cron_time; pub(crate) mod next_schedule_time; pub mod parse_cron; -pub mod plan_ticks; -pub mod previous_cron_time; pub mod renew_lease; pub mod run_scheduler; pub mod scheduler_cycle; diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/next_schedule_time.rs b/crates/sqlbuild-orchestration/src/scheduler/main/next_schedule_time.rs index c4675ccbef..880e4ad7c9 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/next_schedule_time.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/next_schedule_time.rs @@ -5,7 +5,7 @@ use std::ops::Bound::{Excluded, Unbounded}; use chrono::{DateTime, Utc}; use crate::declarations::models::Schedule; -use crate::scheduler::main::next_cron_time::next_cron_time; +use crate::scheduler::main::_next_cron_time::next_cron_time; use crate::scheduler::main::parse_cron::parse_cron; use crate::store::errors::ControlError; diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs index bfc531a1fc..0484e35cac 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs @@ -8,7 +8,7 @@ use proptest::test_runner::{Config, TestCaseError, TestCaseResult, TestRunner}; use proptest::{prop_assert, prop_assert_eq}; use uuid::Uuid; -use crate::scheduler::main::plan_ticks::plan_ticks; +use crate::scheduler::main::_plan_ticks::plan_ticks; use crate::scheduler::models::{CatchUp, Overlap, TickAction, TickDecision, TickPlan, TickState}; use crate::scheduler::tests::test_types::TickTestCase; diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/test_backlog.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/test_backlog.rs index 19798dce2a..ac9985f292 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/tests/test_backlog.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/test_backlog.rs @@ -5,7 +5,7 @@ use std::time::{Duration, Instant}; use crate::declarations::models::{Schedule, TriggerDefinition, TriggerStatus}; use crate::scheduler::_helpers::expansion::expand_schedule; use crate::scheduler::constants::MAX_TICKS_PER_BATCH; -use crate::scheduler::main::plan_ticks::plan_ticks; +use crate::scheduler::main::_plan_ticks::plan_ticks; use crate::scheduler::models::{CatchUp, Overlap, TickDecision, TickState}; use crate::scheduler::tests::helpers::time; use crate::scheduler::tests::test_types::BacklogTestCase; diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/test_cron.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/test_cron.rs index d05d1e0ae1..fac0537e06 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/tests/test_cron.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/test_cron.rs @@ -1,8 +1,8 @@ use chrono::{DateTime, SecondsFormat}; -use crate::scheduler::main::next_cron_time::next_cron_time; +use crate::scheduler::main::_next_cron_time::next_cron_time; +use crate::scheduler::main::_previous_cron_time::previous_cron_time; use crate::scheduler::main::parse_cron::parse_cron; -use crate::scheduler::main::previous_cron_time::previous_cron_time; use crate::scheduler::tests::test_types::{CronTestCase, PreviousCronTestCase}; #[test] diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/test_ticks.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/test_ticks.rs index de1eaeeba9..2aecca4468 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/tests/test_ticks.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/test_ticks.rs @@ -1,6 +1,6 @@ use std::num::NonZeroUsize; -use crate::scheduler::main::plan_ticks::plan_ticks; +use crate::scheduler::main::_plan_ticks::plan_ticks; use crate::scheduler::models::{CatchUp, Overlap}; use crate::scheduler::tests::helpers::{covered_seconds, request_seconds, state, time}; use crate::scheduler::tests::test_types::TickTestCase; From ac3f2c905445a6769713e9f09c01dec137d3920b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 07:41:55 +0000 Subject: [PATCH 27/59] feat: audit native run cancellation atomically Expose owner-only cancellation with stable retry identity. Commit lifecycle facts, projections and security audit together while retaining active physical claims until termination. Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../sqlbuild-cli/src/argv/command_table.json | 65 ++++++++++++++++ .../src/native_commands/_helpers/decision.rs | 9 ++- .../src/native_commands/main/_run_admin.rs | 23 ++++++ .../src/native_commands/models.rs | 5 ++ .../tests/native_commands/test_control.rs | 22 ++++++ .../tests/native_commands/worker/helpers.rs | 75 ++++++++++++++----- .../src/runs/_helpers/changes.rs | 6 +- .../src/runs/main/change_run.rs | 26 ++++++- .../sqlbuild-orchestration/src/runs/models.rs | 4 +- .../tests/api/test_contract.rs | 3 +- .../tests/api/test_worker.rs | 8 +- .../tests/runs/test_lifecycle.rs | 74 ++++++++++++++++-- 12 files changed, 290 insertions(+), 30 deletions(-) diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index cd0b2193e9..86cf2619e4 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -352,6 +352,66 @@ ] } }, + { + "name": "cancel-run", + "parser": { + "prog": "sqb admin cancel-run", + "description": "Request cancellation through privileged database access; the fact, projection and security audit commit together. Active warehouse claims remain held until execution ends.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "RUN_ID", + "required": true, + "help": "run UUID", + "group": 0 + }, + { + "kind": "store", + "options": [ + "--event-id" + ], + "dest": "admin_event_id", + "required": true, + "help": "request UUID; reuse it with identical arguments after an uncertain result", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--reason" + ], + "dest": "admin_reason", + "required": true, + "help": "nonblank reason recorded with the cancellation fact and security audit", + "group": 1 + } + ] + } + }, { "name": "change-trigger", "parser": { @@ -464,6 +524,11 @@ "metavar": "revoke-token", "help": "revoke a credential by UUID" }, + { + "dest": "cancel-run", + "metavar": "cancel-run", + "help": "request run cancellation with an atomic audit and retry identity" + }, { "dest": "change-trigger", "metavar": "change-trigger", diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index 0673c8aa18..a04f2333df 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -67,13 +67,20 @@ pub(crate) fn control_admin(namespace: &Namespace) -> Result { + Some("change-trigger" | "cancel-run") => { let event_id = Uuid::parse_str(&text(namespace, "admin_event_id").unwrap_or_default()) .map_err(|_| ControlError::Invalid("--event-id must be a UUID".into()))?; let reason = text(namespace, "admin_reason").unwrap_or_default(); if reason.trim().is_empty() { return Err(ControlError::Invalid("--reason must not be blank".into())); } + if action.as_deref() == Some("cancel-run") { + return Ok(ControlAdmin::CancelRun { + event_id, + run_id: id, + reason, + }); + } let mut change = json!({ "kind": text(namespace, "admin_trigger_change").unwrap_or_default().replace('-', "_"), }); diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs index 90bc31d618..e51cc02eb3 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs @@ -8,6 +8,8 @@ use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; use sqlbuild_orchestration::credentials::models::AuditContext; use sqlbuild_orchestration::declarations::main::change_trigger::change_trigger; use sqlbuild_orchestration::declarations::models::TriggerChangeRequest; +use sqlbuild_orchestration::runs::main::change_run::change_run; +use sqlbuild_orchestration::runs::models::{RunAuthority, RunChange, RunChangeRequest}; use sqlbuild_orchestration::store::errors::ControlError; use uuid::Uuid; @@ -42,6 +44,27 @@ pub(crate) async fn run_admin(namespace: &Namespace) -> Result { + let applied = change_run( + &store, + RunAuthority::Operator(&context), + &RunChangeRequest { + event_id, + run_id, + claim_id: None, + change: RunChange::CancelRequested { + actor: context.actor.clone(), + reason, + }, + }, + ) + .await?; + Ok(json!({"event_id": event_id, "run_id": run_id, "applied": applied})) + } ControlAdmin::ChangeTrigger { event_id, trigger_id, diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index b7b6f00b4b..a4847c0813 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -15,6 +15,11 @@ pub(crate) enum ControlAdmin { CreatePrincipal(String, PrincipalKind), IssueToken(CredentialScope, DateTime), RevokeToken(Uuid), + CancelRun { + event_id: Uuid, + run_id: Uuid, + reason: String, + }, ChangeTrigger { event_id: Uuid, trigger_id: Uuid, diff --git a/crates/sqlbuild-cli/tests/native_commands/test_control.rs b/crates/sqlbuild-cli/tests/native_commands/test_control.rs index 74eaf195d3..9c4b553eff 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_control.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_control.rs @@ -35,6 +35,28 @@ fn given_a_standalone_binary_when_printing_help_or_rejecting_invalid_input_then_ expected_stdout_prefix: "usage: sqb admin change-trigger ", expected_stderr: "", }, + CommandTestCase { + description: "native cancellation help", + arguments: &["admin", "cancel-run", "--help"], + expected_success: true, + expected_stdout_prefix: "usage: sqb admin cancel-run ", + expected_stderr: "", + }, + CommandTestCase { + description: "blank cancellation reason rejected before store initialization", + arguments: &[ + "admin", + "cancel-run", + "00000000-0000-0000-0000-000000000001", + "--event-id", + "00000000-0000-0000-0000-000000000002", + "--reason", + " ", + ], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: --reason must not be blank\n", + }, CommandTestCase { description: "blank trigger reason rejected before store initialization", arguments: &[ diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs index c951d675f2..dc17641f46 100644 --- a/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs +++ b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs @@ -23,6 +23,7 @@ use sqlbuild_orchestration::worker::main::start_worker::start_worker; use sqlbuild_orchestration::worker::models::WorkerSession; use uuid::Uuid; +use crate::admin_helpers::administer; use crate::test_types::TestControl; use crate::worker::test_types::WorkerControl; @@ -137,25 +138,63 @@ pub(crate) async fn start_claimed_worker( ) } -pub(crate) fn cancel_worker(worker: WorkerControl, _control: &mut TestControl) { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .expect("independent operator runtime"); - runtime - .block_on(change_run( - &worker.store, - RunAuthority::Operator("test-admin"), - &RunChangeRequest { - event_id: Uuid::now_v7(), - run_id: worker.identity.0, - claim_id: Some(worker.identity.1), - change: RunChange::CancelRequested { - actor: "test-admin".into(), - }, +pub(crate) fn cancel_worker(worker: WorkerControl, control: &mut TestControl) { + let event_id = Uuid::now_v7().to_string(); + let run_id = worker.identity.0.to_string(); + let arguments = [ + "cancel-run", + &run_id, + "--event-id", + &event_id, + "--reason", + "Stop the orders build", + ]; + let applied = administer(control, &arguments); + assert_eq!(applied["run_id"], run_id); + assert_eq!(applied["event_id"], event_id); + assert_eq!(applied["applied"], true); + assert_eq!(administer(control, &arguments)["applied"], false); + let connection = + rusqlite::Connection::open(control.directory.path().join(".sqlbuild/control.sqlite3")) + .expect("observe durable cancellation audit"); + connection + .query_row( + "SELECT actor, granting_role, scope_json, before_json, after_json FROM sqb_security_audit WHERE event_id=?1 AND action='run.cancel_requested'", + [&event_id], + |row| { + let actor: String = row.get(0)?; + let role: String = row.get(1)?; + let scope: String = row.get(2)?; + let before: String = row.get(3)?; + let after: String = row.get(4)?; + let scope: serde_json::Value = serde_json::from_str(&scope).expect("audit scope"); + let before: serde_json::Value = serde_json::from_str(&before).expect("before state"); + let after: serde_json::Value = serde_json::from_str(&after).expect("after state"); + assert_eq!(actor, "local-control-store-owner"); + assert_eq!(role, "control-store-owner"); + assert_eq!(scope["run_id"], run_id); + assert_eq!(scope["target"], "development"); + assert!(scope["location_id"].is_string()); + assert_eq!(before["status"], "running"); + assert!(before["cancel_requested_at"].is_null()); + assert_eq!(after["status"], "running"); + assert!(after["cancel_requested_at"].is_string()); + assert_eq!(after["reason"], "Stop the orders build"); + Ok(()) }, - )) - .expect("request cancellation while worker runtime is blocked"); + ) + .expect("same-transaction operator history"); + let claims: i64 = connection + .query_row( + "SELECT COUNT(*) FROM sqb_write_claims WHERE run_id=?1", + [&run_id], + |row| row.get(0), + ) + .expect("physical claims after cancellation request"); + assert_eq!( + claims, 1, + "cancellation is not proof of physical termination" + ); } pub(crate) fn revoke_worker(worker: WorkerControl, _control: &mut TestControl) { diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/changes.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/changes.rs index e9480d69da..6c9fcee455 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/changes.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/changes.rs @@ -55,8 +55,10 @@ pub(crate) fn validate_authority( } ) } - RunAuthority::Operator(actor) => { - matches!(&request.change, RunChange::CancelRequested { actor: recorded } if !actor.trim().is_empty() && actor == recorded) + RunAuthority::Operator(context) => { + matches!(&request.change, RunChange::CancelRequested { actor, reason } + if !context.actor.trim().is_empty() && context.actor == *actor + && !context.granting_role.trim().is_empty() && !reason.trim().is_empty()) } }; if !allowed { diff --git a/crates/sqlbuild-orchestration/src/runs/main/change_run.rs b/crates/sqlbuild-orchestration/src/runs/main/change_run.rs index 02fe0308c1..012ff4dbe2 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/change_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/change_run.rs @@ -2,9 +2,10 @@ use serde_json::json; +use crate::credentials::main::append_audit::append_audit; use crate::runs::_helpers::changes::{validate_authority, validate_change, validate_retry}; use crate::runs::_helpers::publish_change::publish_change; -use crate::runs::models::{ChangingRun, RunAuthority, RunChangeRequest}; +use crate::runs::models::{ChangingRun, RunAuthority, RunChange, RunChangeRequest}; use crate::scheduler::main::check_lease::check_lease; use crate::store::errors::ControlError; use crate::store::main::begin::begin; @@ -50,6 +51,29 @@ pub async fn change_run( )?; validate_change(&state, request)?; let mut transaction = publish_change(transaction, request, &state).await?; + if let (RunAuthority::Operator(context), RunChange::CancelRequested { reason, .. }) = + (authority, &request.change) + { + let mut rows; + (transaction, rows) = query( + transaction, + "SELECT location_id, target, status, cancel_requested_at FROM sqb_runs WHERE run_id=$1", + vec![Parameter::Id(request.run_id)], + ) + .await?; + let after = rows.remove(0); + transaction = append_audit( + transaction, + context, + (request.event_id, "run.cancel_requested"), + ( + json!({"location_id": after["location_id"], "target": after["target"], "run_id": request.run_id}), + json!({"status": state.status, "cancel_requested_at": state.cancel_requested_at}), + json!({"status": after["status"], "cancel_requested_at": after["cancel_requested_at"], "reason": reason}), + ), + ) + .await?; + } if let RunAuthority::Scheduler(lease) = authority { transaction = check_lease(transaction, lease).await?; } diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index c0496bd605..643c126459 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -211,6 +211,8 @@ pub enum RunChange { Recovered, CancelRequested { actor: String, + #[serde(default)] + reason: String, }, Finished { outcome: RunOutcome, @@ -248,7 +250,7 @@ pub struct RunChangeRequest { pub enum RunAuthority<'a> { Scheduler(&'a crate::scheduler::models::SchedulerLease), Worker { run_id: Uuid, claim_id: Uuid }, - Operator(&'a str), + Operator(&'a crate::credentials::models::AuditContext), } #[derive(Clone, Debug, Deserialize)] diff --git a/crates/sqlbuild-orchestration/tests/api/test_contract.rs b/crates/sqlbuild-orchestration/tests/api/test_contract.rs index df890b2024..65ad349b97 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contract.rs @@ -419,6 +419,7 @@ fn given_a_real_http_server_when_locations_and_workers_report_then_scopes_retrie event_id: Uuid::now_v7(), change: RunChange::CancelRequested { actor: "test-admin".into(), + reason: "Stop the orders build".into(), }, ..started.clone() }; @@ -447,7 +448,7 @@ fn given_a_real_http_server_when_locations_and_workers_report_then_scopes_retrie .expect("heartbeat response"), json!({"cancel_requested": false}) ); - change_run(store, RunAuthority::Operator("test-admin"), &cancellation) + change_run(store, RunAuthority::Operator(&audit), &cancellation) .await .expect("operator cancellation"); let response: serde_json::Value = client diff --git a/crates/sqlbuild-orchestration/tests/api/test_worker.rs b/crates/sqlbuild-orchestration/tests/api/test_worker.rs index 07a1beb282..8f4bf9e993 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_worker.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_worker.rs @@ -282,13 +282,19 @@ fn given_a_claimed_run_when_a_worker_connects_then_verification_precedes_start_a .expect("another launched process"); change_run( store, - RunAuthority::Operator("test-operator"), + RunAuthority::Operator(&AuditContext { + actor: "test-operator".into(), + granting_role: "operator".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }), &RunChangeRequest { event_id: Uuid::now_v7(), run_id: cancelled_identity.0, claim_id: Some(cancelled_identity.1), change: RunChange::CancelRequested { actor: "test-operator".into(), + reason: "Stop before execution".into(), }, }, ) diff --git a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs index d06a397736..8dca64c004 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs @@ -1,5 +1,6 @@ use std::collections::BTreeMap; +use sqlbuild_orchestration::credentials::models::AuditContext; use sqlbuild_orchestration::runs::main::change_run::change_run; use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; use sqlbuild_orchestration::runs::main::heartbeat_run::heartbeat_run; @@ -75,9 +76,15 @@ fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_ run_id: Uuid::from_u128(2), claim_id: Uuid::now_v7(), }; + let operator = AuditContext { + actor: "test-operator".into(), + granting_role: "operator".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; assert!(change_run(store, other_worker, &started).await.is_err()); assert!( - change_run(store, RunAuthority::Operator("test-operator"), &started) + change_run(store, RunAuthority::Operator(&operator), &started) .await .is_err() ); @@ -268,6 +275,7 @@ fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_ event_id: Uuid::now_v7(), change: RunChange::CancelRequested { actor: "test-operator".into(), + reason: "Stop the orders build".into(), }, ..started.clone() }; @@ -275,7 +283,7 @@ fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_ change_run(store, worker, &cancel).await.is_err(), "workers cannot impersonate operators" ); - change_run(store, RunAuthority::Operator("test-operator"), &cancel) + change_run(store, RunAuthority::Operator(&operator), &cancel) .await .expect("record cancellation"); assert!( @@ -386,23 +394,79 @@ fn given_unclaimed_work_when_an_operator_cancels_then_a_terminal_fact_prevents_l claim_id: None, change: RunChange::CancelRequested { actor: "test-operator".into(), + reason: "Cancel queued orders".into(), }, }; + let operator = AuditContext { + actor: "test-operator".into(), + granting_role: "operator".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let other = AuditContext { + actor: "another-operator".into(), + ..operator.clone() + }; assert!( - change_run(store, RunAuthority::Operator("another-operator"), &request) + change_run(store, RunAuthority::Operator(&other), &request) .await .is_err() ); + let before_events = read_recall_events(store, 0, 100) + .await + .expect("facts before audit failure"); + let point = TransactionFault { + description: "cancellation audit failure rolls back terminal facts and projections", + table: "sqb_security_audit", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected cancellation audit failure')", + postgres_body: "RAISE EXCEPTION 'injected cancellation audit failure'", + }; + install_fault(store, &point).await; assert!( - change_run(store, RunAuthority::Operator("test-operator"), &request) + change_run(store, RunAuthority::Operator(&operator), &request) + .await + .is_err(), + "{}", + point.description + ); + remove_fault(store, &point).await; + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("cancel rollback") + .remove(0) + .status, + RunStatus::Queued + ); + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("fact rollback"), + before_events + ); + assert!( + change_run(store, RunAuthority::Operator(&operator), &request) .await .expect("cancel before launch") ); assert!( - !change_run(store, RunAuthority::Operator("test-operator"), &request) + !change_run(store, RunAuthority::Operator(&operator), &request) .await .expect("retry cancellation") ); + let mut changed = request.clone(); + changed.change = RunChange::CancelRequested { + actor: operator.actor.clone(), + reason: "Different cancellation reason".into(), + }; + assert!( + change_run(store, RunAuthority::Operator(&operator), &changed) + .await + .is_err(), + "retry identity includes the cancellation reason" + ); let lease = acquire_lease(store, Uuid::now_v7()) .await .expect("elect") From 683a17955ca60750f164dfc3d7e58c01f6417a39 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 07:58:25 +0000 Subject: [PATCH 28/59] feat: queue pinned published jobs with atomic audits Resolve manual requests against the current published job inside one write transaction. Preserve the accepted snapshot and execution inputs across retries after redeployment, and commit queue publication with the trusted owner audit. Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../sqlbuild-cli/src/argv/command_table.json | 94 ++++++++ .../src/native_commands/_helpers/decision.rs | 33 ++- .../src/native_commands/main/_run_admin.rs | 5 + .../src/native_commands/models.rs | 2 + .../tests/native_commands/test_control.rs | 45 ++++ .../tests/native_commands/worker/helpers.rs | 68 +++--- .../src/runs/_helpers/job_request.rs | 63 +++++ .../src/runs/_helpers/mod.rs | 2 + .../src/runs/_helpers/request.rs | 66 ++++++ .../src/runs/main/job_write_set.rs | 30 +++ .../src/runs/main/mod.rs | 2 + .../src/runs/main/request_job_run.rs | 68 ++++++ .../src/runs/main/request_run.rs | 63 +---- .../sqlbuild-orchestration/src/runs/models.rs | 12 + .../src/scheduler/_helpers/persistence.rs | 20 +- crates/sqlbuild-orchestration/tests/runs.rs | 2 + .../tests/runs/test_requests.rs | 219 ++++++++++++++++++ 17 files changed, 691 insertions(+), 103 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/job_request.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/request.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/job_write_set.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/test_requests.rs diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index 86cf2619e4..7d04786726 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -352,6 +352,95 @@ ] } }, + { + "name": "request-run", + "parser": { + "prog": "sqb admin request-run", + "description": "Queue a published job using privileged database access. Selection, image, settings and write claims come from the snapshot; the request and audit commit together.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "LOCATION_ID", + "required": true, + "help": "location UUID", + "group": 0 + }, + { + "kind": "store", + "options": [], + "dest": "admin_job", + "metavar": "JOB", + "required": true, + "help": "published job name", + "group": 0 + }, + { + "kind": "store", + "options": [ + "--target" + ], + "dest": "admin_target", + "required": true, + "help": "published target name", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--run-id" + ], + "dest": "admin_event_id", + "required": true, + "help": "run UUID; reuse it with identical arguments after an uncertain result", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--priority" + ], + "dest": "admin_priority", + "type": "int", + "help": "signed queue priority; higher values run first (default: 0)", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--reason" + ], + "dest": "admin_reason", + "required": true, + "help": "nonblank reason recorded with the security audit", + "group": 1 + } + ] + } + }, { "name": "cancel-run", "parser": { @@ -524,6 +613,11 @@ "metavar": "revoke-token", "help": "revoke a credential by UUID" }, + { + "dest": "request-run", + "metavar": "request-run", + "help": "queue a pinned published job with an atomic security audit" + }, { "dest": "cancel-run", "metavar": "cancel-run", diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index a04f2333df..9e79832001 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -7,6 +7,7 @@ use chrono::{DateTime, Utc}; use serde_json::json; use sqlbuild_orchestration::credentials::models::{CredentialScope, PrincipalKind}; use sqlbuild_orchestration::declarations::models::TriggerChange; +use sqlbuild_orchestration::runs::models::JobRunRequest; use sqlbuild_orchestration::store::errors::ControlError; use uuid::Uuid; @@ -67,13 +68,41 @@ pub(crate) fn control_admin(namespace: &Namespace) -> Result { + Some("change-trigger" | "cancel-run" | "request-run") => { + let identity_option = if action.as_deref() == Some("request-run") { + "--run-id" + } else { + "--event-id" + }; let event_id = Uuid::parse_str(&text(namespace, "admin_event_id").unwrap_or_default()) - .map_err(|_| ControlError::Invalid("--event-id must be a UUID".into()))?; + .map_err(|_| ControlError::Invalid(format!("{identity_option} must be a UUID")))?; let reason = text(namespace, "admin_reason").unwrap_or_default(); if reason.trim().is_empty() { return Err(ControlError::Invalid("--reason must not be blank".into())); } + if action.as_deref() == Some("request-run") { + let target = text(namespace, "admin_target").unwrap_or_default(); + let job_key = text(namespace, "admin_job").unwrap_or_default(); + if target.trim().is_empty() || job_key.trim().is_empty() { + return Err(ControlError::Invalid( + "target and job must not be blank".into(), + )); + } + let priority = match namespace.get("admin_priority") { + Some(Value::Int(value)) => value.parse::().map_err(|_| { + ControlError::Invalid("--priority must fit a signed 64-bit integer".into()) + })?, + _ => 0, + }; + return Ok(ControlAdmin::RequestJobRun(JobRunRequest { + run_id: event_id, + location_id: id, + target, + job_key, + reason, + priority, + })); + } if action.as_deref() == Some("cancel-run") { return Ok(ControlAdmin::CancelRun { event_id, diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs index e51cc02eb3..9c2bbde856 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs @@ -9,6 +9,7 @@ use sqlbuild_orchestration::credentials::models::AuditContext; use sqlbuild_orchestration::declarations::main::change_trigger::change_trigger; use sqlbuild_orchestration::declarations::models::TriggerChangeRequest; use sqlbuild_orchestration::runs::main::change_run::change_run; +use sqlbuild_orchestration::runs::main::request_job_run::request_job_run; use sqlbuild_orchestration::runs::models::{RunAuthority, RunChange, RunChangeRequest}; use sqlbuild_orchestration::store::errors::ControlError; use uuid::Uuid; @@ -44,6 +45,10 @@ pub(crate) async fn run_admin(namespace: &Namespace) -> Result { + let applied = request_job_run(&store, &context, &request).await?; + Ok(json!({"run_id": request.run_id, "applied": applied})) + } ControlAdmin::CancelRun { event_id, run_id, diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index a4847c0813..7d7cbc3df5 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -5,6 +5,7 @@ use std::path::PathBuf; use chrono::{DateTime, Utc}; use sqlbuild_orchestration::credentials::models::{CredentialScope, PrincipalKind}; use sqlbuild_orchestration::declarations::models::TriggerChange; +use sqlbuild_orchestration::runs::models::JobRunRequest; use uuid::Uuid; use crate::argv::models::Namespace; @@ -15,6 +16,7 @@ pub(crate) enum ControlAdmin { CreatePrincipal(String, PrincipalKind), IssueToken(CredentialScope, DateTime), RevokeToken(Uuid), + RequestJobRun(JobRunRequest), CancelRun { event_id: Uuid, run_id: Uuid, diff --git a/crates/sqlbuild-cli/tests/native_commands/test_control.rs b/crates/sqlbuild-cli/tests/native_commands/test_control.rs index 9c4b553eff..2347a94d9a 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_control.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_control.rs @@ -42,6 +42,51 @@ fn given_a_standalone_binary_when_printing_help_or_rejecting_invalid_input_then_ expected_stdout_prefix: "usage: sqb admin cancel-run ", expected_stderr: "", }, + CommandTestCase { + description: "native manual job request help", + arguments: &["admin", "request-run", "--help"], + expected_success: true, + expected_stdout_prefix: "usage: sqb admin request-run ", + expected_stderr: "", + }, + CommandTestCase { + description: "blank job rejected before store initialization", + arguments: &[ + "admin", + "request-run", + "00000000-0000-0000-0000-000000000001", + " ", + "--target", + "development", + "--run-id", + "00000000-0000-0000-0000-000000000002", + "--reason", + "Refresh orders", + ], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: target and job must not be blank\n", + }, + CommandTestCase { + description: "oversized manual queue priority rejected before store initialization", + arguments: &[ + "admin", + "request-run", + "00000000-0000-0000-0000-000000000001", + "daily_orders", + "--target", + "development", + "--run-id", + "00000000-0000-0000-0000-000000000002", + "--priority", + "9223372036854775808", + "--reason", + "Refresh orders", + ], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: --priority must fit a signed 64-bit integer\n", + }, CommandTestCase { description: "blank cancellation reason rejected before store initialization", arguments: &[ diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs index dc17641f46..2948290095 100644 --- a/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs +++ b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs @@ -9,12 +9,8 @@ use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; use sqlbuild_orchestration::runs::main::change_run::change_run; use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; -use sqlbuild_orchestration::runs::main::request_run::request_run; -use sqlbuild_orchestration::runs::models::{ - RunAuthority, RunChange, RunChangeRequest, RunKind, RunOrigin, RunRequest, -}; +use sqlbuild_orchestration::runs::models::{RunAuthority, RunChange, RunChangeRequest}; use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::scheduler::models::Overlap; use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; use sqlbuild_orchestration::store::main::open_store::open_store; @@ -64,33 +60,40 @@ pub(crate) async fn start_claimed_worker( python_source: None, }, )]), - documents: BTreeMap::new(), + documents: BTreeMap::from([( + "jobs".into(), + serde_json::json!({"daily_orders": { + "description": "Build orders", "selection": ["orders"], "overlap": "allow", "settings": {"full_refresh": true} + }}), + )]), }; - let publication = publish_snapshot(&store, &baseline).await.expect("snapshot"); + publish_snapshot(&store, &baseline).await.expect("snapshot"); let run_id = Uuid::now_v7(); - let request = RunRequest { - run_id, - location_id, - target: "development".into(), - snapshot_id: publication.snapshot_id, - image_id: None, - kind: RunKind::Adhoc, - origin: RunOrigin::Manual, - job_key: None, - parent_run_id: None, - root_run_id: run_id, - trigger_id: None, - tick_id: None, - rerun_of_run_id: None, - requested_by: "test-admin".into(), - priority: 0, - selection: ["orders".into()].into(), - write_set: ["orders".into()].into(), - settings: BTreeMap::new(), - concurrency_groups: Default::default(), - overlap: Overlap::Allow, - }; - request_run(&store, &request).await.expect("queue run"); + let location = location_id.to_string(); + let run = run_id.to_string(); + let arguments = [ + "request-run", + &location, + "daily_orders", + "--target", + "development", + "--run-id", + &run, + "--priority", + "-3", + "--reason", + "Refresh orders", + ]; + let requested = administer(control, &arguments); + assert_eq!(requested["run_id"], run); + assert_eq!(requested["applied"], true); + assert_eq!(administer(control, &arguments)["applied"], false); + let connection = rusqlite::Connection::open(&path).expect("audit connection"); + let count: i64 = connection.query_row( + "SELECT COUNT(*) FROM sqb_security_audit WHERE action='job.run_requested' AND actor='local-control-store-owner' AND granting_role='control-store-owner' AND json_extract(scope_json,'$.run_id')=?1", + [&run], |row| row.get(0)).expect("trusted manual request audit"); + assert_eq!(count, 1); + drop(connection); let lease = acquire_lease(&store, Uuid::now_v7()) .await .expect("acquire lease") @@ -99,6 +102,11 @@ pub(crate) async fn start_claimed_worker( .await .expect("claim") .expect("queued work"); + assert_eq!(claim.request.priority, -3); + assert_eq!( + claim.request.settings.get("full_refresh"), + Some(&serde_json::json!(true)) + ); let identity = (run_id, claim.claim_id); let token = issue_token( &store, diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/job_request.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/job_request.rs new file mode 100644 index 0000000000..691436c7c5 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/job_request.rs @@ -0,0 +1,63 @@ +//! Freeze execution inputs from one published job under the caller's write transaction. + +use crate::credentials::models::AuditContext; +use crate::declarations::models::JobDefinition; +use crate::runs::main::job_write_set::job_write_set; +use crate::runs::models::{JobRunRequest, RunKind, RunOrigin, RunRequest}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn frozen_job_request( + transaction: ControlTransaction, + context: &AuditContext, + request: &JobRunRequest, +) -> Result<(ControlTransaction, RunRequest), ControlError> { + let (transaction, jobs) = query( + transaction, + "SELECT j.snapshot_id, j.definition_json, s.image_id FROM sqb_jobs j \ + JOIN sqb_current_snapshots c ON c.location_id=j.location_id AND c.target=j.target AND c.snapshot_id=j.snapshot_id \ + JOIN sqb_snapshots s ON s.snapshot_id=j.snapshot_id WHERE j.location_id=$1 AND j.target=$2 AND j.job_key=$3 AND j.kind='job'", + vec![Parameter::Id(request.location_id), Parameter::Text(request.target.clone()), Parameter::Text(request.job_key.clone())], + ).await?; + let row = jobs.first().ok_or_else(|| { + ControlError::Invalid("published job not found in this location and target".into()) + })?; + let snapshot_id = serde_json::from_value(row["snapshot_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let definition: JobDefinition = serde_json::from_value(row["definition_json"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let image_id = serde_json::from_value(row["image_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let (transaction, write_set) = job_write_set( + transaction, + snapshot_id, + (request.location_id, &request.target, &request.job_key), + ) + .await?; + Ok(( + transaction, + RunRequest { + run_id: request.run_id, + location_id: request.location_id, + target: request.target.clone(), + snapshot_id, + image_id, + kind: RunKind::Job, + origin: RunOrigin::Manual, + job_key: Some(request.job_key.clone()), + parent_run_id: None, + root_run_id: request.run_id, + trigger_id: None, + tick_id: None, + rerun_of_run_id: None, + requested_by: context.actor.clone(), + priority: request.priority, + selection: definition.selection, + write_set, + settings: definition.settings, + concurrency_groups: definition.concurrency_groups, + overlap: definition.overlap, + }, + )) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs index a3adbd20a7..a8a8e0a6f5 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs @@ -1,8 +1,10 @@ pub(crate) mod changes; pub(crate) mod claims; pub(crate) mod events; +pub(crate) mod job_request; pub(crate) mod nodes; pub(crate) mod output; pub(crate) mod projection; pub(crate) mod publish_change; pub(crate) mod recall; +pub(crate) mod request; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/request.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/request.rs new file mode 100644 index 0000000000..b4b727be05 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/request.rs @@ -0,0 +1,66 @@ +//! Validate and publish a frozen request inside the caller's audit transaction. + +use std::collections::BTreeSet; + +use serde_json::json; + +use crate::runs::main::queue_run::queue_run; +use crate::runs::models::RunRequest; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn publish_request( + transaction: ControlTransaction, + request: &RunRequest, +) -> Result<(ControlTransaction, bool), ControlError> { + if request.requested_by.trim().is_empty() + || request.selection.is_empty() + || !request.write_set.is_subset(&request.selection) + { + return Err(ControlError::Invalid( + "run needs an actor, selection and a contained write set".into(), + )); + } + let (transaction, previous) = query( + transaction, + "SELECT request_json FROM sqb_runs WHERE run_id=$1", + vec![Parameter::Id(request.run_id)], + ) + .await?; + if let Some(previous) = previous.first() { + if previous.get("request_json") != Some(&json!(request)) { + return Err(ControlError::Invalid( + "run ID already has different request content".into(), + )); + } + return Ok((transaction, false)); + } + let (transaction, nodes) = query( + transaction, + "SELECT node_key FROM sqb_snapshot_nodes WHERE snapshot_id=$1 AND EXISTS \ + (SELECT 1 FROM sqb_snapshots WHERE snapshot_id=$1 AND location_id=$2 AND target=$3 \ + AND image_id IS NOT DISTINCT FROM $4)", + vec![ + Parameter::Id(request.snapshot_id), + Parameter::Id(request.location_id), + Parameter::Text(request.target.clone()), + Parameter::OptionalText(request.image_id.clone()), + ], + ) + .await?; + let keys: BTreeSet<&str> = nodes + .iter() + .filter_map(|row| row.get("node_key").and_then(serde_json::Value::as_str)) + .collect(); + if request + .selection + .iter() + .any(|key| !keys.contains(key.as_str())) + { + return Err(ControlError::Invalid( + "run selection or image does not match the pinned location and target snapshot".into(), + )); + } + Ok((queue_run(transaction, request).await?, true)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/job_write_set.rs b/crates/sqlbuild-orchestration/src/runs/main/job_write_set.rs new file mode 100644 index 0000000000..1354b035fb --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/job_write_set.rs @@ -0,0 +1,30 @@ +//! Resolve physical writers from the pinned snapshot and its published job membership. + +use std::collections::BTreeSet; + +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn job_write_set( + transaction: ControlTransaction, + snapshot_id: Uuid, + (location_id, target, job_key): (Uuid, &str, &str), +) -> Result<(ControlTransaction, BTreeSet), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT n.node_key FROM sqb_snapshot_nodes n JOIN sqb_job_nodes j ON j.node_key=n.node_key \ + WHERE n.snapshot_id=$1 AND j.location_id=$2 AND j.target=$3 AND j.job_key=$4 AND n.node_kind IN ('model','seed','function')", + vec![Parameter::Id(snapshot_id), Parameter::Id(location_id), Parameter::Text(target.into()), Parameter::Text(job_key.into())]).await?; + Ok(( + transaction, + rows.into_iter() + .filter_map(|row| { + row.get("node_key") + .and_then(serde_json::Value::as_str) + .map(str::to_owned) + }) + .collect(), + )) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/mod.rs b/crates/sqlbuild-orchestration/src/runs/main/mod.rs index da3eca538f..97d86e1251 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/mod.rs @@ -1,11 +1,13 @@ pub mod change_run; pub mod claim_next_run; pub mod heartbeat_run; +pub(crate) mod job_write_set; pub mod list_run_nodes; pub mod list_runs; pub mod monitor_runs; pub mod publish_run_output; pub mod publish_run_recall; pub(crate) mod queue_run; +pub mod request_job_run; pub mod request_run; pub mod run_execution; diff --git a/crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs b/crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs new file mode 100644 index 0000000000..e768a67f4d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs @@ -0,0 +1,68 @@ +//! Direct-store owners queue published jobs without supplying or widening their write scope. + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::AuditContext; +use crate::runs::_helpers::job_request::frozen_job_request; +use crate::runs::_helpers::request::publish_request; +use crate::runs::models::JobRunRequest; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// The context is a trusted control-store owner, not a role supplied by an HTTP caller. +pub async fn request_job_run( + store: &ControlStore, + context: &AuditContext, + request: &JobRunRequest, +) -> Result { + if context.actor.trim().is_empty() + || context.granting_role.trim().is_empty() + || request.target.trim().is_empty() + || request.job_key.trim().is_empty() + || request.reason.trim().is_empty() + { + return Err(ControlError::Invalid( + "job run needs an actor, granting role, target, job and reason".into(), + )); + } + let event_id = Uuid::new_v5(&request.run_id, b"manual-job-request"); + let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, previous) = query( + transaction, + "SELECT a.actor, a.after_json FROM sqb_runs r LEFT JOIN sqb_security_audit a ON a.event_id=$1 WHERE r.run_id=$2", + vec![Parameter::Id(event_id), Parameter::Id(request.run_id)], + ).await?; + if let Some(previous) = previous.first() { + if previous.get("actor") != Some(&json!(context.actor)) + || previous + .get("after_json") + .and_then(|after| after.get("request")) + != Some(&json!(request)) + { + return Err(ControlError::Invalid( + "run ID already has different request content".into(), + )); + } + commit(transaction).await?; + return Ok(false); + } + let (transaction, frozen) = frozen_job_request(transaction, context, request).await?; + let (transaction, _) = publish_request(transaction, &frozen).await?; + let transaction = append_audit( + transaction, + context, + (event_id, "job.run_requested"), + ( + json!({"location_id": request.location_id, "target": request.target, "job_key": request.job_key, "run_id": request.run_id}), + serde_json::Value::Null, + json!({"request": request, "frozen": frozen}), + ), + ).await?; + commit(transaction).await?; + Ok(true) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/request_run.rs b/crates/sqlbuild-orchestration/src/runs/main/request_run.rs index 82616a6ee2..71665e84da 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/request_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/request_run.rs @@ -1,69 +1,16 @@ //! Queue a frozen, already authorized request without recreating work on transport retries. -use std::collections::BTreeSet; - -use serde_json::json; - -use crate::runs::main::queue_run::queue_run; +use crate::runs::_helpers::request::publish_request; use crate::runs::models::RunRequest; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; -use crate::store::main::query::query; -use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::models::{ControlStore, TransactionMode}; /// The API must compute and authorize this selection; callers cannot supply their own write scope. pub async fn request_run(store: &ControlStore, request: &RunRequest) -> Result { - if request.requested_by.trim().is_empty() - || request.selection.is_empty() - || !request.write_set.is_subset(&request.selection) - { - return Err(ControlError::Invalid( - "run needs an actor, selection and a contained write set".into(), - )); - } let transaction = begin(store, TransactionMode::Write).await?; - let (transaction, previous) = query( - transaction, - "SELECT request_json FROM sqb_runs WHERE run_id=$1", - vec![Parameter::Id(request.run_id)], - ) - .await?; - if let Some(previous) = previous.first() { - if previous.get("request_json") != Some(&json!(request)) { - return Err(ControlError::Invalid( - "run ID already has different request content".into(), - )); - } - commit(transaction).await?; - return Ok(false); - } - let (transaction, nodes) = query( - transaction, - "SELECT node_key FROM sqb_snapshot_nodes WHERE snapshot_id=$1 AND EXISTS \ - (SELECT 1 FROM sqb_snapshots WHERE snapshot_id=$1 AND location_id=$2 AND target=$3 \ - AND image_id IS NOT DISTINCT FROM $4)", - vec![ - Parameter::Id(request.snapshot_id), - Parameter::Id(request.location_id), - Parameter::Text(request.target.clone()), - Parameter::OptionalText(request.image_id.clone()), - ], - ) - .await?; - let keys: BTreeSet<&str> = nodes - .iter() - .filter_map(|row| row.get("node_key").and_then(serde_json::Value::as_str)) - .collect(); - if request - .selection - .iter() - .any(|key| !keys.contains(key.as_str())) - { - return Err(ControlError::Invalid( - "run selection or image does not match the pinned location and target snapshot".into(), - )); - } - commit(queue_run(transaction, request).await?).await?; - Ok(true) + let (transaction, applied) = publish_request(transaction, request).await?; + commit(transaction).await?; + Ok(applied) } diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index 643c126459..70a560ab44 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -88,6 +88,18 @@ pub struct RunRequest { pub overlap: crate::scheduler::models::Overlap, } +/// A trusted operator names a published job; execution inputs are never supplied by the caller. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct JobRunRequest { + pub run_id: Uuid, + pub location_id: Uuid, + pub target: String, + pub job_key: String, + pub reason: String, + pub priority: i64, +} + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] pub struct RunRecord { pub run_id: Uuid, diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs index e7c466d77b..e84c4f73b9 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs @@ -1,8 +1,8 @@ //! Deterministic tick insertion gates all run facts; retries cannot queue a second run. use serde_json::json; -use std::collections::BTreeSet; +use crate::runs::main::job_write_set::job_write_set; use crate::runs::main::queue_run::queue_run; use crate::runs::models::{RunKind, RunOrigin, RunRequest}; use crate::scheduler::models::{DueTrigger, SchedulerLease, TickDecision, TickPlan}; @@ -16,18 +16,12 @@ pub(crate) async fn persist_ticks( plan: &TickPlan, ) -> Result { let now = transaction.now; - let (mut transaction, rows) = query(transaction, - "SELECT n.node_key FROM sqb_snapshot_nodes n JOIN sqb_job_nodes j ON j.node_key=n.node_key \ - WHERE n.snapshot_id=$1 AND j.location_id=$2 AND j.target=$3 AND j.job_key=$4 AND n.node_kind IN ('model','seed','function')", - vec![Parameter::Id(trigger.snapshot_id), Parameter::Id(trigger.location_id), Parameter::Text(trigger.target.clone()), Parameter::Text(trigger.job_key.clone())]).await?; - let write_set: BTreeSet = rows - .into_iter() - .filter_map(|row| { - row.get("node_key") - .and_then(serde_json::Value::as_str) - .map(str::to_owned) - }) - .collect(); + let (mut transaction, write_set) = job_write_set( + transaction, + trigger.snapshot_id, + (trigger.location_id, &trigger.target, &trigger.job_key), + ) + .await?; for action in &plan.actions { let (outcome, run, covered) = match action.decision { TickDecision::Requested { run_id } => ("requested", Some(run_id), None), diff --git a/crates/sqlbuild-orchestration/tests/runs.rs b/crates/sqlbuild-orchestration/tests/runs.rs index 2ab841868c..b2b4e61852 100644 --- a/crates/sqlbuild-orchestration/tests/runs.rs +++ b/crates/sqlbuild-orchestration/tests/runs.rs @@ -28,5 +28,7 @@ mod test_faults; mod test_lifecycle; #[path = "runs/test_monitor.rs"] mod test_monitor; +#[path = "runs/test_requests.rs"] +mod test_requests; #[path = "runs/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_requests.rs b/crates/sqlbuild-orchestration/tests/runs/test_requests.rs new file mode 100644 index 0000000000..73424f6c21 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_requests.rs @@ -0,0 +1,219 @@ +//! Published-job requests freeze execution inputs and audit atomically on both databases. + +use std::collections::BTreeMap; + +use serde_json::json; +use sqlbuild_orchestration::credentials::models::AuditContext; +use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; +use sqlbuild_orchestration::runs::main::list_runs::list_runs; +use sqlbuild_orchestration::runs::main::request_job_run::request_job_run; +use sqlbuild_orchestration::runs::models::{JobRunRequest, RunStatus}; +use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; +use uuid::Uuid; + +use crate::fault_helpers::{install_fault, remove_fault}; +use crate::helpers::snapshot; +use crate::location_helpers::create_location; +use crate::store_helpers::{close_test_store, open_test_store, run}; +use crate::store_test_types::Backend; +use crate::test_types::{RunLifecycleTestCase, TransactionFault}; + +#[test] +fn given_a_published_job_when_requested_and_retried_after_redeploy_then_audit_and_frozen_execution_commit_once() + { + let test_cases = [ + RunLifecycleTestCase { + description: "SQLite queues and audits the pinned job exactly once", + backend: Backend::Sqlite, + expected_status: RunStatus::Queued, + }, + RunLifecycleTestCase { + description: "PostgreSQL queues and audits the pinned job exactly once", + backend: Backend::Postgres, + expected_status: RunStatus::Queued, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location_id = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let context = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let mut input = snapshot(location_id); + let mut source = input.nodes["orders"].clone(); + source.kind = "source".into(); + input.nodes.insert("warehouse_orders".into(), source); + input.documents.insert("jobs".into(), json!({"daily_orders": { + "description": "Build orders from the warehouse source", "selection": ["orders", "warehouse_orders"], + "settings": {"full_refresh": true}, "concurrency_groups": ["orders-load"], "overlap": "queue" + }})); + let published = publish_snapshot(store, &input).await.expect("publish job"); + let request = JobRunRequest { + run_id: Uuid::now_v7(), + location_id, + target: "development".into(), + job_key: "daily_orders".into(), + reason: "Refresh orders".into(), + priority: 7, + }; + for invalid in [ + JobRunRequest { + target: "production".into(), + ..request.clone() + }, + JobRunRequest { + location_id: Uuid::now_v7(), + ..request.clone() + }, + JobRunRequest { + job_key: "unknown_job".into(), + ..request.clone() + }, + JobRunRequest { + reason: " ".into(), + ..request.clone() + }, + ] { + assert!(request_job_run(store, &context, &invalid).await.is_err()); + } + let fault = TransactionFault { + description: "manual run audit failure", + table: "sqb_security_audit", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected audit failure')", + postgres_body: "RAISE EXCEPTION 'injected audit failure'", + }; + install_fault(store, &fault).await; + assert!(request_job_run(store, &context, &request).await.is_err()); + remove_fault(store, &fault).await; + assert!( + list_runs(store, (location_id, "development"), 10) + .await + .expect("rolled back projection") + .is_empty() + ); + assert!( + read_recall_events(store, 0, 100) + .await + .expect("rolled back facts") + .is_empty() + ); + let (first, retry) = tokio::join!( + request_job_run(store, &context, &request), + request_job_run(store, &context, &request) + ); + assert_ne!(first.expect("first request"), retry.expect("concurrent retry")); + let facts = read_recall_events(store, 0, 100) + .await + .expect("request facts"); + assert_eq!(facts.len(), 2); + input.generation = 2; + input.image_id = Some("sha256:synthetic-orders-image-2".into()); + input.documents.get_mut("jobs").expect("jobs")["daily_orders"]["selection"] = + json!(["inventory"]); + input.documents.get_mut("jobs").expect("jobs")["daily_orders"]["settings"] = + json!({"full_refresh": false}); + publish_snapshot(store, &input).await.expect("redeploy"); + assert!( + !request_job_run(store, &context, &request) + .await + .expect("retry keeps original snapshot") + ); + let mut changed = request.clone(); + changed.reason = "Different request".into(); + assert!(request_job_run(store, &context, &changed).await.is_err()); + let other_actor = AuditContext { + actor: "other-operator".into(), + ..context.clone() + }; + assert!( + request_job_run(store, &other_actor, &request) + .await + .is_err() + ); + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("no duplicate facts"), + facts + ); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::from([( + "orders-load".into(), + std::num::NonZeroU32::new(1).expect("positive group limit"), + )]); + let claim = claim_next_run(store, &lease, &limits) + .await + .expect("claim") + .expect("queued request"); + assert_eq!(claim.request.run_id, request.run_id); + assert_eq!(claim.request.snapshot_id, published.snapshot_id); + assert_eq!( + claim.request.image_id.as_deref(), + Some("sha256:synthetic-orders-image") + ); + assert_eq!( + claim.request.selection, + ["orders".into(), "warehouse_orders".into()].into() + ); + assert_eq!(claim.request.write_set, ["orders".into()].into()); + assert_eq!( + claim.request.settings, + BTreeMap::from([("full_refresh".into(), json!(true))]) + ); + assert_eq!( + claim.request.concurrency_groups, + ["orders-load".into()].into() + ); + assert_eq!(claim.request.priority, 7); + assert_eq!(claim.request.requested_by, "test-operator"); + let fresh = JobRunRequest { + run_id: Uuid::now_v7(), + ..request.clone() + }; + assert!( + request_job_run(store, &context, &fresh) + .await + .expect("new request uses redeploy") + ); + let runs = list_runs(store, (location_id, "development"), 10) + .await + .expect("queued jobs"); + assert_eq!(runs.len(), 2); + assert_eq!( + runs.iter() + .find(|run| run.run_id == fresh.run_id) + .expect("new run") + .status, + test_case.expected_status, + "{}", + test_case.description + ); + assert_eq!( + runs.iter() + .find(|run| run.run_id == fresh.run_id) + .expect("new run") + .selection_json, + ["inventory".into()].into() + ); + close_test_store(fixture).await; + }); + } +} From 3236ba7561649ef8cba13e1457457dd3c53b8336 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 08:18:39 +0000 Subject: [PATCH 29/59] feat: publish compiled locations through scoped HTTP Compile in the project interpreter and retain native snapshots through publication. Keep JSON results separate from progress and distinguish uncertain acknowledgements from failed result delivery. Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../sqlbuild-cli/src/argv/command_table.json | 88 ++++++++ crates/sqlbuild-cli/src/dispatch/constants.rs | 18 ++ .../src/dispatch/main/command_dispatch.rs | 7 +- .../src/command_connection/constants.rs | 3 +- .../sqlbuild-core/src/progress/constants.rs | 1 + .../src/progress/main/machine_output_mode.rs | 3 +- .../tests/runs/test_requests.rs | 5 +- .../src/bindings/_helpers/cli/dispatch.rs | 2 + .../_helpers/compiled_project/registration.rs | 1 + .../_helpers/compiled_project/snapshot.rs | 33 ++- .../_helpers/location_command/command.rs | 199 ++++++++++++++++++ .../bindings/_helpers/location_command/mod.rs | 1 + .../src/bindings/_helpers/mod.rs | 1 + .../src/bindings/_helpers/native_host/host.rs | 8 + .../bindings/_helpers/records/constants.rs | 2 + fensu.toml | 1 + src/sqlbuild/_native.pyi | 12 ++ .../compiled_project/publication/helpers.py | 8 + .../publication/test_publication.py | 154 ++++++++++++++ 19 files changed, 533 insertions(+), 14 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index 7d04786726..cd3d7e216a 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -78,6 +78,89 @@ "nargs": "A...", "group": 0, "parsers": [ + { + "name": "location", + "parser": { + "prog": "sqb location", + "description": "Compile and publish a project from its own Python environment. Uses SQLBUILD_LOCATION_TOKEN for scoped HTTP access, never the control database. Results are JSON; progress goes to stderr.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": ["-h", "--help"], + "dest": "help", + "nargs": 0, + "default": {"suppress": true}, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "location_command", + "required": true, + "choices": ["publish"], + "help": "compile the project and publish one immutable snapshot", + "group": 0 + }, + { + "kind": "store", + "options": ["--server"], + "dest": "location_server_url", + "required": true, + "help": "control API HTTP or HTTPS origin, without credentials", + "group": 1 + }, + { + "kind": "store", + "options": ["--location-id"], + "dest": "location_id", + "required": true, + "help": "registered location UUID", + "group": 1 + }, + { + "kind": "store", + "options": ["--target"], + "dest": "target", + "required": true, + "help": "configured project target to compile", + "group": 1 + }, + { + "kind": "store", + "options": ["--generation"], + "dest": "location_generation", + "type": "int", + "required": true, + "help": "positive generation newer than the last successful publication", + "group": 1 + }, + { + "kind": "store", + "options": ["--image-id"], + "dest": "location_image_id", + "help": "immutable deployment image identity to pin for workers", + "group": 1 + }, + { + "kind": "store", + "options": ["--project-dir", "--sqb-project-dir"], + "dest": "project_dir", + "default": {"suppress": true}, + "help": "SQLBuild project directory (default: the current directory)", + "group": 1 + } + ] + } + }, { "name": "admin", "parser": { @@ -8274,6 +8357,11 @@ } ], "choices_help": [ + { + "dest": "location", + "metavar": "location", + "help": "compile and publish code-location snapshots over HTTP" + }, { "dest": "admin", "metavar": "admin", diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 550b31ebe3..e224c4f9f4 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -515,6 +515,24 @@ pub(crate) const DEBUG_ROUTE: Route = routed( }, ); +const LOCATION_ARGUMENTS: [Field; 6] = [ + field("project_dir", Source::EffectiveProjectDir), + SELECTED_TARGET, + same("location_id"), + attr("server_url", "location_server_url"), + attr("generation", "location_generation"), + attr("image_id", "location_image_id"), +]; +pub(crate) const LOCATION_ROUTE: Route = routed( + "run_location", + "run_location_command", + &LOCATION_ARGUMENTS, + CallShape { + slot_positional: 0, + implementation_keywords: true, + }, +); + const JANITOR_FIELDS: [Field; 11] = [ PROJECT_DIR, NO_COLOR, diff --git a/crates/sqlbuild-cli/src/dispatch/main/command_dispatch.rs b/crates/sqlbuild-cli/src/dispatch/main/command_dispatch.rs index 29de9f0374..bb91c6290a 100644 --- a/crates/sqlbuild-cli/src/dispatch/main/command_dispatch.rs +++ b/crates/sqlbuild-cli/src/dispatch/main/command_dispatch.rs @@ -6,9 +6,9 @@ use crate::dispatch::_helpers::subcommands::{ use crate::dispatch::constants::{ AUDIT_ROUTE, BUILD_ROUTE, CHECK_ROUTE, CLONE_ROUTE, COMPILE_ROUTE, CONTRACT_ROUTE, COST_ROUTE, DAG_ROUTE, DEBUG_ROUTE, DIFF_ROUTE, FORMAT_ROUTE, FRESHNESS_ROUTE, INIT_ROUTE, JANITOR_ROUTE, - LINEAGE_ROUTE, LOAD_ROUTE, PLAN_ROUTE, PLAYGROUND_ROUTE, QUERY_ROUTE, RECALL_COMMAND_NAME, - RECALL_ROUTE, REFACTOR_ROUTE, REVIEW_ROUTE, RULES_ROUTE, SCOPE_ROUTE, SEED_ROUTE, SKILLS_ROUTE, - TEST_ROUTE, + LINEAGE_ROUTE, LOAD_ROUTE, LOCATION_ROUTE, PLAN_ROUTE, PLAYGROUND_ROUTE, QUERY_ROUTE, + RECALL_COMMAND_NAME, RECALL_ROUTE, REFACTOR_ROUTE, REVIEW_ROUTE, RULES_ROUTE, SCOPE_ROUTE, + SEED_ROUTE, SKILLS_ROUTE, TEST_ROUTE, }; use crate::dispatch::models::{CommandFacts, Dispatch, DispatchFailure, Route}; @@ -55,6 +55,7 @@ pub fn command_dispatch(facts: &CommandFacts) -> Dispatch { RECALL_COMMAND_NAME => run(&RECALL_ROUTE), "cost" => run(&COST_ROUTE), "debug" => run(&DEBUG_ROUTE), + "location" => run(&LOCATION_ROUTE), "janitor" => run(&JANITOR_ROUTE), "init" => run(&INIT_ROUTE), "playground" => run(&PLAYGROUND_ROUTE), diff --git a/crates/sqlbuild-config/src/command_connection/constants.rs b/crates/sqlbuild-config/src/command_connection/constants.rs index f5c5246f56..7a68e32d8a 100644 --- a/crates/sqlbuild-config/src/command_connection/constants.rs +++ b/crates/sqlbuild-config/src/command_connection/constants.rs @@ -24,7 +24,7 @@ pub(crate) const SETTING_SNIPPET_INDENT: &str = " "; /// Two quotes around at least one character. pub(crate) const MIN_QUOTED_IDENTIFIER_LENGTH: usize = 3; /// The warehouse group each CLI command selects a target's default warehouse from. -pub const COMMAND_WAREHOUSE_GROUPS: [(&str, Option<&str>); 31] = [ +pub const COMMAND_WAREHOUSE_GROUPS: [(&str, Option<&str>); 32] = [ ("audit", Some("build")), ("build", Some("build")), ("check", Some("build")), @@ -50,6 +50,7 @@ pub const COMMAND_WAREHOUSE_GROUPS: [(&str, Option<&str>); 31] = [ ("format", None), ("init", None), ("lineage", None), + ("location", None), ("mv", None), ("playground", None), ("rename", None), diff --git a/crates/sqlbuild-core/src/progress/constants.rs b/crates/sqlbuild-core/src/progress/constants.rs index 36ee9d8e7e..bcdf9d0799 100644 --- a/crates/sqlbuild-core/src/progress/constants.rs +++ b/crates/sqlbuild-core/src/progress/constants.rs @@ -110,6 +110,7 @@ pub(crate) const SQL_PREVIEW_LENGTH: usize = 160; pub(crate) const EXECUTION_TITLE: &str = "Execution"; pub(crate) const LINEAGE_COMMAND: &str = "lineage"; pub(crate) const QUERY_COMMAND: &str = "query"; +pub(crate) const LOCATION_COMMAND: &str = "location"; pub(crate) const DBT_COMMAND: &str = "dbt"; pub(crate) const DBT_JSON_OPTION: &str = "--json"; pub(crate) const JSON_FORMAT: &str = "json"; diff --git a/crates/sqlbuild-core/src/progress/main/machine_output_mode.rs b/crates/sqlbuild-core/src/progress/main/machine_output_mode.rs index 7a6c3bc531..1337d46807 100644 --- a/crates/sqlbuild-core/src/progress/main/machine_output_mode.rs +++ b/crates/sqlbuild-core/src/progress/main/machine_output_mode.rs @@ -1,7 +1,7 @@ //! Whether a command's stdout carries machine output, so human progress must use stderr. use crate::progress::constants::{ - DBT_COMMAND, DBT_JSON_OPTION, JSON_FORMAT, LINEAGE_COMMAND, QUERY_COMMAND, + DBT_COMMAND, DBT_JSON_OPTION, JSON_FORMAT, LINEAGE_COMMAND, LOCATION_COMMAND, QUERY_COMMAND, }; use crate::progress::models::OutputModeFacts; @@ -11,6 +11,7 @@ pub fn machine_output_mode(facts: &OutputModeFacts) -> bool { let format = |value: &Option| value.as_deref().unwrap_or_default().to_owned(); match facts.command.as_deref() { _ if facts.json => true, + Some(LOCATION_COMMAND) => true, Some(DBT_COMMAND) => facts.dbt_args.iter().any(|arg| arg == DBT_JSON_OPTION), Some(LINEAGE_COMMAND) => format(&facts.lineage_format) == JSON_FORMAT, Some(QUERY_COMMAND) => format(&facts.query_format) == JSON_FORMAT, diff --git a/crates/sqlbuild-orchestration/tests/runs/test_requests.rs b/crates/sqlbuild-orchestration/tests/runs/test_requests.rs index 73424f6c21..0cf4c4b234 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_requests.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_requests.rs @@ -116,7 +116,10 @@ fn given_a_published_job_when_requested_and_retried_after_redeploy_then_audit_an request_job_run(store, &context, &request), request_job_run(store, &context, &request) ); - assert_ne!(first.expect("first request"), retry.expect("concurrent retry")); + assert_ne!( + first.expect("first request"), + retry.expect("concurrent retry") + ); let facts = read_recall_events(store, 0, 100) .await .expect("request facts"); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index f92b9941c0..1f05467222 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -23,6 +23,7 @@ use crate::bindings::_helpers::init_command::command::run_init_command; use crate::bindings::_helpers::janitor::janitor_command::run_janitor_command; use crate::bindings::_helpers::lineage::lineage_command::run_lineage_command; use crate::bindings::_helpers::load::load_command::run_load_command; +use crate::bindings::_helpers::location_command::command::run_location_command; use crate::bindings::_helpers::plan_command::plan::run_plan_command; use crate::bindings::_helpers::plan_command::rules::run_rules_command; use crate::bindings::_helpers::query_command::dag::run_dag_command; @@ -387,6 +388,7 @@ fn run_implementation<'py>( "run_janitor_command" => run_janitor_command(py, &request()?)?, "run_lineage_command" => run_lineage_command(py, &request()?)?, "run_load_command" => run_load_command(py, &request()?)?, + "run_location_command" => run_location_command(py, options())?, "run_plan_command" => run_plan_command(&request()?)?, "run_playground_command" => run_playground_command(py, &request()?)?, "run_query_command" => run_query_command(py, options())?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs index c342479217..2c8daa3464 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs @@ -7,5 +7,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { crate::bindings::_helpers::compiled_project::project::register(module)?; input_loops::registration::register(module)?; compiled_views::registration::register(module)?; + crate::bindings::_helpers::location_command::command::register(module)?; crate::bindings::_helpers::compiled_project::snapshot::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs index 018f69fb18..a2f44bf10a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs @@ -10,8 +10,9 @@ use pyo3::prelude::{ use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; use serde_json::Value; use sqlbuild_core::models::DiagnosticSeverity; +use sqlbuild_orchestration::api::errors::ClientError; use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; -use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; +use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode, SnapshotRecord}; use crate::bindings::_helpers::compiled_project::snapshot_declarations::{ AuthoredJob, compile_declarations, discover_declarations, @@ -30,6 +31,26 @@ pub(crate) struct NativeOrchestrationSnapshot { pub(crate) input: SnapshotInput, } +impl NativeOrchestrationSnapshot { + /// Native callers retain transport errors so they can report an uncertain acknowledgement. + pub(crate) fn publication( + &self, + py: Python<'_>, + server_url: &str, + bearer: &str, + ) -> PyResult> { + py.detach(|| { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; + let result = runtime.block_on(publish_remote_snapshot(server_url, bearer, &self.input)); + runtime.shutdown_timeout(std::time::Duration::from_secs(1)); + Ok(result) + }) + } +} + #[pymethods] impl NativeOrchestrationSnapshot { /// Serialize only at the public boundary; native transports can use the retained input. @@ -40,14 +61,10 @@ impl NativeOrchestrationSnapshot { /// Publish retained data without projecting it through Python, releasing the GIL during IO. fn publish(&self, py: Python<'_>, server_url: &str, bearer: &str) -> PyResult { + let record = self + .publication(py, server_url, bearer)? + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; py.detach(|| { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; - let result = runtime.block_on(publish_remote_snapshot(server_url, bearer, &self.input)); - runtime.shutdown_timeout(std::time::Duration::from_secs(1)); - let record = result.map_err(|error| PyRuntimeError::new_err(error.to_string()))?; serde_json::to_string(&record) .map_err(|error| PyRuntimeError::new_err(error.to_string())) }) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs new file mode 100644 index 0000000000..001f7211e1 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs @@ -0,0 +1,199 @@ +//! Compile in the location interpreter and publish data through its scoped HTTP credential. + +use std::path::Path; +use std::process::{Command, Stdio}; + +use pyo3::exceptions::{PyRuntimeError, PyValueError}; +use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::PyDict; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::git_revisions::main::head_commit::head_commit; +use sqlbuild_orchestration::snapshots::models::SnapshotInput; + +use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; +use crate::bindings::_helpers::compile_driver::pipeline::compiled_phase; +use crate::bindings::_helpers::compiled_project::snapshot::{ + NativeOrchestrationSnapshot, build_orchestration_snapshot, +}; +use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; +use crate::bindings::_helpers::native_host::host::write_console; +use crate::bindings::_helpers::query_command::project::{ + ProjectAdapter, ProjectInputs, project_adapter, +}; +use crate::bindings::_helpers::records::construct::new_record_with; +use crate::bindings::_helpers::records::fields::field; +use crate::bindings::_helpers::records::ids::RecordId; + +fn status(py: Python<'_>, message: &str) -> PyResult<()> { + write_console(py, "stderr", &format!("{message}\n")) +} + +/// Git is optional in deployed images; absent metadata is not fabricated from the last publication. +fn metadata( + py: Python<'_>, + arguments: &KeywordArguments<'_>, + directory: &Path, +) -> PyResult { + let location: String = arguments.value("location_id")?.extract()?; + let location_id = location + .parse() + .map_err(|_| PyValueError::new_err("location ID must be a UUID"))?; + let target: String = arguments.value("selected_target")?.extract()?; + let generation: i64 = arguments.value("generation")?.extract()?; + if target.trim().is_empty() || generation < 1 { + return Err(PyValueError::new_err( + "target must not be blank and generation must be positive", + )); + } + let (git_revision, git_dirty, git_author) = py.detach(|| -> PyResult<_> { + let revision = head_commit(directory); + let mut dirty = false; + let mut author = None; + if revision.is_some() { + for (is_dirty, options) in [ + (false, vec!["log", "-1", "--format=%an", "HEAD"]), + ( + true, + vec!["status", "--porcelain", "--untracked-files=normal"], + ), + ] { + let output = Command::new("git") + .current_dir(directory) + .args(options) + .stdin(Stdio::null()) + .output()?; + if !output.status.success() { + return Err(PyRuntimeError::new_err( + "could not read location Git metadata", + )); + } + let text = String::from_utf8_lossy(&output.stdout); + if is_dirty { + dirty = !text.trim().is_empty(); + } else { + author = Some(text.trim().to_owned()); + } + } + } + Ok((revision, dirty, author)) + })?; + Ok(SnapshotInput { + location_id, + target, + generation, + git_revision, + git_dirty, + git_author, + image_id: arguments.value("image_id")?.extract()?, + environment_hash: String::new(), + sqb_version: String::new(), + nodes: Default::default(), + documents: Default::default(), + }) +} + +fn compile_location<'py>( + py: Python<'py>, + arguments: &KeywordArguments<'py>, + input: &SnapshotInput, +) -> PyResult { + status(py, "Discovering location project...")?; + let ProjectAdapter { + project_dir, + discovered, + adapter, + } = project_adapter( + py, + Some(arguments.value("project_dir")?), + ProjectInputs::Discovered, + )?; + status(py, "Location project discovered. Compiling snapshot...")?; + let options = new_record_with( + py, + RecordId::CompilePipelineOptions, + &[("selected_target", arguments.value("selected_target")?)], + )?; + let compiled = compiled_phase((&discovered, &adapter), &options, &py.None().into_bound(py))?; + let project = field(&compiled, "project")?; + let view = project.cast::()?.borrow(); + let snapshot = build_orchestration_snapshot( + view, + &serde_json::to_string(input).map_err(|error| PyValueError::new_err(error.to_string()))?, + project_dir.extract()?, + None, + )?; + status(py, "Location snapshot compiled.")?; + Ok(snapshot) +} + +/// One publication is also the isolated compiler boundary a reloading location can invoke. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn run_location_command<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult { + let arguments = KeywordArguments::read( + py, + "run_location_command", + options, + ( + &[ + "project_dir", + "location_id", + "selected_target", + "generation", + "server_url", + ], + &["image_id"], + ), + )?; + let publication = (|| { + let bearer = std::env::var("SQLBUILD_LOCATION_TOKEN").map_err(|_| { + PyValueError::new_err( + "SQLBUILD_LOCATION_TOKEN must contain a UTF-8 location credential", + ) + })?; + if bearer.trim().is_empty() { + return Err(PyValueError::new_err( + "SQLBUILD_LOCATION_TOKEN must not be blank", + )); + } + let directory = arguments + .value("project_dir")? + .extract::()?; + let input = metadata(py, &arguments, &directory)?; + let snapshot = compile_location(py, &arguments, &input)?; + status(py, "Publishing location snapshot...")?; + let record = snapshot.publication( + py, + &arguments.value("server_url")?.extract::()?, + &bearer, + )?.map_err(|error| { + let message = if matches!(error, sqlbuild_orchestration::api::errors::ClientError::Transport(_)) { + format!("Publication outcome is unknown; generation {} may already be published. Check the current snapshot before retrying: {error}", input.generation) + } else { + error.to_string() + }; + PyValueError::new_err(message) + })?; + let output = serde_json::to_string(&record) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + write_console(py, "stdout", &format!("{output}\n")).map_err(|error| { + PyValueError::new_err(format!( + "Snapshot {} was published, but result delivery failed: {error}", + record.snapshot_id + )) + })?; + status(py, "Location snapshot published.")?; + Ok(0) + })(); + if publication.is_err() { + status(py, "Location command failed.")?; + } + publication +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_location_command, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs new file mode 100644 index 0000000000..d5ef1bbf1b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs @@ -0,0 +1 @@ +pub(crate) mod command; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 457ca748d9..732a57220e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -59,6 +59,7 @@ pub(crate) mod janitor; pub(crate) mod lifecycle_events; pub(crate) mod lineage; pub(crate) mod load; +pub(crate) mod location_command; pub(crate) mod macro_bridge; pub(crate) mod manifest; pub(crate) mod microbatches; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs index c2904c48dc..796bb304d7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs @@ -65,6 +65,14 @@ pub(crate) fn entry<'py>(py: Python<'py>, id: impl Into) -> PyResult, stream: &str, text: &str) -> PyResult<()> { + let stream = py.import("sys")?.getattr(stream)?; + stream.call_method1("write", (text,))?; + stream.call_method0("flush")?; + Ok(()) +} + /// `log_debug_event(logger=logging.getLogger(logger), message=message)`. pub(crate) fn log_debug_message(py: Python<'_>, logger: &str, message: &str) -> PyResult<()> { let kwargs: Bound<'_, PyDict> = PyDict::new(py); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index d8e3df098d..6a1c04bac6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -982,6 +982,7 @@ const CLI_ENTRYPOINT_HANDLERS: RecordSpec = RecordSpec { FieldSpec::new("run_refactor", FieldDefault::None), FieldSpec::new("run_recall", FieldDefault::None), FieldSpec::new("run_review", FieldDefault::None), + FieldSpec::new("run_location", FieldDefault::None), ], }; const COMMAND_WAREHOUSE_SCOPE: RecordSpec = RecordSpec { @@ -1560,6 +1561,7 @@ const CLI_COMMAND: EnumSpec = EnumSpec { ("DIFF", "diff"), ("REVIEW", "review"), ("DEBUG", "debug"), + ("LOCATION", "location"), ("LINEAGE", "lineage"), ("QUERY", "query"), ("COST", "cost"), diff --git a/fensu.toml b/fensu.toml index 191fa4912d..104f5bec85 100644 --- a/fensu.toml +++ b/fensu.toml @@ -2654,6 +2654,7 @@ module_layers = [ "lineage/lineage_command=commands", "lineage/lineage_facts=project", "load=commands", + "location_command=commands", "macro_bridge=front", "manifest=compile", "microbatches=plan", diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 27199c0f53..b4579905de 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1769,6 +1769,15 @@ def build_orchestration_snapshot( *, declarations: Sequence[object] | None = None, ) -> NativeOrchestrationSnapshot: ... +def run_location_command( + *, + project_dir: Path, + location_id: str, + selected_target: str, + generation: int, + server_url: str, + image_id: str | None = None, +) -> int: ... class NativeProjectView: @property @@ -7302,6 +7311,7 @@ class CliEntrypointHandlers(NativeRecord): run_refactor: Callable[..., int] | None run_recall: Callable[..., int] | None run_review: Callable[..., int] | None + run_location: Callable[..., int] | None def __init__( self, run_compile: Callable[[CompileCommandRequest], int], @@ -7338,6 +7348,7 @@ class CliEntrypointHandlers(NativeRecord): run_refactor: Callable[..., int] | None = None, run_recall: Callable[..., int] | None = None, run_review: Callable[..., int] | None = None, + run_location: Callable[..., int] | None = None, ) -> None: ... class CommandWarehouseScope(NativeRecord): @@ -8053,6 +8064,7 @@ class CliCommand(StrEnum): DIFF = "diff" REVIEW = "review" DEBUG = "debug" + LOCATION = "location" LINEAGE = "lineage" QUERY = "query" COST = "cost" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index 27b44181af..7639ec285f 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -13,6 +13,7 @@ from collections.abc import Iterator from contextlib import contextmanager from datetime import UTC, datetime, timedelta +from io import StringIO from pathlib import Path from typing import Any from urllib.parse import quote @@ -25,6 +26,13 @@ ) +class RejectedPublicationOutput(StringIO): + """A caller loses only the result stream, after the real API accepts a publication.""" + + def write(self, text: str) -> int: + raise OSError("synthetic publication result delivery failure") + + def api_address(process: subprocess.Popen[bytes], log: Path) -> str: """Wait for bounded native startup, never publishing an unready address.""" deadline: float = time.monotonic() + 10 diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py index 90d620f024..31caf9be2d 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py @@ -1,18 +1,23 @@ """Real compile, native transport, authenticated server and durable publication.""" import json +import socket +import sqlite3 +import subprocess from pathlib import Path from typing import Any from urllib.parse import quote from urllib.request import Request, urlopen import pytest +from _pytest.capture import CaptureResult from sqlbuild._native import ( NativeOrchestrationSnapshot, build_compiled_project, build_orchestration_snapshot, discover_project_inputs, + run_cli, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView @@ -24,6 +29,7 @@ SnapshotPublicationTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + RejectedPublicationOutput, current_snapshot, snapshot_api, ) @@ -120,3 +126,151 @@ def test_given_retained_compiler_data_when_publishing_then_http_updates_atomical assert triggers[0]["definition_json"]["starts_at"] == "2026-07-02T19:47:23.456789Z" assert triggers[0]["definition_json"]["default_status"] == "paused" assert triggers[0]["definition_json"]["catch_up"] == {"all": {"max_runs": 3}} + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotPublicationTestCase( + description="configured development target", + target="development", + expected_generation=2, + expected_nodes=frozenset({"model:orders", "model:stg_orders"}), + ), + SnapshotPublicationTestCase( + description="configured target with encoded path characters", + target="preview #1", + expected_generation=2, + expected_nodes=frozenset({"model:orders", "model:stg_orders"}), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_code_location_when_publishing_from_cli_then_compilation_isolated_from_store_and_failed_edits_keep_last_good( + test_case: SnapshotPublicationTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + project_dir: Path = tmp_path / "project" + files: dict[str, str] = TYPE_PROOF_FILES | { + "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + + f'\n[targets.{json.dumps(test_case.target)}]\nschema = "main"\n', + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job\n" + 'daily_orders = job("daily_orders", select="orders", description="Build orders.")\n' + ), + } + write_files(project_dir=project_dir, files=files) + _ = subprocess.run(["git", "init", "-q", str(project_dir)], check=True, capture_output=True) + _ = subprocess.run( + [ + "git", + "-C", + str(project_dir), + "-c", + "user.name=Orders Contributor", + "-c", + "user.email=orders@example.com", + "commit", + "--allow-empty", + "-qm", + "Initial orders", + ], + check=True, + capture_output=True, + ) + monkeypatch.setenv("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://must-not-open") + with snapshot_api(tmp_path / "control") as api: + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", api.bearer) + arguments: list[str] = [ + "--no-color", + "--project-dir", + str(project_dir), + "location", + "publish", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + test_case.target, + "--image-id", + "sha256:synthetic-orders-image", + ] + assert run_cli([*arguments, "--generation", "1"]) == 0 + first_output: CaptureResult[str] = capsys.readouterr() + first: dict[str, Any] = json.loads(first_output.out) + assert first["generation"] == 1 + assert "Location snapshot published." in first_output.err + initial: dict[str, Any] = current_snapshot(api, test_case.target) + assert frozenset(initial["nodes"]) == test_case.expected_nodes + assert initial["documents"]["jobs"]["daily_orders"]["selection"] == ["model:orders"] + assert not (project_dir / ".sqlbuild/control.sqlite3").exists() + assert not (project_dir / "warehouse.duckdb").exists() + with sqlite3.connect(tmp_path / "control/.sqlbuild/control.sqlite3") as connection: + metadata: tuple[str, int, str, str] | None = connection.execute( + "SELECT git_revision, git_dirty, git_author, image_id FROM sqb_snapshots WHERE snapshot_id=?", + (first["snapshot_id"],), + ).fetchone() + assert metadata is not None + assert len(metadata[0]) == 40 + assert metadata[1:] == (1, "Orders Contributor", "sha256:synthetic-orders-image") + + staging: Path = project_dir / "models/stg_orders.sql" + _ = staging.write_text( + files["models/stg_orders.sql"].replace("CAST(5 AS INTEGER)", "CAST(11 AS INTEGER)") + ) + assert run_cli([*arguments, "--generation", "2"]) == 0 + second_output: CaptureResult[str] = capsys.readouterr() + second: dict[str, Any] = json.loads(second_output.out) + assert second["generation"] == test_case.expected_generation + updated: dict[str, Any] = current_snapshot(api, test_case.target) + assert {change["subject_key"] for change in updated["changes"]} == {"model:stg_orders"} + assert "CAST(11 AS INTEGER)" in updated["nodes"]["model:stg_orders"]["raw_sql"] + + _ = staging.write_text("SELECT FROM\n") + assert run_cli([*arguments, "--generation", "3"]) != 0 + failed_output: CaptureResult[str] = capsys.readouterr() + assert not failed_output.out + assert "Location command failed." in failed_output.err + assert current_snapshot(api, test_case.target)["record"] == second + + _ = staging.write_text(files["models/stg_orders.sql"]) + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", "invalid-credential") + assert run_cli([*arguments, "--generation", "3"]) != 0 + rejected_output: CaptureResult[str] = capsys.readouterr() + assert not rejected_output.out + assert "HTTP 401" in rejected_output.err + assert api.bearer not in "".join( + ( + first_output.out, + first_output.err, + second_output.out, + second_output.err, + failed_output.err, + rejected_output.err, + ) + ) + assert current_snapshot(api, test_case.target)["record"] == second + + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", api.bearer) + with socket.socket() as unavailable: + unavailable.bind(("127.0.0.1", 0)) + disconnected: list[str] = list(arguments) + disconnected[arguments.index("--server") + 1] = ( + f"http://127.0.0.1:{unavailable.getsockname()[1]}" + ) + assert run_cli([*disconnected, "--generation", "3"]) != 0 + transport_output: CaptureResult[str] = capsys.readouterr() + assert not transport_output.out + assert "generation 3 may already be published" in transport_output.err + assert current_snapshot(api, test_case.target)["record"] == second + + with monkeypatch.context() as result_scope: + result_scope.setattr("sys.stdout", RejectedPublicationOutput()) + assert run_cli([*arguments, "--generation", "3"]) != 0 + delivery_output: CaptureResult[str] = capsys.readouterr() + assert "was published, but result delivery failed" in delivery_output.err + assert current_snapshot(api, test_case.target)["record"]["generation"] == 3 From 131445d8f1c2bf08535c0f9b4fad100517de0f0a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 08:37:48 +0000 Subject: [PATCH 30/59] feat: supervise isolated code location reloads Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- Cargo.lock | 1 + Cargo.toml | 1 + crates/sqlbuild-cli/Cargo.toml | 1 + .../sqlbuild-cli/src/argv/command_table.json | 7 +- crates/sqlbuild-cli/src/dispatch/constants.rs | 3 +- .../src/launcher/_helpers/dispatch.rs | 6 + .../src/native_commands/_helpers/execution.rs | 119 ++++++++++- .../src/native_commands/_helpers/paths.rs | 54 +++++ .../src/native_commands/main/_run_location.rs | 91 +++++++++ .../src/native_commands/main/mod.rs | 2 + .../main/parse_native_invocation.rs | 16 +- .../main/run_location_invocation.rs | 48 +++++ .../src/native_commands/models.rs | 10 + .../src/snapshots/main/mod.rs | 1 + .../snapshots/main/next_remote_generation.rs | 36 ++++ .../_helpers/location_command/command.rs | 17 +- .../_helpers/location_command/constants.rs | 2 + .../bindings/_helpers/location_command/mod.rs | 1 + src/sqlbuild/_native.pyi | 1 + .../publication/_test_types.py | 15 ++ .../compiled_project/publication/helpers.py | 55 ++++++ .../publication/test_reload.py | 187 ++++++++++++++++++ 22 files changed, 658 insertions(+), 16 deletions(-) create mode 100644 crates/sqlbuild-cli/src/native_commands/main/_run_location.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/main/run_location_invocation.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/main/next_remote_generation.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py diff --git a/Cargo.lock b/Cargo.lock index ad63bc0ef9..dbef530346 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2498,6 +2498,7 @@ dependencies = [ "reqwest", "rusqlite", "rustix", + "same-file", "serde", "serde_json", "sqlbuild-build-features", diff --git a/Cargo.toml b/Cargo.toml index b28edb4dda..01ca1f28c3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -128,6 +128,7 @@ chrono-tz = "0.10" croner = "=3.0.1" flate2 = "1" proptest = "1" +same-file = "1" semver = "1" # Pin SQLx's driver crates together: its umbrella also resolves an incompatible SQLite engine. sqlx-core = { version = "=0.8.6", default-features = false, features = ["_rt-tokio", "_tls-rustls-ring-webpki", "chrono", "uuid", "json"] } diff --git a/crates/sqlbuild-cli/Cargo.toml b/crates/sqlbuild-cli/Cargo.toml index fd9d3be80e..d5e621fd80 100644 --- a/crates/sqlbuild-cli/Cargo.toml +++ b/crates/sqlbuild-cli/Cargo.toml @@ -37,6 +37,7 @@ rustix.workspace = true [target.'cfg(windows)'.dependencies] ctrlc.workspace = true +same-file.workspace = true [dev-dependencies] reqwest.workspace = true diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index cd3d7e216a..f63070d273 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -106,8 +106,8 @@ "options": [], "dest": "location_command", "required": true, - "choices": ["publish"], - "help": "compile the project and publish one immutable snapshot", + "choices": ["publish", "serve"], + "help": "publish one snapshot or watch and reload the project", "group": 0 }, { @@ -139,8 +139,7 @@ "options": ["--generation"], "dest": "location_generation", "type": "int", - "required": true, - "help": "positive generation newer than the last successful publication", + "help": "required for publish; serve derives the next generation from the server", "group": 1 }, { diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index e224c4f9f4..2f6ce26c7f 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -515,13 +515,14 @@ pub(crate) const DEBUG_ROUTE: Route = routed( }, ); -const LOCATION_ARGUMENTS: [Field; 6] = [ +const LOCATION_ARGUMENTS: [Field; 7] = [ field("project_dir", Source::EffectiveProjectDir), SELECTED_TARGET, same("location_id"), attr("server_url", "location_server_url"), attr("generation", "location_generation"), attr("image_id", "location_image_id"), + attr("action", "location_command"), ]; pub(crate) const LOCATION_ROUTE: Route = routed( "run_location", diff --git a/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs b/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs index 441901c2b6..ff3bfcc8ea 100644 --- a/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs +++ b/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs @@ -12,6 +12,7 @@ use crate::launcher::constants::{ }; use crate::native_commands::main::parse_native_invocation::parse_native_invocation; use crate::native_commands::main::run_control_invocation::run_control_invocation; +use crate::native_commands::main::run_location_invocation::run_location_invocation; use crate::native_commands::main::run_native_invocation::run_native_invocation; use crate::native_commands::models::NativeParse; use crate::replay::main::serve_reused_compile::serve_reused_compile; @@ -55,6 +56,11 @@ pub(crate) fn launch(arguments: Vec, started: (Instant, SystemTime)) - report_missing_python_cli(&program); return HANDOFF_FAILURE_EXIT_CODE; }; + if let Ok(NativeParse::Invocation { namespace, .. }) = &native + && let Some(code) = run_location_invocation(namespace, &script) + { + return code; + } let mut handed: Vec = Vec::with_capacity(arguments.len() + 1); let record: Option<(PathBuf, String)> = launch_record(&program, &script, &arguments); if let Some((path, key)) = &record diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 0a84181be4..66135cf49c 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -2,37 +2,144 @@ use std::io; use std::path::{Path, PathBuf}; +use std::process::Stdio; use std::time::SystemTime; +use sqlbuild_orchestration::api::errors::ClientError; +use sqlbuild_orchestration::snapshots::main::next_remote_generation::next_remote_generation; use sqlbuild_orchestration::store::errors::ControlError; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::open_store::open_store; use sqlbuild_orchestration::store::models::ControlStore; +use tokio::process::{Child, Command}; use tokio::sync::watch; use tokio::task::JoinSet; -use crate::argv::models::{Namespace, Stream}; +use crate::argv::models::{Namespace, Stream, Value}; use crate::compute_logs::constants::DEFAULT_RETENTION_COUNT; use crate::compute_logs::main::abandon_capture::abandon_capture; use crate::compute_logs::main::finish_capture::finish_capture; use crate::compute_logs::main::open_log_root::open_log_root; use crate::compute_logs::main::start_capture::start_capture; use crate::compute_logs::models::{CaptureEnd, CaptureStart, OpenCapture}; -use crate::native_commands::_helpers::decision::control_database; +use crate::native_commands::_helpers::decision::{control_database, text}; use crate::native_commands::_helpers::init::run_init; use crate::native_commands::_helpers::package::installed_package; use crate::native_commands::_helpers::paths::project_location; use crate::native_commands::_helpers::project_facts::has_no_runtime_extensions; use crate::native_commands::_helpers::rules::run_rules; use crate::native_commands::_helpers::skills::{run_skills, skill_freshness_notice}; -use crate::native_commands::_helpers::terminal::{finish_lost_stdout, write_output}; -use crate::native_commands::constants::{UNCAUGHT_EXCEPTION_STATUS, UNKNOWN_COMMAND}; +use crate::native_commands::_helpers::terminal::{ + finish_lost_stdout, terminal_columns, write_output, +}; +use crate::native_commands::constants::{ + COLUMNS_VARIABLE, FALLBACK_COLUMNS, UNCAUGHT_EXCEPTION_STATUS, UNKNOWN_COMMAND, +}; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::main::_run_cost::run_cost; use crate::native_commands::models::{ - CommandOutput, InstalledPackage, LostStdout, NativeCommand, NativeInvocation, ProjectLocation, + CommandOutput, InstalledPackage, LocationServe, LostStdout, NativeCommand, NativeInvocation, + ProjectLocation, }; +pub(crate) fn location_options(namespace: &Namespace) -> Result { + if namespace + .get("location_generation") + .is_some_and(|value| *value != Value::None) + { + return Err(ControlError::Invalid( + "serve derives --generation from the server; do not supply it".into(), + )); + } + let project_dir = text(namespace, "project_dir") + .map_or_else(std::env::current_dir, |directory| { + std::fs::canonicalize(directory) + })?; + if !project_dir.is_dir() { + return Err(ControlError::Invalid("project directory must exist".into())); + } + let location_id = text(namespace, "location_id") + .unwrap_or_default() + .parse() + .map_err(|_| ControlError::Invalid("location ID must be a UUID".into()))?; + let target = text(namespace, "target").unwrap_or_default(); + if target.trim().is_empty() { + return Err(ControlError::Invalid("target must not be blank".into())); + } + Ok(LocationServe { + project_dir, + server_url: text(namespace, "location_server_url").unwrap_or_default(), + location_id, + target, + image_id: text(namespace, "location_image_id"), + }) +} + +/// Reconcile from the server after restarts or lost acknowledgements; transient outages retry. +pub(crate) async fn location_generation( + options: &LocationServe, + bearer: &str, +) -> Result, ControlError> { + match next_remote_generation( + &options.server_url, + bearer, + options.location_id, + &options.target, + ) + .await + { + Ok(generation) => Ok(Some(generation)), + Err(error) => { + if matches!(&error, ClientError::Configuration(_)) + || matches!(&error, ClientError::Rejected(status) if matches!(status.as_u16(), 401 | 403)) + { + return Err(ControlError::Invalid(error.to_string())); + } + drop(write_output(&[( + Stream::Stderr, + &format!( + "Location generation lookup failed; retaining last good snapshot and retrying: {error}\n" + ), + )])); + Ok(None) + } + } +} + +/// The installed interpreter owns user imports; the supervisor never imports Python. +pub(crate) fn publish_child( + options: &LocationServe, + script: &Path, + generation: i64, +) -> io::Result { + let mut command = Command::new(script); + command + .arg("--project-dir") + .arg(&options.project_dir) + .args([ + "location", + "publish", + "--server", + &options.server_url, + "--location-id", + ]) + .arg(options.location_id.to_string()) + .args(["--target", &options.target, "--generation"]) + .arg(generation.to_string()); + if let Some(image) = &options.image_id { + command.args(["--image-id", image]); + } + if terminal_columns().is_err() { + command.env(COLUMNS_VARIABLE, FALLBACK_COLUMNS.to_string()); + } + command + .stdin(Stdio::null()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .kill_on_drop(true) + .spawn() +} + /// Open and migrate the privileged store, reporting progress without contaminating JSON stdout. pub(crate) async fn open_control_store( namespace: &Namespace, @@ -48,7 +155,7 @@ pub(crate) async fn open_control_store( Ok(store) } -async fn shutdown_signal() -> Result<(), ControlError> { +pub(crate) async fn shutdown_signal() -> Result<(), ControlError> { #[cfg(unix)] { let mut terminate = diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs index 302e8a5a66..565ecf7f95 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs @@ -2,10 +2,64 @@ use std::path::{Component, Path, PathBuf}; +use sqlbuild_cache::digest::constants::{ + EXCLUDED_DIRECTORIES, EXCLUDED_ROOT_DIRECTORIES, PRESENCE_ONLY_FILE_SUFFIXES, +}; +use sqlbuild_cache::project_snapshot::main::snapshot_project_files::snapshot_project_files; +use sqlbuild_cache::project_snapshot::models::{PathStamp, SnapshotRules}; +use sqlbuild_core::git_revisions::main::head_commit::head_commit; use sqlbuild_rules::project_tree::main::project_real_path::project_real_path; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::models::ProjectLocation; +use crate::replay::main::output_file_identities::output_file_identities; + +/// Observe the same project inputs as compiler reuse, excluding redirected command output. +pub(crate) fn location_rules() -> SnapshotRules { + SnapshotRules { + excluded: EXCLUDED_DIRECTORIES + .iter() + .map(ToString::to_string) + .collect(), + excluded_root: EXCLUDED_ROOT_DIRECTORIES + .iter() + .map(ToString::to_string) + .collect(), + presence_suffixes: PRESENCE_ONLY_FILE_SUFFIXES + .iter() + .map(ToString::to_string) + .collect(), + output_files: output_file_identities(), + } +} + +/// Git commits matter to publication metadata even when no working-tree file changes. +pub(crate) fn location_inputs( + root: &Path, + rules: &SnapshotRules, +) -> (Vec, Option) { + let mut stamps = snapshot_project_files(root, rules); + #[cfg(windows)] + { + let output: Vec = + [same_file::Handle::stdout(), same_file::Handle::stderr()] + .into_iter() + .flatten() + .filter(|handle| { + handle + .as_file() + .metadata() + .is_ok_and(|metadata| metadata.is_file()) + }) + .collect(); + stamps.retain(|stamp| { + same_file::Handle::from_path(root.join(&stamp.relative_path)) + .map_or(true, |handle| !output.contains(&handle)) + }); + } + stamps.sort_by(|left, right| left.relative_path.cmp(&right.relative_path)); + (stamps, head_commit(root)) +} /// `Path(text)`: repeated separators, `.` parts and trailing separators dropped. pub(crate) fn pathlib_path(text: &Path) -> PathBuf { diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs new file mode 100644 index 0000000000..017c4fd008 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs @@ -0,0 +1,91 @@ +//! Coalesce source edits and retain the last good publication across isolated compiler failures. + +use std::path::Path; +use std::time::Duration; + +use sqlbuild_orchestration::store::errors::ControlError; +use tokio::time::{Instant, MissedTickBehavior}; + +use crate::argv::models::Stream; +use crate::native_commands::_helpers::execution::{ + location_generation, publish_child, shutdown_signal, +}; +use crate::native_commands::_helpers::paths::{location_inputs, location_rules}; +use crate::native_commands::_helpers::terminal::write_output; +use crate::native_commands::models::LocationServe; + +pub(super) async fn run_location( + options: LocationServe, + script: &Path, + bearer: &str, +) -> Result<(), ControlError> { + let rules = location_rules(); + let mut observed = location_inputs(&options.project_dir, &rules); + let mut attempted = None; + let mut changed_at = Instant::now(); + let mut retry_at = None; + let mut clock = tokio::time::interval(Duration::from_millis(250)); + clock.set_missed_tick_behavior(MissedTickBehavior::Skip); + let stopped = shutdown_signal(); + tokio::pin!(stopped); + drop(write_output(&[( + Stream::Stderr, + "Starting code location; watching project inputs...\n", + )])); + loop { + tokio::select! { + result = &mut stopped => return result, + _ = clock.tick() => {}, + } + let current = location_inputs(&options.project_dir, &rules); + if observed != current { + observed = current; + changed_at = Instant::now(); + } + if attempted.as_ref() == Some(&observed) + && retry_at.is_none_or(|retry| Instant::now() < retry) + { + continue; + } + if attempted.is_some() && changed_at.elapsed() < Duration::from_millis(250) { + continue; + } + attempted = Some(observed.clone()); + let generation = tokio::select! { + result = &mut stopped => return result, + generation = location_generation(&options, bearer) => generation?, + }; + let generation = match generation { + Some(generation) => generation, + None => { + retry_at = Some(Instant::now() + Duration::from_secs(5)); + continue; + } + }; + drop(write_output(&[( + Stream::Stderr, + &format!("Compiling location generation {generation} in an isolated interpreter...\n"), + )])); + let mut child = publish_child(&options, script, generation)?; + let result = tokio::select! { + result = &mut stopped => { + child.kill().await?; + return result; + }, + result = child.wait() => result?, + }; + if result.success() { + retry_at = None; + drop(write_output(&[( + Stream::Stderr, + "Code location ready; watching for changes.\n", + )])); + } else { + retry_at = Some(Instant::now() + Duration::from_secs(30)); + drop(write_output(&[( + Stream::Stderr, + "Location reload failed; retaining last good snapshot. Retrying on change or in 30 seconds.\n", + )])); + } + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/mod.rs b/crates/sqlbuild-cli/src/native_commands/main/mod.rs index e93969bdc4..ac5f9232e5 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/mod.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/mod.rs @@ -2,6 +2,8 @@ pub(super) mod _cost_report_options; pub(super) mod _run_admin; pub(super) mod _run_control; pub(super) mod _run_cost; +pub(super) mod _run_location; pub(crate) mod parse_native_invocation; pub(crate) mod run_control_invocation; +pub(crate) mod run_location_invocation; pub mod run_native_invocation; diff --git a/crates/sqlbuild-cli/src/native_commands/main/parse_native_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/parse_native_invocation.rs index 707115b4f9..8c2056d976 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/parse_native_invocation.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/parse_native_invocation.rs @@ -5,11 +5,11 @@ use std::ffi::OsString; use crate::argv::main::parse_cli_arguments::parse_cli_arguments; use crate::argv::models::{ArgvOutcome, ArgvSettings, Namespace}; use crate::native_commands::_helpers::argv_host::{NativeArgvHost, namespace_defaults}; -use crate::native_commands::_helpers::decision::review_json_failure; +use crate::native_commands::_helpers::decision::{review_json_failure, text}; use crate::native_commands::_helpers::terminal::{ finish_rejected_parse, stdout_supports_color, terminal_columns, }; -use crate::native_commands::constants::NO_COLOR_OPTION; +use crate::native_commands::constants::{FALLBACK_COLUMNS, NO_COLOR_OPTION}; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::models::NativeParse; @@ -21,13 +21,23 @@ pub(crate) fn parse_native_invocation(arguments: &[OsString]) -> Result Option { + if text(namespace, "command").as_deref() != Some("location") + || text(namespace, "location_command").as_deref() != Some("serve") + { + return None; + } + let result = (|| { + let options = location_options(namespace)?; + let bearer = std::env::var("SQLBUILD_LOCATION_TOKEN").map_err(|_| { + std::io::Error::other( + "SQLBUILD_LOCATION_TOKEN must contain a UTF-8 location credential", + ) + })?; + if bearer.trim().is_empty() { + return Err(std::io::Error::other("SQLBUILD_LOCATION_TOKEN must not be blank").into()); + } + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build()?; + runtime.block_on(run_location(options, script, &bearer)) + })(); + Some(match result { + Ok(()) => { + drop(write_output(&[( + Stream::Stderr, + "Code location stopped successfully\n", + )])); + 0 + } + Err(error) => { + drop(write_output(&[( + Stream::Stderr, + &format!("Code location failed: {error}\n"), + )])); + 1 + } + }) +} diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 7d7cbc3df5..7be0c55c84 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -126,6 +126,16 @@ pub(crate) struct ProjectLocation { pub(crate) resolved: PathBuf, } +/// A long-running location owns no control-store credentials or executable user objects. +#[derive(Debug)] +pub(crate) struct LocationServe { + pub(crate) project_dir: PathBuf, + pub(crate) server_url: String, + pub(crate) location_id: uuid::Uuid, + pub(crate) target: String, + pub(crate) image_id: Option, +} + /// What Python's buffered stdout does with output its reader no longer takes. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum LostStdout { diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs index 8411aaf37d..56f1b75755 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs @@ -1,4 +1,5 @@ pub mod current_snapshot; +pub mod next_remote_generation; pub mod publish_remote_snapshot; pub mod publish_snapshot; pub mod verify_execution; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/next_remote_generation.rs b/crates/sqlbuild-orchestration/src/snapshots/main/next_remote_generation.rs new file mode 100644 index 0000000000..f92dabe4fa --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/next_remote_generation.rs @@ -0,0 +1,36 @@ +//! Reconcile publication generations from the authenticated server, including lost acknowledgements. + +use uuid::Uuid; + +use crate::api::errors::ClientError; +use crate::api::main::client_response::client_response; +use crate::api::main::connect_client::connect_client; +use crate::snapshots::models::CurrentSnapshot; + +pub async fn next_remote_generation( + server_url: &str, + bearer: &str, + location_id: Uuid, + target: &str, +) -> Result { + let connection = connect_client(server_url, bearer)?; + let url = connection.endpoint(&[ + "api", + "locations", + &location_id.to_string(), + "targets", + target, + "snapshots", + ])?; + match client_response::(connection.client.get(url)).await { + Ok(current) => current + .record + .generation + .checked_add(1) + .ok_or(ClientError::Configuration( + "snapshot generation is exhausted", + )), + Err(ClientError::Rejected(status)) if status == reqwest::StatusCode::NOT_FOUND => Ok(1), + Err(error) => Err(error), + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs index 001f7211e1..1ec1b8a2ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs @@ -16,6 +16,7 @@ use crate::bindings::_helpers::compiled_project::snapshot::{ NativeOrchestrationSnapshot, build_orchestration_snapshot, }; use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; +use crate::bindings::_helpers::location_command::constants::PUBLISH_ACTION; use crate::bindings::_helpers::native_host::host::write_console; use crate::bindings::_helpers::query_command::project::{ ProjectAdapter, ProjectInputs, project_adapter, @@ -39,7 +40,10 @@ fn metadata( .parse() .map_err(|_| PyValueError::new_err("location ID must be a UUID"))?; let target: String = arguments.value("selected_target")?.extract()?; - let generation: i64 = arguments.value("generation")?.extract()?; + let generation = arguments + .value("generation")? + .extract::>()? + .ok_or_else(|| PyValueError::new_err("publish requires --generation"))?; if target.trim().is_empty() || generation < 1 { return Err(PyValueError::new_err( "target must not be blank and generation must be positive", @@ -145,10 +149,19 @@ pub(crate) fn run_location_command<'py>( "generation", "server_url", ], - &["image_id"], + &["image_id", "action"], ), )?; let publication = (|| { + if arguments + .value("action")? + .extract::>()? + .is_some_and(|action| action != PUBLISH_ACTION) + { + return Err(PyValueError::new_err( + "location serve must run through the native sqb executable", + )); + } let bearer = std::env::var("SQLBUILD_LOCATION_TOKEN").map_err(|_| { PyValueError::new_err( "SQLBUILD_LOCATION_TOKEN must contain a UTF-8 location credential", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs new file mode 100644 index 0000000000..dc0a1d6e2d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs @@ -0,0 +1,2 @@ +/// Location actions accepted by the interpreter-side one-shot publisher. +pub(crate) const PUBLISH_ACTION: &str = "publish"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs index d5ef1bbf1b..8aa0ebb43e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs @@ -1 +1,2 @@ pub(crate) mod command; +mod constants; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index b4579905de..b235862344 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1777,6 +1777,7 @@ def run_location_command( generation: int, server_url: str, image_id: str | None = None, + action: str = "publish", ) -> int: ... class NativeProjectView: diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 63a996d791..be66d72ee4 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -31,3 +31,18 @@ class SnapshotWorkerTestCase: original_sql: str replacement_sql: str expected_error: str + + +@dataclass(frozen=True) +class LocationReloadTestCase: + description: str + target: str + expected_generation: int + expected_stop_code: int + + +@dataclass(frozen=True) +class LocationShutdownTestCase: + description: str + signal: int + expected_exit_code: int diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index 7639ec285f..994e90c8cc 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -16,6 +16,7 @@ from io import StringIO from pathlib import Path from typing import Any +from urllib.error import HTTPError from urllib.parse import quote from urllib.request import Request, urlopen from uuid import uuid4 @@ -33,6 +34,60 @@ def write(self, text: str) -> int: raise OSError("synthetic publication result delivery failure") +@contextmanager +def location_process( + arguments: list[str], log: Path, publications: Path +) -> Iterator[subprocess.Popen[bytes]]: + """Own and reap the supervisor even when an assertion interrupts a reload.""" + with log.open("ab") as stderr, publications.open("ab") as stdout: + process: subprocess.Popen[bytes] = subprocess.Popen(arguments, stderr=stderr, stdout=stdout) + try: + yield process + finally: + process.terminate() + try: + _ = process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + _ = process.wait(timeout=5) + + +def wait_for_generation( + api: SnapshotApi, target: str, generation: int, process: subprocess.Popen[bytes], log: Path +) -> dict[str, Any]: + deadline: float = time.monotonic() + 15 + current: dict[str, Any] = {"record": {"generation": 0}} + while current["record"]["generation"] < generation: + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, ( + f"Generation {generation} was not published: {log.read_text()}" + ) + try: + current = current_snapshot(api, target) + except HTTPError as error: + assert error.code == 404 + time.sleep(0.02) + assert current["record"]["generation"] == generation, log.read_text() + return current + + +def wait_for_log(process: subprocess.Popen[bytes], log: Path, text: str) -> None: + deadline: float = time.monotonic() + 15 + while text not in log.read_text(): + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + time.sleep(0.02) + + +def wait_for_interpreter(process: subprocess.Popen[bytes], pidfile: Path, log: Path) -> int: + deadline: float = time.monotonic() + 15 + while not pidfile.exists(): + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + time.sleep(0.02) + return int(pidfile.read_text()) + + def api_address(process: subprocess.Popen[bytes], log: Path) -> str: """Wait for bounded native startup, never publishing an unready address.""" deadline: float = time.monotonic() + 10 diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py new file mode 100644 index 0000000000..d2cbbe1f13 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py @@ -0,0 +1,187 @@ +"""A native supervisor reloads real project interpreters through the authenticated HTTP API.""" + +import json +import os +import signal +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import TYPE_PROOF_FILES +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + LocationReloadTestCase, + LocationShutdownTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + current_snapshot, + location_process, + snapshot_api, + wait_for_generation, + wait_for_interpreter, + wait_for_log, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + + +@pytest.mark.parametrize( + "test_case", + [ + LocationReloadTestCase("ordinary target", "development", 5, int(sys.platform == "win32")), + LocationReloadTestCase("encoded target", "preview #1", 5, int(sys.platform == "win32")), + ], + ids=lambda case: case.description, +) +def test_given_a_served_location_when_inputs_change_or_restart_then_generations_reload_without_losing_last_good( + test_case: LocationReloadTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project: Path = tmp_path / "project" + declaration: str = ( + "from sqlbuild.orchestration import job\n" + 'daily_orders = job("daily_orders", select="orders", description="Build orders.")\n' + ) + files: dict[str, str] = TYPE_PROOF_FILES | { + "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + + f'\n[targets.{json.dumps(test_case.target)}]\nschema = "main"\n', + "orchestration/orders.py": declaration, + } + write_files(project_dir=project, files=files) + _ = subprocess.run(["git", "init", "-q", str(project)], check=True, capture_output=True) + monkeypatch.setenv("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://must-not-open") + with snapshot_api(tmp_path / "control") as api: + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", api.bearer) + # A presentation-only width fallback must not bypass the native-only supervisor. + monkeypatch.setenv("COLUMNS", "999999999999999999999999999999") + arguments: list[str] = [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + test_case.target, + ] + log: Path = project / "location.log" + publications: Path = project / "publications.jsonl" + with location_process(arguments, log, publications) as process: + first: dict[str, Any] = wait_for_generation(api, test_case.target, 1, process, log) + staging: Path = project / "models/stg_orders.sql" + for value in (11, 12, 13): + _ = staging.write_text( + files["models/stg_orders.sql"].replace( + "CAST(5 AS INTEGER)", f"CAST({value} AS INTEGER)" + ) + ) + second: dict[str, Any] = wait_for_generation(api, test_case.target, 2, process, log) + assert second["record"]["snapshot_id"] != first["record"]["snapshot_id"] + assert "CAST(13 AS INTEGER)" in second["nodes"]["model:stg_orders"]["raw_sql"] + assert {change["subject_key"] for change in second["changes"]} == {"model:stg_orders"} + + _ = (project / "orchestration/orders.py").write_text("invalid Python declaration !\n") + wait_for_log(process, log, "Location reload failed; retaining last good snapshot.") + assert current_snapshot(api, test_case.target)["record"] == second["record"] + _ = (project / "orchestration/orders.py").write_text( + declaration.replace("Build orders.", "Refresh order totals.") + ) + third: dict[str, Any] = wait_for_generation(api, test_case.target, 3, process, log) + assert ( + third["documents"]["jobs"]["daily_orders"]["description"] == "Refresh order totals." + ) + _ = subprocess.run( + [ + "git", + "-C", + str(project), + "-c", + "user.name=Orders Contributor", + "-c", + "user.email=orders@example.com", + "commit", + "--allow-empty", + "-qm", + "Record orders", + ], + check=True, + capture_output=True, + ) + committed: dict[str, Any] = wait_for_generation(api, test_case.target, 4, process, log) + assert committed["nodes"] == third["nodes"] + process.terminate() + assert process.wait(timeout=5) == test_case.expected_stop_code + with location_process(arguments, log, publications) as process: + restarted: dict[str, Any] = wait_for_generation( + api, test_case.target, test_case.expected_generation, process, log + ) + assert restarted["nodes"] == third["nodes"] + # Redirected output and compiler cache writes must not feed the input watcher. + time.sleep(1) + assert current_snapshot(api, test_case.target)["record"] == restarted["record"] + process.terminate() + assert process.wait(timeout=5) == test_case.expected_stop_code + records: list[dict[str, Any]] = [ + json.loads(line) for line in publications.read_text().splitlines() + ] + assert [record["generation"] for record in records] == [1, 2, 3, 4, 5] + assert api.bearer not in log.read_text() + publications.read_text() + assert not (project / ".sqlbuild/control.sqlite3").exists() + assert not (project / "warehouse.duckdb").exists() + + +@pytest.mark.skipif(sys.platform == "win32", reason="Unix signal and process ownership contract") +@pytest.mark.parametrize( + "test_case", + [ + LocationShutdownTestCase("terminate compiling interpreter", signal.SIGTERM, 0), + LocationShutdownTestCase("interrupt compiling interpreter", signal.SIGINT, 0), + ], + ids=lambda case: case.description, +) +def test_given_a_compiling_location_when_stopped_then_its_interpreter_is_reaped( + test_case: LocationShutdownTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project: Path = tmp_path / "project" + child_pid: Path = tmp_path / "interpreter.pid" + write_files( + project_dir=project, + files=TYPE_PROOF_FILES + | { + "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + + '\n[targets.development]\nschema = "main"\n', + "orchestration/slow.py": ( + "import os, time\nfrom pathlib import Path\n" + f"Path({str(child_pid)!r}).write_text(str(os.getpid()))\n" + "time.sleep(60)\n" + ), + }, + ) + with snapshot_api(tmp_path / "control") as api: + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", api.bearer) + log: Path = tmp_path / "location.log" + arguments: list[str] = [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + ] + with location_process(arguments, log, tmp_path / "publications.jsonl") as process: + interpreter: int = wait_for_interpreter(process, child_pid, log) + process.send_signal(test_case.signal) + assert process.wait(timeout=5) == test_case.expected_exit_code + with pytest.raises(ProcessLookupError): + os.kill(interpreter, 0) From b99909a09c2fdaf52869708e9241f469c3069828 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 09:18:25 +0000 Subject: [PATCH 31/59] feat: report scoped code location health Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../src/native_commands/_helpers/execution.rs | 104 ++++++++- .../src/native_commands/_helpers/paths.rs | 94 +++++++- .../src/native_commands/main/_run_location.rs | 106 +++++++--- .../src/native_commands/models.rs | 12 ++ .../debug_report/main/format_debug_json.rs | 11 + .../sqlbuild-core/src/debug_report/models.rs | 2 + .../src/debug_report/tests/helpers.rs | 1 + .../debug_report/tests/test_debug_report.rs | 2 +- .../src/api/_helpers/location_health.rs | 45 ++++ .../src/api/_helpers/mod.rs | 1 + .../src/api/main/serve_api.rs | 6 +- .../sqlbuild-orchestration/src/api/models.rs | 4 +- .../platform/_helpers/location_heartbeat.rs | 50 +++++ .../src/platform/_helpers/mod.rs | 1 + .../src/platform/main/location_health.rs | 63 ++++++ .../src/platform/main/mod.rs | 3 + .../platform/main/report_location_health.rs | 104 +++++++++ .../main/report_remote_location_health.rs | 21 ++ .../src/platform/models.rs | 25 +++ .../src/store/_helpers/migration.rs | 11 +- .../src/store/location_instances.sql | 9 + .../src/store/main/migrate_store.rs | 20 +- crates/sqlbuild-orchestration/tests/api.rs | 2 + .../tests/api/test_location_health.rs | 200 ++++++++++++++++++ .../tests/api/test_types.rs | 6 + .../tests/snapshots/helpers.rs | 21 +- .../_helpers/command_runtime/connection.rs | 65 +++++- .../_helpers/debug_command/run_debug.rs | 5 +- .../publication/_test_types.py | 12 ++ .../compiled_project/publication/helpers.py | 32 +++ .../publication/test_reload.py | 154 +++++++++++++- 31 files changed, 1119 insertions(+), 73 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/location_health.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/_helpers/location_heartbeat.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/location_health.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/report_location_health.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/report_remote_location_health.rs create mode 100644 crates/sqlbuild-orchestration/src/store/location_instances.sql create mode 100644 crates/sqlbuild-orchestration/tests/api/test_location_health.rs diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 66135cf49c..30241401e9 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -1,19 +1,25 @@ //! Running a Python-free command: its checks, then its output with the observed services. +use std::future::Future; use std::io; use std::path::{Path, PathBuf}; use std::process::Stdio; -use std::time::SystemTime; +use std::time::{Duration, SystemTime}; use sqlbuild_orchestration::api::errors::ClientError; +use sqlbuild_orchestration::platform::main::report_remote_location_health::report_remote_location_health; +use sqlbuild_orchestration::platform::models::{LocationHealth, LocationHealthReport}; use sqlbuild_orchestration::snapshots::main::next_remote_generation::next_remote_generation; use sqlbuild_orchestration::store::errors::ControlError; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::open_store::open_store; use sqlbuild_orchestration::store::models::ControlStore; +use tokio::io::AsyncReadExt; use tokio::process::{Child, Command}; use tokio::sync::watch; use tokio::task::JoinSet; +use tokio::time::MissedTickBehavior; +use uuid::Uuid; use crate::argv::models::{Namespace, Stream, Value}; use crate::compute_logs::constants::DEFAULT_RETENTION_COUNT; @@ -25,7 +31,7 @@ use crate::compute_logs::models::{CaptureEnd, CaptureStart, OpenCapture}; use crate::native_commands::_helpers::decision::{control_database, text}; use crate::native_commands::_helpers::init::run_init; use crate::native_commands::_helpers::package::installed_package; -use crate::native_commands::_helpers::paths::project_location; +use crate::native_commands::_helpers::paths::{location_database_files, project_location}; use crate::native_commands::_helpers::project_facts::has_no_runtime_extensions; use crate::native_commands::_helpers::rules::run_rules; use crate::native_commands::_helpers::skills::{run_skills, skill_freshness_notice}; @@ -42,6 +48,43 @@ use crate::native_commands::models::{ ProjectLocation, }; +/// Health IO remains native even while a user-code child holds its interpreter's GIL. +pub(crate) async fn report_location_health( + options: &LocationServe, + bearer: &str, + session_id: Uuid, + health: LocationHealth, +) { + let report = LocationHealthReport { + event_id: Uuid::now_v7(), + session_id, + target: options.target.clone(), + health, + }; + if let Err(error) = + report_remote_location_health((&options.server_url, bearer), options.location_id, &report) + .await + { + drop(write_output(&[( + Stream::Stderr, + &format!("Location health report failed: {error}\n"), + )])); + } +} + +pub(crate) async fn location_heartbeats( + options: &LocationServe, + bearer: &str, + session_id: Uuid, +) -> Result<(), ControlError> { + let mut clock = tokio::time::interval(Duration::from_secs(30)); + clock.set_missed_tick_behavior(MissedTickBehavior::Skip); + loop { + clock.tick().await; + report_location_health(options, bearer, session_id, LocationHealth::Heartbeat {}).await; + } +} + pub(crate) fn location_options(namespace: &Namespace) -> Result { if namespace .get("location_generation") @@ -112,10 +155,8 @@ pub(crate) fn publish_child( script: &Path, generation: i64, ) -> io::Result { - let mut command = Command::new(script); + let mut command = location_child(options, script); command - .arg("--project-dir") - .arg(&options.project_dir) .args([ "location", "publish", @@ -129,15 +170,62 @@ pub(crate) fn publish_child( if let Some(image) = &options.image_id { command.args(["--image-id", image]); } + command.stdout(Stdio::inherit()).spawn() +} + +/// Capture the ordinary debug check's exact local file paths without contaminating publication stdout. +pub(crate) async fn check_location_connection( + options: &LocationServe, + script: &Path, + stopped: impl Future>, +) -> Result)>, ControlError> { + let mut child = location_child(options, script) + .args(["debug", "--target", &options.target, "--json"]) + .stdout(Stdio::piped()) + .spawn()?; + let mut stdout = child + .stdout + .take() + .ok_or_else(|| ControlError::Invalid("location debug output is unavailable".into()))?; + let mut output: Vec = Vec::new(); + tokio::pin!(stopped); + let connected = tokio::select! { + result = &mut stopped => { + child.kill().await?; + result?; + return Ok(None); + }, + result = async { tokio::try_join!(child.wait(), stdout.read_to_end(&mut output)) } => result?.0.success(), + }; + let files: Vec = match serde_json::from_slice::(&output) + .and_then(|report| serde_json::from_value(report["local_files"].clone())) + { + Ok(files) => files, + Err(_) => { + drop(write_output(&[( + Stream::Stderr, + "Location connection check did not produce a valid report; retaining the published snapshot.\n", + )])); + return Ok(Some((false, Vec::new()))); + } + }; + Ok(Some(( + connected, + location_database_files(&options.project_dir, files), + ))) +} + +fn location_child(options: &LocationServe, script: &Path) -> Command { + let mut command = Command::new(script); + command.arg("--project-dir").arg(&options.project_dir); if terminal_columns().is_err() { command.env(COLUMNS_VARIABLE, FALLBACK_COLUMNS.to_string()); } command .stdin(Stdio::null()) - .stdout(Stdio::inherit()) .stderr(Stdio::inherit()) - .kill_on_drop(true) - .spawn() + .kill_on_drop(true); + command } /// Open and migrate the privileged store, reporting progress without contaminating JSON stdout. diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs index 565ecf7f95..e51e498caa 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs @@ -1,6 +1,7 @@ //! `pathlib.Path` construction and `Path.resolve()` over native paths. use std::path::{Component, Path, PathBuf}; +use std::time::Duration; use sqlbuild_cache::digest::constants::{ EXCLUDED_DIRECTORIES, EXCLUDED_ROOT_DIRECTORIES, PRESENCE_ONLY_FILE_SUFFIXES, @@ -9,13 +10,48 @@ use sqlbuild_cache::project_snapshot::main::snapshot_project_files::snapshot_pro use sqlbuild_cache::project_snapshot::models::{PathStamp, SnapshotRules}; use sqlbuild_core::git_revisions::main::head_commit::head_commit; use sqlbuild_rules::project_tree::main::project_real_path::project_real_path; +use tokio::time::Instant; use crate::native_commands::errors::PythonNeeded; -use crate::native_commands::models::ProjectLocation; +use crate::native_commands::models::{LocationWatch, ProjectLocation}; use crate::replay::main::output_file_identities::output_file_identities; /// Observe the same project inputs as compiler reuse, excluding redirected command output. -pub(crate) fn location_rules() -> SnapshotRules { +pub(crate) fn location_watch(root: &Path) -> LocationWatch { + let rules = location_rules(); + LocationWatch { + observed: location_inputs(root, &rules, &[]), + rules, + local_files: Vec::new(), + attempted: None, + changed_at: tokio::time::Instant::now(), + retry_at: None, + } +} + +/// Coalesce source edits and retry deadlines without replacing the retained attempted input. +pub(crate) fn observe_location_inputs( + root: &Path, + mut watch: LocationWatch, +) -> (LocationWatch, bool) { + let current = location_inputs(root, &watch.rules, &watch.local_files); + if watch.observed != current { + watch.observed = current; + watch.changed_at = Instant::now(); + } + if watch.attempted.as_ref() == Some(&watch.observed) + && watch.retry_at.is_none_or(|retry| Instant::now() < retry) + { + return (watch, false); + } + if watch.attempted.is_some() && watch.changed_at.elapsed() < Duration::from_millis(250) { + return (watch, false); + } + watch.attempted = Some(watch.observed.clone()); + (watch, true) +} + +fn location_rules() -> SnapshotRules { SnapshotRules { excluded: EXCLUDED_DIRECTORIES .iter() @@ -34,11 +70,19 @@ pub(crate) fn location_rules() -> SnapshotRules { } /// Git commits matter to publication metadata even when no working-tree file changes. -pub(crate) fn location_inputs( +fn location_inputs( root: &Path, rules: &SnapshotRules, + local_files: &[PathBuf], ) -> (Vec, Option) { let mut stamps = snapshot_project_files(root, rules); + stamps.retain(|stamp| { + !rules + .presence_suffixes + .iter() + .any(|suffix| stamp.relative_path.ends_with(suffix)) + }); + stamps = exclude_location_files(stamps, root, local_files); #[cfg(windows)] { let output: Vec = @@ -61,6 +105,50 @@ pub(crate) fn location_inputs( (stamps, head_commit(root)) } +/// Apply resolved exclusions to retained observations, never a fresh snapshot of possible source edits. +pub(crate) fn exclude_location_databases( + root: &Path, + files: Vec, + mut watch: LocationWatch, +) -> LocationWatch { + watch.local_files.extend(files); + watch.local_files.sort(); + watch.local_files.dedup(); + watch.observed.0 = exclude_location_files(watch.observed.0, root, &watch.local_files); + watch.attempted = watch.attempted.map(|(stamps, commit)| { + ( + exclude_location_files(stamps, root, &watch.local_files), + commit, + ) + }); + watch +} + +fn exclude_location_files( + mut stamps: Vec, + root: &Path, + local_files: &[PathBuf], +) -> Vec { + stamps.retain(|stamp| !local_files.contains(&root.join(&stamp.relative_path))); + stamps +} + +pub(crate) fn location_database_files(root: &Path, files: Vec) -> Vec { + let mut paths: Vec = Vec::new(); + for file in files { + let authored = pathlib_path(&root.join(file)); + let resolved = project_real_path(&authored); + for path in [authored, resolved] { + for suffix in ["", ".wal", "-wal", "-shm", "-journal"] { + let mut file = path.as_os_str().to_os_string(); + file.push(suffix); + paths.push(PathBuf::from(file)); + } + } + } + paths +} + /// `Path(text)`: repeated separators, `.` parts and trailing separators dropped. pub(crate) fn pathlib_path(text: &Path) -> PathBuf { let path: PathBuf = text diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs index 017c4fd008..b7bff31ea3 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs @@ -1,16 +1,22 @@ //! Coalesce source edits and retain the last good publication across isolated compiler failures. -use std::path::Path; +use std::future::Future; +use std::path::{Path, PathBuf}; use std::time::Duration; +use sqlbuild_orchestration::platform::models::LocationHealth; use sqlbuild_orchestration::store::errors::ControlError; use tokio::time::{Instant, MissedTickBehavior}; +use uuid::Uuid; use crate::argv::models::Stream; use crate::native_commands::_helpers::execution::{ - location_generation, publish_child, shutdown_signal, + check_location_connection, location_generation, location_heartbeats, publish_child, + report_location_health, shutdown_signal, +}; +use crate::native_commands::_helpers::paths::{ + exclude_location_databases, location_watch, observe_location_inputs, }; -use crate::native_commands::_helpers::paths::{location_inputs, location_rules}; use crate::native_commands::_helpers::terminal::write_output; use crate::native_commands::models::LocationServe; @@ -19,11 +25,20 @@ pub(super) async fn run_location( script: &Path, bearer: &str, ) -> Result<(), ControlError> { - let rules = location_rules(); - let mut observed = location_inputs(&options.project_dir, &rules); - let mut attempted = None; - let mut changed_at = Instant::now(); - let mut retry_at = None; + let session_id = Uuid::now_v7(); + tokio::select! { + result = watch_location(&options, script, bearer, session_id) => result, + result = location_heartbeats(&options, bearer, session_id) => result, + } +} + +async fn watch_location( + options: &LocationServe, + script: &Path, + bearer: &str, + session_id: Uuid, +) -> Result<(), ControlError> { + let mut watch = location_watch(&options.project_dir); let mut clock = tokio::time::interval(Duration::from_millis(250)); clock.set_missed_tick_behavior(MissedTickBehavior::Skip); let stopped = shutdown_signal(); @@ -37,28 +52,19 @@ pub(super) async fn run_location( result = &mut stopped => return result, _ = clock.tick() => {}, } - let current = location_inputs(&options.project_dir, &rules); - if observed != current { - observed = current; - changed_at = Instant::now(); - } - if attempted.as_ref() == Some(&observed) - && retry_at.is_none_or(|retry| Instant::now() < retry) - { - continue; - } - if attempted.is_some() && changed_at.elapsed() < Duration::from_millis(250) { + let ready; + (watch, ready) = observe_location_inputs(&options.project_dir, watch); + if !ready { continue; } - attempted = Some(observed.clone()); let generation = tokio::select! { result = &mut stopped => return result, - generation = location_generation(&options, bearer) => generation?, + generation = location_generation(options, bearer) => generation?, }; let generation = match generation { Some(generation) => generation, None => { - retry_at = Some(Instant::now() + Duration::from_secs(5)); + watch.retry_at = Some(Instant::now() + Duration::from_secs(5)); continue; } }; @@ -66,7 +72,7 @@ pub(super) async fn run_location( Stream::Stderr, &format!("Compiling location generation {generation} in an isolated interpreter...\n"), )])); - let mut child = publish_child(&options, script, generation)?; + let mut child = publish_child(options, script, generation)?; let result = tokio::select! { result = &mut stopped => { child.kill().await?; @@ -75,13 +81,24 @@ pub(super) async fn run_location( result = child.wait() => result?, }; if result.success() { - retry_at = None; - drop(write_output(&[( - Stream::Stderr, - "Code location ready; watching for changes.\n", - )])); + watch.retry_at = None; + let Some(files) = check_published_location( + options, + script, + (bearer, session_id, generation), + stopped.as_mut(), + ) + .await? + else { + return Ok(()); + }; + watch = exclude_location_databases(&options.project_dir, files, watch); } else { - retry_at = Some(Instant::now() + Duration::from_secs(30)); + watch.retry_at = Some(Instant::now() + Duration::from_secs(30)); + tokio::select! { + result = &mut stopped => return result, + () = report_location_health(options, bearer, session_id, LocationHealth::CompileFailed { generation }) => {}, + } drop(write_output(&[( Stream::Stderr, "Location reload failed; retaining last good snapshot. Retrying on change or in 30 seconds.\n", @@ -89,3 +106,34 @@ pub(super) async fn run_location( } } } + +async fn check_published_location( + options: &LocationServe, + script: &Path, + (bearer, session_id, generation): (&str, Uuid, i64), + stopped: impl Future>, +) -> Result>, ControlError> { + drop(write_output(&[( + Stream::Stderr, + "Checking location connection with sqb debug...\n", + )])); + tokio::pin!(stopped); + let Some((connected, files)) = + check_location_connection(options, script, stopped.as_mut()).await? + else { + return Ok(None); + }; + tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + () = report_location_health(options, bearer, session_id, LocationHealth::Connection { generation, healthy: connected }) => {}, + } + drop(write_output(&[( + Stream::Stderr, + if connected { + "Code location ready; connection check passed; watching for changes.\n" + } else { + "Code location published; connection check failed; watching for changes.\n" + }, + )])); + Ok(Some(files)) +} diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 7be0c55c84..f3ae56696a 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -3,6 +3,7 @@ use std::path::PathBuf; use chrono::{DateTime, Utc}; +use sqlbuild_cache::project_snapshot::models::{PathStamp, SnapshotRules}; use sqlbuild_orchestration::credentials::models::{CredentialScope, PrincipalKind}; use sqlbuild_orchestration::declarations::models::TriggerChange; use sqlbuild_orchestration::runs::models::JobRunRequest; @@ -136,6 +137,17 @@ pub(crate) struct LocationServe { pub(crate) image_id: Option, } +/// Retained source observations survive compilation; warehouse exclusions never replace them. +#[derive(Debug)] +pub(crate) struct LocationWatch { + pub(crate) rules: SnapshotRules, + pub(crate) local_files: Vec, + pub(crate) observed: (Vec, Option), + pub(crate) attempted: Option<(Vec, Option)>, + pub(crate) changed_at: tokio::time::Instant, + pub(crate) retry_at: Option, +} + /// What Python's buffered stdout does with output its reader no longer takes. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum LostStdout { diff --git a/crates/sqlbuild-core/src/debug_report/main/format_debug_json.rs b/crates/sqlbuild-core/src/debug_report/main/format_debug_json.rs index 04e8d3084f..9eb5e8408f 100644 --- a/crates/sqlbuild-core/src/debug_report/main/format_debug_json.rs +++ b/crates/sqlbuild-core/src/debug_report/main/format_debug_json.rs @@ -43,6 +43,17 @@ pub fn format_debug_json(report: &DebugReport) -> String { ), ("providers".to_owned(), lines_value(&report.providers)), ("connection".to_owned(), lines_value(&report.connection)), + ( + "local_files".to_owned(), + JsonValue::Array( + report + .local_files + .iter() + .cloned() + .map(JsonValue::String) + .collect(), + ), + ), ]); let options: StdlibJsonOptions = StdlibJsonOptions { sort_keys: true, diff --git a/crates/sqlbuild-core/src/debug_report/models.rs b/crates/sqlbuild-core/src/debug_report/models.rs index c0ec5075af..144079f7d9 100644 --- a/crates/sqlbuild-core/src/debug_report/models.rs +++ b/crates/sqlbuild-core/src/debug_report/models.rs @@ -38,6 +38,8 @@ pub struct DebugReport { pub configuration: Vec, pub providers: Vec, pub connection: Vec, + /// Exact local database paths, not the shortened display values in connection lines. + pub local_files: Vec, } impl DebugReport { diff --git a/crates/sqlbuild-core/src/debug_report/tests/helpers.rs b/crates/sqlbuild-core/src/debug_report/tests/helpers.rs index 623f516cd6..240efc55d5 100644 --- a/crates/sqlbuild-core/src/debug_report/tests/helpers.rs +++ b/crates/sqlbuild-core/src/debug_report/tests/helpers.rs @@ -2,6 +2,7 @@ use crate::debug_report::models::{ConnectionValue, DebugLine, DebugReport, Debug pub(super) fn sample_report() -> DebugReport { DebugReport { + local_files: vec!["/repo/warehouse.db".into()], runtime: vec![DebugLine::plain("sqlbuild version", "0.2.1")], configuration: vec![DebugLine::checked( "project file", diff --git a/crates/sqlbuild-core/src/debug_report/tests/test_debug_report.rs b/crates/sqlbuild-core/src/debug_report/tests/test_debug_report.rs index 7a2122f2d7..5fb74ee026 100644 --- a/crates/sqlbuild-core/src/debug_report/tests/test_debug_report.rs +++ b/crates/sqlbuild-core/src/debug_report/tests/test_debug_report.rs @@ -18,7 +18,7 @@ fn given_debug_report_when_formatting_then_matches_python_outputs() { description: "renders diagnostics sections and json sections", report: sample_report(), expected_text: "\nSQLBuild Diagnostics\n\nRuntime:\n sqlbuild version: 0.2.1\n\nConfiguration:\n project file: /repo/sqlbuild_project.toml [OK found and valid]\n\nProviders:\n providers: 1 [OK discovered]\n marker_provider: providers/marker.py:MarkerProvider [OK valid settings]\n\nConnection:\n token: ****\n connection test: [ERROR authentication failed]\n query test: [SKIP connection failed]\n\n", - expected_json: r#"{"configuration": [{"label": "project file", "message": "/repo/sqlbuild_project.toml", "status": "OK", "status_message": "found and valid"}], "connection": [{"label": "token", "message": "****"}, {"label": "connection test", "message": "", "status": "ERROR", "status_message": "authentication failed"}, {"label": "query test", "message": "", "status": "SKIP", "status_message": "connection failed"}], "providers": [{"label": "providers", "message": "1", "status": "OK", "status_message": "discovered"}, {"label": "marker_provider", "message": "providers/marker.py:MarkerProvider", "status": "OK", "status_message": "valid settings"}], "runtime": [{"label": "sqlbuild version", "message": "0.2.1"}], "success": false}"#, + expected_json: r#"{"configuration": [{"label": "project file", "message": "/repo/sqlbuild_project.toml", "status": "OK", "status_message": "found and valid"}], "connection": [{"label": "token", "message": "****"}, {"label": "connection test", "message": "", "status": "ERROR", "status_message": "authentication failed"}, {"label": "query test", "message": "", "status": "SKIP", "status_message": "connection failed"}], "local_files": ["/repo/warehouse.db"], "providers": [{"label": "providers", "message": "1", "status": "OK", "status_message": "discovered"}, {"label": "marker_provider", "message": "providers/marker.py:MarkerProvider", "status": "OK", "status_message": "valid settings"}], "runtime": [{"label": "sqlbuild version", "message": "0.2.1"}], "success": false}"#, expected_color_fragments: &[ "\x1b[34m\x1b[1mSQLBuild Diagnostics\x1b[0m", "\x1b[1mRuntime:\x1b[0m", diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/location_health.rs b/crates/sqlbuild-orchestration/src/api/_helpers/location_health.rs new file mode 100644 index 0000000000..60a407233c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/location_health.rs @@ -0,0 +1,45 @@ +//! Scope both observations and reads before exposing any location history. + +use axum::extract::{Path, State}; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::_helpers::authentication::require_location; +use crate::api::errors::ApiError; +use crate::api::models::MutationResponse; +use crate::credentials::models::TokenIdentity; +use crate::platform::main::location_health::location_health; +use crate::platform::main::report_location_health::report_location_health; +use crate::platform::models::{LocationHealthObservation, LocationHealthReport}; +use crate::store::models::ControlStore; + +#[utoipa::path(post, path = "/api/locations/{location_id}/health", + params(("location_id" = Uuid, Path)), request_body = LocationHealthReport, + responses((status = 200, body = MutationResponse), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized")), + security(("bearer" = [])))] +pub(crate) async fn report( + State(store): State, + Extension(identity): Extension, + Path(location_id): Path, + Json(report): Json, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + Ok(Json(MutationResponse { + applied: report_location_health(&store, location_id, &report).await?, + })) +} + +#[utoipa::path(get, path = "/api/locations/{location_id}/health", + params(("location_id" = Uuid, Path)), + responses((status = 200, body = Vec), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized")), + security(("bearer" = [])))] +pub(crate) async fn recent( + State(store): State, + Extension(identity): Extension, + Path(location_id): Path, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + Ok(Json(location_health(&store, location_id).await?)) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs index ed6ed88be8..4b3f4f829e 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod authentication; pub(crate) mod client; +pub(crate) mod location_health; pub(crate) mod run_reads; pub(crate) mod runs; pub(crate) mod snapshots; diff --git a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs index 52c2efba75..0a7ba5c5c2 100644 --- a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs +++ b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs @@ -5,7 +5,7 @@ use axum::routing::{get, post}; use axum::{Json, Router, middleware}; use crate::api::_helpers::authentication::authenticate; -use crate::api::_helpers::{run_reads, runs, snapshots, trigger_reads}; +use crate::api::_helpers::{location_health, run_reads, runs, snapshots, trigger_reads}; use crate::api::constants::BODY_BYTES; use crate::api::main::openapi_document::openapi_document; use crate::store::errors::ControlError; @@ -19,6 +19,10 @@ pub async fn serve_api( ) -> Result<(), ControlError> { migrate_store(&store).await?; let protected = Router::new() + .route( + "/api/locations/{location_id}/health", + get(location_health::recent).post(location_health::report), + ) .route( "/api/locations/{location_id}/targets/{target}/snapshots", get(snapshots::current).post(snapshots::publish), diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index c2604cc5c5..5e4837fa3b 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -2,10 +2,12 @@ use serde::{Deserialize, Serialize}; -use crate::api::_helpers::{run_reads, runs, snapshots, trigger_reads}; +use crate::api::_helpers::{location_health, run_reads, runs, snapshots, trigger_reads}; #[derive(Debug, utoipa::OpenApi)] #[openapi(paths( + location_health::report, + location_health::recent, snapshots::publish, snapshots::current, runs::execution, diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/location_heartbeat.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/location_heartbeat.rs new file mode 100644 index 0000000000..a13e3fabb1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/location_heartbeat.rs @@ -0,0 +1,50 @@ +//! A heartbeat replaces one instance's liveness, never appending an execution fact. + +use serde_json::json; +use uuid::Uuid; + +use crate::platform::models::LocationHealthReport; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn location_heartbeat( + transaction: ControlTransaction, + location_id: Uuid, + report: &LocationHealthReport, +) -> Result<(ControlTransaction, bool), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT instance.location_id AS instance_location, instance.health_json AS report \ + FROM sqb_locations location LEFT JOIN sqb_location_instances instance ON instance.instance_id=$1 \ + WHERE location.location_id=$2 AND location.archived_at IS NULL", + vec![Parameter::Id(report.session_id), Parameter::Id(location_id)]).await?; + let row = rows + .first() + .ok_or_else(|| ControlError::Invalid("active location is required".into()))?; + if let Some(previous) = row.get("report").filter(|value| !value.is_null()) { + let previous: LocationHealthReport = serde_json::from_value(previous.clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if row.get("instance_location") != Some(&json!(location_id)) + || previous.target != report.target + { + return Err(ControlError::Invalid( + "location instance belongs to another scope".into(), + )); + } + if previous == *report { + return Ok((transaction, false)); + } + if previous.event_id == report.event_id { + return Err(ControlError::Invalid( + "location health event ID already has different content".into(), + )); + } + } + let now = transaction.now; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_location_instances (instance_id, location_id, target, connected_at, last_seen_at, health_json) \ + VALUES ($1, $2, $3, $4, $4, $5) ON CONFLICT (instance_id) DO UPDATE SET last_seen_at=$4, health_json=$5", + vec![Parameter::Id(report.session_id), Parameter::Id(location_id), Parameter::Text(report.target.clone()), + Parameter::Time(now), Parameter::Json(json!(report))]).await?; + Ok((transaction, true)) +} diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs index b604790017..e32efd2a5f 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs @@ -1,2 +1,3 @@ +pub(crate) mod location_heartbeat; pub(crate) mod observations; pub(crate) mod watchdog; diff --git a/crates/sqlbuild-orchestration/src/platform/main/location_health.rs b/crates/sqlbuild-orchestration/src/platform/main/location_health.rs new file mode 100644 index 0000000000..977a61e6c4 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/location_health.rs @@ -0,0 +1,63 @@ +//! Read bounded recent observations and instance liveness; consumers derive freshness. + +use uuid::Uuid; + +use crate::platform::models::LocationHealthObservation; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn location_health( + store: &ControlStore, + location_id: Uuid, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let today = transaction.now.date_naive(); + let yesterday = today - chrono::TimeDelta::days(1); + let (transaction, rows) = query( + transaction, + "SELECT received_at, payload FROM (\ + SELECT * FROM (SELECT occurred_at AS received_at, invocation_sequence, payload FROM recall_events \ + WHERE invocation_id=$1 AND event_type='location_health' ORDER BY invocation_sequence DESC LIMIT 100) current_day \ + UNION ALL \ + SELECT * FROM (SELECT occurred_at AS received_at, invocation_sequence, payload FROM recall_events \ + WHERE invocation_id=$2 AND event_type='location_health' ORDER BY invocation_sequence DESC LIMIT 100) previous_day\ + ) observations ORDER BY received_at DESC, invocation_sequence DESC LIMIT 100", + vec![ + Parameter::Text(format!("location/{location_id}/{today}")), + Parameter::Text(format!("location/{location_id}/{yesterday}")), + ], + ) + .await?; + let mut observations: Vec = rows + .into_iter() + .map(|mut row| { + let report = row + .remove("payload") + .and_then(|mut payload| payload.get_mut("report").map(serde_json::Value::take)) + .ok_or_else(|| ControlError::Invalid("location health report is missing".into()))?; + row.insert("report".into(), report); + decode_row(row) + }) + .collect::>()?; + let (transaction, instances) = query( + transaction, + "SELECT last_seen_at AS received_at, health_json AS report FROM sqb_location_instances \ + WHERE location_id=$1 ORDER BY last_seen_at DESC LIMIT 100", + vec![Parameter::Id(location_id)], + ) + .await?; + observations.extend( + instances + .into_iter() + .map(decode_row) + .collect::, _>>()?, + ); + observations.sort_by_key(|observation| std::cmp::Reverse(observation.received_at)); + observations.truncate(100); + commit(transaction).await?; + Ok(observations) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/mod.rs b/crates/sqlbuild-orchestration/src/platform/main/mod.rs index bb879e253f..a633f3e7ce 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/mod.rs @@ -1 +1,4 @@ +pub mod location_health; +pub mod report_location_health; +pub mod report_remote_location_health; pub mod run_watchdog; diff --git a/crates/sqlbuild-orchestration/src/platform/main/report_location_health.rs b/crates/sqlbuild-orchestration/src/platform/main/report_location_health.rs new file mode 100644 index 0000000000..389d80fa18 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/report_location_health.rs @@ -0,0 +1,104 @@ +//! Append scoped location observations without turning health into mutable lifecycle state. + +use serde_json::json; +use uuid::Uuid; + +use crate::platform::_helpers::location_heartbeat::location_heartbeat; +use crate::platform::models::{LocationHealth, LocationHealthReport}; +use crate::store::errors::ControlError; +use crate::store::main::append_control_event::append_control_event; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlEvent, ControlStore, Parameter, TransactionMode}; + +pub async fn report_location_health( + store: &ControlStore, + location_id: Uuid, + report: &LocationHealthReport, +) -> Result { + if report.target.trim().is_empty() { + return Err(ControlError::Invalid( + "location health target is invalid".into(), + )); + } + let generation = match report.health { + LocationHealth::Heartbeat {} => None, + LocationHealth::CompileFailed { generation } + | LocationHealth::Connection { generation, .. } => Some(generation), + }; + if generation.is_some_and(|generation| generation < 1) { + return Err(ControlError::Invalid( + "location health generation must be positive".into(), + )); + } + let transaction = begin(store, TransactionMode::Write).await?; + if generation.is_none() { + let (transaction, applied) = location_heartbeat(transaction, location_id, report).await?; + commit(transaction).await?; + return Ok(applied); + } + let invocation_id = format!("location/{location_id}/{}", transaction.now.date_naive()); + let event_id = Uuid::new_v5(&location_id, report.event_id.as_bytes()); + let payload = json!({"location_id": location_id, "report": report}); + let (transaction, previous) = query( + transaction, + "SELECT payload FROM recall_events WHERE event_id=$1 LIMIT 1", + vec![Parameter::Text(event_id.to_string())], + ) + .await?; + if let Some(previous) = previous.first() { + if previous.get("payload") != Some(&payload) { + return Err(ControlError::Invalid( + "location health event ID already has different content".into(), + )); + } + commit(transaction).await?; + return Ok(false); + } + let (transaction, location) = query( + transaction, + "SELECT location_id FROM sqb_locations WHERE location_id=$1 AND archived_at IS NULL", + vec![Parameter::Id(location_id)], + ) + .await?; + if location.is_empty() { + return Err(ControlError::Invalid("active location is required".into())); + } + let transaction = if let LocationHealth::Connection { generation, .. } = report.health { + let (transaction, snapshot) = query( + transaction, + "SELECT snapshot_id FROM sqb_snapshots WHERE location_id=$1 AND target=$2 AND generation=$3", + vec![Parameter::Id(location_id), Parameter::Text(report.target.clone()), Parameter::Integer(generation)], + ).await?; + if snapshot.is_empty() { + return Err(ControlError::Invalid( + "connection health requires a published generation".into(), + )); + } + transaction + } else { + transaction + }; + let occurred_at = transaction.now; + let (transaction, outcome) = append_control_event( + transaction, + ControlEvent { + event_id, + invocation_id, + event_type: "location_health", + producer: "location", + occurred_at, + run_id: None, + payload, + }, + ) + .await?; + if outcome.conflict.is_some() { + return Err(ControlError::Invalid( + "location health conflicts with a durable fact".into(), + )); + } + commit(transaction).await?; + Ok(true) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/report_remote_location_health.rs b/crates/sqlbuild-orchestration/src/platform/main/report_remote_location_health.rs new file mode 100644 index 0000000000..5963cf20b6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/report_remote_location_health.rs @@ -0,0 +1,21 @@ +//! The code location has only its scoped HTTP credential, never a store handle. + +use uuid::Uuid; + +use crate::api::errors::ClientError; +use crate::api::main::client_response::client_response; +use crate::api::main::connect_client::connect_client; +use crate::api::models::MutationResponse; +use crate::platform::models::LocationHealthReport; + +pub async fn report_remote_location_health( + (server_url, bearer): (&str, &str), + location_id: Uuid, + report: &LocationHealthReport, +) -> Result { + let connection = connect_client(server_url, bearer)?; + let url = connection.endpoint(&["api", "locations", &location_id.to_string(), "health"])?; + let response: MutationResponse = + client_response(connection.client.post(url).json(report)).await?; + Ok(response.applied) +} diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index 0f3bb0325c..052d9bf775 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -6,6 +6,31 @@ use uuid::Uuid; use crate::declarations::models::TriggerDefinition; +/// Locations report bounded facts, never connection strings or user exception text. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum LocationHealth { + Heartbeat {}, + CompileFailed { generation: i64 }, + Connection { generation: i64, healthy: bool }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct LocationHealthReport { + pub event_id: Uuid, + pub session_id: Uuid, + pub target: String, + pub health: LocationHealth, +} + +/// Receipt time belongs to the server; consumers derive freshness rather than storing availability. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct LocationHealthObservation { + pub received_at: DateTime, + pub report: LocationHealthReport, +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "kind", rename_all = "snake_case")] pub enum PlatformEvent { diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index a29a62c75f..e2933cdb64 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -7,11 +7,12 @@ use crate::store::models::ControlStore; use crate::store::types::Row; pub(crate) const SCHEMA: &str = include_str!("../schema.sql"); +pub(crate) const LOCATION_INSTANCES: &str = include_str!("../location_instances.sql"); -pub(crate) fn verify_migrations(rows: &[Row], digest: &str) -> Result { +pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) .map_err(|error| ControlError::Schema(error.to_string()))?; - for row in rows { + for (index, row) in rows.iter().enumerate() { let minimum = row .get("min_server_version") .and_then(|value| value.as_str()) @@ -20,9 +21,11 @@ pub(crate) fn verify_migrations(rows: &[Row], digest: &str) -> Result current - || row.get("checksum").and_then(|value| value.as_str()) != Some(digest) + || row.get("checksum").and_then(|value| value.as_str()) + != digests.get(index).map(String::as_str) + || index >= digests.len() { return Err(ControlError::Schema( "unknown migration, checksum mismatch, or minimum server version not met".into(), diff --git a/crates/sqlbuild-orchestration/src/store/location_instances.sql b/crates/sqlbuild-orchestration/src/store/location_instances.sql new file mode 100644 index 0000000000..04b3e8dfd5 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/location_instances.sql @@ -0,0 +1,9 @@ +CREATE TABLE sqb_location_instances ( + instance_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + connected_at {time} NOT NULL, + last_seen_at {time} NOT NULL, + health_json {json} NOT NULL +); +CREATE INDEX sqb_location_instances_location_seen ON sqb_location_instances(location_id, last_seen_at); diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index fca5fe59c0..9063a5fedd 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -3,7 +3,7 @@ use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; -use crate::store::_helpers::migration::{SCHEMA, render, verify_migrations}; +use crate::store::_helpers::migration::{LOCATION_INSTANCES, SCHEMA, render, verify_migrations}; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; use crate::store::main::begin::begin; @@ -25,7 +25,9 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { .await?; let recall_statements = recall_schema(store); let digest = sha256_hex(format!("{SCHEMA}\n{}", recall_statements.join(";\n")).as_bytes()); - let current = verify_migrations(&rows, &digest)?; + let location_digest = sha256_hex(LOCATION_INSTANCES.as_bytes()); + let digests = [digest.clone(), location_digest.clone()]; + let current = verify_migrations(&rows, &digests)?; if rows.is_empty() { for statement in SCHEMA .split(';') @@ -46,5 +48,19 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { VALUES (1, 'control_store', $1, $2, $3)", vec![Parameter::Text(digest), Parameter::Text(current.to_string()), Parameter::Time(now)]).await?; } + if rows.len() < digests.len() { + for statement in LOCATION_INSTANCES + .split(';') + .map(str::trim) + .filter(|statement| !statement.is_empty()) + { + (transaction, _) = query(transaction, &render(store, statement), vec![]).await?; + } + let now = transaction.now; + (transaction, _) = query(transaction, + "INSERT INTO sqb_schema_migrations (version, name, checksum, min_server_version, applied_at) \ + VALUES (2, 'location_instances', $1, $2, $3)", + vec![Parameter::Text(location_digest), Parameter::Text(current.to_string()), Parameter::Time(now)]).await?; + } commit(transaction).await } diff --git a/crates/sqlbuild-orchestration/tests/api.rs b/crates/sqlbuild-orchestration/tests/api.rs index f72b32d6fb..55a6bce444 100644 --- a/crates/sqlbuild-orchestration/tests/api.rs +++ b/crates/sqlbuild-orchestration/tests/api.rs @@ -14,6 +14,8 @@ mod store_helpers; mod store_test_types; #[path = "api/test_contract.rs"] mod test_contract; +#[path = "api/test_location_health.rs"] +mod test_location_health; #[path = "api/test_reads.rs"] mod test_reads; #[path = "api/test_types.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_health.rs b/crates/sqlbuild-orchestration/tests/api/test_location_health.rs new file mode 100644 index 0000000000..38928875b7 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_location_health.rs @@ -0,0 +1,200 @@ +use chrono::Utc; +use reqwest::StatusCode; +use serde_json::json; +use sqlbuild_orchestration::credentials::main::issue_token::issue_token; +use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; +use sqlbuild_orchestration::platform::main::report_remote_location_health::report_remote_location_health; +use sqlbuild_orchestration::platform::models::{ + LocationHealth, LocationHealthObservation, LocationHealthReport, +}; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; +use uuid::Uuid; + +use crate::helpers::{start_api, stop_api}; +use crate::location_helpers::create_location; +use crate::run_helpers::snapshot; +use crate::store_helpers::{close_test_store, open_test_store, run}; +use crate::store_test_types::Backend; +use crate::test_types::LocationHealthTestCase; + +#[test] +fn given_scoped_location_observations_when_retried_or_read_then_history_and_authority_are_preserved() + { + let test_cases = [ + LocationHealthTestCase { + description: "SQLite health history", + backend: Backend::Sqlite, + expected_observations: 2, + }, + LocationHealthTestCase { + description: "PostgreSQL health history", + backend: Backend::Postgres, + expected_observations: 2, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location_id = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "inventory", "test-admin") + .await + .expect("other location"); + let token = issue_token( + store, + &AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }, + CredentialScope::Location { location_id }, + Utc::now() + chrono::Duration::hours(1), + ) + .await + .expect("location token"); + let api = start_api(store).await; + let client = reqwest::Client::new(); + let url = format!("{}/api/locations/{location_id}/health", api.url); + let other_url = format!("{}/api/locations/{other}/health", api.url); + let heartbeat = LocationHealthReport { + event_id: Uuid::now_v7(), + session_id: Uuid::now_v7(), + target: "preview #1".into(), + health: LocationHealth::Heartbeat {}, + }; + let before = Utc::now(); + assert!( + report_remote_location_health((&api.url, &token.bearer), location_id, &heartbeat) + .await + .expect("register") + ); + assert!( + !report_remote_location_health((&api.url, &token.bearer), location_id, &heartbeat) + .await + .expect("exact retry") + ); + let changed = LocationHealthReport { + target: "another target".into(), + ..heartbeat.clone() + }; + let response = client + .post(&url) + .bearer_auth(&token.bearer) + .json(&changed) + .send() + .await + .expect("changed retry"); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let response = client + .post(&other_url) + .bearer_auth(&token.bearer) + .json(&heartbeat) + .send() + .await + .expect("foreign observation"); + assert_eq!(response.status(), StatusCode::FORBIDDEN); + let response = client + .get(&other_url) + .bearer_auth(&token.bearer) + .send() + .await + .expect("foreign history"); + assert_eq!(response.status(), StatusCode::FORBIDDEN); + let response = client + .post(&url) + .header("Content-Type", "application/json") + .body("invalid") + .send() + .await + .expect("anonymous report"); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + let response = client.post(&url).bearer_auth(&token.bearer).json(&json!({ + "event_id": Uuid::now_v7(), "session_id": heartbeat.session_id, "target": heartbeat.target, + "health": { "kind": "heartbeat", "unexpected": "must not disappear" }, + })).send().await.expect("strict health schema"); + assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY); + let heartbeat = LocationHealthReport { + event_id: Uuid::now_v7(), + ..heartbeat + }; + assert!( + report_remote_location_health((&api.url, &token.bearer), location_id, &heartbeat) + .await + .expect("refresh existing instance liveness") + ); + let connection = LocationHealthReport { + event_id: Uuid::now_v7(), + health: LocationHealth::Connection { + generation: 1, + healthy: false, + }, + ..heartbeat.clone() + }; + let response = client + .post(&url) + .bearer_auth(&token.bearer) + .json(&connection) + .send() + .await + .expect("unpublished health"); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let mut input = snapshot(location_id); + input.target.clone_from(&heartbeat.target); + publish_snapshot(store, &input) + .await + .expect("published generation"); + assert!( + report_remote_location_health((&api.url, &token.bearer), location_id, &connection) + .await + .expect("connection observation") + ); + let response = client + .get(&url) + .bearer_auth(&token.bearer) + .send() + .await + .expect("health history"); + assert_eq!(response.headers()["cache-control"], "no-store"); + let observations: Vec = + response.json().await.expect("typed health history"); + assert_eq!( + observations.len(), + test_case.expected_observations, + "{}: rejected and duplicate reports append nothing", + test_case.description + ); + assert_eq!(observations[0].report, connection); + assert_eq!(observations[1].report, heartbeat); + let events = read_recall_events(store, 0, 100) + .await + .expect("read durable events"); + assert_eq!( + events + .iter() + .filter( + |event| event.get("event_type").and_then(serde_json::Value::as_str) + == Some("location_health") + ) + .count(), + 1, + "heartbeats only update liveness; connection observations remain durable facts" + ); + assert!( + observations + .iter() + .all(|observation| observation.received_at >= before + && observation.received_at <= Utc::now()) + ); + stop_api(api).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index 550618fdb8..59c0610602 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -33,3 +33,9 @@ pub(crate) struct TestApi { pub(crate) shutdown: tokio::sync::oneshot::Sender<()>, pub(crate) task: tokio::task::JoinHandle>, } + +pub(crate) struct LocationHealthTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_observations: usize, +} diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index 9030c6f077..ab2594e4e4 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -241,6 +241,25 @@ pub(super) async fn sqlite_failed_publish_and_schema_floor() -> i64 { .await .expect("publish good snapshot"); let connection = rusqlite::Connection::open(&path).expect("open fault injection connection"); + connection + .execute_batch( + "DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version=2", + ) + .expect("reconstruct the previous schema without changing existing facts"); + migrate_store(&store) + .await + .expect("upgrade the previous schema"); + migrate_store(&store) + .await + .expect("repeat the migration without changes"); + assert_eq!( + current_snapshot(&store, location, "development") + .await + .expect("read upgraded store") + .expect("snapshot survives upgrade") + .record, + good + ); // Fail after immutable bodies were inserted but before switching the current pointer. connection.execute_batch("CREATE TRIGGER fail_snapshot_switch BEFORE UPDATE ON sqb_current_snapshots BEGIN SELECT RAISE(ABORT, 'synthetic publication failure'); END").expect("install publication failure"); input.generation = 2; @@ -279,7 +298,7 @@ pub(super) async fn sqlite_failed_publish_and_schema_floor() -> i64 { }, SchemaRejectionCase { description: "migration version is unknown", - update_sql: "UPDATE sqb_schema_migrations SET version=2", + update_sql: "UPDATE sqb_schema_migrations SET version=999 WHERE version=2", }, ] { connection diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs index 8a824ae6e6..0f6a0d3127 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs @@ -189,18 +189,8 @@ fn anchor_attach_paths(config: &Bound<'_, PyDict>, project_dir: &Bound<'_, PyAny anchored.push(entry); continue; }; - let local_file: bool = match mapping.get_item(ATTACH_TYPE_KEY)? { - None => true, - Some(kind) => kind - .extract::() - .is_ok_and(|kind| LOCAL_ATTACH_TYPES.contains(&kind.to_ascii_lowercase().as_str())), - }; let copy: Bound<'_, PyDict> = mapping.copy()?; - if let Some(path_value) = mapping.get_item(ATTACH_PATH_KEY)? - && local_file - && !path_value.extract::().is_ok_and(|text| { - text.contains(URL_SCHEME_SEPARATOR) || text.starts_with(MOTHERDUCK_PATH_PREFIX) - }) + if let Some(path_value) = local_attach_file(mapping)? && let Some(file) = anchored_file_path(&path_value, project_dir)? { copy.set_item(ATTACH_PATH_KEY, file)?; @@ -210,6 +200,59 @@ fn anchor_attach_paths(config: &Bound<'_, PyDict>, project_dir: &Bound<'_, PyAny config.set_item(ATTACH_KEY, pyo3::types::PyList::new(py, anchored)?) } +fn local_attach_file<'py>(mapping: &Bound<'py, PyDict>) -> PyResult>> { + let local = match mapping.get_item(ATTACH_TYPE_KEY)? { + None => true, + Some(kind) => kind + .extract::() + .is_ok_and(|kind| LOCAL_ATTACH_TYPES.contains(&kind.to_ascii_lowercase().as_str())), + }; + Ok(mapping.get_item(ATTACH_PATH_KEY)?.filter(|value| { + local + && !value.extract::().is_ok_and(|text| { + text.contains(URL_SCHEME_SEPARATOR) || text.starts_with(MOTHERDUCK_PATH_PREFIX) + }) + })) +} + +/// Use the resolved adapter connection, including local attachments, not authored TOML guesses. +pub(crate) fn local_database_files( + adapter: &str, + config: &Bound<'_, PyDict>, +) -> PyResult> { + let Some((_, key)) = LOCAL_DATABASE_PATH_KEYS + .iter() + .find(|(name, _)| *name == adapter) + else { + return Ok(Vec::new()); + }; + let mut files: Vec = Vec::new(); + if let Some(value) = config.get_item(*key)? + && let Ok(file) = value.extract::() + && !file.is_empty() + && file != DUCKDB_MEMORY_DATABASE + && !file.contains(URL_SCHEME_SEPARATOR) + && !file.starts_with(MOTHERDUCK_PATH_PREFIX) + { + files.push(file); + } + if adapter == DUCKDB_ADAPTER + && let Some(entries) = config.get_item(ATTACH_KEY)? + && let Ok(entries) = entries.try_iter() + { + for entry in entries { + if let Ok(mapping) = entry?.cast_into::() + && let Some(value) = local_attach_file(&mapping)? + && let Ok(file) = value.extract::() + && file != DUCKDB_MEMORY_DATABASE + { + files.push(file); + } + } + } + Ok(files) +} + /// Resolve dbt profiles, relative DuckDB paths, and the command group's warehouse. #[pyfunction] #[pyo3(signature = (**options))] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs index ae20cf9408..f3a130215b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs @@ -28,7 +28,9 @@ use crate::bindings::_helpers::boundary::python_calls::{ }; use crate::bindings::_helpers::cli::failure_cause::{FailureCause, report_failure_cause}; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; -use crate::bindings::_helpers::command_runtime::connection::project_connection_config; +use crate::bindings::_helpers::command_runtime::connection::{ + local_database_files, project_connection_config, +}; use crate::bindings::_helpers::command_runtime::runtime_errors::{ expected_resolution_error, resolution_message, }; @@ -357,6 +359,7 @@ fn build_report( ConnectionCheck::Skipped }; Ok(DebugReport { + local_files: local_database_files(&facts.adapter_name, &connection_config)?, runtime: debug_runtime_lines(&runtime(py)?), configuration: debug_configuration_lines(&facts, &warehouses), providers: debug_provider_lines(&providers(&context.discovered_inputs)?), diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index be66d72ee4..eebedd8538 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -39,6 +39,8 @@ class LocationReloadTestCase: target: str expected_generation: int expected_stop_code: int + database: str + expected_connection: bool @dataclass(frozen=True) @@ -46,3 +48,13 @@ class LocationShutdownTestCase: description: str signal: int expected_exit_code: int + minimum_heartbeats: int + blocked_command: str + + +@dataclass(frozen=True) +class LocationDatabaseTestCase: + description: str + database: str + expected_generation: int + connection_toml: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index 994e90c8cc..efa304cafd 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -79,6 +79,38 @@ def wait_for_log(process: subprocess.Popen[bytes], log: Path, text: str) -> None time.sleep(0.02) +def wait_for_location_health( + api: SnapshotApi, + target: str, + health: dict[str, Any], + process: subprocess.Popen[bytes], + log: Path, + *, + minimum_reports: int = 1, +) -> dict[str, Any]: + deadline: float = time.monotonic() + 15 + 30 * (minimum_reports - 1) + request: Request = Request( + f"{api.url}/api/locations/{api.location_id}/health", + headers={"Authorization": f"Bearer {api.bearer}"}, + ) + key: tuple[str, str] = (target, json.dumps(health, sort_keys=True)) + observations: dict[tuple[str, str], dict[str, dict[str, Any]]] = {} + while len(observations.get(key, ())) < minimum_reports: + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + with urlopen(request, timeout=2) as response: + reports: list[dict[str, Any]] = json.load(response) + for report in reversed(reports): + event_id: str = report["report"]["event_id"] + observed_key: tuple[str, str] = ( + report["report"]["target"], + json.dumps(report["report"]["health"], sort_keys=True), + ) + observations.setdefault(observed_key, {})[event_id] = report + time.sleep(0.02) + return list(observations[key].values())[-1] + + def wait_for_interpreter(process: subprocess.Popen[bytes], pidfile: Path, log: Path) -> int: deadline: float = time.monotonic() + 15 while not pidfile.exists(): diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py index d2cbbe1f13..26892bc6c0 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py @@ -9,10 +9,12 @@ from pathlib import Path from typing import Any +import duckdb import pytest from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import TYPE_PROOF_FILES from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + LocationDatabaseTestCase, LocationReloadTestCase, LocationShutdownTestCase, ) @@ -22,6 +24,7 @@ snapshot_api, wait_for_generation, wait_for_interpreter, + wait_for_location_health, wait_for_log, ) from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files @@ -30,8 +33,22 @@ @pytest.mark.parametrize( "test_case", [ - LocationReloadTestCase("ordinary target", "development", 5, int(sys.platform == "win32")), - LocationReloadTestCase("encoded target", "preview #1", 5, int(sys.platform == "win32")), + LocationReloadTestCase( + "healthy connection", + "development", + 5, + int(sys.platform == "win32"), + "warehouse.duckdb", + True, + ), + LocationReloadTestCase( + "failed connection with encoded target", + "preview #1", + 5, + int(sys.platform == "win32"), + "missing/warehouse.duckdb", + False, + ), ], ids=lambda case: case.description, ) @@ -45,7 +62,7 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ ) files: dict[str, str] = TYPE_PROOF_FILES | { "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] - + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + + f"\n[connections.local]\ndatabase = {json.dumps(test_case.database)}\n" + f'\n[targets.{json.dumps(test_case.target)}]\nschema = "main"\n', "orchestration/orders.py": declaration, } @@ -73,6 +90,17 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ publications: Path = project / "publications.jsonl" with location_process(arguments, log, publications) as process: first: dict[str, Any] = wait_for_generation(api, test_case.target, 1, process, log) + heartbeat: dict[str, Any] = wait_for_location_health( + api, test_case.target, {"kind": "heartbeat"}, process, log + ) + initial_health: dict[str, Any] = wait_for_location_health( + api, + test_case.target, + {"kind": "connection", "generation": 1, "healthy": test_case.expected_connection}, + process, + log, + ) + assert initial_health["report"]["session_id"] == heartbeat["report"]["session_id"] staging: Path = project / "models/stg_orders.sql" for value in (11, 12, 13): _ = staging.write_text( @@ -87,6 +115,9 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ _ = (project / "orchestration/orders.py").write_text("invalid Python declaration !\n") wait_for_log(process, log, "Location reload failed; retaining last good snapshot.") + _ = wait_for_location_health( + api, test_case.target, {"kind": "compile_failed", "generation": 3}, process, log + ) assert current_snapshot(api, test_case.target)["record"] == second["record"] _ = (project / "orchestration/orders.py").write_text( declaration.replace("Build orders.", "Refresh order totals.") @@ -121,6 +152,17 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ api, test_case.target, test_case.expected_generation, process, log ) assert restarted["nodes"] == third["nodes"] + restarted_health: dict[str, Any] = wait_for_location_health( + api, + test_case.target, + {"kind": "connection", "generation": 5, "healthy": test_case.expected_connection}, + process, + log, + ) + assert ( + restarted_health["report"]["session_id"] != initial_health["report"]["session_id"] + ) + assert restarted_health["received_at"] >= initial_health["received_at"] # Redirected output and compiler cache writes must not feed the input watcher. time.sleep(1) assert current_snapshot(api, test_case.target)["record"] == restarted["record"] @@ -132,19 +174,28 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ assert [record["generation"] for record in records] == [1, 2, 3, 4, 5] assert api.bearer not in log.read_text() + publications.read_text() assert not (project / ".sqlbuild/control.sqlite3").exists() - assert not (project / "warehouse.duckdb").exists() + assert (project / "warehouse.duckdb").exists() == test_case.expected_connection @pytest.mark.skipif(sys.platform == "win32", reason="Unix signal and process ownership contract") @pytest.mark.parametrize( "test_case", [ - LocationShutdownTestCase("terminate compiling interpreter", signal.SIGTERM, 0), - LocationShutdownTestCase("interrupt compiling interpreter", signal.SIGINT, 0), + LocationShutdownTestCase( + "terminate compiling interpreter after repeated heartbeats", + signal.SIGTERM, + 0, + 2, + "publish", + ), + LocationShutdownTestCase("interrupt compiling interpreter", signal.SIGINT, 0, 1, "publish"), + LocationShutdownTestCase( + "terminate connection checking interpreter", signal.SIGTERM, 0, 1, "debug" + ), ], ids=lambda case: case.description, ) -def test_given_a_compiling_location_when_stopped_then_its_interpreter_is_reaped( +def test_given_a_busy_location_when_stopped_then_its_interpreter_is_reaped( test_case: LocationShutdownTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: project: Path = tmp_path / "project" @@ -156,10 +207,11 @@ def test_given_a_compiling_location_when_stopped_then_its_interpreter_is_reaped( "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + '\n[targets.development]\nschema = "main"\n', - "orchestration/slow.py": ( - "import os, time\nfrom pathlib import Path\n" - f"Path({str(child_pid)!r}).write_text(str(os.getpid()))\n" - "time.sleep(60)\n" + "adapters/slow.py": ( + "import os, sys, time\nfrom pathlib import Path\n" + f"if {test_case.blocked_command!r} in sys.argv:\n" + f" Path({str(child_pid)!r}).write_text(str(os.getpid()))\n" + " time.sleep(60)\n" ), }, ) @@ -181,7 +233,87 @@ def test_given_a_compiling_location_when_stopped_then_its_interpreter_is_reaped( ] with location_process(arguments, log, tmp_path / "publications.jsonl") as process: interpreter: int = wait_for_interpreter(process, child_pid, log) + _ = wait_for_location_health( + api, + "development", + {"kind": "heartbeat"}, + process, + log, + minimum_reports=test_case.minimum_heartbeats, + ) process.send_signal(test_case.signal) assert process.wait(timeout=5) == test_case.expected_exit_code with pytest.raises(ProcessLookupError): os.kill(interpreter, 0) + + +@pytest.mark.parametrize( + "test_case", + [ + LocationDatabaseTestCase( + "custom database extension", "warehouse.db", 1, 'database = "warehouse.db"' + ), + LocationDatabaseTestCase( + "extensionless database", "warehouse", 1, 'database = "warehouse"' + ), + LocationDatabaseTestCase( + "custom attached database", + "inventory.db", + 1, + 'database = ":memory:"\nattach = [{path = "inventory.db", alias = "inventory"}]', + ), + ], + ids=lambda case: case.description, +) +def test_given_a_location_using_a_local_database_when_it_is_created_and_written_then_only_source_edits_reload( + test_case: LocationDatabaseTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project: Path = tmp_path / "project" + write_files( + project_dir=project, + files=TYPE_PROOF_FILES + | { + "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + + f"\n[connections.local]\n{test_case.connection_toml}\n" + + '\n[targets.development]\nschema = "main"\n', + }, + ) + database: Path = project / test_case.database + assert not database.exists() + with snapshot_api(tmp_path / "control") as api: + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", api.bearer) + log: Path = tmp_path / "location.log" + arguments: list[str] = [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + ] + with location_process(arguments, log, tmp_path / "publications.jsonl") as process: + baseline: dict[str, Any] = wait_for_generation( + api, "development", test_case.expected_generation, process, log + ) + _ = wait_for_location_health( + api, + "development", + {"kind": "connection", "generation": 1, "healthy": True}, + process, + log, + ) + assert database.is_file() + with duckdb.connect(str(database)) as connection: + _ = connection.execute( + "CREATE TABLE orders (order_id INTEGER); INSERT INTO orders VALUES (17), (23)" + ) + assert connection.execute("SELECT sum(order_id) FROM orders").fetchone() == (40,) + time.sleep(1) + assert current_snapshot(api, "development")["record"] == baseline["record"] + time.sleep(1) + assert current_snapshot(api, "development")["record"] == baseline["record"] From 4a8c898183216f8d68c1598ed56fbe77444e1664 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 09:51:54 +0000 Subject: [PATCH 32/59] feat: keep code location compilers warm Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../sqlbuild-cli/src/argv/command_table.json | 4 +- .../src/native_commands/_helpers/execution.rs | 94 ++++++++++++++++--- .../main/_location_compiler.rs | 83 ++++++++++++++++ .../src/native_commands/main/_run_location.rs | 37 ++++---- .../src/native_commands/main/mod.rs | 1 + .../src/native_commands/models.rs | 19 ++++ .../src/bindings/_helpers/cli/entry.rs | 29 +++++- .../bindings/_helpers/cli/observability.rs | 17 +++- .../_helpers/debug_command/run_debug.rs | 27 ++++-- .../_helpers/location_command/command.rs | 94 ++++++++++++------- .../bindings/_helpers/location_command/mod.rs | 1 + .../_helpers/location_command/runtime.rs | 50 ++++++++++ .../src/bindings/_helpers/native_host/host.rs | 7 ++ .../publication/test_reload.py | 58 +++++++++--- 14 files changed, 426 insertions(+), 95 deletions(-) create mode 100644 crates/sqlbuild-cli/src/native_commands/main/_location_compiler.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index f63070d273..1dd2277e11 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -106,8 +106,8 @@ "options": [], "dest": "location_command", "required": true, - "choices": ["publish", "serve"], - "help": "publish one snapshot or watch and reload the project", + "choices": ["publish", "serve", "runtime"], + "help": "publish one snapshot, watch and reload, or run the supervisor's stdio compiler", "group": 0 }, { diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 30241401e9..a9d41344e3 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -10,13 +10,14 @@ use sqlbuild_orchestration::api::errors::ClientError; use sqlbuild_orchestration::platform::main::report_remote_location_health::report_remote_location_health; use sqlbuild_orchestration::platform::models::{LocationHealth, LocationHealthReport}; use sqlbuild_orchestration::snapshots::main::next_remote_generation::next_remote_generation; +use sqlbuild_orchestration::snapshots::models::SnapshotRecord; use sqlbuild_orchestration::store::errors::ControlError; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::open_store::open_store; use sqlbuild_orchestration::store::models::ControlStore; -use tokio::io::AsyncReadExt; +use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt}; use tokio::process::{Child, Command}; -use tokio::sync::watch; +use tokio::sync::{mpsc, oneshot, watch}; use tokio::task::JoinSet; use tokio::time::MissedTickBehavior; use uuid::Uuid; @@ -44,10 +45,82 @@ use crate::native_commands::constants::{ use crate::native_commands::errors::PythonNeeded; use crate::native_commands::main::_run_cost::run_cost; use crate::native_commands::models::{ - CommandOutput, InstalledPackage, LocationServe, LostStdout, NativeCommand, NativeInvocation, - ProjectLocation, + CommandOutput, InstalledPackage, LocationCompileRequest, LocationCompiler, LocationServe, + LostStdout, NativeCommand, NativeInvocation, ProjectLocation, }; +/// Request one publication without transferring interpreter ownership to a cancellable future. +pub(crate) async fn compile_generation( + requests: &mpsc::Sender, + generation: i64, + sources: &[sqlbuild_cache::project_snapshot::models::PathStamp], + stopped: impl Future>, +) -> Result, ControlError> { + let (reply, publication) = oneshot::channel(); + requests + .send(LocationCompileRequest { + generation, + python_sources: sources + .iter() + .filter(|stamp| stamp.relative_path.ends_with(".py")) + .cloned() + .collect(), + reply, + }) + .await + .map_err(|_| ControlError::Invalid("location compiler stopped".into()))?; + tokio::pin!(stopped); + let result = tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + result = publication => result.map_err(|_| ControlError::Invalid("location compiler stopped before replying".into()))?, + }; + let record = match result { + Ok(record) => record, + Err(error) => { + drop(write_output(&[( + Stream::Stderr, + &format!( + "Location compiler failed; generation {generation} may already be published: {error}\n" + ), + )])); + None + } + }; + let published = record.is_some(); + if let Some(record) = record { + let output = serde_json::to_string(&record) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if write_output(&[(Stream::Stdout, &format!("{output}\n"))]).is_some() { + return Err(ControlError::Invalid(format!( + "Snapshot {} was published, but result delivery failed", + record.snapshot_id + ))); + } + } + Ok(Some(published)) +} + +/// The child is retained by its owning task while this exchange can be cancelled. +pub(crate) async fn exchange_location_generation( + compiler: &mut LocationCompiler, + generation: i64, +) -> Result, ControlError> { + compiler + .stdin + .write_all(format!("{generation}\n").as_bytes()) + .await?; + compiler.stdin.flush().await?; + let mut line = String::new(); + if compiler.stdout.read_line(&mut line).await? == 0 { + return Err(ControlError::Invalid( + "location interpreter exited before replying".into(), + )); + } + serde_json::from_str(&line).map_err(|_| { + ControlError::Invalid("location interpreter returned an invalid publication record".into()) + }) +} + /// Health IO remains native even while a user-code child holds its interpreter's GIL. pub(crate) async fn report_location_health( options: &LocationServe, @@ -150,27 +223,22 @@ pub(crate) async fn location_generation( } /// The installed interpreter owns user imports; the supervisor never imports Python. -pub(crate) fn publish_child( - options: &LocationServe, - script: &Path, - generation: i64, -) -> io::Result { +pub(crate) fn location_runtime_child(options: &LocationServe, script: &Path) -> io::Result { let mut command = location_child(options, script); command .args([ "location", - "publish", + "runtime", "--server", &options.server_url, "--location-id", ]) .arg(options.location_id.to_string()) - .args(["--target", &options.target, "--generation"]) - .arg(generation.to_string()); + .args(["--target", &options.target]); if let Some(image) = &options.image_id { command.args(["--image-id", image]); } - command.stdout(Stdio::inherit()).spawn() + command.stdin(Stdio::piped()).stdout(Stdio::piped()).spawn() } /// Capture the ordinary debug check's exact local file paths without contaminating publication stdout. diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_compiler.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_compiler.rs new file mode 100644 index 0000000000..2a20ef09b2 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_compiler.rs @@ -0,0 +1,83 @@ +//! One task owns a warm interpreter; cancelled requests and source restarts always reap it. + +use std::path::Path; + +use sqlbuild_orchestration::store::errors::ControlError; +use tokio::io::BufReader; +use tokio::sync::mpsc; + +use crate::native_commands::_helpers::execution::{ + exchange_location_generation, location_runtime_child, +}; +use crate::native_commands::models::{LocationCompileRequest, LocationCompiler, LocationServe}; + +pub(crate) async fn run_location_compiler( + options: &LocationServe, + script: &Path, + mut requests: mpsc::Receiver, +) -> Result<(), ControlError> { + let mut compiler: Option = None; + let result = async { + while let Some(mut request) = requests.recv().await { + if compiler + .as_ref() + .is_some_and(|compiler| compiler.python_sources != request.python_sources) + { + stop_compiler(compiler.take()).await?; + } + if compiler.is_none() { + compiler = Some(open_compiler(options, script, request.python_sources)?); + } + let active = compiler + .as_mut() + .ok_or_else(|| ControlError::Invalid("location compiler is unavailable".into()))?; + let result = tokio::select! { + _ = request.reply.closed() => break, + result = exchange_location_generation(active, request.generation) => result, + }; + let failed = result.is_err(); + let _ = request + .reply + .send(result.map_err(|error| error.to_string())); + if failed { + stop_compiler(compiler.take()).await?; + } + } + Ok(()) + } + .await; + stop_compiler(compiler).await?; + result +} + +fn open_compiler( + options: &LocationServe, + script: &Path, + python_sources: Vec, +) -> Result { + let mut child = location_runtime_child(options, script)?; + let stdin = child + .stdin + .take() + .ok_or_else(|| ControlError::Invalid("location interpreter input is unavailable".into()))?; + let stdout = child.stdout.take().ok_or_else(|| { + ControlError::Invalid("location interpreter output is unavailable".into()) + })?; + Ok(LocationCompiler { + child, + stdin, + stdout: BufReader::new(stdout), + python_sources, + }) +} + +async fn stop_compiler(compiler: Option) -> Result<(), ControlError> { + if let Some(mut compiler) = compiler { + if compiler.child.try_wait()?.is_none() { + compiler.child.kill().await?; + } else { + compiler.child.wait().await?; + } + } + Ok(()) +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs index b7bff31ea3..0c5060dab2 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs @@ -6,19 +6,21 @@ use std::time::Duration; use sqlbuild_orchestration::platform::models::LocationHealth; use sqlbuild_orchestration::store::errors::ControlError; +use tokio::sync::mpsc; use tokio::time::{Instant, MissedTickBehavior}; use uuid::Uuid; use crate::argv::models::Stream; use crate::native_commands::_helpers::execution::{ - check_location_connection, location_generation, location_heartbeats, publish_child, + check_location_connection, compile_generation, location_generation, location_heartbeats, report_location_health, shutdown_signal, }; use crate::native_commands::_helpers::paths::{ exclude_location_databases, location_watch, observe_location_inputs, }; use crate::native_commands::_helpers::terminal::write_output; -use crate::native_commands::models::LocationServe; +use crate::native_commands::main::_location_compiler::run_location_compiler; +use crate::native_commands::models::{LocationCompileRequest, LocationServe}; pub(super) async fn run_location( options: LocationServe, @@ -26,17 +28,23 @@ pub(super) async fn run_location( bearer: &str, ) -> Result<(), ControlError> { let session_id = Uuid::now_v7(); - tokio::select! { - result = watch_location(&options, script, bearer, session_id) => result, - result = location_heartbeats(&options, bearer, session_id) => result, - } + let (requests, receiver) = mpsc::channel(1); + let serving = async { + tokio::select! { + result = watch_location(&options, script, (bearer, session_id), requests) => result, + result = location_heartbeats(&options, bearer, session_id) => result, + } + }; + let (serving, compiler) = + tokio::join!(serving, run_location_compiler(&options, script, receiver)); + serving.and(compiler) } async fn watch_location( options: &LocationServe, script: &Path, - bearer: &str, - session_id: Uuid, + (bearer, session_id): (&str, Uuid), + requests: mpsc::Sender, ) -> Result<(), ControlError> { let mut watch = location_watch(&options.project_dir); let mut clock = tokio::time::interval(Duration::from_millis(250)); @@ -72,15 +80,12 @@ async fn watch_location( Stream::Stderr, &format!("Compiling location generation {generation} in an isolated interpreter...\n"), )])); - let mut child = publish_child(options, script, generation)?; - let result = tokio::select! { - result = &mut stopped => { - child.kill().await?; - return result; - }, - result = child.wait() => result?, + let Some(published) = + compile_generation(&requests, generation, &watch.observed.0, stopped.as_mut()).await? + else { + return Ok(()); }; - if result.success() { + if published { watch.retry_at = None; let Some(files) = check_published_location( options, diff --git a/crates/sqlbuild-cli/src/native_commands/main/mod.rs b/crates/sqlbuild-cli/src/native_commands/main/mod.rs index ac5f9232e5..66a0b76d88 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/mod.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/mod.rs @@ -1,4 +1,5 @@ pub(super) mod _cost_report_options; +pub(super) mod _location_compiler; pub(super) mod _run_admin; pub(super) mod _run_control; pub(super) mod _run_cost; diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index f3ae56696a..152ede9b5e 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -148,6 +148,25 @@ pub(crate) struct LocationWatch { pub(crate) retry_at: Option, } +/// A bounded supervisor request; dropping the reply cancels and reaps an active compiler. +#[derive(Debug)] +pub(crate) struct LocationCompileRequest { + pub(crate) generation: i64, + pub(crate) python_sources: Vec, + pub(crate) reply: tokio::sync::oneshot::Sender< + Result, String>, + >, +} + +/// Only this task owns the isolated interpreter and its private stdio protocol. +#[derive(Debug)] +pub(crate) struct LocationCompiler { + pub(crate) child: tokio::process::Child, + pub(crate) stdin: tokio::process::ChildStdin, + pub(crate) stdout: tokio::io::BufReader, + pub(crate) python_sources: Vec, +} + /// What Python's buffered stdout does with output its reader no longer takes. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum LostStdout { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index 3962b79206..8abf67a87f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -122,9 +122,7 @@ fn print_object(value: &Bound<'_, PyAny>, stream: Option<&str>, end: Option<&str if let Some(end) = end { keywords.set_item("end", end)?; } - py.import("builtins")? - .getattr("print")? - .call((value,), Some(&keywords))?; + imported(py, "builtins", "print")?.call((value,), Some(&keywords))?; Ok(()) } @@ -766,6 +764,31 @@ fn parsed_invocation<'py>( Ok(exit_code) }; } + let namespace = args.getattr("__dict__")?.cast_into::()?; + let command: Option = namespace + .get_item("command")? + .map(|value| value.extract()) + .transpose()?; + let location_command: Option = namespace + .get_item("location_command")? + .map(|value| value.extract()) + .transpose()?; + if command.as_deref() == Some("location") && location_command.as_deref() == Some("runtime") { + return within(&host::redirect_stdout_to_stderr(py)?, |_| { + execute_parsed_invocation(args, handlers, identity, use_color) + }); + } + execute_parsed_invocation(args, handlers, identity, use_color) +} + +/// Runtime stream routing must cover extension discovery, invocation scopes and their teardown. +fn execute_parsed_invocation<'py>( + args: Bound<'py, PyAny>, + handlers: Option>, + identity: &Bound<'py, PyAny>, + use_color: bool, +) -> PyResult> { + let py = args.py(); let facts: CommandFacts = command_facts(&args)?; let policy: InvocationPolicy = invocation_policy(&facts); let recall: InvocationRecall = if policy.recalled { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs index 6a26b63e16..0f48c13014 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs @@ -15,7 +15,7 @@ use sqlbuild_cli::dispatch::models::{ExportSummary, ExporterLoss}; use sqlbuild_core::event_export::constants::DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS; use sqlbuild_host::entries::models::{ExecutorHost, UserCodeHost}; -use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, keywords, within}; use crate::bindings::_helpers::build_progress::progress::NativeProgressProjector; use crate::bindings::_helpers::build_progress::progress::machine_output_mode; use crate::bindings::_helpers::cli::launch_state::withhold_launch_record; @@ -337,10 +337,17 @@ impl Observed { ) -> PyResult<()> { let mut command_output_sinks: bool = false; if native_field(args, "command")?.ne(LINEAGE_COMMAND)? { - let extensions: Bound<'_, PyAny> = call_with( - &host::entry(py, UserCodeHost::DiscoverRuntimeExtensions)?, - &[("project_dir", project_dir.clone())], - )?; + let discover = || { + call_with( + &host::entry(py, UserCodeHost::DiscoverRuntimeExtensions)?, + &[("project_dir", project_dir.clone())], + ) + }; + let extensions: Bound<'_, PyAny> = if machine_output_mode(args)? { + within(&host::redirect_stdout_to_stderr(py)?, |_| discover())? + } else { + discover()? + }; let lifecycle_replacement: Bound<'_, PyAny> = native_field(&extensions, "recall_lifecycle_sink")?; let output_replacement: Bound<'_, PyAny> = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs index f3a130215b..c0e0910921 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs @@ -41,6 +41,7 @@ use crate::bindings::_helpers::command_runtime::warehouses::{ use crate::bindings::_helpers::compile_attachment::target::build_effective_target_namespace; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; +use crate::bindings::_helpers::native_host::host::redirect_stdout_to_stderr; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::records::construct::record_class; use crate::bindings::_helpers::records::fields::field as native_field; @@ -384,15 +385,23 @@ pub(crate) fn run_debug_command( } else { project_dir.extract()? }; - let report: DebugReport = build_report( - py, - &project_dir, - ( - arguments.value("selected_target")?, - !arguments.flag("no_connection")?, - ), - )?; - let text: String = if arguments.flag("json_output")? { + let json_output = arguments.flag("json_output")?; + let build = || { + build_report( + py, + &project_dir, + ( + arguments.value("selected_target")?, + !arguments.flag("no_connection")?, + ), + ) + }; + let report: DebugReport = if json_output { + within(&redirect_stdout_to_stderr(py)?, |_| build())? + } else { + build()? + }; + let text: String = if json_output { format_debug_json(&report) + "\n" } else { let use_color: bool = !arguments.flag("no_color")? && stdout_supports_color(py)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs index 1ec1b8a2ee..e5673f87ea 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs @@ -8,7 +8,7 @@ use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Py use pyo3::types::PyDict; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::git_revisions::main::head_commit::head_commit; -use sqlbuild_orchestration::snapshots::models::SnapshotInput; +use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotRecord}; use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::compile_driver::pipeline::compiled_phase; @@ -17,6 +17,7 @@ use crate::bindings::_helpers::compiled_project::snapshot::{ }; use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; use crate::bindings::_helpers::location_command::constants::PUBLISH_ACTION; +use crate::bindings::_helpers::location_command::runtime::run_runtime; use crate::bindings::_helpers::native_host::host::write_console; use crate::bindings::_helpers::query_command::project::{ ProjectAdapter, ProjectInputs, project_adapter, @@ -130,14 +131,11 @@ fn compile_location<'py>( Ok(snapshot) } -/// One publication is also the isolated compiler boundary a reloading location can invoke. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn run_location_command<'py>( +pub(super) fn location_arguments<'py>( py: Python<'py>, options: Option>, -) -> PyResult { - let arguments = KeywordArguments::read( +) -> PyResult> { + KeywordArguments::read( py, "run_location_command", options, @@ -151,34 +149,37 @@ pub(crate) fn run_location_command<'py>( ], &["image_id", "action"], ), - )?; - let publication = (|| { - if arguments - .value("action")? - .extract::>()? - .is_some_and(|action| action != PUBLISH_ACTION) - { - return Err(PyValueError::new_err( - "location serve must run through the native sqb executable", - )); - } - let bearer = std::env::var("SQLBUILD_LOCATION_TOKEN").map_err(|_| { - PyValueError::new_err( - "SQLBUILD_LOCATION_TOKEN must contain a UTF-8 location credential", - ) - })?; - if bearer.trim().is_empty() { - return Err(PyValueError::new_err( - "SQLBUILD_LOCATION_TOKEN must not be blank", - )); - } - let directory = arguments - .value("project_dir")? - .extract::()?; - let input = metadata(py, &arguments, &directory)?; - let snapshot = compile_location(py, &arguments, &input)?; - status(py, "Publishing location snapshot...")?; - let record = snapshot.publication( + ) +} + +pub(super) fn publish_location( + py: Python<'_>, + arguments: &KeywordArguments<'_>, +) -> PyResult { + if arguments + .value("action")? + .extract::>()? + .is_some_and(|action| action != PUBLISH_ACTION) + { + return Err(PyValueError::new_err( + "location serve must run through the native sqb executable", + )); + } + let bearer = std::env::var("SQLBUILD_LOCATION_TOKEN").map_err(|_| { + PyValueError::new_err("SQLBUILD_LOCATION_TOKEN must contain a UTF-8 location credential") + })?; + if bearer.trim().is_empty() { + return Err(PyValueError::new_err( + "SQLBUILD_LOCATION_TOKEN must not be blank", + )); + } + let directory = arguments + .value("project_dir")? + .extract::()?; + let input = metadata(py, arguments, &directory)?; + let snapshot = compile_location(py, arguments, &input)?; + status(py, "Publishing location snapshot...")?; + let record = snapshot.publication( py, &arguments.value("server_url")?.extract::()?, &bearer, @@ -190,6 +191,28 @@ pub(crate) fn run_location_command<'py>( }; PyValueError::new_err(message) })?; + status(py, "Location snapshot published.")?; + Ok(record) +} + +/// One-shot publication and the supervisor's private stdio runtime share the same compiler path. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn run_location_command<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult { + let arguments = location_arguments(py, options.clone())?; + if arguments + .value("action")? + .extract::>()? + .as_deref() + == Some("runtime") + { + return run_runtime(py, options.unwrap_or_else(|| PyDict::new(py))); + } + let publication = (|| { + let record = publish_location(py, &arguments)?; let output = serde_json::to_string(&record) .map_err(|error| PyValueError::new_err(error.to_string()))?; write_console(py, "stdout", &format!("{output}\n")).map_err(|error| { @@ -198,7 +221,6 @@ pub(crate) fn run_location_command<'py>( record.snapshot_id )) })?; - status(py, "Location snapshot published.")?; Ok(0) })(); if publication.is_err() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs index 8aa0ebb43e..f80ce86450 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs @@ -1,2 +1,3 @@ pub(crate) mod command; mod constants; +mod runtime; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs new file mode 100644 index 0000000000..6a0505dbda --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs @@ -0,0 +1,50 @@ +//! A location keeps its interpreter warm; only the supervisor writes generation requests. + +use std::io::{self, Write}; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; + +use crate::bindings::_helpers::boundary::python_calls::within; +use crate::bindings::_helpers::location_command::command::{location_arguments, publish_location}; +use crate::bindings::_helpers::native_host::host::{redirect_stdout_to_stderr, write_console}; + +pub(super) fn run_runtime(py: Python<'_>, options: Bound<'_, PyDict>) -> PyResult { + options.set_item("action", "publish")?; + let redirect = redirect_stdout_to_stderr(py)?; + within(&redirect, |_| { + loop { + let line = py.detach(|| -> io::Result> { + let mut line = String::new(); + Ok((io::stdin().read_line(&mut line)? != 0).then_some(line)) + })?; + let Some(line) = line else { + break; + }; + let generation: i64 = serde_json::from_str(&line) + .map_err(|_| PyValueError::new_err("runtime expects a JSON generation per line"))?; + options.set_item("generation", generation)?; + let arguments = location_arguments(py, Some(options.clone()))?; + let record = match publish_location(py, &arguments) { + Ok(record) => Some(record), + Err(error) => { + write_console( + py, + "stderr", + &format!("Location compilation/publication failed: {error}\n"), + )?; + None + } + }; + let output = serde_json::to_string(&record) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + py.detach(|| -> io::Result<()> { + let mut stdout = io::stdout().lock(); + writeln!(stdout, "{output}")?; + stdout.flush() + })?; + } + Ok(0) + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs index 796bb304d7..babcc38c91 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs @@ -73,6 +73,13 @@ pub(crate) fn write_console(py: Python<'_>, stream: &str, text: &str) -> PyResul Ok(()) } +/// Keep user prints off machine-readable stdout while running project code. +pub(crate) fn redirect_stdout_to_stderr(py: Python<'_>) -> PyResult> { + py.import("contextlib")? + .getattr("redirect_stdout")? + .call1((py.import("sys")?.getattr("stderr")?,)) +} + /// `log_debug_event(logger=logging.getLogger(logger), message=message)`. pub(crate) fn log_debug_message(py: Python<'_>, logger: &str, message: &str) -> PyResult<()> { let kwargs: Bound<'_, PyDict> = PyDict::new(py); diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py index 26892bc6c0..abc6cdea92 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py @@ -36,7 +36,7 @@ LocationReloadTestCase( "healthy connection", "development", - 5, + 7, int(sys.platform == "win32"), "warehouse.duckdb", True, @@ -44,7 +44,7 @@ LocationReloadTestCase( "failed connection with encoded target", "preview #1", - 5, + 7, int(sys.platform == "win32"), "missing/warehouse.duckdb", False, @@ -56,8 +56,12 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ test_case: LocationReloadTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: project: Path = tmp_path / "project" + compiler_pid: Path = tmp_path / "compiler.pid" declaration: str = ( "from sqlbuild.orchestration import job\n" + "import os\nfrom pathlib import Path\n" + f"Path({str(compiler_pid)!r}).write_text(str(os.getpid()))\n" + 'print("Orders declaration loaded.")\n' 'daily_orders = job("daily_orders", select="orders", description="Build orders.")\n' ) files: dict[str, str] = TYPE_PROOF_FILES | { @@ -65,6 +69,7 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ + f"\n[connections.local]\ndatabase = {json.dumps(test_case.database)}\n" + f'\n[targets.{json.dumps(test_case.target)}]\nschema = "main"\n', "orchestration/orders.py": declaration, + "sinks/orders.py": 'print("Loading orders sink", flush=True)\n', } write_files(project_dir=project, files=files) _ = subprocess.run(["git", "init", "-q", str(project)], check=True, capture_output=True) @@ -90,6 +95,7 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ publications: Path = project / "publications.jsonl" with location_process(arguments, log, publications) as process: first: dict[str, Any] = wait_for_generation(api, test_case.target, 1, process, log) + first_interpreter: int = int(compiler_pid.read_text()) heartbeat: dict[str, Any] = wait_for_location_health( api, test_case.target, {"kind": "heartbeat"}, process, log ) @@ -110,19 +116,34 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ ) second: dict[str, Any] = wait_for_generation(api, test_case.target, 2, process, log) assert second["record"]["snapshot_id"] != first["record"]["snapshot_id"] + assert int(compiler_pid.read_text()) == first_interpreter assert "CAST(13 AS INTEGER)" in second["nodes"]["model:stg_orders"]["raw_sql"] assert {change["subject_key"] for change in second["changes"]} == {"model:stg_orders"} - _ = (project / "orchestration/orders.py").write_text("invalid Python declaration !\n") - wait_for_log(process, log, "Location reload failed; retaining last good snapshot.") + _ = staging.write_text("SELECT FROM invalid SQL !\n") _ = wait_for_location_health( api, test_case.target, {"kind": "compile_failed", "generation": 3}, process, log ) assert current_snapshot(api, test_case.target)["record"] == second["record"] + _ = staging.write_text( + files["models/stg_orders.sql"].replace("CAST(5 AS INTEGER)", "CAST(14 AS INTEGER)") + ) + recovered: dict[str, Any] = wait_for_generation(api, test_case.target, 3, process, log) + assert int(compiler_pid.read_text()) == first_interpreter + assert "CAST(14 AS INTEGER)" in recovered["nodes"]["model:stg_orders"]["raw_sql"] + + _ = (project / "orchestration/orders.py").write_text("invalid Python declaration !\n") + wait_for_log(process, log, "Location reload failed; retaining last good snapshot.") + _ = wait_for_location_health( + api, test_case.target, {"kind": "compile_failed", "generation": 4}, process, log + ) + assert current_snapshot(api, test_case.target)["record"] == recovered["record"] _ = (project / "orchestration/orders.py").write_text( declaration.replace("Build orders.", "Refresh order totals.") ) - third: dict[str, Any] = wait_for_generation(api, test_case.target, 3, process, log) + third: dict[str, Any] = wait_for_generation(api, test_case.target, 4, process, log) + replacement_interpreter: int = int(compiler_pid.read_text()) + assert replacement_interpreter != first_interpreter assert ( third["documents"]["jobs"]["daily_orders"]["description"] == "Refresh order totals." ) @@ -143,19 +164,34 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ check=True, capture_output=True, ) - committed: dict[str, Any] = wait_for_generation(api, test_case.target, 4, process, log) + committed: dict[str, Any] = wait_for_generation(api, test_case.target, 5, process, log) + assert int(compiler_pid.read_text()) == replacement_interpreter assert committed["nodes"] == third["nodes"] + _ = (project / "sqlbuild_project.toml").write_text( + files["sqlbuild_project.toml"].replace('schema = "main"', 'schema = "reporting"') + ) + configured: dict[str, Any] = wait_for_generation(api, test_case.target, 6, process, log) + assert int(compiler_pid.read_text()) == replacement_interpreter + assert ( + configured["nodes"]["model:orders"]["definition"]["destination"]["schema"] + == "reporting" + ) + assert ( + configured["nodes"]["model:orders"]["raw_sql"] + == committed["nodes"]["model:orders"]["raw_sql"] + ) process.terminate() assert process.wait(timeout=5) == test_case.expected_stop_code with location_process(arguments, log, publications) as process: restarted: dict[str, Any] = wait_for_generation( api, test_case.target, test_case.expected_generation, process, log ) - assert restarted["nodes"] == third["nodes"] + assert int(compiler_pid.read_text()) != replacement_interpreter + assert restarted["nodes"] == configured["nodes"] restarted_health: dict[str, Any] = wait_for_location_health( api, test_case.target, - {"kind": "connection", "generation": 5, "healthy": test_case.expected_connection}, + {"kind": "connection", "generation": 7, "healthy": test_case.expected_connection}, process, log, ) @@ -171,7 +207,7 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ records: list[dict[str, Any]] = [ json.loads(line) for line in publications.read_text().splitlines() ] - assert [record["generation"] for record in records] == [1, 2, 3, 4, 5] + assert [record["generation"] for record in records] == [1, 2, 3, 4, 5, 6, 7] assert api.bearer not in log.read_text() + publications.read_text() assert not (project / ".sqlbuild/control.sqlite3").exists() assert (project / "warehouse.duckdb").exists() == test_case.expected_connection @@ -186,9 +222,9 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ signal.SIGTERM, 0, 2, - "publish", + "runtime", ), - LocationShutdownTestCase("interrupt compiling interpreter", signal.SIGINT, 0, 1, "publish"), + LocationShutdownTestCase("interrupt compiling interpreter", signal.SIGINT, 0, 1, "runtime"), LocationShutdownTestCase( "terminate connection checking interpreter", signal.SIGTERM, 0, 1, "debug" ), From 3bd848a8dc242c25cc2621a973f81c0b59250b8f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 10:15:02 +0000 Subject: [PATCH 33/59] feat: process scoped outbound location work Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../sqlbuild-cli/src/argv/command_table.json | 96 ++++++ .../src/native_commands/_helpers/decision.rs | 21 ++ .../src/native_commands/_helpers/paths.rs | 2 +- .../native_commands/main/_location_work.rs | 135 ++++++++ .../src/native_commands/main/_run_admin.rs | 9 + .../src/native_commands/main/_run_location.rs | 41 ++- .../src/native_commands/main/mod.rs | 1 + .../src/native_commands/models.rs | 3 + .../tests/native_commands/test_control.rs | 24 ++ .../src/api/_helpers/location_work.rs | 65 ++++ .../src/api/_helpers/mod.rs | 1 + .../src/api/main/serve_api.rs | 12 +- .../sqlbuild-orchestration/src/api/models.rs | 7 +- .../src/platform/_helpers/location_work.rs | 40 +++ .../src/platform/_helpers/mod.rs | 2 + .../src/platform/_helpers/work_client.rs | 51 +++ .../src/platform/main/claim_location_work.rs | 63 ++++ .../platform/main/complete_location_work.rs | 57 ++++ .../platform/main/connect_location_work.rs | 21 ++ .../src/platform/main/location_work.rs | 20 ++ .../src/platform/main/mod.rs | 5 + .../platform/main/request_location_work.rs | 81 +++++ .../src/platform/models.rs | 79 +++++ .../src/store/_helpers/migration.rs | 1 + .../src/store/location_work.sql | 20 ++ .../src/store/main/migrate_store.rs | 24 +- crates/sqlbuild-orchestration/tests/api.rs | 4 + .../tests/api/test_location_work.rs | 302 ++++++++++++++++++ .../tests/api/test_types.rs | 6 + .../tests/api/work/helpers.rs | 20 ++ .../tests/snapshots/helpers.rs | 3 +- .../publication/_test_types.py | 7 + .../compiled_project/publication/helpers.py | 19 ++ .../compiled_project/publication/test_work.py | 158 +++++++++ 34 files changed, 1384 insertions(+), 16 deletions(-) create mode 100644 crates/sqlbuild-cli/src/native_commands/main/_location_work.rs create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/location_work.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/_helpers/location_work.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/connect_location_work.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/location_work.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs create mode 100644 crates/sqlbuild-orchestration/src/store/location_work.sql create mode 100644 crates/sqlbuild-orchestration/tests/api/test_location_work.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/work/helpers.rs create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_work.py diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index 1dd2277e11..d86b9c0ac6 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -523,6 +523,92 @@ ] } }, + { + "name": "check-location", + "parser": { + "prog": "sqb admin check-location", + "description": "Request an outbound connection check on a published location using privileged database access.", + "groups": [ + {"title": "positional arguments"}, + {"title": "options"} + ], + "actions": [ + { + "kind": "help", + "options": ["-h", "--help"], + "dest": "help", + "nargs": 0, + "default": {"suppress": true}, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "LOCATION_ID", + "required": true, + "help": "location UUID", + "group": 0 + }, + { + "kind": "store", + "options": ["--target"], + "dest": "admin_target", + "required": true, + "help": "published target name", + "group": 1 + }, + { + "kind": "store", + "options": ["--request-id"], + "dest": "admin_event_id", + "required": true, + "help": "request UUID; reuse it with identical arguments after an uncertain result", + "group": 1 + }, + { + "kind": "store", + "options": ["--reason"], + "dest": "admin_reason", + "required": true, + "help": "nonblank reason recorded with the security audit", + "group": 1 + } + ] + } + }, + { + "name": "location-work", + "parser": { + "prog": "sqb admin location-work", + "description": "Read an outbound location request and its immutable result using privileged database access.", + "groups": [ + {"title": "positional arguments"}, + {"title": "options"} + ], + "actions": [ + { + "kind": "help", + "options": ["-h", "--help"], + "dest": "help", + "nargs": 0, + "default": {"suppress": true}, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "REQUEST_ID", + "required": true, + "help": "location work request UUID", + "group": 0 + } + ] + } + }, { "name": "cancel-run", "parser": { @@ -705,6 +791,16 @@ "metavar": "cancel-run", "help": "request run cancellation with an atomic audit and retry identity" }, + { + "dest": "check-location", + "metavar": "check-location", + "help": "request an outbound location connection check with an atomic audit" + }, + { + "dest": "location-work", + "metavar": "location-work", + "help": "read a location work request and its result" + }, { "dest": "change-trigger", "metavar": "change-trigger", diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index 9e79832001..aae4693e44 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -7,6 +7,7 @@ use chrono::{DateTime, Utc}; use serde_json::json; use sqlbuild_orchestration::credentials::models::{CredentialScope, PrincipalKind}; use sqlbuild_orchestration::declarations::models::TriggerChange; +use sqlbuild_orchestration::platform::models::{LocationWork, LocationWorkRequest}; use sqlbuild_orchestration::runs::models::JobRunRequest; use sqlbuild_orchestration::store::errors::ControlError; use uuid::Uuid; @@ -68,6 +69,26 @@ pub(crate) fn control_admin(namespace: &Namespace) -> Result Ok(ControlAdmin::LocationWork(id)), + Some("check-location") => { + let request_id = + Uuid::parse_str(&text(namespace, "admin_event_id").unwrap_or_default()) + .map_err(|_| ControlError::Invalid("--request-id must be a UUID".into()))?; + let target = text(namespace, "admin_target").unwrap_or_default(); + let reason = text(namespace, "admin_reason").unwrap_or_default(); + if target.trim().is_empty() || reason.trim().is_empty() { + return Err(ControlError::Invalid( + "target and reason must not be blank".into(), + )); + } + Ok(ControlAdmin::RequestLocationWork(LocationWorkRequest { + request_id, + location_id: id, + target, + reason, + work: LocationWork::CheckConnection {}, + })) + } Some("change-trigger" | "cancel-run" | "request-run") => { let identity_option = if action.as_deref() == Some("request-run") { "--run-id" diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs index e51e498caa..78b96ec94b 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs @@ -70,7 +70,7 @@ fn location_rules() -> SnapshotRules { } /// Git commits matter to publication metadata even when no working-tree file changes. -fn location_inputs( +pub(crate) fn location_inputs( root: &Path, rules: &SnapshotRules, local_files: &[PathBuf], diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs new file mode 100644 index 0000000000..281b8bb686 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs @@ -0,0 +1,135 @@ +//! Named outbound work shares the watcher's generation boundary, never its interpreter. + +use std::future::Future; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use chrono::Utc; +use sqlbuild_orchestration::api::errors::ClientError; +use sqlbuild_orchestration::platform::models::{ + LocationWork, LocationWorkClient, LocationWorkOutcome, LocationWorkPoll, LocationWorkResult, +}; +use sqlbuild_orchestration::store::errors::ControlError; +use tokio::time::Instant; + +use crate::argv::models::Stream; +use crate::native_commands::_helpers::execution::check_location_connection; +use crate::native_commands::_helpers::paths::location_inputs; +use crate::native_commands::_helpers::terminal::write_output; +use crate::native_commands::models::{LocationServe, LocationWatch}; + +/// Retry the exact completed result after uncertain delivery; never rerun it to obtain a new result. +pub(super) async fn run_location_work( + (options, script): (&LocationServe, &Path), + (client, poll, watch): (&LocationWorkClient, &LocationWorkPoll, &LocationWatch), + generation: Option, + stopped: impl Future>, +) -> Result>, ControlError> { + tokio::pin!(stopped); + let claim = tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + result = client.poll(poll) => match result { + Ok(claim) => claim, + Err(error) => { + work_error(&error)?; + return Ok(Some(Vec::new())); + }, + }, + }; + let Some(claim) = claim else { + return Ok(Some(Vec::new())); + }; + let deadline = Instant::now() + + (claim.lease_expires_at - Utc::now()) + .to_std() + .unwrap_or_default(); + let mut files: Vec = Vec::new(); + let outcome = if generation == Some(claim.generation) { + drop(write_output(&[( + Stream::Stderr, + "Checking requested location connection...\n", + )])); + let mut shutting_down = false; + let expires = async { + tokio::select! { + result = &mut stopped => { shutting_down = true; result }, + () = tokio::time::sleep_until(deadline) => Ok(()), + } + }; + let checked = match claim.request.work { + LocationWork::CheckConnection {} => { + check_location_connection(options, script, expires).await? + } + }; + let Some((healthy, local_files)) = checked else { + drop(write_output(&[( + Stream::Stderr, + "Location work stopped before completion; no result reported.\n", + )])); + return Ok((!shutting_down).then_some(files)); + }; + files = local_files; + let mut expected = watch.observed.clone(); + expected + .0 + .retain(|stamp| !files.contains(&options.project_dir.join(&stamp.relative_path))); + let mut excluded = watch.local_files.clone(); + excluded.extend(files.clone()); + if location_inputs(&options.project_dir, &watch.rules, &excluded) == expected { + LocationWorkOutcome::Connection { healthy } + } else { + LocationWorkOutcome::Superseded {} + } + } else { + LocationWorkOutcome::Superseded {} + }; + let result = LocationWorkResult { + claim_id: claim.claim_id, + session_id: poll.session_id, + outcome, + }; + while Instant::now() < deadline { + let completed = tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + () = tokio::time::sleep_until(deadline) => break, + result = client.complete(claim.request.request_id, &result) => result, + }; + match completed { + Ok(_) => { + drop(write_output(&[( + Stream::Stderr, + "Location work completed.\n", + )])); + return Ok(Some(files)); + } + Err(error) => { + work_error(&error)?; + if matches!(error, ClientError::Rejected(status) if status.is_client_error()) { + break; + } + } + } + tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + () = tokio::time::sleep(Duration::from_secs(1)) => {}, + } + } + drop(write_output(&[( + Stream::Stderr, + "Location work acknowledgement expired; check the server result before retrying.\n", + )])); + Ok(Some(files)) +} + +fn work_error(error: &ClientError) -> Result<(), ControlError> { + if matches!(error, ClientError::Configuration(_)) + || matches!(error, ClientError::Rejected(status) if matches!(status.as_u16(), 401 | 403)) + { + return Err(ControlError::Invalid(error.to_string())); + } + drop(write_output(&[( + Stream::Stderr, + &format!("Location work request failed: {error}\n"), + )])); + Ok(()) +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs index 9c2bbde856..7cfcd0074f 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs @@ -8,6 +8,8 @@ use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; use sqlbuild_orchestration::credentials::models::AuditContext; use sqlbuild_orchestration::declarations::main::change_trigger::change_trigger; use sqlbuild_orchestration::declarations::models::TriggerChangeRequest; +use sqlbuild_orchestration::platform::main::location_work::location_work; +use sqlbuild_orchestration::platform::main::request_location_work::request_location_work; use sqlbuild_orchestration::runs::main::change_run::change_run; use sqlbuild_orchestration::runs::main::request_job_run::request_job_run; use sqlbuild_orchestration::runs::models::{RunAuthority, RunChange, RunChangeRequest}; @@ -49,6 +51,13 @@ pub(crate) async fn run_admin(namespace: &Namespace) -> Result { + let applied = request_location_work(&store, &context, &request).await?; + Ok(json!({"request_id": request.request_id, "applied": applied})) + } + ControlAdmin::LocationWork(request_id) => { + Ok(json!(location_work(&store, request_id).await?)) + } ControlAdmin::CancelRun { event_id, run_id, diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs index 0c5060dab2..7fc5eb449e 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs @@ -4,7 +4,10 @@ use std::future::Future; use std::path::{Path, PathBuf}; use std::time::Duration; -use sqlbuild_orchestration::platform::models::LocationHealth; +use sqlbuild_orchestration::platform::main::connect_location_work::connect_location_work; +use sqlbuild_orchestration::platform::models::{ + LocationHealth, LocationWorkClient, LocationWorkPoll, +}; use sqlbuild_orchestration::store::errors::ControlError; use tokio::sync::mpsc; use tokio::time::{Instant, MissedTickBehavior}; @@ -20,6 +23,7 @@ use crate::native_commands::_helpers::paths::{ }; use crate::native_commands::_helpers::terminal::write_output; use crate::native_commands::main::_location_compiler::run_location_compiler; +use crate::native_commands::main::_location_work::run_location_work; use crate::native_commands::models::{LocationCompileRequest, LocationServe}; pub(super) async fn run_location( @@ -28,10 +32,16 @@ pub(super) async fn run_location( bearer: &str, ) -> Result<(), ControlError> { let session_id = Uuid::now_v7(); + let client = connect_location_work((&options.server_url, bearer), options.location_id) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let poll = LocationWorkPoll { + session_id, + target: options.target.clone(), + }; let (requests, receiver) = mpsc::channel(1); let serving = async { tokio::select! { - result = watch_location(&options, script, (bearer, session_id), requests) => result, + result = watch_location(&options, script, (bearer, &client, &poll), requests) => result, result = location_heartbeats(&options, bearer, session_id) => result, } }; @@ -43,10 +53,12 @@ pub(super) async fn run_location( async fn watch_location( options: &LocationServe, script: &Path, - (bearer, session_id): (&str, Uuid), + (bearer, client, poll): (&str, &LocationWorkClient, &LocationWorkPoll), requests: mpsc::Sender, ) -> Result<(), ControlError> { let mut watch = location_watch(&options.project_dir); + let mut work_at = Instant::now() + Duration::from_secs(2); + let mut published_generation = None; let mut clock = tokio::time::interval(Duration::from_millis(250)); clock.set_missed_tick_behavior(MissedTickBehavior::Skip); let stopped = shutdown_signal(); @@ -63,8 +75,26 @@ async fn watch_location( let ready; (watch, ready) = observe_location_inputs(&options.project_dir, watch); if !ready { + if Instant::now() >= work_at + && published_generation.is_some() + && watch.attempted.as_ref() == Some(&watch.observed) + { + let Some(files) = run_location_work( + (options, script), + (client, poll, &watch), + published_generation, + stopped.as_mut(), + ) + .await? + else { + return Ok(()); + }; + watch = exclude_location_databases(&options.project_dir, files, watch); + work_at = Instant::now() + Duration::from_secs(2); + } continue; } + published_generation = None; let generation = tokio::select! { result = &mut stopped => return result, generation = location_generation(options, bearer) => generation?, @@ -87,10 +117,11 @@ async fn watch_location( }; if published { watch.retry_at = None; + published_generation = Some(generation); let Some(files) = check_published_location( options, script, - (bearer, session_id, generation), + (bearer, poll.session_id, generation), stopped.as_mut(), ) .await? @@ -102,7 +133,7 @@ async fn watch_location( watch.retry_at = Some(Instant::now() + Duration::from_secs(30)); tokio::select! { result = &mut stopped => return result, - () = report_location_health(options, bearer, session_id, LocationHealth::CompileFailed { generation }) => {}, + () = report_location_health(options, bearer, poll.session_id, LocationHealth::CompileFailed { generation }) => {}, } drop(write_output(&[( Stream::Stderr, diff --git a/crates/sqlbuild-cli/src/native_commands/main/mod.rs b/crates/sqlbuild-cli/src/native_commands/main/mod.rs index 66a0b76d88..a1333f51a6 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/mod.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/mod.rs @@ -1,5 +1,6 @@ pub(super) mod _cost_report_options; pub(super) mod _location_compiler; +pub(super) mod _location_work; pub(super) mod _run_admin; pub(super) mod _run_control; pub(super) mod _run_cost; diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 152ede9b5e..619784b8bd 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -6,6 +6,7 @@ use chrono::{DateTime, Utc}; use sqlbuild_cache::project_snapshot::models::{PathStamp, SnapshotRules}; use sqlbuild_orchestration::credentials::models::{CredentialScope, PrincipalKind}; use sqlbuild_orchestration::declarations::models::TriggerChange; +use sqlbuild_orchestration::platform::models::LocationWorkRequest; use sqlbuild_orchestration::runs::models::JobRunRequest; use uuid::Uuid; @@ -18,6 +19,8 @@ pub(crate) enum ControlAdmin { IssueToken(CredentialScope, DateTime), RevokeToken(Uuid), RequestJobRun(JobRunRequest), + RequestLocationWork(LocationWorkRequest), + LocationWork(Uuid), CancelRun { event_id: Uuid, run_id: Uuid, diff --git a/crates/sqlbuild-cli/tests/native_commands/test_control.rs b/crates/sqlbuild-cli/tests/native_commands/test_control.rs index 2347a94d9a..215540fdb2 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_control.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_control.rs @@ -49,6 +49,30 @@ fn given_a_standalone_binary_when_printing_help_or_rejecting_invalid_input_then_ expected_stdout_prefix: "usage: sqb admin request-run ", expected_stderr: "", }, + CommandTestCase { + description: "native location work help", + arguments: &["admin", "check-location", "--help"], + expected_success: true, + expected_stdout_prefix: "usage: sqb admin check-location ", + expected_stderr: "", + }, + CommandTestCase { + description: "blank location work reason rejected before store initialization", + arguments: &[ + "admin", + "check-location", + "00000000-0000-0000-0000-000000000001", + "--target", + "development", + "--request-id", + "00000000-0000-0000-0000-000000000002", + "--reason", + " ", + ], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: target and reason must not be blank\n", + }, CommandTestCase { description: "blank job rejected before store initialization", arguments: &[ diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/location_work.rs b/crates/sqlbuild-orchestration/src/api/_helpers/location_work.rs new file mode 100644 index 0000000000..6e48859e78 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/location_work.rs @@ -0,0 +1,65 @@ +//! Locations may poll and complete only their own work; enqueueing stays a trusted-owner action. + +use axum::extract::{Path, State}; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::_helpers::authentication::require_location; +use crate::api::errors::ApiError; +use crate::api::models::MutationResponse; +use crate::credentials::models::TokenIdentity; +use crate::platform::main::claim_location_work::claim_location_work; +use crate::platform::main::complete_location_work::complete_location_work; +use crate::platform::main::location_work::location_work; +use crate::platform::models::{ + LocationWorkClaim, LocationWorkPoll, LocationWorkRecord, LocationWorkResult, +}; +use crate::store::models::ControlStore; + +#[utoipa::path(post, path = "/api/locations/{location_id}/work/poll", + params(("location_id" = Uuid, Path)), request_body = LocationWorkPoll, + responses((status = 200, body = Option), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized")), + security(("bearer" = [])))] +pub(crate) async fn poll( + State(store): State, + Extension(identity): Extension, + Path(location_id): Path, + Json(poll): Json, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + Ok(Json(claim_location_work(&store, location_id, &poll).await?)) +} + +#[utoipa::path(post, path = "/api/locations/{location_id}/work/{request_id}", + params(("location_id" = Uuid, Path), ("request_id" = Uuid, Path)), request_body = LocationWorkResult, + responses((status = 200, body = MutationResponse), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized"), + (status = 409, description = "Work claim expired or was replaced")), security(("bearer" = [])))] +pub(crate) async fn complete( + State(store): State, + Extension(identity): Extension, + Path((location_id, request_id)): Path<(Uuid, Uuid)>, + Json(result): Json, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + Ok(Json(MutationResponse { + applied: complete_location_work(&store, (location_id, request_id), &result).await?, + })) +} + +#[utoipa::path(get, path = "/api/locations/{location_id}/work/{request_id}", + params(("location_id" = Uuid, Path), ("request_id" = Uuid, Path)), + responses((status = 200, body = Option), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized")), + security(("bearer" = [])))] +pub(crate) async fn result( + State(store): State, + Extension(identity): Extension, + Path((location_id, request_id)): Path<(Uuid, Uuid)>, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + Ok(Json(location_work(&store, request_id).await?.filter( + |record| record.request_json.location_id == location_id, + ))) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs index 4b3f4f829e..996c7546bb 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod authentication; pub(crate) mod client; pub(crate) mod location_health; +pub(crate) mod location_work; pub(crate) mod run_reads; pub(crate) mod runs; pub(crate) mod snapshots; diff --git a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs index 0a7ba5c5c2..f2ce27445f 100644 --- a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs +++ b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs @@ -5,7 +5,9 @@ use axum::routing::{get, post}; use axum::{Json, Router, middleware}; use crate::api::_helpers::authentication::authenticate; -use crate::api::_helpers::{location_health, run_reads, runs, snapshots, trigger_reads}; +use crate::api::_helpers::{ + location_health, location_work, run_reads, runs, snapshots, trigger_reads, +}; use crate::api::constants::BODY_BYTES; use crate::api::main::openapi_document::openapi_document; use crate::store::errors::ControlError; @@ -19,6 +21,14 @@ pub async fn serve_api( ) -> Result<(), ControlError> { migrate_store(&store).await?; let protected = Router::new() + .route( + "/api/locations/{location_id}/work/poll", + post(location_work::poll), + ) + .route( + "/api/locations/{location_id}/work/{request_id}", + get(location_work::result).post(location_work::complete), + ) .route( "/api/locations/{location_id}/health", get(location_health::recent).post(location_health::report), diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index 5e4837fa3b..d5070b7bb9 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -2,12 +2,17 @@ use serde::{Deserialize, Serialize}; -use crate::api::_helpers::{location_health, run_reads, runs, snapshots, trigger_reads}; +use crate::api::_helpers::{ + location_health, location_work, run_reads, runs, snapshots, trigger_reads, +}; #[derive(Debug, utoipa::OpenApi)] #[openapi(paths( location_health::report, location_health::recent, + location_work::poll, + location_work::complete, + location_work::result, snapshots::publish, snapshots::current, runs::execution, diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/location_work.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/location_work.rs new file mode 100644 index 0000000000..c523d62210 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/location_work.rs @@ -0,0 +1,40 @@ +//! Request retries and completion share the same immutable result reader and instance scope. + +use uuid::Uuid; + +use crate::platform::models::StoredLocationWork; +use crate::store::errors::ControlError; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn read_work( + transaction: ControlTransaction, + request_id: Uuid, +) -> Result<(ControlTransaction, Option), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT w.request_json, w.requested_by, w.requested_at, w.expires_at, w.generation, \ + w.claim_id, w.session_id, w.claimed_until, r.result_json, r.completed_at \ + FROM sqb_location_work w LEFT JOIN sqb_location_work_results r ON r.request_id=w.request_id \ + WHERE w.request_id=$1", vec![Parameter::Id(request_id)]).await?; + Ok(( + transaction, + rows.into_iter().next().map(decode_row).transpose()?, + )) +} + +pub(crate) async fn require_instance( + transaction: ControlTransaction, + (location_id, session_id, target): (Uuid, Uuid, &str), +) -> Result { + let (transaction, rows) = query(transaction, + "SELECT i.instance_id FROM sqb_location_instances i JOIN sqb_locations l ON l.location_id=i.location_id \ + WHERE i.instance_id=$1 AND i.location_id=$2 AND i.target=$3 AND l.archived_at IS NULL", + vec![Parameter::Id(session_id), Parameter::Id(location_id), Parameter::Text(target.into())]).await?; + if rows.is_empty() { + return Err(ControlError::Invalid( + "a registered instance of this location and target is required".into(), + )); + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs index e32efd2a5f..8f5f6285fa 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs @@ -1,3 +1,5 @@ pub(crate) mod location_heartbeat; +pub(crate) mod location_work; pub(crate) mod observations; pub(crate) mod watchdog; +pub(crate) mod work_client; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs new file mode 100644 index 0000000000..fe1e104915 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs @@ -0,0 +1,51 @@ +//! The authenticated client verifies every leased request's location, target and instance. + +use uuid::Uuid; + +use crate::api::errors::ClientError; +use crate::api::main::client_response::client_response; +use crate::api::models::MutationResponse; +use crate::platform::models::{ + LocationWorkClaim, LocationWorkClient, LocationWorkPoll, LocationWorkResult, +}; + +impl LocationWorkClient { + pub async fn poll( + &self, + poll: &LocationWorkPoll, + ) -> Result, ClientError> { + let claim: Option = client_response( + self.connection + .client + .post(format!("{}poll", self.work_url)) + .json(poll), + ) + .await?; + if claim.as_ref().is_some_and(|claim| { + claim.request.location_id != self.location_id + || claim.request.target != poll.target + || claim.session_id != poll.session_id + }) { + return Err(ClientError::Configuration( + "work response does not match this location, target and instance", + )); + } + Ok(claim) + } + + /// Retain the same request, claim and result when retrying an uncertain acknowledgement. + pub async fn complete( + &self, + request_id: Uuid, + result: &LocationWorkResult, + ) -> Result { + let response: MutationResponse = client_response( + self.connection + .client + .post(format!("{}{request_id}", self.work_url)) + .json(result), + ) + .await?; + Ok(response.applied) + } +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs new file mode 100644 index 0000000000..fd24042528 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs @@ -0,0 +1,63 @@ +//! Serialized leases fence results from replaced location instances and uncertain poll retries. + +use chrono::Duration; +use uuid::Uuid; + +use crate::platform::_helpers::location_work::{read_work, require_instance}; +use crate::platform::models::{LocationWorkClaim, LocationWorkPoll}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn claim_location_work( + store: &ControlStore, + location_id: Uuid, + poll: &LocationWorkPoll, +) -> Result, ControlError> { + let transaction = require_instance( + begin(store, TransactionMode::Write).await?, + (location_id, poll.session_id, &poll.target), + ) + .await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "SELECT w.request_id FROM sqb_location_work w WHERE w.location_id=$1 AND w.target=$2 \ + AND w.expires_at>$3 AND NOT EXISTS (SELECT 1 FROM sqb_location_work_results r WHERE r.request_id=w.request_id) \ + AND (w.claimed_until IS NULL OR w.claimed_until<=$3 OR w.session_id=$4) \ + ORDER BY w.requested_at, w.request_id LIMIT 1", + vec![Parameter::Id(location_id), Parameter::Text(poll.target.clone()), Parameter::Time(now), Parameter::Id(poll.session_id)]).await?; + let Some(row) = rows.first() else { + commit(transaction).await?; + return Ok(None); + }; + let request_id = serde_json::from_value(row["request_id"].clone()) + .map_err(|error| ControlError::Invalid(format!("invalid work identity: {error}")))?; + let (transaction, stored) = read_work(transaction, request_id).await?; + let stored = stored.ok_or_else(|| ControlError::Invalid("location work disappeared".into()))?; + let retained = stored.session_id == Some(poll.session_id) + && stored.claimed_until.is_some_and(|until| until > now); + let (claim_id, lease_expires_at) = if retained { + ( + stored.claim_id.ok_or(ControlError::Fenced)?, + stored.claimed_until.ok_or(ControlError::Fenced)?, + ) + } else { + ( + Uuid::now_v7(), + (now + Duration::seconds(60)).min(stored.record.expires_at), + ) + }; + let (transaction, _) = query(transaction, + "UPDATE sqb_location_work SET claim_id=$1, session_id=$2, claimed_until=$3 WHERE request_id=$4", + vec![Parameter::Id(claim_id), Parameter::Id(poll.session_id), Parameter::Time(lease_expires_at), Parameter::Id(request_id)]).await?; + commit(transaction).await?; + Ok(Some(LocationWorkClaim { + request: stored.record.request_json, + generation: stored.record.generation, + claim_id, + session_id: poll.session_id, + lease_expires_at, + })) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs new file mode 100644 index 0000000000..0f1888ed8b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs @@ -0,0 +1,57 @@ +//! A completion is immutable; only the current unexpired claim may publish its result. + +use serde_json::json; +use uuid::Uuid; + +use crate::platform::_helpers::location_work::{read_work, require_instance}; +use crate::platform::models::LocationWorkResult; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn complete_location_work( + store: &ControlStore, + (location_id, request_id): (Uuid, Uuid), + result: &LocationWorkResult, +) -> Result { + let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, stored) = read_work(transaction, request_id).await?; + let stored = stored + .filter(|stored| stored.record.request_json.location_id == location_id) + .ok_or_else(|| ControlError::Invalid("work is not assigned to this location".into()))?; + let transaction = require_instance( + transaction, + ( + location_id, + result.session_id, + &stored.record.request_json.target, + ), + ) + .await?; + if let Some(previous) = stored.record.result_json { + if previous != *result { + return Err(ControlError::Invalid( + "location work already has a different result".into(), + )); + } + commit(transaction).await?; + return Ok(false); + } + if stored.claim_id != Some(result.claim_id) + || stored.session_id != Some(result.session_id) + || stored + .claimed_until + .is_none_or(|until| until <= transaction.now) + || stored.record.expires_at <= transaction.now + { + return Err(ControlError::Fenced); + } + let now = transaction.now; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_location_work_results (request_id,result_json,completed_at) VALUES ($1,$2,$3)", + vec![Parameter::Id(request_id), Parameter::Json(json!(result)), Parameter::Time(now)]).await?; + commit(transaction).await?; + Ok(true) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/connect_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/connect_location_work.rs new file mode 100644 index 0000000000..a0df47b88d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/connect_location_work.rs @@ -0,0 +1,21 @@ +//! Reuse one credential-bearing pool for the location's outbound work channel. + +use uuid::Uuid; + +use crate::api::errors::ClientError; +use crate::api::main::connect_client::connect_client; +use crate::platform::models::LocationWorkClient; + +pub fn connect_location_work( + (server_url, bearer): (&str, &str), + location_id: Uuid, +) -> Result { + let connection = connect_client(server_url, bearer)?; + let work_url = + connection.endpoint(&["api", "locations", &location_id.to_string(), "work", ""])?; + Ok(LocationWorkClient { + connection, + work_url, + location_id, + }) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/location_work.rs new file mode 100644 index 0000000000..f71f848d79 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/location_work.rs @@ -0,0 +1,20 @@ +//! Read one request and its durable result without persisting a derived pending/expired status. + +use uuid::Uuid; + +use crate::platform::_helpers::location_work::read_work; +use crate::platform::models::LocationWorkRecord; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn location_work( + store: &ControlStore, + request_id: Uuid, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, record) = read_work(transaction, request_id).await?; + commit(transaction).await?; + Ok(record.map(|stored| stored.record)) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/mod.rs b/crates/sqlbuild-orchestration/src/platform/main/mod.rs index a633f3e7ce..095c42262e 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/mod.rs @@ -1,4 +1,9 @@ +pub mod claim_location_work; +pub mod complete_location_work; +pub mod connect_location_work; pub mod location_health; +pub mod location_work; pub mod report_location_health; pub mod report_remote_location_health; +pub mod request_location_work; pub mod run_watchdog; diff --git a/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs new file mode 100644 index 0000000000..d03b92dce4 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs @@ -0,0 +1,81 @@ +//! Only trusted store owners enqueue named work; retries preserve the original generation/deadline. + +use chrono::Duration; +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::AuditContext; +use crate::platform::_helpers::location_work::read_work; +use crate::platform::models::LocationWorkRequest; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn request_location_work( + store: &ControlStore, + context: &AuditContext, + request: &LocationWorkRequest, +) -> Result { + if request.target.trim().is_empty() + || request.reason.trim().is_empty() + || context.actor.trim().is_empty() + { + return Err(ControlError::Invalid( + "location work needs a target, reason and trusted actor".into(), + )); + } + let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, previous) = read_work(transaction, request.request_id).await?; + if let Some(previous) = previous { + if previous.record.request_json != *request || previous.record.requested_by != context.actor + { + return Err(ControlError::Invalid( + "location work ID already has different content or actor".into(), + )); + } + commit(transaction).await?; + return Ok(false); + } + let (transaction, rows) = query(transaction, + "SELECT s.snapshot_id, s.generation FROM sqb_current_snapshots c \ + JOIN sqb_snapshots s ON s.snapshot_id=c.snapshot_id JOIN sqb_locations l ON l.location_id=c.location_id \ + WHERE c.location_id=$1 AND c.target=$2 AND l.archived_at IS NULL", + vec![Parameter::Id(request.location_id), Parameter::Text(request.target.clone())]).await?; + let snapshot = rows + .first() + .ok_or_else(|| ControlError::Invalid("location work requires a published target".into()))?; + let snapshot_id = serde_json::from_value(snapshot["snapshot_id"].clone()) + .map_err(|error| ControlError::Invalid(format!("invalid snapshot identity: {error}")))?; + let generation = snapshot["generation"] + .as_i64() + .ok_or_else(|| ControlError::Invalid("invalid snapshot generation".into()))?; + let now = transaction.now; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_location_work \ + (request_id,location_id,target,snapshot_id,generation,request_json,requested_by,requested_at,expires_at) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)", + vec![Parameter::Id(request.request_id), Parameter::Id(request.location_id), Parameter::Text(request.target.clone()), + Parameter::Id(snapshot_id), Parameter::Integer(generation), Parameter::Json(json!(request)), + Parameter::Text(context.actor.clone()), Parameter::Time(now), Parameter::Time(now + Duration::minutes(5))]).await?; + commit( + append_audit( + transaction, + context, + ( + Uuid::new_v5(&request.request_id, b"location-work-requested"), + "location.work_requested", + ), + ( + json!({"location_id": request.location_id, "target": request.target}), + json!(null), + json!(request), + ), + ) + .await?, + ) + .await?; + Ok(true) +} diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index 052d9bf775..29f902dff0 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -4,8 +4,17 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use uuid::Uuid; +use crate::api::models::ControlClient; use crate::declarations::models::TriggerDefinition; +/// One pooled outbound connection; the location never receives a store handle. +#[derive(Clone, Debug)] +pub struct LocationWorkClient { + pub(crate) connection: ControlClient, + pub(crate) work_url: reqwest::Url, + pub(crate) location_id: Uuid, +} + /// Locations report bounded facts, never connection strings or user exception text. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] @@ -31,6 +40,76 @@ pub struct LocationHealthObservation { pub report: LocationHealthReport, } +/// Only named location operations cross this boundary, never arbitrary SQL or Python. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum LocationWork { + CheckConnection {}, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct LocationWorkRequest { + pub request_id: Uuid, + pub location_id: Uuid, + pub target: String, + pub reason: String, + pub work: LocationWork, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct LocationWorkPoll { + pub session_id: Uuid, + pub target: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct LocationWorkClaim { + pub request: LocationWorkRequest, + pub generation: i64, + pub claim_id: Uuid, + pub session_id: Uuid, + pub lease_expires_at: DateTime, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum LocationWorkOutcome { + Connection { healthy: bool }, + Superseded {}, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct LocationWorkResult { + pub claim_id: Uuid, + pub session_id: Uuid, + pub outcome: LocationWorkOutcome, +} + +/// Expiry is derived from the deadline; only requests, leases and completed results are stored. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct LocationWorkRecord { + pub request_json: LocationWorkRequest, + pub requested_by: String, + pub requested_at: DateTime, + pub expires_at: DateTime, + pub generation: i64, + pub result_json: Option, + pub completed_at: Option>, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct StoredLocationWork { + #[serde(flatten)] + pub(crate) record: LocationWorkRecord, + pub(crate) claim_id: Option, + pub(crate) session_id: Option, + pub(crate) claimed_until: Option>, +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "kind", rename_all = "snake_case")] pub enum PlatformEvent { diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index e2933cdb64..cd44c4e59a 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -8,6 +8,7 @@ use crate::store::types::Row; pub(crate) const SCHEMA: &str = include_str!("../schema.sql"); pub(crate) const LOCATION_INSTANCES: &str = include_str!("../location_instances.sql"); +pub(crate) const LOCATION_WORK: &str = include_str!("../location_work.sql"); pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) diff --git a/crates/sqlbuild-orchestration/src/store/location_work.sql b/crates/sqlbuild-orchestration/src/store/location_work.sql new file mode 100644 index 0000000000..2f5274b305 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/location_work.sql @@ -0,0 +1,20 @@ +CREATE TABLE sqb_location_work ( + request_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + snapshot_id {uuid} NOT NULL REFERENCES sqb_snapshots(snapshot_id), + generation BIGINT NOT NULL, + request_json {json} NOT NULL, + requested_by TEXT NOT NULL, + requested_at {time} NOT NULL, + expires_at {time} NOT NULL, + claim_id {uuid}, + session_id {uuid} REFERENCES sqb_location_instances(instance_id), + claimed_until {time} +); +CREATE INDEX sqb_location_work_pending ON sqb_location_work(location_id, target, requested_at); +CREATE TABLE sqb_location_work_results ( + request_id {uuid} PRIMARY KEY REFERENCES sqb_location_work(request_id), + result_json {json} NOT NULL, + completed_at {time} NOT NULL +); diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index 9063a5fedd..31fefb1c4e 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -3,7 +3,9 @@ use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; -use crate::store::_helpers::migration::{LOCATION_INSTANCES, SCHEMA, render, verify_migrations}; +use crate::store::_helpers::migration::{ + LOCATION_INSTANCES, LOCATION_WORK, SCHEMA, render, verify_migrations, +}; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; use crate::store::main::begin::begin; @@ -25,8 +27,12 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { .await?; let recall_statements = recall_schema(store); let digest = sha256_hex(format!("{SCHEMA}\n{}", recall_statements.join(";\n")).as_bytes()); - let location_digest = sha256_hex(LOCATION_INSTANCES.as_bytes()); - let digests = [digest.clone(), location_digest.clone()]; + let additions = [ + ("location_instances", LOCATION_INSTANCES), + ("location_work", LOCATION_WORK), + ]; + let mut digests = vec![digest.clone()]; + digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); let current = verify_migrations(&rows, &digests)?; if rows.is_empty() { for statement in SCHEMA @@ -48,8 +54,11 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { VALUES (1, 'control_store', $1, $2, $3)", vec![Parameter::Text(digest), Parameter::Text(current.to_string()), Parameter::Time(now)]).await?; } - if rows.len() < digests.len() { - for statement in LOCATION_INSTANCES + for (index, (name, sql)) in additions.into_iter().enumerate() { + if index + 1 < rows.len() { + continue; + } + for statement in sql .split(';') .map(str::trim) .filter(|statement| !statement.is_empty()) @@ -59,8 +68,9 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { let now = transaction.now; (transaction, _) = query(transaction, "INSERT INTO sqb_schema_migrations (version, name, checksum, min_server_version, applied_at) \ - VALUES (2, 'location_instances', $1, $2, $3)", - vec![Parameter::Text(location_digest), Parameter::Text(current.to_string()), Parameter::Time(now)]).await?; + VALUES ($1, $2, $3, $4, $5)", + vec![Parameter::Integer(index as i64 + 2), Parameter::Text(name.into()), + Parameter::Text(digests[index + 1].clone()), Parameter::Text(current.to_string()), Parameter::Time(now)]).await?; } commit(transaction).await } diff --git a/crates/sqlbuild-orchestration/tests/api.rs b/crates/sqlbuild-orchestration/tests/api.rs index 55a6bce444..0b7016321c 100644 --- a/crates/sqlbuild-orchestration/tests/api.rs +++ b/crates/sqlbuild-orchestration/tests/api.rs @@ -16,9 +16,13 @@ mod store_test_types; mod test_contract; #[path = "api/test_location_health.rs"] mod test_location_health; +#[path = "api/test_location_work.rs"] +mod test_location_work; #[path = "api/test_reads.rs"] mod test_reads; #[path = "api/test_types.rs"] mod test_types; #[path = "api/test_worker.rs"] mod test_worker; +#[path = "api/work/helpers.rs"] +mod work_helpers; diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs new file mode 100644 index 0000000000..52b883742f --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs @@ -0,0 +1,302 @@ +use chrono::{Duration, Utc}; +use reqwest::StatusCode; +use sqlbuild_orchestration::api::errors::ClientError; +use sqlbuild_orchestration::credentials::main::issue_token::issue_token; +use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; +use sqlbuild_orchestration::platform::main::connect_location_work::connect_location_work; +use sqlbuild_orchestration::platform::main::location_work::location_work; +use sqlbuild_orchestration::platform::main::report_location_health::report_location_health; +use sqlbuild_orchestration::platform::main::request_location_work::request_location_work; +use sqlbuild_orchestration::platform::models::{ + LocationHealth, LocationHealthReport, LocationWork, LocationWorkOutcome, LocationWorkPoll, + LocationWorkRecord, LocationWorkRequest, LocationWorkResult, +}; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use uuid::Uuid; + +use crate::helpers::{start_api, stop_api}; +use crate::location_helpers::create_location; +use crate::run_helpers::snapshot; +use crate::store_helpers::{close_test_store, open_test_store, run}; +use crate::store_test_types::Backend; +use crate::test_types::LocationWorkTestCase; +use crate::work_helpers::execute; + +#[test] +fn given_outbound_work_when_claimed_retried_or_replaced_then_scope_and_immutable_results_are_enforced() + { + let test_cases = [ + LocationWorkTestCase { + description: "SQLite outbound work", + backend: Backend::Sqlite, + expected_generation: 1, + }, + LocationWorkTestCase { + description: "PostgreSQL outbound work", + backend: Backend::Postgres, + expected_generation: 1, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("fresh migration"); + // Exercise both deployed predecessors, including checksums and repeated startup. + execute(store, "DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DELETE FROM sqb_schema_migrations WHERE version=3").await; + migrate_store(store).await.expect("schema 2 upgrade"); + execute(store, "DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2").await; + migrate_store(store).await.expect("schema 1 upgrade"); + migrate_store(store).await.expect("repeat migration"); + let location_id = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "inventory", "test-admin") + .await + .expect("other location"); + let audit = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let request = LocationWorkRequest { + request_id: Uuid::now_v7(), + location_id, + target: "preview #1".into(), + reason: "Check orders connectivity".into(), + work: LocationWork::CheckConnection {}, + }; + assert!( + request_location_work(store, &audit, &request) + .await + .is_err(), + "unpublished target" + ); + let mut input = snapshot(location_id); + input.target.clone_from(&request.target); + publish_snapshot(store, &input).await.expect("publish"); + let invalid_audit = AuditContext { + granting_role: String::new(), + ..audit.clone() + }; + assert!( + request_location_work(store, &invalid_audit, &request) + .await + .is_err() + ); + assert!( + location_work(store, request.request_id) + .await + .expect("rolled back read") + .is_none() + ); + assert!( + request_location_work(store, &audit, &request) + .await + .expect("enqueue") + ); + let original = location_work(store, request.request_id) + .await + .expect("read") + .expect("request"); + input.generation = 2; + publish_snapshot(store, &input).await.expect("republish"); + assert!( + !request_location_work(store, &audit, &request) + .await + .expect("retry after redeploy") + ); + assert_eq!( + location_work(store, request.request_id) + .await + .expect("read"), + Some(original.clone()) + ); + let changed = LocationWorkRequest { + reason: "Different request".into(), + ..request.clone() + }; + assert!( + request_location_work(store, &audit, &changed) + .await + .is_err() + ); + let changed_actor = AuditContext { + actor: "another-admin".into(), + ..audit.clone() + }; + assert!( + request_location_work(store, &changed_actor, &request) + .await + .is_err() + ); + let token = issue_token( + store, + &audit, + CredentialScope::Location { location_id }, + Utc::now() + Duration::hours(1), + ) + .await + .expect("token"); + let api = start_api(store).await; + let worker = + connect_location_work((&api.url, &token.bearer), location_id).expect("work client"); + let poll = LocationWorkPoll { + session_id: Uuid::now_v7(), + target: request.target.clone(), + }; + assert!(worker.poll(&poll).await.is_err(), "register before polling"); + for session_id in [poll.session_id, Uuid::nil()] { + report_location_health( + store, + location_id, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id, + target: poll.target.clone(), + health: LocationHealth::Heartbeat {}, + }, + ) + .await + .expect("register instance"); + } + let wrong_target = LocationWorkPoll { + target: "another target".into(), + ..poll.clone() + }; + assert!(worker.poll(&wrong_target).await.is_err()); + let foreign = + connect_location_work((&api.url, &token.bearer), other).expect("foreign client"); + assert!(matches!( + foreign.poll(&poll).await, + Err(ClientError::Rejected(StatusCode::FORBIDDEN)) + )); + let client = reqwest::Client::new(); + let url = format!("{}/api/locations/{location_id}/work", api.url); + let response = client + .post(format!("{url}/poll")) + .header("Content-Type", "application/json") + .body("invalid") + .send() + .await + .expect("anonymous"); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + let claim = worker.poll(&poll).await.expect("poll").expect("claim"); + assert_eq!( + claim.generation, test_case.expected_generation, + "{}: request remains pinned despite redeployment", + test_case.description + ); + assert_eq!(claim.request, request); + assert_eq!( + worker.poll(&poll).await.expect("lost poll response retry"), + Some(claim.clone()) + ); + let replacement = LocationWorkPoll { + session_id: Uuid::nil(), + ..poll.clone() + }; + assert!( + worker + .poll(&replacement) + .await + .expect("another instance") + .is_none() + ); + let stale = LocationWorkResult { + claim_id: claim.claim_id, + session_id: poll.session_id, + outcome: LocationWorkOutcome::Connection { healthy: false }, + }; + execute( + store, + "UPDATE sqb_location_work SET claimed_until='2000-01-01T00:00:00Z'", + ) + .await; + assert!(matches!( + worker.complete(request.request_id, &stale).await, + Err(ClientError::Rejected(StatusCode::CONFLICT)) + )); + let reclaimed = worker + .poll(&replacement) + .await + .expect("reclaim") + .expect("new claim"); + assert_ne!(claim.claim_id, reclaimed.claim_id); + assert!(matches!( + worker.complete(request.request_id, &stale).await, + Err(ClientError::Rejected(StatusCode::CONFLICT)) + )); + let result = LocationWorkResult { + claim_id: reclaimed.claim_id, + session_id: replacement.session_id, + outcome: LocationWorkOutcome::Superseded {}, + }; + assert!( + worker + .complete(request.request_id, &result) + .await + .expect("complete") + ); + execute(store, "UPDATE sqb_location_work SET expires_at='2000-01-01T00:00:00Z', claimed_until='2000-01-01T00:00:00Z'").await; + assert!( + !worker + .complete(request.request_id, &result) + .await + .expect("lost acknowledgement retry after expiry") + ); + let changed_result = LocationWorkResult { + outcome: LocationWorkOutcome::Connection { healthy: true }, + ..result.clone() + }; + assert!( + worker + .complete(request.request_id, &changed_result) + .await + .is_err() + ); + assert!( + worker + .poll(&poll) + .await + .expect("completed excluded") + .is_none() + ); + let response = client + .get(format!("{url}/{}", request.request_id)) + .bearer_auth(&token.bearer) + .send() + .await + .expect("result read"); + assert_eq!(response.headers()["cache-control"], "no-store"); + let record: LocationWorkRecord = response.json().await.expect("typed result"); + assert_eq!(record.result_json, Some(result)); + assert!(record.completed_at.is_some()); + let next = LocationWorkRequest { + request_id: Uuid::now_v7(), + ..request + }; + request_location_work(store, &audit, &next) + .await + .expect("another request"); + execute( + store, + "UPDATE sqb_location_work SET expires_at='2000-01-01T00:00:00Z'", + ) + .await; + assert!( + worker + .poll(&poll) + .await + .expect("expired excluded") + .is_none() + ); + stop_api(api).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index 59c0610602..cf088cb040 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -39,3 +39,9 @@ pub(crate) struct LocationHealthTestCase { pub(crate) backend: Backend, pub(crate) expected_observations: usize, } + +pub(crate) struct LocationWorkTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_generation: i64, +} diff --git a/crates/sqlbuild-orchestration/tests/api/work/helpers.rs b/crates/sqlbuild-orchestration/tests/api/work/helpers.rs new file mode 100644 index 0000000000..32a7a3cbd3 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/work/helpers.rs @@ -0,0 +1,20 @@ +//! Fault injection changes only disposable test stores, never product time or lease semantics. + +use sqlbuild_orchestration::store::models::ControlStore; + +pub(crate) async fn execute(store: &ControlStore, sql: &str) { + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!("SQLite store") + }; + rusqlite::Connection::open(path) + .expect("fault connection") + .execute_batch(sql) + .expect("inject SQLite fault"); + return; + }; + sqlx_core::raw_sql::raw_sql(sql) + .execute(pool) + .await + .expect("inject PostgreSQL fault"); +} diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index ab2594e4e4..e486bb4182 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -243,7 +243,8 @@ pub(super) async fn sqlite_failed_publish_and_schema_floor() -> i64 { let connection = rusqlite::Connection::open(&path).expect("open fault injection connection"); connection .execute_batch( - "DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version=2", + "DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; \ + DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2", ) .expect("reconstruct the previous schema without changing existing facts"); migrate_store(&store) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index eebedd8538..66bcebbc27 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -58,3 +58,10 @@ class LocationDatabaseTestCase: database: str expected_generation: int connection_toml: str + + +@dataclass(frozen=True) +class LocationWorkTestCase: + description: str + database: str + expected_connection: bool diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index efa304cafd..d10a50b37c 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -120,6 +120,25 @@ def wait_for_interpreter(process: subprocess.Popen[bytes], pidfile: Path, log: P return int(pidfile.read_text()) +def wait_for_location_work( + api: SnapshotApi, request_id: str, process: subprocess.Popen[bytes], log: Path +) -> dict[str, Any]: + """Observe completion through the scoped API rather than private coordination tables.""" + request: Request = Request( + f"{api.url}/api/locations/{api.location_id}/work/{request_id}", + headers={"Authorization": f"Bearer {api.bearer}"}, + ) + deadline: float = time.monotonic() + 15 + result: dict[str, Any] = {"result_json": None} + while result["result_json"] is None: + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + with urlopen(request, timeout=2) as response: + result = json.load(response) + time.sleep(0.02) + return result + + def api_address(process: subprocess.Popen[bytes], log: Path) -> str: """Wait for bounded native startup, never publishing an unready address.""" deadline: float = time.monotonic() + 10 diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_work.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_work.py new file mode 100644 index 0000000000..502613c572 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_work.py @@ -0,0 +1,158 @@ +"""Trusted owner requests cross the real HTTP channel to an isolated DuckDB connection check.""" + +import json +import os +import subprocess +import sys +from pathlib import Path +from typing import Any +from uuid import uuid4 + +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import TYPE_PROOF_FILES +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + LocationWorkTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + location_process, + snapshot_api, + wait_for_generation, + wait_for_interpreter, + wait_for_location_health, + wait_for_location_work, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + + +@pytest.mark.parametrize( + "test_case", + [ + LocationWorkTestCase("healthy requested check", "warehouse.db", True), + LocationWorkTestCase("failed requested check", "missing/warehouse.db", False), + ], + ids=lambda case: case.description, +) +def test_given_a_published_location_when_an_owner_requests_work_then_results_return_outbound_and_stale_work_is_not_executed( + test_case: LocationWorkTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project: Path = tmp_path / "project" + blocked: Path = tmp_path / "block-debug" + child_pid: Path = tmp_path / "debug.pid" + write_files( + project_dir=project, + files=TYPE_PROOF_FILES + | { + "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + + f"\n[connections.local]\ndatabase = {json.dumps(test_case.database)}\n" + + '\n[targets."preview #1"]\nschema = "main"\n', + "adapters/slow.py": ( + "import os, sys, time\nfrom pathlib import Path\n" + f"if 'debug' in sys.argv and Path({str(blocked)!r}).exists():\n" + f" Path({str(child_pid)!r}).write_text(str(os.getpid()))\n" + f" while Path({str(blocked)!r}).exists():\n" + " time.sleep(0.02)\n" + ), + }, + ) + executable: str = str(Path(sys.executable).with_name("sqb")) + with snapshot_api(tmp_path / "control") as api: + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", api.bearer) + # The serving location must never open the owner's database. + monkeypatch.setenv("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://must-not-open") + owner_environment: dict[str, str] = os.environ | { + "SQLBUILD_CONTROL_DATABASE_URL": str(tmp_path / "control/.sqlbuild/control.sqlite3") + } + arguments: list[str] = [ + executable, + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "preview #1", + ] + request_id: str = str(uuid4()) + enqueue: list[str] = [ + executable, + "admin", + "check-location", + api.location_id, + "--target", + "preview #1", + "--request-id", + request_id, + "--reason", + "Check orders connectivity", + ] + log: Path = tmp_path / "location.log" + publications: Path = tmp_path / "publications.jsonl" + with location_process(arguments, log, publications) as process: + _ = wait_for_generation(api, "preview #1", 1, process, log) + _ = wait_for_location_health( + api, + "preview #1", + {"kind": "connection", "generation": 1, "healthy": test_case.expected_connection}, + process, + log, + ) + first: subprocess.CompletedProcess[str] = subprocess.run( + enqueue, env=owner_environment, capture_output=True, text=True, check=True + ) + assert json.loads(first.stdout) == {"request_id": request_id, "applied": True} + retry: subprocess.CompletedProcess[str] = subprocess.run( + enqueue, env=owner_environment, capture_output=True, text=True, check=True + ) + assert json.loads(retry.stdout) == {"request_id": request_id, "applied": False} + result: dict[str, Any] = wait_for_location_work(api, request_id, process, log) + assert result["generation"] == 1 + assert result["result_json"]["outcome"] == { + "kind": "connection", + "healthy": test_case.expected_connection, + } + read: subprocess.CompletedProcess[str] = subprocess.run( + [executable, "admin", "location-work", request_id], + env=owner_environment, + capture_output=True, + text=True, + check=True, + ) + assert json.loads(read.stdout) == result + process.terminate() + _ = process.wait(timeout=5) + stale_id: str = str(uuid4()) + enqueue[enqueue.index(request_id)] = stale_id + _ = subprocess.run( + enqueue, env=owner_environment, capture_output=True, text=True, check=True + ) + with location_process(arguments, log, publications) as process: + _ = wait_for_generation(api, "preview #1", 2, process, log) + result = wait_for_location_work(api, stale_id, process, log) + assert result["generation"] == 1 + assert result["result_json"]["outcome"] == {"kind": "superseded"} + changed_id: str = str(uuid4()) + enqueue[enqueue.index(stale_id)] = changed_id + blocked.touch() + _ = subprocess.run( + enqueue, env=owner_environment, capture_output=True, text=True, check=True + ) + _ = wait_for_interpreter(process, child_pid, log) + _ = (project / "models/stg_orders.sql").write_text( + TYPE_PROOF_FILES["models/stg_orders.sql"].replace( + "CAST(5 AS INTEGER)", "CAST(17 AS INTEGER)" + ) + ) + blocked.unlink() + result = wait_for_location_work(api, changed_id, process, log) + assert result["generation"] == 2 + assert result["result_json"]["outcome"] == {"kind": "superseded"} + _ = wait_for_generation(api, "preview #1", 3, process, log) + assert [ + json.loads(line)["generation"] for line in publications.read_text().splitlines() + ] == [1, 2, 3] + assert api.bearer not in log.read_text() + publications.read_text() + assert not (project / ".sqlbuild/control.sqlite3").exists() From d339392bee5a22ca62bdf694e9f09d95b4d12fe9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 12:53:04 +0000 Subject: [PATCH 34/59] feat: execute scheduled jobs with supervised local workers Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- Cargo.toml | 2 +- .../rpt_edge_scheduled_function.sql | 7 + .../sqlbuild-cli/src/argv/command_table.json | 30 +- crates/sqlbuild-cli/src/dispatch/constants.rs | 4 +- crates/sqlbuild-cli/src/lib.rs | 1 + .../src/native_commands/_helpers/execution.rs | 14 +- .../src/native_commands/main/_run_control.rs | 9 +- .../src/workers/_helpers/configuration.rs | 40 +++ .../src/workers/_helpers/local.rs | 296 ++++++++++++++++++ .../sqlbuild-cli/src/workers/_helpers/mod.rs | 3 + .../src/workers/_helpers/shutdown.rs | 148 +++++++++ crates/sqlbuild-cli/src/workers/main/mod.rs | 1 + .../src/workers/main/open_local_workers.rs | 18 ++ crates/sqlbuild-cli/src/workers/mod.rs | 5 + crates/sqlbuild-cli/src/workers/models.rs | 47 +++ .../tests/native_commands/worker/helpers.rs | 13 +- .../src/api/_helpers/authentication.rs | 41 ++- .../src/api/_helpers/run_reads.rs | 99 +++++- .../src/api/_helpers/trigger_reads.rs | 49 +-- .../src/api/main/serve_api.rs | 8 + .../sqlbuild-orchestration/src/api/models.rs | 23 ++ .../src/runs/main/claim_next_run.rs | 6 +- .../src/runs/main/job_write_set.rs | 2 +- .../src/runs/main/mod.rs | 1 + .../src/runs/main/observe_run.rs | 24 ++ .../sqlbuild-orchestration/src/runs/models.rs | 10 + .../src/scheduler/main/run_scheduler.rs | 29 +- .../src/scheduler/mod.rs | 2 +- .../src/scheduler/types.rs | 19 ++ .../src/store/main/list_recall_invocations.rs | 62 ++++ .../src/store/main/mod.rs | 1 + .../tests/api/test_contract.rs | 2 +- .../tests/api/test_reads.rs | 166 +++++++++- .../tests/api/test_worker.rs | 4 +- .../tests/credentials/test_contract.rs | 2 +- .../tests/runs/monitor/helpers.rs | 2 +- .../tests/runs/test_claims.rs | 124 +++++++- .../tests/runs/test_coalesce.rs | 2 +- .../tests/runs/test_faults.rs | 8 +- .../tests/runs/test_lifecycle.rs | 10 +- .../tests/runs/test_monitor.rs | 4 +- .../tests/runs/test_requests.rs | 2 +- .../tests/runs/test_types.rs | 7 + .../tests/scheduler/ticks/test_loop.rs | 9 +- .../tests/scheduler/ticks/test_types.rs | 20 ++ .../_helpers/boundary/config_values.rs | 32 ++ .../_helpers/build_command/command.rs | 28 +- .../_helpers/build_command/execution.rs | 4 + .../_helpers/build_command/invocation.rs | 4 + .../build_progress/build_callbacks.rs | 22 +- .../src/bindings/_helpers/cli/dispatch.rs | 15 +- .../src/bindings/_helpers/cli/entry.rs | 44 ++- .../_helpers/compiled_project/constants.rs | 5 + .../bindings/_helpers/compiled_project/mod.rs | 2 + .../compiled_project/snapshot_execution.rs | 247 +++++++++++++++ .../compiled_project/snapshot_worker.rs | 58 +++- .../_helpers/location_command/command.rs | 47 ++- .../bindings/_helpers/location_command/mod.rs | 1 + .../_helpers/location_command/worker.rs | 154 +++++++++ .../src/bindings/_helpers/recall/session.rs | 106 +++++-- .../bindings/_helpers/rules/rules_config.rs | 28 +- .../publication/_test_types.py | 27 +- .../compiled_project/publication/helpers.py | 188 ++++++++++- .../publication/test_worker.py | 275 ++++++++++++++++ 64 files changed, 2463 insertions(+), 200 deletions(-) create mode 100644 benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_scheduled_function.sql create mode 100644 crates/sqlbuild-cli/src/workers/_helpers/configuration.rs create mode 100644 crates/sqlbuild-cli/src/workers/_helpers/local.rs create mode 100644 crates/sqlbuild-cli/src/workers/_helpers/mod.rs create mode 100644 crates/sqlbuild-cli/src/workers/_helpers/shutdown.rs create mode 100644 crates/sqlbuild-cli/src/workers/main/mod.rs create mode 100644 crates/sqlbuild-cli/src/workers/main/open_local_workers.rs create mode 100644 crates/sqlbuild-cli/src/workers/mod.rs create mode 100644 crates/sqlbuild-cli/src/workers/models.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/observe_run.rs create mode 100644 crates/sqlbuild-orchestration/src/store/main/list_recall_invocations.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs diff --git a/Cargo.toml b/Cargo.toml index 01ca1f28c3..61061e23b9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -115,7 +115,7 @@ errno = "0.3" windows-sys = { version = "0.61", default-features = false, features = ["Win32_System_Diagnostics_Debug"] } # Recall's local store: SQLite compiled in, so every build has the same engine version. rusqlite = { version = "0.37", features = ["bundled", "column_decltype"] } -rustix = { version = "1.1.4", default-features = false, features = ["fs", "std", "termios"] } +rustix = { version = "1.1.4", default-features = false, features = ["fs", "process", "std", "termios"] } saphyr-parser = "0.1" regex = "1" reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } diff --git a/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_scheduled_function.sql b/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_scheduled_function.sql new file mode 100644 index 0000000000..5e33820e53 --- /dev/null +++ b/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_scheduled_function.sql @@ -0,0 +1,7 @@ +MODEL ( + description 'A scheduled model whose implicit scalar-function writer must be claimed.', + tags [edge_case], + materialized table, +); + +SELECT __udf("edge_next_quantity")(CAST(23 AS INTEGER)) AS quantity diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index d86b9c0ac6..08bbcf87c6 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -106,7 +106,7 @@ "options": [], "dest": "location_command", "required": true, - "choices": ["publish", "serve", "runtime"], + "choices": ["publish", "serve", "runtime", "execute"], "help": "publish one snapshot, watch and reload, or run the supervisor's stdio compiler", "group": 0 }, @@ -149,6 +149,20 @@ "help": "immutable deployment image identity to pin for workers", "group": 1 }, + { + "kind": "store", + "options": ["--run-id"], + "dest": "location_run_id", + "help": "frozen run UUID for execute; requires SQLBUILD_RUN_TOKEN", + "group": 1 + }, + { + "kind": "store", + "options": ["--claim-id"], + "dest": "location_claim_id", + "help": "active claim UUID for execute", + "group": 1 + }, { "kind": "store", "options": ["--project-dir", "--sqb-project-dir"], @@ -849,6 +863,13 @@ "help": "environment variable holding the control database URL (default: SQLBUILD_CONTROL_DATABASE_URL; SQLite when unset)", "group": 1 }, + { + "kind": "store", + "options": ["--launchers"], + "dest": "control_launchers", + "help": "installation-owned JSON file of local location/target launchers and concurrency limits", + "group": 1 + }, { "kind": "store", "options": [ @@ -934,6 +955,13 @@ "default": "SQLBUILD_CONTROL_DATABASE_URL", "help": "environment variable holding the control database URL (default: SQLBUILD_CONTROL_DATABASE_URL; SQLite when unset)", "group": 1 + }, + { + "kind": "store", + "options": ["--launchers"], + "dest": "control_launchers", + "help": "installation-owned JSON file of local location/target launchers and concurrency limits", + "group": 1 } ] } diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 2f6ce26c7f..247242fd45 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -515,7 +515,7 @@ pub(crate) const DEBUG_ROUTE: Route = routed( }, ); -const LOCATION_ARGUMENTS: [Field; 7] = [ +const LOCATION_ARGUMENTS: [Field; 9] = [ field("project_dir", Source::EffectiveProjectDir), SELECTED_TARGET, same("location_id"), @@ -523,6 +523,8 @@ const LOCATION_ARGUMENTS: [Field; 7] = [ attr("generation", "location_generation"), attr("image_id", "location_image_id"), attr("action", "location_command"), + attr("run_id", "location_run_id"), + attr("claim_id", "location_claim_id"), ]; pub(crate) const LOCATION_ROUTE: Route = routed( "run_location", diff --git a/crates/sqlbuild-cli/src/lib.rs b/crates/sqlbuild-cli/src/lib.rs index 538dec3263..5364b9c483 100644 --- a/crates/sqlbuild-cli/src/lib.rs +++ b/crates/sqlbuild-cli/src/lib.rs @@ -10,3 +10,4 @@ pub mod dispatch; pub mod launcher; pub mod native_commands; pub mod replay; +pub(crate) mod workers; diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index a9d41344e3..41b1df3e3f 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -29,7 +29,7 @@ use crate::compute_logs::main::finish_capture::finish_capture; use crate::compute_logs::main::open_log_root::open_log_root; use crate::compute_logs::main::start_capture::start_capture; use crate::compute_logs::models::{CaptureEnd, CaptureStart, OpenCapture}; -use crate::native_commands::_helpers::decision::{control_database, text}; +use crate::native_commands::_helpers::decision::{control_database, flag, text}; use crate::native_commands::_helpers::init::run_init; use crate::native_commands::_helpers::package::installed_package; use crate::native_commands::_helpers::paths::{location_database_files, project_location}; @@ -48,6 +48,8 @@ use crate::native_commands::models::{ CommandOutput, InstalledPackage, LocationCompileRequest, LocationCompiler, LocationServe, LostStdout, NativeCommand, NativeInvocation, ProjectLocation, }; +use crate::workers::main::open_local_workers::open_local_workers; +use crate::workers::models::LocalWorkers; /// Request one publication without transferring interpreter ownership to a cancellable future. pub(crate) async fn compile_generation( @@ -311,6 +313,16 @@ pub(crate) async fn open_control_store( Ok(store) } +/// Decode privileged CLI configuration before handing process ownership to the workers domain. +pub(crate) fn open_control_workers(namespace: &Namespace) -> Result { + open_local_workers( + text(namespace, "control_launchers").as_deref(), + text(namespace, "control_database_env") + .unwrap_or_else(|| "SQLBUILD_CONTROL_DATABASE_URL".into()), + !flag(namespace, "control_no_scheduler"), + ) +} + pub(crate) async fn shutdown_signal() -> Result<(), ControlError> { #[cfg(unix)] { diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs index 1f39c05c8b..1ca4f4f3b1 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs @@ -10,7 +10,9 @@ use uuid::Uuid; use crate::argv::models::{Namespace, Stream}; use crate::native_commands::_helpers::decision::{control_address, flag}; -use crate::native_commands::_helpers::execution::{open_control_store, supervise_control_roles}; +use crate::native_commands::_helpers::execution::{ + open_control_store, open_control_workers, supervise_control_roles, +}; use crate::native_commands::_helpers::terminal::write_output; async fn stopped(mut shutdown: watch::Receiver) { @@ -27,6 +29,7 @@ pub(crate) async fn run_control( ) -> Result<(), ControlError> { let address = control_address(namespace, standalone)?; let store = open_control_store(namespace).await?; + let workers = open_control_workers(namespace)?; let (stop, shutdown) = watch::channel(false); let (_changes, changes) = watch::channel(()); let mut tasks = tokio::task::JoinSet::new(); @@ -66,8 +69,8 @@ pub(crate) async fn run_control( run_scheduler( &scheduler_store, SchedulerState::new(Uuid::now_v7()), - shutdown, - changes, + (shutdown, changes), + workers, ) .await }); diff --git a/crates/sqlbuild-cli/src/workers/_helpers/configuration.rs b/crates/sqlbuild-cli/src/workers/_helpers/configuration.rs new file mode 100644 index 0000000000..bf486eeb40 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/_helpers/configuration.rs @@ -0,0 +1,40 @@ +//! Reject ambiguous or nonexistent installation-owned launcher scopes before claiming any run. + +use std::collections::BTreeSet; + +use sqlbuild_orchestration::store::errors::ControlError; +use uuid::Uuid; + +use crate::workers::models::LocalLaunchers; + +pub(in crate::workers) fn read_configuration( + path: Option<&str>, + scheduler_enabled: bool, +) -> Result { + let configuration: LocalLaunchers = match path { + Some(path) => serde_json::from_slice(&std::fs::read(path)?) + .map_err(|error| ControlError::Invalid(format!("launcher configuration: {error}")))?, + None => LocalLaunchers { + local: Vec::new(), + concurrency_limits: Default::default(), + }, + }; + if !scheduler_enabled && !configuration.local.is_empty() { + return Err(ControlError::Invalid( + "launchers require the embedded scheduler".into(), + )); + } + let mut scopes: BTreeSet<(Uuid, &str)> = BTreeSet::new(); + for location in &configuration.local { + if location.target.trim().is_empty() + || !location.project_dir.is_absolute() + || !location.project_dir.is_dir() + || !location.executable.is_absolute() + || !location.executable.is_file() + || !scopes.insert((location.location_id, location.target.as_str())) + { + return Err(ControlError::Invalid("local launchers require unique location/target scopes and existing absolute project/executable paths".into())); + } + } + Ok(configuration) +} diff --git a/crates/sqlbuild-cli/src/workers/_helpers/local.rs b/crates/sqlbuild-cli/src/workers/_helpers/local.rs new file mode 100644 index 0000000000..49bbbe75af --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/_helpers/local.rs @@ -0,0 +1,296 @@ +//! The privileged control role owns and reaps local children; workers receive claim-scoped HTTP. + +use std::io::Write; +use std::process::{ExitStatus, Stdio}; +use std::time::{Duration, Instant}; + +use chrono::{TimeDelta, Utc}; +use sqlbuild_core::command_output::constants::INVOCATION_CONTEXT_ENV; +use sqlbuild_orchestration::credentials::main::issue_token::issue_token; +use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; +use sqlbuild_orchestration::runs::main::change_run::change_run; +use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; +use sqlbuild_orchestration::runs::main::observe_run::observe_run; +use sqlbuild_orchestration::runs::models::{ + RunAuthority, RunChange, RunChangeRequest, RunClaim, RunOutcome, RunStatus, +}; +use sqlbuild_orchestration::scheduler::models::SchedulerLease; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::models::ControlStore; +use tokio::process::{Child, Command}; +use uuid::Uuid; + +use crate::workers::models::{LocalLauncher, LocalWorker, LocalWorkers}; + +impl sqlbuild_orchestration::scheduler::types::RunLauncher for LocalWorkers { + async fn poll( + &mut self, + store: &ControlStore, + lease: Option<&SchedulerLease>, + ) -> Result<(), ControlError> { + for index in (0..self.active.len()).rev() { + let worker = &mut self.active[index]; + let mut state = observe_run(store, worker.claim.request.run_id).await?; + if let Some(child) = worker.child.as_mut() { + worker.exit = worker.exit.or(child.try_wait()?); + } + if worker.exit.is_none() + && let Some(child) = worker.child.as_mut() + { + let now = state.observed_at; + let not_started = matches!(state.status, RunStatus::Queued | RunStatus::Launched) + && state + .claimed_at + .is_some_and(|claimed| now - claimed > TimeDelta::minutes(2)); + let silent = state + .last_heartbeat_at + .is_some_and(|last| now - last > TimeDelta::minutes(10)); + if (state.cancel_requested_at.is_some() || not_started || silent) + && worker.stopping.is_none() + { + terminate(child)?; + worker.stopping = Some(Instant::now()); + } + if worker + .stopping + .is_some_and(|at| at.elapsed() >= Duration::from_secs(60)) + { + child.kill().await?; + worker.exit = child.try_wait()?; + state = observe_run(store, worker.claim.request.run_id).await?; + } + } + if worker.exit.is_some() || worker.child.is_none() { + if !terminal(state.status) { + let Some(lease) = lease else { + continue; + }; + let outcome = if state.cancel_requested_at.is_some() { + RunOutcome::Cancelled + } else { + RunOutcome::Failed + }; + publish( + store, + lease, + &worker.claim, + ( + "local-exit", + RunChange::Finished { + outcome, + reason: "local worker exited; physical child has been reaped" + .into(), + exit_code: worker.exit.map(exit_code), + }, + ), + ) + .await?; + } + self.active.swap_remove(index); + } + } + let Some(lease) = lease else { + return Ok(()); + }; + for location in &self.configuration.local { + let running = self + .active + .iter() + .filter(|worker| { + worker.claim.request.location_id == location.location_id + && worker.claim.request.target == location.target + }) + .count(); + for _ in running..location.max_workers.get() { + let Some(claim) = claim_next_run( + store, + lease, + &self.configuration.concurrency_limits, + Some((location.location_id, &location.target)), + ) + .await? + else { + break; + }; + let index = self.active.len(); + self.active.push(LocalWorker { + claim, + child: None, + stopping: None, + exit: None, + }); + let worker = &mut self.active[index]; + let claim = &worker.claim; + if claim.request.image_id.is_some() { + publish(store, lease, claim, ("local-image-rejected", RunChange::Finished { outcome: RunOutcome::Failed, reason: "image-pinned runs require a Docker launcher, not a local interpreter".into(), exit_code: None })).await?; + self.active.swap_remove(index); + continue; + } + let context = AuditContext { + actor: format!("scheduler/{}", lease.holder), + granting_role: "scheduler".into(), + request_id: claim.claim_id, + ip_address: None, + }; + let credential = issue_token( + store, + &context, + CredentialScope::Run { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }, + Utc::now() + TimeDelta::days(7), + ) + .await?; + publish( + store, + lease, + claim, + ("local-launched", RunChange::Launched { container_id: None }), + ) + .await?; + match spawn(location, claim, &credential.bearer, &self.control_variable) { + Ok(child) => { + drop(writeln!( + std::io::stderr().lock(), + "Worker launched for run {}", + claim.request.run_id + )); + worker.child = Some(child); + } + Err(_) => { + publish( + store, + lease, + claim, + ( + "local-spawn-failed", + RunChange::Finished { + outcome: RunOutcome::Failed, + reason: "local worker process could not start".into(), + exit_code: None, + }, + ), + ) + .await?; + self.active.swap_remove(index); + } + } + } + } + Ok(()) + } + + /// Never release claims until children have physically stopped, even if shutdown loses its fence. + async fn shutdown( + &mut self, + store: &ControlStore, + lease: Option<&SchedulerLease>, + ) -> Result<(), ControlError> { + self.shutdown_workers(store, lease).await + } +} + +fn spawn( + location: &LocalLauncher, + claim: &RunClaim, + bearer: &str, + control_variable: &str, +) -> Result { + Ok(Command::new(&location.executable) + .args([ + "location", + "execute", + "--server", + &location.server_url, + "--location-id", + &location.location_id.to_string(), + "--target", + &location.target, + "--run-id", + &claim.request.run_id.to_string(), + "--claim-id", + &claim.claim_id.to_string(), + "--project-dir", + ]) + .arg(&location.project_dir) + .current_dir(&location.project_dir) + .env_remove(control_variable) + .env_remove("SQLBUILD_CONTROL_DATABASE_URL") + .env_remove("SQLBUILD_LOCATION_TOKEN") + .env("SQLBUILD_RUN_TOKEN", bearer) + .env( + INVOCATION_CONTEXT_ENV, + serde_json::json!({ + "run_id": claim.request.run_id, + "integration": { + "name": "sqlbuild", + "run_id": claim.request.run_id, + "job_name": claim.request.job_key, + }, + }) + .to_string(), + ) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .kill_on_drop(true) + .spawn()?) +} + +pub(in crate::workers) async fn publish( + store: &ControlStore, + lease: &SchedulerLease, + claim: &RunClaim, + (event, change): (&str, RunChange), +) -> Result<(), ControlError> { + change_run( + store, + RunAuthority::Scheduler(lease), + &RunChangeRequest { + event_id: Uuid::new_v5(&claim.claim_id, event.as_bytes()), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change, + }, + ) + .await?; + Ok(()) +} + +pub(in crate::workers) fn terminal(status: RunStatus) -> bool { + !matches!( + status, + RunStatus::Queued | RunStatus::Launched | RunStatus::Running | RunStatus::Unresponsive + ) +} + +pub(in crate::workers) fn exit_code(exit: ExitStatus) -> i64 { + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt; + i64::from( + exit.code() + .unwrap_or_else(|| 128 + exit.signal().unwrap_or(0)), + ) + } + #[cfg(not(unix))] + i64::from(exit.code().unwrap_or(1)) +} + +pub(in crate::workers) fn terminate(child: &mut Child) -> Result<(), ControlError> { + #[cfg(unix)] + if let Some(id) = child.id() { + let raw = i32::try_from(id) + .map_err(|error| ControlError::Invalid(format!("invalid child process ID: {error}")))?; + let pid = rustix::process::Pid::from_raw(raw) + .ok_or_else(|| ControlError::Invalid("invalid child process ID".into()))?; + match rustix::process::kill_process(pid, rustix::process::Signal::TERM) { + Ok(()) | Err(rustix::io::Errno::SRCH) => {} + Err(error) => return Err(std::io::Error::from(error).into()), + } + } + #[cfg(not(unix))] + let _ = child; + Ok(()) +} diff --git a/crates/sqlbuild-cli/src/workers/_helpers/mod.rs b/crates/sqlbuild-cli/src/workers/_helpers/mod.rs new file mode 100644 index 0000000000..16f45af3e7 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(super) mod configuration; +mod local; +mod shutdown; diff --git a/crates/sqlbuild-cli/src/workers/_helpers/shutdown.rs b/crates/sqlbuild-cli/src/workers/_helpers/shutdown.rs new file mode 100644 index 0000000000..24ab13b5cf --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/_helpers/shutdown.rs @@ -0,0 +1,148 @@ +//! Keep leadership alive while reaping children, then release only successfully reported claims. + +use std::io::Write; +use std::time::{Duration, Instant}; + +use sqlbuild_orchestration::runs::main::observe_run::observe_run; +use sqlbuild_orchestration::runs::models::{RunChange, RunOutcome}; +use sqlbuild_orchestration::scheduler::constants::RENEW_INTERVAL_SECONDS; +use sqlbuild_orchestration::scheduler::main::renew_lease::renew_lease; +use sqlbuild_orchestration::scheduler::models::SchedulerLease; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::models::ControlStore; + +use crate::workers::_helpers::local::{exit_code, publish, terminal, terminate}; +use crate::workers::models::{LocalWorker, LocalWorkers}; + +impl LocalWorker { + async fn reap( + &mut self, + store: &ControlStore, + lease: Option<&SchedulerLease>, + ) -> Result<(), ControlError> { + let Some(child) = self.child.as_mut() else { + return Ok(()); + }; + if self.exit.is_some() { + return Ok(()); + } + if self.stopping.is_none() { + if terminate(child).is_err() { + child.kill().await?; + } + self.stopping = Some(Instant::now()); + } + let deadline = self.stopping.unwrap_or_else(Instant::now) + Duration::from_secs(60); + let mut renew = tokio::time::interval(Duration::from_secs(RENEW_INTERVAL_SECONDS)); + loop { + tokio::select! { + exit = child.wait() => { + self.exit = Some(exit?); + return Ok(()); + } + _ = tokio::time::sleep_until(deadline.into()) => { + child.kill().await?; + self.exit = Some(child.wait().await?); + return Ok(()); + } + _ = renew.tick(), if lease.is_some() => { + if let Some(lease) = lease + && renew_lease(store, lease).await.is_err() + { + drop(writeln!(std::io::stderr().lock(), "Scheduler lease renewal failed during worker shutdown")); + } + } + } + } + } + + async fn finish_shutdown( + &mut self, + store: &ControlStore, + lease: Option<&SchedulerLease>, + ) -> Result<(), ControlError> { + self.reap(store, lease).await?; + let state = observe_run(store, self.claim.request.run_id).await?; + if !terminal(state.status) { + let lease = lease.ok_or(ControlError::Fenced)?; + let reason = if self.child.is_none() { + "local worker launch preparation failed before a child started" + } else { + "scheduler stopped its local worker and reaped the child" + }; + publish( + store, + lease, + &self.claim, + ( + "local-shutdown", + RunChange::Finished { + outcome: if state.cancel_requested_at.is_some() { + RunOutcome::Cancelled + } else { + RunOutcome::Failed + }, + reason: reason.into(), + exit_code: self.exit.map(exit_code), + }, + ), + ) + .await?; + } + Ok(()) + } +} + +impl LocalWorkers { + pub(in crate::workers) async fn shutdown_workers( + &mut self, + store: &ControlStore, + lease: Option<&SchedulerLease>, + ) -> Result<(), ControlError> { + let mut retrying = false; + let mut renew = tokio::time::interval(Duration::from_secs(RENEW_INTERVAL_SECONDS)); + while !self.active.is_empty() { + let mut failure = None; + for index in (0..self.active.len()).rev() { + match self.active[index].finish_shutdown(store, lease).await { + Ok(()) => { + self.active.swap_remove(index); + } + Err(ControlError::Fenced) => failure = Some(ControlError::Fenced), + Err(error) => { + failure.get_or_insert(error); + } + } + } + if matches!(failure, Some(ControlError::Fenced)) { + return Err(ControlError::Fenced); + } + if failure.is_some() { + if !retrying { + drop(writeln!( + std::io::stderr().lock(), + "Retrying worker shutdown publication; stopped workers and their claims remain owned" + )); + retrying = true; + } + tokio::select! { + _ = tokio::time::sleep(Duration::from_secs(1)) => {}, + _ = renew.tick(), if lease.is_some() => { + if let Some(lease) = lease + && matches!(renew_lease(store, lease).await, Err(ControlError::Fenced)) + { + return Err(ControlError::Fenced); + } + } + } + } + } + if retrying { + drop(writeln!( + std::io::stderr().lock(), + "Worker shutdown publication recovered; claims released" + )); + } + Ok(()) + } +} diff --git a/crates/sqlbuild-cli/src/workers/main/mod.rs b/crates/sqlbuild-cli/src/workers/main/mod.rs new file mode 100644 index 0000000000..abf03f2f04 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/main/mod.rs @@ -0,0 +1 @@ +pub(crate) mod open_local_workers; diff --git a/crates/sqlbuild-cli/src/workers/main/open_local_workers.rs b/crates/sqlbuild-cli/src/workers/main/open_local_workers.rs new file mode 100644 index 0000000000..33803a125e --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/main/open_local_workers.rs @@ -0,0 +1,18 @@ +//! Open the installation's local process owner before accepting scheduler work. + +use sqlbuild_orchestration::store::errors::ControlError; + +use crate::workers::_helpers::configuration::read_configuration; +use crate::workers::models::LocalWorkers; + +pub(crate) fn open_local_workers( + configuration_path: Option<&str>, + control_variable: String, + scheduler_enabled: bool, +) -> Result { + Ok(LocalWorkers { + configuration: read_configuration(configuration_path, scheduler_enabled)?, + active: Vec::new(), + control_variable, + }) +} diff --git a/crates/sqlbuild-cli/src/workers/mod.rs b/crates/sqlbuild-cli/src/workers/mod.rs new file mode 100644 index 0000000000..ec01b51f5c --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/mod.rs @@ -0,0 +1,5 @@ +//! Native process owners launch claim-scoped workers without transferring control-store access. + +mod _helpers; +pub(crate) mod main; +pub(crate) mod models; diff --git a/crates/sqlbuild-cli/src/workers/models.rs b/crates/sqlbuild-cli/src/workers/models.rs new file mode 100644 index 0000000000..6fb061ad4d --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/models.rs @@ -0,0 +1,47 @@ +//! Installation launch configuration and the physical children retained until reaping. + +use std::collections::BTreeMap; +use std::num::{NonZeroU32, NonZeroUsize}; +use std::path::PathBuf; +use std::process::ExitStatus; +use std::time::Instant; + +use sqlbuild_orchestration::runs::models::RunClaim; +use tokio::process::Child; +use uuid::Uuid; + +#[derive(Debug, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct LocalLaunchers { + pub(super) local: Vec, + #[serde(default)] + pub(super) concurrency_limits: BTreeMap, +} + +/// No executable, project path or API address is supplied by an untrusted run request. +#[derive(Debug, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct LocalLauncher { + pub(super) location_id: Uuid, + pub(super) target: String, + pub(super) project_dir: PathBuf, + pub(super) executable: PathBuf, + pub(super) server_url: String, + pub(super) max_workers: NonZeroUsize, +} + +#[derive(Debug)] +pub(super) struct LocalWorker { + pub(super) claim: RunClaim, + pub(super) child: Option, + pub(super) stopping: Option, + pub(super) exit: Option, +} + +/// Physical ownership survives leadership loss; only the current fence changes durable lifecycle. +#[derive(Debug)] +pub(crate) struct LocalWorkers { + pub(super) configuration: LocalLaunchers, + pub(super) active: Vec, + pub(super) control_variable: String, +} diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs index 2948290095..4340e30294 100644 --- a/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs +++ b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs @@ -98,10 +98,15 @@ pub(crate) async fn start_claimed_worker( .await .expect("acquire lease") .expect("leader"); - let claim = claim_next_run(&store, &lease, &BTreeMap::new()) - .await - .expect("claim") - .expect("queued work"); + let claim = claim_next_run( + &store, + &lease, + &BTreeMap::new(), + Some((location_id, "development")), + ) + .await + .expect("claim") + .expect("queued work"); assert_eq!(claim.request.priority, -3); assert_eq!( claim.request.settings.get("full_refresh"), diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication.rs index c7f6dd2e06..70c607a995 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication.rs @@ -7,9 +7,11 @@ use axum::response::Response; use uuid::Uuid; use crate::api::errors::ApiError; +use crate::api::models::ReadResource; use crate::credentials::main::authenticate_token::authenticate_token; use crate::credentials::models::{CredentialScope, TokenIdentity}; -use crate::store::models::ControlStore; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; pub(crate) async fn authenticate( State(store): State, @@ -51,3 +53,40 @@ pub(crate) fn require_location( } Ok(()) } + +/// A guessed resource ID never widens the already authenticated location and target. +pub(crate) async fn require_resource( + store: &ControlStore, + (location_id, target): (Uuid, &str), + (resource, id): (ReadResource, Uuid), +) -> Result<(), ApiError> { + let (statement, missing) = match resource { + ReadResource::Run => ( + "SELECT run_id FROM sqb_runs WHERE run_id=$1 AND location_id=$2 AND target=$3", + "run not found", + ), + ReadResource::Trigger => ( + "SELECT trigger_id FROM sqb_triggers WHERE trigger_id=$1 AND location_id=$2 AND target=$3", + "trigger not found", + ), + }; + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query( + transaction, + statement, + vec![ + Parameter::Id(id), + Parameter::Id(location_id), + Parameter::Text(target.into()), + ], + ) + .await?; + commit(transaction).await?; + if rows.is_empty() { + return Err(ApiError { + status: StatusCode::NOT_FOUND, + message: missing.into(), + }); + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/run_reads.rs b/crates/sqlbuild-orchestration/src/api/_helpers/run_reads.rs index c8c002e3d0..06ee0574ab 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/run_reads.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/run_reads.rs @@ -5,17 +5,20 @@ use axum::http::StatusCode; use axum::{Extension, Json}; use uuid::Uuid; -use crate::api::_helpers::authentication::require_location; +use crate::api::_helpers::authentication::{require_location, require_resource}; use crate::api::errors::ApiError; -use crate::api::models::ReadQuery; +use crate::api::models::{ReadQuery, ReadResource, RunRecallPath, RunRecallQuery}; use crate::credentials::models::TokenIdentity; use crate::runs::main::list_run_nodes::list_run_nodes; use crate::runs::main::list_runs::list_runs; use crate::runs::models::{RunNodeRecord, RunRecord}; use crate::store::main::begin::begin; use crate::store::main::commit::commit; +use crate::store::main::list_recall_invocations::list_recall_invocations; use crate::store::main::query::query; +use crate::store::main::read_recall_view::read_recall_view; use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::types::Row; #[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/runs", params(("location_id" = Uuid, Path), ("target" = String, Path), @@ -45,23 +48,85 @@ pub(crate) async fn nodes( Path((location_id, target, run_id)): Path<(Uuid, String, Uuid)>, ) -> Result>, ApiError> { require_location(&identity, location_id)?; - let transaction = begin(&store, TransactionMode::Read).await?; - let (transaction, rows) = query( - transaction, - "SELECT run_id FROM sqb_runs WHERE run_id=$1 AND location_id=$2 AND target=$3", - vec![ - Parameter::Id(run_id), - Parameter::Id(location_id), - Parameter::Text(target), - ], - ) - .await?; - commit(transaction).await?; - if rows.is_empty() { + require_resource(&store, (location_id, &target), (ReadResource::Run, run_id)).await?; + Ok(Json(list_run_nodes(&store, run_id).await?)) +} + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/runs/{run_id}/invocations", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("run_id" = Uuid, Path), + ("limit" = Option, Query, description = "Maximum invocation namespaces; defaults to 100, capped at 1000")), + responses((status = 200, body = Vec), (status = 401, description = "Invalid credential"), + (status = 403, description = "Location not authorized"), (status = 404, description = "Run not found")), + security(("bearer" = [])))] +pub(crate) async fn invocations( + State(store): State, + Extension(identity): Extension, + Path((location_id, target, run_id)): Path<(Uuid, String, Uuid)>, + Query(options): Query, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + require_resource(&store, (location_id, &target), (ReadResource::Run, run_id)).await?; + let namespaces = run_namespaces(&store, run_id).await?; + Ok(Json( + list_recall_invocations(&store, &namespaces, options.limit.unwrap_or(100)).await?, + )) +} + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/runs/{run_id}/recall/{view}", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("run_id" = Uuid, Path), + ("view" = String, Path), ("invocation" = String, Query, description = "Exact scheduler or worker invocation ID"), + ("limit" = Option, Query, description = "Maximum rows; defaults to 100, capped at 1000")), + responses((status = 200, body = Vec), (status = 400, description = "Unknown Recall view"), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized"), + (status = 404, description = "Run or invocation not found")), security(("bearer" = [])))] +pub(crate) async fn recall( + State(store): State, + Extension(identity): Extension, + Path(RunRecallPath { + location_id, + target, + run_id, + view, + }): Path, + Query(options): Query, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + require_resource(&store, (location_id, &target), (ReadResource::Run, run_id)).await?; + let namespaces = run_namespaces(&store, run_id).await?; + if !namespaces.iter().any(|namespace| { + options.invocation == *namespace || options.invocation.starts_with(&format!("{namespace}/")) + }) { return Err(ApiError { status: StatusCode::NOT_FOUND, - message: "run not found".into(), + message: "run invocation not found".into(), }); } - Ok(Json(list_run_nodes(&store, run_id).await?)) + Ok(Json( + read_recall_view( + &store, + (&view, &options.invocation), + options.limit.unwrap_or(100), + ) + .await?, + )) +} + +/// Immutable claim evidence remains available after terminal publication removes liveness. +async fn run_namespaces(store: &ControlStore, run_id: Uuid) -> Result, ApiError> { + let scheduler = format!("scheduler/{run_id}"); + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, claims) = query( + transaction, + "SELECT payload FROM recall_events WHERE invocation_id=$1 AND event_type='run_claimed' ORDER BY invocation_sequence DESC LIMIT 1", + vec![Parameter::Text(scheduler.clone())], + ).await?; + commit(transaction).await?; + let mut namespaces = vec![scheduler]; + if let Some(claim_id) = claims + .first() + .and_then(|row| row["payload"]["claim"]["claim_id"].as_str()) + { + namespaces.push(format!("worker/{claim_id}")); + } + Ok(namespaces) } diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/trigger_reads.rs b/crates/sqlbuild-orchestration/src/api/_helpers/trigger_reads.rs index ec5dfa88eb..bcc5fd5bdd 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/trigger_reads.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/trigger_reads.rs @@ -1,23 +1,19 @@ //! Trigger definitions and immutable history are visible only within the authenticated location. use axum::extract::{Path, Query, State}; -use axum::http::StatusCode; use axum::{Extension, Json}; use uuid::Uuid; -use crate::api::_helpers::authentication::require_location; +use crate::api::_helpers::authentication::{require_location, require_resource}; use crate::api::errors::ApiError; -use crate::api::models::ReadQuery; +use crate::api::models::{ReadQuery, ReadResource}; use crate::credentials::models::TokenIdentity; use crate::declarations::main::list_trigger_events::list_trigger_events; use crate::declarations::main::list_triggers::list_triggers; use crate::declarations::models::{TriggerChangeRecord, TriggerRecord}; use crate::scheduler::main::list_ticks::list_ticks; use crate::scheduler::models::TickRecord; -use crate::store::main::begin::begin; -use crate::store::main::commit::commit; -use crate::store::main::query::query; -use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::models::ControlStore; #[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/triggers", params(("location_id" = Uuid, Path), ("target" = String, Path)), @@ -45,7 +41,12 @@ pub(crate) async fn events( Query(options): Query, ) -> Result>, ApiError> { require_location(&identity, location_id)?; - require_trigger(&store, (location_id, &target, trigger_id)).await?; + require_resource( + &store, + (location_id, &target), + (ReadResource::Trigger, trigger_id), + ) + .await?; Ok(Json( list_trigger_events(&store, trigger_id, options.limit.unwrap_or(100)).await?, )) @@ -64,33 +65,13 @@ pub(crate) async fn ticks( Query(options): Query, ) -> Result>, ApiError> { require_location(&identity, location_id)?; - require_trigger(&store, (location_id, &target, trigger_id)).await?; + require_resource( + &store, + (location_id, &target), + (ReadResource::Trigger, trigger_id), + ) + .await?; Ok(Json( list_ticks(&store, trigger_id, options.limit.unwrap_or(100)).await?, )) } - -async fn require_trigger( - store: &ControlStore, - (location_id, target, trigger_id): (Uuid, &str, Uuid), -) -> Result<(), ApiError> { - let transaction = begin(store, TransactionMode::Read).await?; - let (transaction, rows) = query( - transaction, - "SELECT trigger_id FROM sqb_triggers WHERE trigger_id=$1 AND location_id=$2 AND target=$3", - vec![ - Parameter::Id(trigger_id), - Parameter::Id(location_id), - Parameter::Text(target.into()), - ], - ) - .await?; - commit(transaction).await?; - if rows.is_empty() { - return Err(ApiError { - status: StatusCode::NOT_FOUND, - message: "trigger not found".into(), - }); - } - Ok(()) -} diff --git a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs index f2ce27445f..a08c8a3130 100644 --- a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs +++ b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs @@ -45,6 +45,14 @@ pub async fn serve_api( "/api/locations/{location_id}/targets/{target}/runs/{run_id}/nodes", get(run_reads::nodes), ) + .route( + "/api/locations/{location_id}/targets/{target}/runs/{run_id}/invocations", + get(run_reads::invocations), + ) + .route( + "/api/locations/{location_id}/targets/{target}/runs/{run_id}/recall/{view}", + get(run_reads::recall), + ) .route( "/api/locations/{location_id}/targets/{target}/triggers", get(trigger_reads::current), diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index d5070b7bb9..b39ad9fd6c 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -22,6 +22,8 @@ use crate::api::_helpers::{ runs::recall, run_reads::recent, run_reads::nodes, + run_reads::invocations, + run_reads::recall, trigger_reads::current, trigger_reads::events, trigger_reads::ticks @@ -34,6 +36,27 @@ pub(crate) struct ReadQuery { pub(crate) limit: Option, } +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RunRecallQuery { + pub(crate) invocation: String, + pub(crate) limit: Option, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct RunRecallPath { + pub(crate) location_id: uuid::Uuid, + pub(crate) target: String, + pub(crate) run_id: uuid::Uuid, + pub(crate) view: String, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) enum ReadResource { + Run, + Trigger, +} + #[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] pub struct MutationResponse { pub applied: bool, diff --git a/crates/sqlbuild-orchestration/src/runs/main/claim_next_run.rs b/crates/sqlbuild-orchestration/src/runs/main/claim_next_run.rs index 3a86ee6d75..9bf328e7ed 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/claim_next_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/claim_next_run.rs @@ -24,6 +24,7 @@ pub async fn claim_next_run( store: &ControlStore, lease: &SchedulerLease, limits: &BTreeMap, + scope: Option<(Uuid, &str)>, ) -> Result, ControlError> { let mut transaction = check_lease(begin(store, TransactionMode::Write).await?, lease).await?; let locking = if matches!(transaction.connection, Some(Connection::Postgres(_))) { @@ -37,11 +38,14 @@ pub async fn claim_next_run( (transaction, rows) = query(transaction, &format!("SELECT r.request_json, r.queued_at FROM sqb_runs r WHERE r.status='queued' AND r.cancel_requested_at IS NULL \ AND NOT EXISTS (SELECT 1 FROM sqb_run_liveness l WHERE l.run_id=r.run_id) \ + AND ($4 IS NULL OR (r.location_id=$4 AND r.target=$5)) \ AND ($1 IS NULL OR r.priority<$1 OR (r.priority=$1 AND (r.queued_at>$2 OR (r.queued_at=$2 AND r.run_id>$3)))) \ ORDER BY r.priority DESC, r.queued_at, r.run_id LIMIT 100{locking}"), vec![after.as_ref().map_or(Parameter::Null(ColumnType::Integer), |value| Parameter::Integer(value.request_json.priority)), Parameter::OptionalTime(after.as_ref().map(|value| value.queued_at)), - Parameter::OptionalId(after.as_ref().map(|value| value.request_json.run_id))]).await?; + Parameter::OptionalId(after.as_ref().map(|value| value.request_json.run_id)), + Parameter::OptionalId(scope.map(|(location, _)| location)), + Parameter::OptionalText(scope.map(|(_, target)| target.to_owned()))]).await?; if rows.is_empty() { commit(check_lease(transaction, lease).await?).await?; return Ok(None); diff --git a/crates/sqlbuild-orchestration/src/runs/main/job_write_set.rs b/crates/sqlbuild-orchestration/src/runs/main/job_write_set.rs index 1354b035fb..101f419a95 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/job_write_set.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/job_write_set.rs @@ -15,7 +15,7 @@ pub(crate) async fn job_write_set( ) -> Result<(ControlTransaction, BTreeSet), ControlError> { let (transaction, rows) = query(transaction, "SELECT n.node_key FROM sqb_snapshot_nodes n JOIN sqb_job_nodes j ON j.node_key=n.node_key \ - WHERE n.snapshot_id=$1 AND j.location_id=$2 AND j.target=$3 AND j.job_key=$4 AND n.node_kind IN ('model','seed','function')", + WHERE n.snapshot_id=$1 AND j.location_id=$2 AND j.target=$3 AND j.job_key=$4 AND n.node_kind IN ('model','seed','udf','table_fn')", vec![Parameter::Id(snapshot_id), Parameter::Id(location_id), Parameter::Text(target.into()), Parameter::Text(job_key.into())]).await?; Ok(( transaction, diff --git a/crates/sqlbuild-orchestration/src/runs/main/mod.rs b/crates/sqlbuild-orchestration/src/runs/main/mod.rs index 97d86e1251..a7989d799a 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/mod.rs @@ -5,6 +5,7 @@ pub(crate) mod job_write_set; pub mod list_run_nodes; pub mod list_runs; pub mod monitor_runs; +pub mod observe_run; pub mod publish_run_output; pub mod publish_run_recall; pub(crate) mod queue_run; diff --git a/crates/sqlbuild-orchestration/src/runs/main/observe_run.rs b/crates/sqlbuild-orchestration/src/runs/main/observe_run.rs new file mode 100644 index 0000000000..f18e11f7a6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/observe_run.rs @@ -0,0 +1,24 @@ +//! Observe one run and its current claim using the same database-clock snapshot. + +use uuid::Uuid; + +use crate::runs::models::RunObservation; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, decode_row::decode_row, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn observe_run( + store: &ControlStore, + run_id: Uuid, +) -> Result { + let transaction = begin(store, TransactionMode::Read).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, "SELECT r.status, r.cancel_requested_at, l.claimed_at, l.last_heartbeat_at, $2 AS observed_at FROM sqb_runs r LEFT JOIN sqb_run_liveness l ON l.run_id=r.run_id WHERE r.run_id=$1", vec![Parameter::Id(run_id), Parameter::Time(now)]).await?; + let state = decode_row( + rows.into_iter() + .next() + .ok_or_else(|| ControlError::Invalid("launched run is missing".into()))?, + )?; + commit(transaction).await?; + Ok(state) +} diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index 70a560ab44..34444c32e8 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -307,3 +307,13 @@ pub(crate) struct ClaimMetadata { pub(crate) claimed_at: DateTime, pub(crate) sqb_version: String, } + +/// A physical launcher uses the database clock and current liveness, never an old run event. +#[derive(Debug, Deserialize)] +pub struct RunObservation { + pub status: RunStatus, + pub cancel_requested_at: Option>, + pub claimed_at: Option>, + pub last_heartbeat_at: Option>, + pub observed_at: DateTime, +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs b/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs index 29c8180b65..b4fd01d5e5 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs @@ -4,6 +4,7 @@ use tokio::sync::watch; use crate::scheduler::main::scheduler_cycle::scheduler_cycle; use crate::scheduler::models::SchedulerState; +use crate::scheduler::types::RunLauncher; use crate::store::errors::ControlError; use crate::store::models::ControlStore; @@ -11,17 +12,25 @@ use crate::store::models::ControlStore; pub async fn run_scheduler( store: &ControlStore, mut state: SchedulerState, - mut shutdown: watch::Receiver, - mut changes: watch::Receiver<()>, + (mut shutdown, mut changes): (watch::Receiver, watch::Receiver<()>), + mut launcher: impl RunLauncher, ) -> Result<(), ControlError> { - while !*shutdown.borrow() { - let (next_state, cycle) = scheduler_cycle(store, state).await?; - state = next_state; - tokio::select! { - _ = tokio::time::sleep(cycle.wait_for) => {}, - result = changes.changed() => { if result.is_err() { break; } }, - result = shutdown.changed() => { if result.is_err() { break; } }, + let mut last_lease = None; + let result = async { + while !*shutdown.borrow() { + let (next_state, cycle) = scheduler_cycle(store, state).await?; + state = next_state; + last_lease = state.lease.clone(); + launcher.poll(store, state.lease.as_ref()).await?; + tokio::select! { + _ = tokio::time::sleep(cycle.wait_for.min(std::time::Duration::from_secs(1))) => {}, + result = changes.changed() => { if result.is_err() { break; } }, + result = shutdown.changed() => { if result.is_err() { break; } }, + } } + Ok(()) } - Ok(()) + .await; + let stopped = launcher.shutdown(store, last_lease.as_ref()).await; + result.and(stopped) } diff --git a/crates/sqlbuild-orchestration/src/scheduler/mod.rs b/crates/sqlbuild-orchestration/src/scheduler/mod.rs index 3978df0a4a..cf840ff555 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/mod.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/mod.rs @@ -4,7 +4,7 @@ pub(crate) mod _helpers; pub mod constants; pub mod main; pub mod models; -pub(crate) mod types; +pub mod types; #[cfg(test)] mod tests; diff --git a/crates/sqlbuild-orchestration/src/scheduler/types.rs b/crates/sqlbuild-orchestration/src/scheduler/types.rs index 274b453740..53e9334d97 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/types.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/types.rs @@ -2,5 +2,24 @@ use chrono::{DateTime, Utc}; +use crate::scheduler::models::SchedulerLease; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; + +/// Physical process ownership survives leadership loss, but durable writes require a live fence. +pub trait RunLauncher { + fn poll( + &mut self, + store: &ControlStore, + lease: Option<&SchedulerLease>, + ) -> impl Future> + Send; + + fn shutdown( + &mut self, + store: &ControlStore, + lease: Option<&SchedulerLease>, + ) -> impl Future> + Send; +} + /// Expanded wall-clock times and the first omitted time, if the batch was bounded. pub(crate) type ScheduleExpansion = (Vec>, Option>); diff --git a/crates/sqlbuild-orchestration/src/store/main/list_recall_invocations.rs b/crates/sqlbuild-orchestration/src/store/main/list_recall_invocations.rs new file mode 100644 index 0000000000..ced2c62b2c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/list_recall_invocations.rs @@ -0,0 +1,62 @@ +//! Discover invocation namespaces from retained Recall facts, including output-only attempts. + +use sqlbuild_recall::schema::constants::INVOCATION_TABLES; +use sqlbuild_recall::schema::main::table_spec::table_spec; + +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::types::Row; + +/// The caller authorizes exact namespace roots; prefix matching escapes LIKE patterns and includes the slash boundary. +pub async fn list_recall_invocations( + store: &ControlStore, + namespaces: &[String], + limit: u32, +) -> Result, ControlError> { + if namespaces.is_empty() || limit == 0 { + return Ok(Vec::new()); + } + let mut parameters: Vec = Vec::new(); + let predicates = namespaces + .iter() + .map(|namespace| { + let exact = parameters.len() + 1; + parameters.push(Parameter::Text(namespace.clone())); + let pattern = namespace + .replace('!', "!!") + .replace('%', "!%") + .replace('_', "!_"); + parameters.push(Parameter::Text(format!("{pattern}/%"))); + format!( + "invocation_id=${exact} OR invocation_id LIKE ${} ESCAPE '!'", + exact + 1 + ) + }) + .collect::>() + .join(" OR "); + let tables = INVOCATION_TABLES + .iter() + .map(|table| { + format!( + "SELECT invocation_id, invocation_started_at FROM {} WHERE {predicates}", + table_spec(*table).name, + ) + }) + .collect::>() + .join(" UNION ALL "); + parameters.push(Parameter::Integer(i64::from(limit.min(1000)))); + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query( + transaction, + &format!( + "SELECT invocation_id, MIN(invocation_started_at) AS invocation_started_at \ + FROM ({tables}) AS records GROUP BY invocation_id ORDER BY invocation_id LIMIT ${}", + parameters.len() + ), + parameters, + ) + .await?; + commit(transaction).await?; + Ok(rows) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/mod.rs b/crates/sqlbuild-orchestration/src/store/main/mod.rs index de522badbc..715e39afaf 100644 --- a/crates/sqlbuild-orchestration/src/store/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/store/main/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod append_recall; pub(crate) mod begin; pub(crate) mod commit; pub(crate) mod decode_row; +pub mod list_recall_invocations; pub mod migrate_store; pub mod open_store; pub mod publish_recall; diff --git a/crates/sqlbuild-orchestration/tests/api/test_contract.rs b/crates/sqlbuild-orchestration/tests/api/test_contract.rs index 65ad349b97..9b4b904f29 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contract.rs @@ -183,7 +183,7 @@ fn given_a_real_http_server_when_locations_and_workers_report_then_scopes_retrie .await .expect("elect") .expect("leader"); - let claim = claim_next_run(store, &lease, &BTreeMap::new()) + let claim = claim_next_run(store, &lease, &BTreeMap::new(), None) .await .expect("claim") .expect("work"); diff --git a/crates/sqlbuild-orchestration/tests/api/test_reads.rs b/crates/sqlbuild-orchestration/tests/api/test_reads.rs index 6849aae676..598dbd7496 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_reads.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_reads.rs @@ -13,16 +13,24 @@ use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope, use sqlbuild_orchestration::declarations::main::change_trigger::change_trigger; use sqlbuild_orchestration::declarations::main::list_triggers::list_triggers; use sqlbuild_orchestration::declarations::models::{TriggerChange, TriggerChangeRequest}; +use sqlbuild_orchestration::runs::main::change_run::change_run; use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; use sqlbuild_orchestration::runs::main::request_run::request_run; +use sqlbuild_orchestration::runs::models::{RunAuthority, RunChange, RunChangeRequest, RunOutcome}; use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; use sqlbuild_orchestration::scheduler::main::evaluate_trigger::evaluate_trigger; use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use sqlbuild_orchestration::store::main::publish_recall::publish_recall; +use sqlbuild_orchestration::worker::main::connect_worker::connect_worker; +use sqlbuild_recall::records::main::output_row::output_row; +use sqlbuild_recall::records::models::{OutputStream, RowEnvelope}; use uuid::Uuid; use crate::helpers::{read_json, start_api, stop_api}; use crate::location_helpers::create_location; +use crate::output_helpers::chunk; +use crate::recall_helpers::recall_contract; use crate::run_helpers::{request, snapshot}; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; @@ -145,10 +153,15 @@ fn given_published_locations_and_scheduled_work_when_read_through_http_then_reso ) .await .expect("principal token"); - let claim = claim_next_run(store, &lease, &BTreeMap::new()) - .await - .expect("claim") - .expect("work"); + let claim = claim_next_run( + store, + &lease, + &BTreeMap::new(), + Some((location_id, "development")), + ) + .await + .expect("claim") + .expect("work"); let worker = issue_token( store, &audit, @@ -174,13 +187,123 @@ fn given_published_locations_and_scheduled_work_when_read_through_http_then_reso let trigger_path = format!("{prefix}/triggers"); let event_path = format!("{trigger_path}/{trigger_id}/events"); let tick_path = format!("{trigger_path}/{trigger_id}/ticks"); + let invocation_path = format!("{prefix}/runs/{}/invocations", claim.request.run_id); + let recall_path = format!( + "{prefix}/runs/{}/recall/events?invocation=worker/{}/execution", + claim.request.run_id, claim.claim_id + ); let paths = [ &run_path, &node_path, &trigger_path, &event_path, &tick_path, + &invocation_path, + &recall_path, ]; + let connection = connect_worker( + &api.url, + &worker.bearer, + (claim.request.run_id, claim.claim_id), + ) + .expect("worker HTTP connection"); + recall_contract(store, &connection, claim.claim_id).await; + let output = chunk(0, OutputStream::Stdout, b"orders worker output"); + connection + .publish_output(std::slice::from_ref(&output)) + .await + .expect("output-only namespace"); + let neighboring = RowEnvelope { + invocation_id: format!("worker/{}-other", claim.claim_id), + invocation_started_at: "2026-01-02T03:04:05Z".into(), + producer_version: "test".into(), + }; + publish_recall(store, &[output_row(&neighboring, &output)]) + .await + .expect("neighboring namespace"); + let invocations = read_json(&client, (&invocation_path, &token.bearer)).await; + let invocation_ids = invocations + .as_array() + .expect("run invocations") + .iter() + .map(|row| { + row["invocation_id"] + .as_str() + .expect("invocation ID") + .to_owned() + }) + .collect::>(); + assert_eq!( + invocation_ids, + vec![ + format!("scheduler/{}", claim.request.run_id), + format!("worker/{}", claim.claim_id), + format!("worker/{}/another-invocation", claim.claim_id), + format!("worker/{}/backlog", claim.claim_id), + format!("worker/{}/execution", claim.claim_id), + format!("worker/{}/native-writer", claim.claim_id), + ] + ); + let evidence = read_json(&client, (&recall_path, &token.bearer)).await; + assert_eq!(evidence.as_array().expect("worker events").len(), 1); + assert_eq!(evidence[0]["duration_ms"], json!(12.75)); + assert_eq!(evidence[0]["row_count"], json!(9)); + assert_eq!( + read_json(&client, (&format!("{recall_path}&limit=0"), &token.bearer)).await, + json!([]) + ); + for path in [ + recall_path.replace( + &format!("worker/{}/execution", claim.claim_id), + &format!("scheduler/{}", foreign_run.run_id), + ), + recall_path.replace( + &format!("worker/{}/execution", claim.claim_id), + &format!("worker/{}-other/execution", claim.claim_id), + ), + recall_path.replace( + &format!("/runs/{}/", claim.request.run_id), + &format!("/runs/{}/", manual_ids[0]), + ), + recall_path.replace( + &format!("/runs/{}/", claim.request.run_id), + &format!("/runs/{}/", foreign_run.run_id), + ), + recall_path.replace("/development/", "/validation/"), + ] { + assert_eq!( + client + .get(path) + .bearer_auth(&token.bearer) + .send() + .await + .expect("Recall scope denial") + .status(), + StatusCode::NOT_FOUND + ); + } + assert_eq!( + client + .get(recall_path.replace("/recall/events?", "/recall/unknown?")) + .bearer_auth(&token.bearer) + .send() + .await + .expect("unknown view") + .status(), + StatusCode::BAD_REQUEST + ); + let scheduler_path = format!( + "{prefix}/runs/{}/recall/events?invocation=scheduler/{}", + claim.request.run_id, claim.request.run_id + ); + let history = read_json(&client, (&scheduler_path, &token.bearer)).await; + assert!( + history + .as_array() + .expect("run history") + .iter() + .any(|row| row["event_type"] == "run_claimed") + ); let runs = read_json(&client, (&run_path, &token.bearer)).await; assert_eq!( runs.as_array().expect("runs").len(), @@ -214,7 +337,7 @@ fn given_published_locations_and_scheduled_work_when_read_through_http_then_reso assert_eq!(ticks[0]["outcome"], "requested"); assert_eq!(ticks[0]["scheduled_for"], "2020-01-03T00:00:00Z"); assert_eq!(ticks[1]["outcome"], "covered"); - for path in [&run_path, &event_path, &tick_path] { + for path in [&run_path, &event_path, &tick_path, &invocation_path] { assert_eq!( read_json(&client, (&format!("{path}?limit=1"), &token.bearer)) .await @@ -275,9 +398,14 @@ fn given_published_locations_and_scheduled_work_when_read_through_http_then_reso node_path.replace(&manual_ids[0].to_string(), &foreign_run.run_id.to_string()), event_path.replace(&trigger_id.to_string(), &foreign_trigger.to_string()), tick_path.replace(&trigger_id.to_string(), &foreign_trigger.to_string()), + invocation_path.replace( + &claim.request.run_id.to_string(), + &foreign_run.run_id.to_string(), + ), node_path.replace("/development/", "/validation/"), event_path.replace("/development/", "/validation/"), tick_path.replace("/development/", "/validation/"), + invocation_path.replace("/development/", "/validation/"), ] { assert_eq!( client @@ -322,6 +450,8 @@ fn given_published_locations_and_scheduled_work_when_read_through_http_then_reso for path in [ "/api/locations/{location_id}/targets/{target}/runs", "/api/locations/{location_id}/targets/{target}/runs/{run_id}/nodes", + "/api/locations/{location_id}/targets/{target}/runs/{run_id}/invocations", + "/api/locations/{location_id}/targets/{target}/runs/{run_id}/recall/{view}", "/api/locations/{location_id}/targets/{target}/triggers", "/api/locations/{location_id}/targets/{target}/triggers/{trigger_id}/events", "/api/locations/{location_id}/targets/{target}/triggers/{trigger_id}/ticks", @@ -345,6 +475,32 @@ fn given_published_locations_and_scheduled_work_when_read_through_http_then_reso events ); assert_eq!(read_json(&client, (&tick_path, &token.bearer)).await, ticks); + change_run( + store, + RunAuthority::Scheduler(&lease), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change: RunChange::Finished { + outcome: RunOutcome::Failed, + reason: "Stop scoped read fixture".into(), + exit_code: Some(1), + }, + }, + ) + .await + .expect("terminalize worker and release liveness"); + assert_eq!( + read_json(&client, (&invocation_path, &token.bearer)).await, + invocations, + "invocation discovery uses retained facts, not active liveness" + ); + assert_eq!( + read_json(&client, (&recall_path, &token.bearer)).await, + evidence, + "historical Recall remains authorized after the active claim is removed" + ); revoke_token(store, &audit, token.identity.token_id) .await .expect("revoke location"); diff --git a/crates/sqlbuild-orchestration/tests/api/test_worker.rs b/crates/sqlbuild-orchestration/tests/api/test_worker.rs index 8f4bf9e993..60d5932881 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_worker.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_worker.rs @@ -68,7 +68,7 @@ fn given_a_claimed_run_when_a_worker_connects_then_verification_precedes_start_a .await .expect("acquire") .expect("leader"); - let claim = claim_next_run(store, &lease, &BTreeMap::new()) + let claim = claim_next_run(store, &lease, &BTreeMap::new(), None) .await .expect("claim") .expect("work"); @@ -247,7 +247,7 @@ fn given_a_claimed_run_when_a_worker_connects_then_verification_precedes_start_a &request((location, publication.snapshot_id), 2, 0, &["orders"]), ) .await; - let cancelled_claim = claim_next_run(store, &lease, &BTreeMap::new()) + let cancelled_claim = claim_next_run(store, &lease, &BTreeMap::new(), None) .await .expect("claim another run") .expect("queued work"); diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs index 3c3c62f087..67526d5c57 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs @@ -191,7 +191,7 @@ fn given_scoped_tokens_when_revoked_or_the_run_ends_then_all_replicas_reject_the .await .expect("elect") .expect("leader"); - let claim = claim_next_run(store, &lease, &BTreeMap::new()) + let claim = claim_next_run(store, &lease, &BTreeMap::new(), None) .await .expect("claim") .expect("work"); diff --git a/crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs index 7f90e2f877..ec318744e7 100644 --- a/crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs @@ -20,7 +20,7 @@ pub(crate) async fn start_run( input: &RunRequest, ) -> RunClaim { queue(store, input).await; - let claim = claim_next_run(store, lease, &BTreeMap::new()) + let claim = claim_next_run(store, lease, &BTreeMap::new(), None) .await .expect("claim queued run") .expect("run available"); diff --git a/crates/sqlbuild-orchestration/tests/runs/test_claims.rs b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs index fdf5e34d4e..43619dfc46 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_claims.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs @@ -16,7 +16,103 @@ use crate::replica_helpers::second_store; use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; use crate::store_test_types::Backend; use crate::terminal_helpers::finish; -use crate::test_types::{BacklogClaimTestCase, GroupLimitTestCase, PriorityClaimsTestCase}; +use crate::test_types::{ + BacklogClaimTestCase, GroupLimitTestCase, PriorityClaimsTestCase, ScopedClaimTestCase, +}; + +#[test] +fn given_higher_priority_work_for_other_scopes_when_a_local_launcher_claims_then_only_its_location_and_target_are_reserved() + { + let test_cases = [ + ScopedClaimTestCase { + description: "SQLite local dispatch cannot reserve another location or target", + backend: Backend::Sqlite, + expected_selected_run: 1, + expected_unscoped_run: 3, + }, + ScopedClaimTestCase { + description: "PostgreSQL local dispatch cannot reserve another location or target", + backend: Backend::Postgres, + expected_selected_run: 1, + expected_unscoped_run: 3, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "inventory", "test-admin") + .await + .expect("other location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("snapshot"); + let other_published = publish_snapshot(store, &snapshot(other)) + .await + .expect("other location snapshot"); + let mut secondary = snapshot(location); + secondary.target = "secondary".into(); + let secondary = publish_snapshot(store, &secondary) + .await + .expect("other target snapshot"); + let wanted = request((location, published.snapshot_id), 1, 0, &["orders"]); + let mut other_target = request((location, secondary.snapshot_id), 2, 10, &["orders"]); + other_target.target = "secondary".into(); + let other_location = request((other, other_published.snapshot_id), 3, 20, &["orders"]); + for input in [&wanted, &other_target, &other_location] { + queue(store, input).await; + } + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let claim = claim_next_run( + store, + &lease, + &BTreeMap::new(), + Some((location, "development")), + ) + .await + .expect("scoped claim") + .expect("matching work"); + assert_eq!( + claim.request.run_id, + Uuid::from_u128(test_case.expected_selected_run), + "{}", + test_case.description + ); + assert!( + claim_next_run( + store, + &lease, + &BTreeMap::new(), + Some((location, "development")) + ) + .await + .expect("scope exhausted") + .is_none() + ); + finish(store, &lease, &claim).await; + let next = claim_next_run(store, &lease, &BTreeMap::new(), None) + .await + .expect("unrestricted claim") + .expect("other work was not reserved"); + assert_eq!( + next.request.run_id, + Uuid::from_u128(test_case.expected_unscoped_run), + "{}", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} #[test] fn given_overlapping_priority_queues_when_two_launchers_claim_then_conflicts_rollback_and_terminal_facts_release_claims() @@ -65,8 +161,8 @@ fn given_overlapping_priority_queues_when_two_launchers_claim_then_conflicts_rol let second = second_store(store).await; let limits = BTreeMap::new(); let (first, other) = tokio::join!( - claim_next_run(store, &lease, &limits), - claim_next_run(&second, &lease, &limits) + claim_next_run(store, &lease, &limits, None), + claim_next_run(&second, &lease, &limits, None) ); let first = first.expect("first claimant").expect("claim available"); let other = other @@ -87,14 +183,14 @@ fn given_overlapping_priority_queues_when_two_launchers_claim_then_conflicts_rol test_case.description ); assert!( - claim_next_run(store, &lease, &limits) + claim_next_run(store, &lease, &limits, None) .await .expect("blocked run stays queued") .is_none() ); queue(store, &request(scope, 4, 9, &["customers", "orders"])).await; queue(store, &request(scope, 5, 0, &["customers"])).await; - let independent = claim_next_run(store, &lease, &limits) + let independent = claim_next_run(store, &lease, &limits, None) .await .expect("try partially conflicting candidate") .expect("independent work proceeds"); @@ -109,7 +205,7 @@ fn given_overlapping_priority_queues_when_two_launchers_claim_then_conflicts_rol .expect("orders owner"); finish(store, &lease, orders).await; finish(store, &lease, &independent).await; - let next = claim_next_run(store, &lease, &limits) + let next = claim_next_run(store, &lease, &limits, None) .await .expect("terminal cleanup committed") .expect("next priority run claims both nodes"); @@ -193,23 +289,23 @@ fn given_named_group_limits_when_capacity_is_lowered_then_active_high_slots_stil queue(store, &input).await; } assert!( - claim_next_run(store, &lease, &BTreeMap::new()) + claim_next_run(store, &lease, &BTreeMap::new(), None) .await .is_err(), "unknown groups cannot silently become unlimited" ); - let first = claim_next_run(store, &lease, &limits) + let first = claim_next_run(store, &lease, &limits, None) .await .expect("first slot") .expect("available"); - let second = claim_next_run(store, &lease, &limits) + let second = claim_next_run(store, &lease, &limits, None) .await .expect("second slot") .expect("available"); assert_eq!(first.request.run_id, Uuid::from_u128(1)); assert_eq!(second.request.run_id, Uuid::from_u128(2)); assert!( - claim_next_run(store, &lease, &limits) + claim_next_run(store, &lease, &limits, None) .await .expect("group full") .is_none() @@ -220,13 +316,13 @@ fn given_named_group_limits_when_capacity_is_lowered_then_active_high_slots_stil NonZeroU32::new(1).expect("nonzero lowered limit"), ); assert!( - claim_next_run(store, &lease, &limits) + claim_next_run(store, &lease, &limits, None) .await .expect("high slot counts after limit reduction") .is_none() ); finish(store, &lease, &second).await; - let third = claim_next_run(store, &lease, &limits) + let third = claim_next_run(store, &lease, &limits, None) .await .expect("released group") .expect("third queued run"); @@ -276,7 +372,7 @@ fn given_more_than_one_page_of_blocked_runs_when_claiming_then_lower_priority_in .expect("leader"); let limits = BTreeMap::new(); queue(store, &request(scope, 1, 0, &["orders"])).await; - claim_next_run(store, &lease, &limits) + claim_next_run(store, &lease, &limits, None) .await .expect("claim blocker") .expect("available"); @@ -292,7 +388,7 @@ fn given_more_than_one_page_of_blocked_runs_when_claiming_then_lower_priority_in } queue(store, &request(scope, 105, -1, &["customers"])).await; let started = std::time::Instant::now(); - let independent = claim_next_run(store, &lease, &limits) + let independent = claim_next_run(store, &lease, &limits, None) .await .expect("scan past a full blocked page") .expect("independent work"); diff --git a/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs b/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs index b38db00540..3149b775d9 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs @@ -63,7 +63,7 @@ fn given_coalescing_triggers_when_a_previous_run_has_been_claimed_then_new_ticks evaluate_trigger(store, &lease, trigger) .await .expect("first tick"); - let claim = claim_next_run(store, &lease, &BTreeMap::new()) + let claim = claim_next_run(store, &lease, &BTreeMap::new(), None) .await .expect("claim first run") .expect("scheduled run"); diff --git a/crates/sqlbuild-orchestration/tests/runs/test_faults.rs b/crates/sqlbuild-orchestration/tests/runs/test_faults.rs index d32ffd0472..00d20c38c3 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_faults.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_faults.rs @@ -88,7 +88,7 @@ fn given_failures_during_claim_publication_when_retried_then_claims_slots_livene }; install_fault(store, &point).await; assert!( - claim_next_run(store, &lease, &limits).await.is_err(), + claim_next_run(store, &lease, &limits, None).await.is_err(), "{} at {table}", point.description ); @@ -117,14 +117,14 @@ fn given_failures_during_claim_publication_when_retried_then_claims_slots_livene install_fault(store, &expiry).await; assert!( matches!( - claim_next_run(store, &lease, &limits).await, + claim_next_run(store, &lease, &limits, None).await, Err(ControlError::Fenced) ), "{}", expiry.description ); remove_fault(store, &expiry).await; - let claim = claim_next_run(store, &lease, &limits) + let claim = claim_next_run(store, &lease, &limits, None) .await .expect("retry complete claim") .expect("no leaked claims or slots"); @@ -136,7 +136,7 @@ fn given_failures_during_claim_publication_when_retried_then_claims_slots_livene test_case.description ); assert!( - claim_next_run(store, &lease, &limits) + claim_next_run(store, &lease, &limits, None) .await .expect("already claimed run is not relaunched") .is_none() diff --git a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs index 8dca64c004..5c23383e12 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs @@ -58,7 +58,7 @@ fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_ queue(store, &request(scope, 1, 2, &["customers", "orders"])).await; queue(store, &request(scope, 2, 1, &["customers", "orders"])).await; let limits = BTreeMap::new(); - let claim = claim_next_run(store, &lease, &limits) + let claim = claim_next_run(store, &lease, &limits, None) .await .expect("claim") .expect("available"); @@ -246,7 +246,7 @@ fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_ before ); assert!( - claim_next_run(store, &lease, &limits) + claim_next_run(store, &lease, &limits, None) .await .expect("claims retained") .is_none() @@ -292,7 +292,7 @@ fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_ .expect("heartbeat reports pending cancellation") ); assert!( - claim_next_run(store, &lease, &limits) + claim_next_run(store, &lease, &limits, None) .await .expect("cancel request does not release an executing run's claims") .is_none() @@ -345,7 +345,7 @@ fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_ assert_eq!(nodes[0].status, "success"); assert_eq!(nodes[0].rows_affected, Some(17)); assert_eq!(nodes[1].error_code.as_deref(), Some("E100")); - let following = claim_next_run(store, &lease, &limits) + let following = claim_next_run(store, &lease, &limits, None) .await .expect("claim following run") .expect("released capacity"); @@ -472,7 +472,7 @@ fn given_unclaimed_work_when_an_operator_cancels_then_a_terminal_fact_prevents_l .expect("elect") .expect("leader"); assert!( - claim_next_run(store, &lease, &BTreeMap::new()) + claim_next_run(store, &lease, &BTreeMap::new(), None) .await .expect("cancelled work cannot launch") .is_none() diff --git a/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs b/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs index e567d9d25b..228dd090a2 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs @@ -80,7 +80,7 @@ fn given_silent_and_live_workers_when_the_scheduler_monitors_then_fenced_atomic_ start_run(store, &lease, &request(scope, 2, 0, &["customers"])).await; age_heartbeat(store, Uuid::from_u128(2), 40).await; queue(store, &request(scope, 3, 0, &["inventory"])).await; - let launching = claim_next_run(store, &lease, &BTreeMap::new()) + let launching = claim_next_run(store, &lease, &BTreeMap::new(), None) .await .expect("claim") .expect("launching"); @@ -247,7 +247,7 @@ fn given_silent_and_live_workers_when_the_scheduler_monitors_then_fenced_atomic_ .expect("takeover") .expect("successor"); assert!( - claim_next_run(store, &successor, &BTreeMap::new()) + claim_next_run(store, &successor, &BTreeMap::new(), None) .await .expect("write claims remain held after heartbeat loss and takeover") .is_none() diff --git a/crates/sqlbuild-orchestration/tests/runs/test_requests.rs b/crates/sqlbuild-orchestration/tests/runs/test_requests.rs index 0cf4c4b234..226cb57a5b 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_requests.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_requests.rs @@ -162,7 +162,7 @@ fn given_a_published_job_when_requested_and_retried_after_redeploy_then_audit_an "orders-load".into(), std::num::NonZeroU32::new(1).expect("positive group limit"), )]); - let claim = claim_next_run(store, &lease, &limits) + let claim = claim_next_run(store, &lease, &limits, None) .await .expect("claim") .expect("queued request"); diff --git a/crates/sqlbuild-orchestration/tests/runs/test_types.rs b/crates/sqlbuild-orchestration/tests/runs/test_types.rs index f0112b2903..0a21b7f2ef 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_types.rs @@ -5,6 +5,13 @@ use sqlbuild_orchestration::runs::models::RunStatus; pub(crate) use crate::fault_types::TransactionFault; use crate::store_test_types::Backend; +pub(crate) struct ScopedClaimTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_selected_run: u128, + pub(crate) expected_unscoped_run: u128, +} + pub(crate) struct PriorityClaimsTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs index fbe1a63283..e62bf51595 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs @@ -15,7 +15,7 @@ use crate::replica_helpers::second_store; use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; use crate::store_test_types::Backend; use crate::ticks::helpers::snapshot; -use crate::ticks::test_types::TickTransactionTestCase; +use crate::ticks::test_types::{IdleLauncher, TickTransactionTestCase}; #[test] fn given_embedded_and_standalone_schedulers_when_taking_over_then_only_one_drains_due_work_and_idle_shutdown_is_prompt() @@ -85,7 +85,12 @@ fn given_embedded_and_standalone_schedulers_when_taking_over_then_only_one_drain assert_eq!(runs.len(), test_case.expected_runs); let (stop, shutdown) = tokio::sync::watch::channel(false); let (_changed, changes) = tokio::sync::watch::channel(()); - let mut running = Box::pin(run_scheduler(&second, second_state, shutdown, changes)); + let mut running = Box::pin(run_scheduler( + &second, + second_state, + (shutdown, changes), + IdleLauncher, + )); tokio::select! { result = &mut running => panic!("loop ended before shutdown: {result:?}"), _ = tokio::time::sleep(Duration::from_millis(50)) => {}, diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs index 95953810a5..5f10deea0d 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs @@ -15,3 +15,23 @@ pub(crate) struct TriggerControlsTestCase { pub(crate) backend: Backend, pub(crate) expected_events: usize, } + +pub(crate) struct IdleLauncher; + +impl sqlbuild_orchestration::scheduler::types::RunLauncher for IdleLauncher { + async fn poll( + &mut self, + _store: &sqlbuild_orchestration::store::models::ControlStore, + _lease: Option<&sqlbuild_orchestration::scheduler::models::SchedulerLease>, + ) -> Result<(), sqlbuild_orchestration::store::errors::ControlError> { + Ok(()) + } + + async fn shutdown( + &mut self, + _store: &sqlbuild_orchestration::store::models::ControlStore, + _lease: Option<&sqlbuild_orchestration::scheduler::models::SchedulerLease>, + ) -> Result<(), sqlbuild_orchestration::store::errors::ControlError> { + Ok(()) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/boundary/config_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/boundary/config_values.rs index 5f2f304c6b..0a444023a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/boundary/config_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/boundary/config_values.rs @@ -104,3 +104,35 @@ pub(crate) fn config_value_to_python(py: Python<'_>, value: ConfigValue) -> PyRe } } } + +/// JSON-backed configuration keeps integer widths and recursively constructs native Python values. +pub(crate) fn json_value_to_python<'py>( + py: Python<'py>, + value: &serde_json::Value, +) -> PyResult> { + use serde_json::Value; + match value { + Value::Null => Ok(py.None().into_bound(py)), + Value::Bool(value) => value.into_bound_py_any(py), + Value::String(value) => value.into_bound_py_any(py), + Value::Number(value) => match (value.as_i64(), value.as_u64()) { + (Some(value), _) => value.into_bound_py_any(py), + (None, Some(value)) => value.into_bound_py_any(py), + (None, None) => value.as_f64().unwrap_or(f64::NAN).into_bound_py_any(py), + }, + Value::Array(items) => { + let values = items + .iter() + .map(|item| json_value_to_python(py, item)) + .collect::>>()?; + Ok(PyList::new(py, values)?.into_any()) + } + Value::Object(entries) => { + let table = PyDict::new(py); + for (key, item) in entries { + table.set_item(key, json_value_to_python(py, item)?)?; + } + Ok(table.into_any()) + } + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs index 8dbd45566b..3f45271352 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs @@ -1,5 +1,6 @@ //! `sqb build`: resolve, compile, gate, execute and report one direct build. +use std::sync::Arc; use std::time::Instant; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -17,6 +18,7 @@ use crate::bindings::_helpers::build_command::planning::{ compile_build_plan, enforce_build_plan_policies, run_defer_clone_prephase, write_compile_target, write_plan_text, }; +use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; use crate::bindings::_helpers::diagnostics::trackers::BuildPhaseTimingTracker; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::fields::field as native_field; @@ -38,6 +40,14 @@ fn run_with_providers(context: &BuildContext<'_>) -> PyResult { } fn run_planned_build(context: &BuildContext<'_>, providers: &Bound<'_, PyAny>) -> PyResult { + if let Some(worker) = &context.worker { + worker.check_cancelled(context.py())?; + if !context.get("effective_defer_clone_from")?.is_none() { + return Err(pyo3::exceptions::PyValueError::new_err( + "worker defer-clone requires claimed boundary writers and is not yet supported", + )); + } + } run_defer_clone_prephase(context)?; let cache_scope: Bound<'_, PyAny> = call_with( &host::entry(context.py(), AdapterHost::RelationMetadataCacheScope)?, @@ -49,6 +59,9 @@ fn run_planned_build(context: &BuildContext<'_>, providers: &Bound<'_, PyAny>) - /// Plan and run the build, sharing one relation metadata cache from planning to the run's end. fn run_cached_build(context: &BuildContext<'_>, providers: &Bound<'_, PyAny>) -> PyResult { let pipeline: Bound<'_, PyAny> = compile_build_plan(context)?; + if let Some(worker) = &context.worker { + worker.enforce_plan_writers(&pipeline)?; + } write_plan_text(context, &pipeline)?; enforce_build_plan_policies(context, &pipeline)?; write_compile_target(context, &pipeline)?; @@ -74,9 +87,11 @@ fn run_within_limits( request: &Bound<'_, PyAny>, tracker: &Bound<'_, PyAny>, started: Instant, + worker: Option>, ) -> PyResult { let py: Python<'_> = request.py(); - let context: BuildContext<'_> = resolve_build_context(request, tracker, started)?; + let mut context: BuildContext<'_> = resolve_build_context(request, tracker, started)?; + context.worker = worker; let limits: Bound<'_, PyAny> = context.get("execution_limits")?; let deadline: Bound<'_, PyAny> = call_with( &host::entry(py, ExecutorHost::ExecutionDeadlineScope)?, @@ -97,6 +112,15 @@ fn run_within_limits( #[pyfunction] #[pyo3(signature = (*, request))] pub(crate) fn run_build_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { + run_build(py, request, None) +} + +/// A claim owner survives planning, execution and connection cleanup before terminal reporting. +pub(crate) fn run_build( + py: Python<'_>, + request: &Bound<'_, PyAny>, + worker: Option>, +) -> PyResult { let tracker: Bound<'_, PyAny> = BuildPhaseTimingTracker::new(py, None) .and_then(|value| pyo3::Bound::new(py, value)) .map(pyo3::Bound::into_any)?; @@ -107,7 +131,7 @@ pub(crate) fn run_build_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> P within(&reporting, |_| { within(&tracker.call_method0("scope")?, |_| { let started: Instant = Instant::now(); - run_within_limits(request, &tracker, started).inspect_err(|_| { + run_within_limits(request, &tracker, started, worker.clone()).inspect_err(|_| { let verbose: bool = native_field(request, "verbose") .and_then(|value| value.is_truthy()) .unwrap_or(false) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index f0a0f55d12..41e53c84da 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -225,6 +225,10 @@ pub(crate) fn prepare_build_execution<'py>( ], )?), ) + .map(|mut value| { + value.worker = context.worker.clone(); + value + }) .and_then(|value| pyo3::Bound::new(py, value)) .map(pyo3::Bound::into_any)?; let concurrency = context.concurrency()?.limit; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs index 1f1b394cb0..a2a5d45df6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs @@ -1,5 +1,6 @@ //! Resolve discovery, target, adapter, limits, connection and reporters for one build. +use std::sync::Arc; use std::time::Instant; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; @@ -19,6 +20,7 @@ use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_settings_config; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_name; @@ -38,6 +40,7 @@ pub(crate) struct BuildContext<'py> { pub(crate) verbose: bool, pub(crate) started: Instant, pub(crate) tracker: Bound<'py, PyAny>, + pub(crate) worker: Option>, } impl<'py> BuildContext<'py> { @@ -241,5 +244,6 @@ pub(crate) fn resolve_build_context<'py>( verbose: native_field(request, "verbose")?.is_truthy()? || debug, started, tracker: tracker.clone(), + worker: None, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs index ec6bbad153..62e6f13740 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs @@ -1,7 +1,7 @@ //! Live progress rows, completion rows and runtime diagnostics of `sqb build`. use std::collections::HashMap; -use std::sync::{Mutex, MutexGuard, PoisonError}; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; use std::time::Instant; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -31,6 +31,7 @@ use crate::bindings::_helpers::build_progress::terminal::{ consume_terminal, flush, join_spinner, projector, release, shared_lines, start_spinner, terminal_columns, tty_flag, write, }; +use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::replaced_keywords; use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; @@ -85,11 +86,8 @@ fn sql_test_resource_id(result: &Bound<'_, PyAny>) -> PyResult { } /// The resource name and projector identity of a completion. -fn resource_identity( - result: &Bound<'_, PyAny>, - kind: ResultKind, -) -> PyResult> { - Ok(match kind { +pub(crate) fn resource_identity(result: &Bound<'_, PyAny>) -> PyResult> { + Ok(match result_kind(result)? { ResultKind::Model => { let name: String = text(result, "model_name")?; Some((name.clone(), format!("model:{name}"))) @@ -141,6 +139,7 @@ struct CallbackState { /// Encapsulates live build progress output state and callbacks. #[pyclass(module = "sqlbuild._native", frozen)] pub(crate) struct BuildProgressCallbacks { + pub(crate) worker: Option>, plan: Py, stream: Py, lines: Py, @@ -453,6 +452,7 @@ impl BuildProgressCallbacks { } let lines: Bound<'_, PyAny> = shared_lines(py)?; Ok(Self { + worker: None, plan: plan.unbind(), is_tty: tty_flag(&stdout)? && !debug, stream: stream.unbind(), @@ -484,6 +484,9 @@ impl BuildProgressCallbacks { ) -> PyResult<()> { let py: Python<'_> = slf.py(); let callbacks: &Self = slf.get(); + if let Some(worker) = &callbacks.worker { + worker.start(py, &name, resource_kind)?; + } if let Some(projector) = &callbacks.projector { let options: Bound<'_, PyDict> = PyDict::new(py); options.set_item("resource_name", name)?; @@ -579,10 +582,15 @@ impl BuildProgressCallbacks { fn on_node_complete(slf: &Bound<'_, Self>, node_result: &Bound<'_, PyAny>) -> PyResult<()> { let py: Python<'_> = slf.py(); let callbacks: &Self = slf.get(); + if let Some(worker) = &callbacks.worker + && let Some((_, key)) = resource_identity(node_result)? + { + worker.completed(node_result, &key)?; + } let kind: ResultKind = result_kind(node_result)?; let mut result: Bound<'_, PyAny> = node_result.clone(); if let Some(projector) = &callbacks.projector - && let Some((name, identity)) = resource_identity(node_result, kind)? + && let Some((name, identity)) = resource_identity(node_result)? { let Some(duration_ms) = consume_terminal(projector.bind(py), &name, Some(&identity))? else { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index 1f05467222..17c4dea5e1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -23,12 +23,13 @@ use crate::bindings::_helpers::init_command::command::run_init_command; use crate::bindings::_helpers::janitor::janitor_command::run_janitor_command; use crate::bindings::_helpers::lineage::lineage_command::run_lineage_command; use crate::bindings::_helpers::load::load_command::run_load_command; -use crate::bindings::_helpers::location_command::command::run_location_command; +use crate::bindings::_helpers::location_command::command::run_location; use crate::bindings::_helpers::plan_command::plan::run_plan_command; use crate::bindings::_helpers::plan_command::rules::run_rules_command; use crate::bindings::_helpers::query_command::dag::run_dag_command; use crate::bindings::_helpers::query_command::query::run_query_command; use crate::bindings::_helpers::recall::command::run_recall_command; +use crate::bindings::_helpers::recall::session::InvocationRecall; use crate::bindings::_helpers::records::construct::{new_record_with, record_class}; use crate::bindings::_helpers::refactor::refactor_command::run_refactor_command; use crate::bindings::_helpers::review_command::command::run_review_command; @@ -313,6 +314,7 @@ fn call_route<'py>( invocation: &Invocation<'py>, route: &Route, handler: Option>, + recall: &InvocationRecall, ) -> PyResult> { let py: Python<'py> = invocation.py(); let mut positional: Vec> = Vec::new(); @@ -340,7 +342,7 @@ fn call_route<'py>( } match handler { Some(handler) => handler.call(PyTuple::new(py, positional)?, Some(&keywords)), - None => run_implementation(route.implementation, (positional, keywords)), + None => run_implementation(route.implementation, (positional, keywords), recall), } } @@ -359,6 +361,7 @@ fn argument<'py>( fn run_implementation<'py>( name: &str, arguments: (Vec>, Bound<'py, PyDict>), + recall: &InvocationRecall, ) -> PyResult> { let py: Python<'py> = arguments.1.py(); let request = || -> PyResult> { @@ -388,7 +391,7 @@ fn run_implementation<'py>( "run_janitor_command" => run_janitor_command(py, &request()?)?, "run_lineage_command" => run_lineage_command(py, &request()?)?, "run_load_command" => run_load_command(py, &request()?)?, - "run_location_command" => run_location_command(py, options())?, + "run_location_command" => run_location(py, options(), Some(recall))?, "run_plan_command" => run_plan_command(&request()?)?, "run_playground_command" => run_playground_command(py, &request()?)?, "run_query_command" => run_query_command(py, options())?, @@ -419,6 +422,7 @@ fn run_implementation<'py>( fn routed_command<'py>( args: &Bound<'py, PyAny>, handlers: Option<&Bound<'py, PyAny>>, + recall: &InvocationRecall, ) -> PyResult> { let py: Python<'py> = args.py(); let raw_project_dir: Bound<'py, PyAny> = native_field(args, "project_dir")?; @@ -458,7 +462,7 @@ fn routed_command<'py>( if let Some(failure) = failure { return Err(user_error(py, failure)); } - call_route(&invocation, route, handler) + call_route(&invocation, route, handler, recall) } } } @@ -467,6 +471,7 @@ fn routed_command<'py>( pub(crate) fn dispatch_command<'py>( args: &Bound<'py, PyAny>, handlers: Option>, + recall: &InvocationRecall, ) -> PyResult> { let py: Python<'py> = args.py(); let warehouse: Bound<'py, PyAny> = native_field(args, "warehouse")?; @@ -495,7 +500,7 @@ pub(crate) fn dispatch_command<'py>( )?, )? .into_any(); - within(&scope, |_| routed_command(args, handlers.as_ref())) + within(&scope, |_| routed_command(args, handlers.as_ref(), recall)) } /// The parsed values that select a route, read from the namespace. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index 8abf67a87f..4b4c372cee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -17,6 +17,7 @@ use sqlbuild_cli::dispatch::main::invocation_policy::invocation_policy; use sqlbuild_cli::dispatch::models::{CommandFacts, CommandFailure, InvocationPolicy}; use sqlbuild_host::entries::models::{ExecutorHost, FacadeHost}; +use sqlbuild_orchestration::runs::models::RunOutcome; use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, keywords, within}; use crate::bindings::_helpers::build_progress::transient_lines::shared_transient_line_coordinator; @@ -545,7 +546,11 @@ fn observed_dispatch<'py>( if let Some(session) = &invocation.recall.session { session.publish_context(false); } - return dispatch_command(&invocation.args, invocation.handlers.clone()); + return dispatch_command( + &invocation.args, + invocation.handlers.clone(), + &invocation.recall, + ); } let command: String = invocation.command_name(); drop(take_failure_cause()); @@ -601,11 +606,14 @@ fn observed_dispatch<'py>( "invocation_started", started_payload()?, )?)?; - let dispatched: PyResult> = - match dispatch_command(&invocation.args, invocation.handlers.clone()) { - Ok(_) if terminated() => Err(CommandTerminated::new_err(TERMINATED_MESSAGE)), - dispatched => dispatched, - }; + let dispatched: PyResult> = match dispatch_command( + &invocation.args, + invocation.handlers.clone(), + &invocation.recall, + ) { + Ok(_) if terminated() => Err(CommandTerminated::new_err(TERMINATED_MESSAGE)), + dispatched => dispatched, + }; match dispatched { Ok(exit_code) => { let payload: Bound<'py, PyDict> = PyDict::new(py); @@ -864,6 +872,30 @@ fn execute_parsed_invocation<'py>( Ok(_) | Err(_) => None, }; let finished: PyResult<()> = finish_invocation_recall(py, &recall, exit_status); + let worker = recall + .worker + .lock() + .map_err(|_| { + pyo3::exceptions::PyRuntimeError::new_err("worker completion lock is unavailable") + })? + .take(); + if let Some(worker) = worker { + let code = match &finished { + Ok(()) => exit_status.unwrap_or(1), + Err(_) => 1, + }; + let outcome = worker.finish(py, i32::try_from(code).unwrap_or(1))?; + host::write_console( + py, + "stderr", + &format!("Worker execution completed: {}\n", outcome.as_str()), + )?; + if !matches!(outcome, RunOutcome::Success | RunOutcome::Warn) { + dispatched?; + finished?; + return exit(py, code.max(1)); + } + } let exit_code: Bound<'py, PyAny> = dispatched?; finished?; Ok(exit_code) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs new file mode 100644 index 0000000000..a472d4d696 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs @@ -0,0 +1,5 @@ +//! Frozen execution protocol values and bounded acknowledgement retries. + +pub(crate) const SKIP_ACTION: &str = "skip"; +pub(crate) const NO_CHANGE_REASON: &str = "no_change"; +pub(crate) const REPORT_ATTEMPTS: usize = 3; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs index 7ae7f4d521..2101d9ab7d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod attachment_inputs; +pub(crate) mod constants; pub(crate) mod project; pub(crate) mod registration; pub(crate) mod rows; @@ -6,6 +7,7 @@ pub(crate) mod schema_facts; pub(crate) mod snapshot; pub(crate) mod snapshot_boundary; pub(crate) mod snapshot_declarations; +pub(crate) mod snapshot_execution; pub(crate) mod snapshot_models; pub(crate) mod snapshot_resources; pub(crate) mod snapshot_worker; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs new file mode 100644 index 0000000000..57d0f43e90 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs @@ -0,0 +1,247 @@ +//! Frozen selection, executor callbacks and completion belong to one verified worker build. + +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; + +use pyo3::exceptions::{PyKeyboardInterrupt, PyRuntimeError, PyValueError}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyStringMethods; +use sqlbuild_core::models::ResourceKind; +use sqlbuild_orchestration::runs::models::{NodeOutcome, RunChange, RunExecution, RunOutcome}; + +use crate::bindings::_helpers::compiled_project::constants::{NO_CHANGE_REASON, SKIP_ACTION}; +use crate::bindings::_helpers::compiled_project::snapshot_worker::NativeOrchestrationWorker; +use crate::bindings::_helpers::records::fields::field; + +/// This transient owner records only acknowledged outcomes; durable lifecycle remains on the API. +#[derive(Debug)] +pub(crate) struct WorkerBuild { + pub(crate) worker: Py, + execution: RunExecution, + completed: Mutex>, +} + +impl WorkerBuild { + pub(crate) fn new(worker: &Bound<'_, NativeOrchestrationWorker>) -> PyResult> { + Ok(Arc::new(Self { + execution: worker.get().execution(worker.py())?, + worker: worker.clone().unbind(), + completed: Mutex::default(), + })) + } + + pub(crate) fn check_cancelled(&self, py: Python<'_>) -> PyResult<()> { + if self.worker.bind(py).get().cancel_requested(py) { + return Err(PyKeyboardInterrupt::new_err("run cancellation requested")); + } + Ok(()) + } + + /// Reject an implicitly expanded write set before any executor connection or schema change. + pub(crate) fn enforce_plan_writers( + &self, + pipeline: &Bound<'_, PyAny>, + ) -> Result<(), pyo3::PyErr> { + self.check_cancelled(pipeline.py())?; + let plan = field(pipeline, "plan_output")?; + for collection in [ + "model_entries", + "seed_entries", + "function_entries", + "source_load_entries", + ] { + for entry in field(&plan, collection)?.try_iter()? { + let entry = entry?; + let identity = field(&entry, "key")?; + let kind = field(&identity, "resource_type")? + .str()? + .to_str()? + .to_owned(); + let key = format!("{kind}:{}", field(&identity, "name")?.str()?); + if !self.execution.claim.request.write_set.contains(&key) { + return Err(PyValueError::new_err(format!( + "planned writer {key} is outside the frozen run write set" + ))); + } + if kind == ResourceKind::Model.as_str() + && field(&entry, "action")?.str()?.to_str()? == SKIP_ACTION + { + let outcome = if field(&entry, "reason")?.str()?.to_str()? == NO_CHANGE_REASON { + NodeOutcome::Success + } else { + NodeOutcome::Skipped + }; + self.complete(pipeline.py(), &key, (outcome, None, None))?; + } + } + } + Ok(()) + } + + fn complete( + &self, + py: Python<'_>, + key: &str, + (outcome, rows_affected, error_code): (NodeOutcome, Option, Option), + ) -> PyResult<()> { + if !self.execution.claim.request.selection.contains(key) { + return Ok(()); + } + if self + .completed + .lock() + .map_err(|_| poisoned())? + .contains_key(key) + { + return Ok(()); + } + self.worker.bind(py).get().change( + py, + &format!("node-finished/{key}"), + RunChange::NodeFinished { + node_key: key.into(), + outcome, + rows_affected, + error_code, + skipped_because_of: None, + }, + )?; + self.completed + .lock() + .map_err(|_| poisoned())? + .insert(key.into(), outcome); + Ok(()) + } + + pub(crate) fn start( + &self, + py: Python<'_>, + name: &str, + resource_kind: &Bound<'_, PyAny>, + ) -> PyResult<()> { + self.check_cancelled(py)?; + let kind = field(resource_kind, "value")?.extract::()?; + let kind = match kind.as_str() { + "table" | "view" | "snapshot" | "custom" => "model", + kind => kind, + }; + let key = format!("{kind}:{name}"); + if self.execution.claim.request.selection.contains(&key) { + self.worker.bind(py).get().change( + py, + &format!("node-started/{key}"), + RunChange::NodeStarted { node_key: key }, + )?; + } + Ok(()) + } + + /// Existing native progress callbacks publish selected-node evidence without Python relays. + pub(crate) fn completed(&self, result: &Bound<'_, PyAny>, key: &str) -> PyResult<()> { + if self.execution.claim.request.selection.contains(key) { + let status: String = field(result, "status")?.str()?.extract()?; + let outcome = match status.as_str() { + "success" => { + if result.hasattr("warning_messages")? + && field(result, "warning_messages")?.is_truthy()? + { + NodeOutcome::Warn + } else { + NodeOutcome::Success + } + } + "failed" => { + if self + .worker + .bind(result.py()) + .get() + .refresh_cancellation(result.py())? + { + NodeOutcome::Cancelled + } else { + NodeOutcome::Failed + } + } + "skipped" => NodeOutcome::Skipped, + _ => { + return Err(PyValueError::new_err(format!( + "unsupported worker node outcome {status}" + ))); + } + }; + let rows = match result.getattr_opt("rows_affected")? { + Some(value) => value.extract()?, + None => None, + }; + let error = match result.getattr_opt("error_code")? { + Some(value) => value.extract()?, + None => None, + }; + self.complete(result.py(), key, (outcome, rows, error))?; + } + Ok(()) + } + + /// Never call missing selected nodes successful merely because the executor returned zero. + pub(crate) fn outcome(&self, py: Python<'_>, exit_code: i32) -> PyResult { + let cancelled = self.worker.bind(py).get().refresh_cancellation(py)?; + for key in &self.execution.claim.request.selection { + self.complete( + py, + key, + ( + if cancelled { + NodeOutcome::Cancelled + } else { + NodeOutcome::Skipped + }, + None, + None, + ), + )?; + } + let outcome = { + let completed = self.completed.lock().map_err(|_| poisoned())?; + let good = completed + .values() + .any(|value| matches!(value, NodeOutcome::Success | NodeOutcome::Warn)); + let bad = completed.values().any(|value| { + matches!( + value, + NodeOutcome::Failed | NodeOutcome::Skipped | NodeOutcome::Cancelled + ) + }); + if cancelled { + RunOutcome::Cancelled + } else if bad && good { + RunOutcome::Partial + } else if bad || exit_code != 0 { + RunOutcome::Failed + } else if completed.values().any(|value| *value == NodeOutcome::Warn) { + RunOutcome::Warn + } else { + RunOutcome::Success + } + }; + Ok(outcome) + } + + /// Terminal publication follows executor teardown and the invocation's bounded Recall drain. + pub(crate) fn finish(&self, py: Python<'_>, exit_code: i32) -> PyResult { + let outcome = self.outcome(py, exit_code)?; + self.worker.bind(py).get().change( + py, + "worker-finished", + RunChange::Finished { + outcome, + reason: "worker executor completed and closed its connections".into(), + exit_code: Some(i64::from(exit_code)), + }, + )?; + Ok(outcome) + } +} + +fn poisoned() -> pyo3::PyErr { + PyRuntimeError::new_err("worker build outcome lock is unavailable") +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_worker.rs index 966c3709e2..5c5da15456 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_worker.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_worker.rs @@ -11,6 +11,7 @@ use sqlbuild_orchestration::worker::main::start_worker::start_worker; use sqlbuild_orchestration::worker::models::{WorkerClient, WorkerSession}; use uuid::Uuid; +use crate::bindings::_helpers::compiled_project::constants::REPORT_ATTEMPTS; use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSnapshot; struct WorkerOwner { @@ -26,6 +27,61 @@ pub(crate) struct NativeOrchestrationWorker { } impl NativeOrchestrationWorker { + /// SIGTERM may arrive before the periodic heartbeat observes the durable cancellation fact. + pub(crate) fn refresh_cancellation(&self, py: Python<'_>) -> PyResult { + py.detach(|| { + let owner = self.owner.lock().map_err(|_| unavailable())?; + let owner = owner.as_ref().ok_or_else(unavailable)?; + if owner.session.cancel_requested() { + return Ok(true); + } + owner + .runtime + .block_on(owner.client.heartbeat()) + .map_err(|error| PyRuntimeError::new_err(error.to_string())) + }) + } + + /// Native execution reads the retained request, never caller-provided selection or settings. + pub(crate) fn execution( + &self, + py: Python<'_>, + ) -> PyResult { + py.detach(|| { + let owner = self.owner.lock().map_err(|_| unavailable())?; + Ok(owner + .as_ref() + .ok_or_else(unavailable)? + .session + .execution + .clone()) + }) + } + + /// A bounded retry keeps the exact event identity across an uncertain acknowledgement. + pub(crate) fn change(&self, py: Python<'_>, key: &str, change: RunChange) -> PyResult<()> { + py.detach(|| { + let owner = self.owner.lock().map_err(|_| unavailable())?; + let owner = owner.as_ref().ok_or_else(unavailable)?; + let event_id = Uuid::new_v5(&owner.session.execution.claim.claim_id, key.as_bytes()); + owner.runtime.block_on(async { + let mut attempts = 0; + loop { + attempts += 1; + match owner.client.report(event_id, change.clone()).await { + Ok(_) => return Ok(()), + Err(sqlbuild_orchestration::worker::errors::WorkerError::Transport(_)) + if attempts < REPORT_ATTEMPTS => + { + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + } + Err(error) => return Err(PyRuntimeError::new_err(error.to_string())), + } + } + }) + }) + } + /// Verify this retained baseline before the caller applies run settings or opens a warehouse. pub(crate) fn start( snapshot: &NativeOrchestrationSnapshot, @@ -77,7 +133,7 @@ impl NativeOrchestrationWorker { } /// Cancellation remains sticky; a closed worker cannot authorize further work. - fn cancel_requested(&self, py: Python<'_>) -> bool { + pub(crate) fn cancel_requested(&self, py: Python<'_>) -> bool { py.detach(|| { self.owner.lock().map_or(true, |owner| { owner diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs index e5673f87ea..fdd7526b6b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs @@ -4,7 +4,7 @@ use std::path::Path; use std::process::{Command, Stdio}; use pyo3::exceptions::{PyRuntimeError, PyValueError}; -use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::PyDict; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::git_revisions::main::head_commit::head_commit; @@ -18,10 +18,12 @@ use crate::bindings::_helpers::compiled_project::snapshot::{ use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; use crate::bindings::_helpers::location_command::constants::PUBLISH_ACTION; use crate::bindings::_helpers::location_command::runtime::run_runtime; +use crate::bindings::_helpers::location_command::worker::execute_worker; use crate::bindings::_helpers::native_host::host::write_console; use crate::bindings::_helpers::query_command::project::{ ProjectAdapter, ProjectInputs, project_adapter, }; +use crate::bindings::_helpers::recall::session::InvocationRecall; use crate::bindings::_helpers::records::construct::new_record_with; use crate::bindings::_helpers::records::fields::field; use crate::bindings::_helpers::records::ids::RecordId; @@ -41,9 +43,11 @@ fn metadata( .parse() .map_err(|_| PyValueError::new_err("location ID must be a UUID"))?; let target: String = arguments.value("selected_target")?.extract()?; + let action = arguments.value("action")?.extract::>()?; let generation = arguments .value("generation")? .extract::>()? + .or_else(|| (action.as_deref() == Some("execute")).then_some(1)) .ok_or_else(|| PyValueError::new_err("publish requires --generation"))?; if target.trim().is_empty() || generation < 1 { return Err(PyValueError::new_err( @@ -101,7 +105,7 @@ fn compile_location<'py>( py: Python<'py>, arguments: &KeywordArguments<'py>, input: &SnapshotInput, -) -> PyResult { +) -> PyResult<(NativeOrchestrationSnapshot, Bound<'py, PyAny>)> { status(py, "Discovering location project...")?; let ProjectAdapter { project_dir, @@ -128,7 +132,7 @@ fn compile_location<'py>( None, )?; status(py, "Location snapshot compiled.")?; - Ok(snapshot) + Ok((snapshot, discovered)) } pub(super) fn location_arguments<'py>( @@ -147,7 +151,7 @@ pub(super) fn location_arguments<'py>( "generation", "server_url", ], - &["image_id", "action"], + &["image_id", "action", "run_id", "claim_id"], ), ) } @@ -177,7 +181,7 @@ pub(super) fn publish_location( .value("project_dir")? .extract::()?; let input = metadata(py, arguments, &directory)?; - let snapshot = compile_location(py, arguments, &input)?; + let (snapshot, _) = compile_location(py, arguments, &input)?; status(py, "Publishing location snapshot...")?; let record = snapshot.publication( py, @@ -201,8 +205,41 @@ pub(super) fn publish_location( pub(crate) fn run_location_command<'py>( py: Python<'py>, options: Option>, +) -> PyResult { + run_location(py, options, None) +} + +/// CLI dispatch retains the verified worker through invocation history delivery. +pub(crate) fn run_location<'py>( + py: Python<'py>, + options: Option>, + recall: Option<&InvocationRecall>, ) -> PyResult { let arguments = location_arguments(py, options.clone())?; + if arguments + .value("action")? + .extract::>()? + .as_deref() + == Some("execute") + { + let directory = arguments + .value("project_dir")? + .extract::()?; + let input = metadata(py, &arguments, &directory)?; + let bearer = std::env::var("SQLBUILD_RUN_TOKEN") + .map_err(|_| PyValueError::new_err("execute requires SQLBUILD_RUN_TOKEN"))?; + let identity = ( + arguments.value("run_id")?.extract::()?, + arguments.value("claim_id")?.extract::()?, + ); + let (snapshot, discovered) = compile_location(py, &arguments, &input)?; + return execute_worker( + py, + (&snapshot, discovered), + &arguments, + (&bearer, identity, recall), + ); + } if arguments .value("action")? .extract::>()? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs index f80ce86450..63f708bfc2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod command; mod constants; mod runtime; +mod worker; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs new file mode 100644 index 0000000000..472717ab03 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -0,0 +1,154 @@ +//! Execute only the API's frozen request after compiling and verifying the unmodified baseline. + +use std::sync::{Arc, mpsc}; +use std::time::Duration; + +use pyo3::exceptions::{PyRuntimeError, PyValueError}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_orchestration::runs::models::RunOutcome; + +use crate::bindings::_helpers::boundary::config_values::json_value_to_python; +use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::build_command::command::run_build; +use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSnapshot; +use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; +use crate::bindings::_helpers::compiled_project::snapshot_worker::NativeOrchestrationWorker; +use crate::bindings::_helpers::native_host::host::{entry, write_console}; +use crate::bindings::_helpers::recall::session::InvocationRecall; +use crate::bindings::_helpers::records::construct::new_record; +use crate::bindings::_helpers::records::ids::RecordId; + +fn build_request<'py>( + py: Python<'py>, + discovered: Bound<'py, PyAny>, + arguments: &KeywordArguments<'py>, + worker: &Bound<'py, NativeOrchestrationWorker>, +) -> PyResult> { + let execution = worker.get().execution(py)?; + let frozen = execution.claim.request; + for name in frozen.settings.keys() { + if !matches!( + name.as_str(), + "full_refresh" + | "fail_fast" + | "concurrency" + | "cli_vars" + | "run_tests" + | "run_audits" + | "load_sources" + | "reload_sources" + | "include_python" + | "allow_snapshot_full_refresh" + | "allow_snapshot_schema_change" + | "allow_table_type_downgrade" + | "allow_retention_decrease" + | "allow_missing_migration_origin" + | "max_microbatches" + | "test_timeout" + ) { + return Err(PyValueError::new_err(format!( + "unsupported worker build setting {name}" + ))); + } + } + let settings = PyDict::new(py); + for (name, value) in &frozen.settings { + settings.set_item(name, json_value_to_python(py, value)?)?; + } + settings.set_item("project_dir", arguments.value("project_dir")?)?; + settings.set_item("selected_target", frozen.target)?; + let mut selectors: Vec = Vec::new(); + for key in &frozen.selection { + let (_, name) = key + .split_once(':') + .ok_or_else(|| PyValueError::new_err("frozen run contains an invalid node key"))?; + selectors.push(name.to_owned()); + } + settings.set_item("select", selectors)?; + settings.set_item("discovered_inputs", discovered)?; + settings.set_item("no_color", true)?; + settings.set_item("json_output", true)?; + new_record(py, RecordId::BuildCommandRequest, (), Some(&settings)) +} + +/// The heartbeat is native; only driver cancellation crosses the executor host boundary. +fn run_cancellable( + py: Python<'_>, + worker: &Arc, + request: &Bound<'_, PyAny>, +) -> PyResult { + let state = Arc::clone(worker); + let (stop, stopping) = mpsc::channel::<()>(); + let thread = std::thread::Builder::new() + .name("sqb-worker-cancel".into()) + .spawn(move || { + while matches!( + stopping.recv_timeout(Duration::from_millis(100)), + Err(mpsc::RecvTimeoutError::Timeout) + ) { + let _ = Python::attach(|py| -> PyResult<()> { + if state.worker.bind(py).get().cancel_requested(py) { + call_with(&entry(py, ExecutorHost::CancelRunningStatements)?, &[])?; + } + Ok(()) + }); + } + }) + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; + let result = run_build(py, request, Some(Arc::clone(worker))); + drop(stop); + py.detach(|| thread.join()) + .map_err(|_| PyRuntimeError::new_err("worker cancellation thread failed"))?; + result +} + +pub(super) fn execute_worker<'py>( + py: Python<'py>, + (snapshot, discovered): (&NativeOrchestrationSnapshot, Bound<'py, PyAny>), + arguments: &KeywordArguments<'py>, + (bearer, identity, recall): (&str, (String, String), Option<&InvocationRecall>), +) -> PyResult { + write_console(py, "stderr", "Verifying frozen run inputs...\n")?; + let owner = Bound::new( + py, + NativeOrchestrationWorker::start( + snapshot, + py, + (&arguments.value("server_url")?.extract::()?, bearer), + identity, + ) + .map_err(|error| PyValueError::new_err(error.to_string()))?, + )?; + let worker = WorkerBuild::new(&owner)?; + if let Some(recall) = recall { + *recall + .worker + .lock() + .map_err(|_| PyRuntimeError::new_err("worker completion lock is unavailable"))? = + Some(Arc::clone(&worker)); + } + let result = (|| { + let request = build_request(py, discovered, arguments, &owner)?; + write_console( + py, + "stderr", + "Run inputs verified. Executing frozen selection...\n", + )?; + run_cancellable(py, &worker, &request) + })(); + let code = result.as_ref().copied().unwrap_or(1); + let outcome = if recall.is_some() { + worker.outcome(py, code)? + } else { + worker.finish(py, code)? + }; + result.map(|code| { + if matches!(outcome, RunOutcome::Success | RunOutcome::Warn) { + code + } else { + code.max(1) + } + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs index 85714984a5..43b2fafcfa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs @@ -36,6 +36,7 @@ use sqlbuild_recall::writer::models::{ DeliverySummary, RecallSession, SessionConfig, SessionStore, SetupOutcome, }; +use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::output_capture::records::invocation_context; use crate::bindings::_helpers::recall::clickhouse_store::{ @@ -56,6 +57,8 @@ use crate::bindings::_helpers::records::fields::field as native_field; /// What the invocation records: its session, when enabled, and the resolved target and adapter. #[derive(Clone, Debug, Default)] pub(crate) struct InvocationRecall { + /// The verified worker remains owned until Recall has drained and its claim can be released. + pub(crate) worker: Arc>>>, pub(crate) session: Option>, pub(crate) target: Option, pub(crate) adapter: Option, @@ -409,35 +412,38 @@ fn quarantined_notice(path: &Path) -> Option { fn started_session( py: Python<'_>, (args, identity, facts): (&Bound<'_, PyAny>, &Bound<'_, PyAny>, &CommandFacts), - (settings, project_dir): (&RecallSettings, &Path), + (settings, project_dir, hosted): (&RecallSettings, &Path, Option), ) -> PyResult>> { - let store: SessionStore = match &settings.store { - StoreLocation::Sqlite { path } => { - notify(py, quarantined_notice(path)); - if let Some(reason) = store_location_problem(path) { - notify(py, permission_notice((settings, project_dir), &reason)); + let store: SessionStore = match hosted { + Some(store) => store, + None => match &settings.store { + StoreLocation::Sqlite { path } => { + notify(py, quarantined_notice(path)); + if let Some(reason) = store_location_problem(path) { + notify(py, permission_notice((settings, project_dir), &reason)); + return Ok(None); + } + SessionStore::Sqlite(path.clone()) + } + StoreLocation::ClickHouse { + config, database, .. + } => SessionStore::Hosted(Arc::new(ClickHouseStoreOpener { + location: ClickHouseLocation::new(config, database, settings.retention_days), + retention_notice: notice_due, + project_dir: project_dir.to_path_buf(), + information: Mutex::default(), + })), + StoreLocation::Unsupported { + connection, + adapter, + } => { + notify( + py, + unsupported_notice((settings, project_dir), (connection, adapter)), + ); return Ok(None); } - SessionStore::Sqlite(path.clone()) - } - StoreLocation::ClickHouse { - config, database, .. - } => SessionStore::Hosted(Arc::new(ClickHouseStoreOpener { - location: ClickHouseLocation::new(config, database, settings.retention_days), - retention_notice: notice_due, - project_dir: project_dir.to_path_buf(), - information: Mutex::default(), - })), - StoreLocation::Unsupported { - connection, - adapter, - } => { - notify( - py, - unsupported_notice((settings, project_dir), (connection, adapter)), - ); - return Ok(None); - } + }, }; let Some(invocation_id) = text_value(&native_field(identity, "invocation_id")?)? else { return Ok(None); @@ -462,13 +468,44 @@ fn started_session( max_output_bytes: settings.max_output_bytes, heartbeat_interval: heartbeat_interval(), setup_budget: Duration::ZERO, - maintenance: true, + maintenance: !worker_invocation(args, facts)?, defer_context: true, audit_evidence: settings.audit_evidence, }); Ok(Some(Arc::new(session))) } +fn worker_invocation(args: &Bound<'_, PyAny>, facts: &CommandFacts) -> PyResult { + Ok(facts.command.as_deref() == Some("location") + && attribute(args, "location_command")? + .extract::>()? + .as_deref() + == Some("execute")) +} + +fn worker_store(args: &Bound<'_, PyAny>) -> PyResult { + let identity = ( + attribute(args, "location_run_id")? + .extract::()? + .parse() + .map_err(|_| PyValueError::new_err("run ID must be a UUID"))?, + attribute(args, "location_claim_id")? + .extract::()? + .parse() + .map_err(|_| PyValueError::new_err("claim ID must be a UUID"))?, + ); + let bearer = std::env::var("SQLBUILD_RUN_TOKEN") + .map_err(|_| PyValueError::new_err("execute requires SQLBUILD_RUN_TOKEN"))?; + let url = attribute(args, "location_server_url")?.extract::()?; + args.py().detach(|| { + sqlbuild_orchestration::worker::main::connect_worker::connect_worker( + &url, &bearer, identity, + ) + .map(|client| client.recall_store()) + .map_err(|error| PyValueError::new_err(error.to_string())) + }) +} + /// Begin recording when Recall is enabled, without waiting on the store. Only the write connection is resolved; a write connection without a read connection is kept for the command to fail with, and other unresolvable settings record nothing. pub(crate) fn start_invocation_recall( py: Python<'_>, @@ -482,7 +519,7 @@ pub(crate) fn start_invocation_recall( selected_target: text_value(&args.getattr("target")?)?, purpose: StorePurpose::Record, }; - let settings: RecallSettings = match resolve_recall_settings(&request) { + let mut settings: RecallSettings = match resolve_recall_settings(&request) { Ok(settings) => settings, Err(error) if error.invalid_configuration => { return Ok(InvocationRecall { @@ -492,14 +529,25 @@ pub(crate) fn start_invocation_recall( } Err(_unreachable_store) => return Ok(InvocationRecall::default()), }; + let hosted = if worker_invocation(args, facts)? { + settings.enabled = true; + Some(worker_store(args)?) + } else { + None + }; let session: Option> = if settings.enabled && settings.project_name.is_some() { - started_session(py, (args, identity, facts), (&settings, &project_dir))? + started_session( + py, + (args, identity, facts), + (&settings, &project_dir, hosted), + )? } else { None }; let audit_evidence: AuditEvidence = settings.audit_evidence; Ok(InvocationRecall { + worker: Arc::default(), session, target: settings.target.clone(), adapter: settings.project_adapter.clone(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs index a59d0e0da2..349e4ebf1c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs @@ -12,6 +12,7 @@ use sqlbuild_rules::configuration::errors::RulesConfigLoadError; use sqlbuild_rules::configuration::main::load_rules_config::load_rules_config as load_config; use sqlbuild_rules::models::RulesConfig; +use crate::bindings::_helpers::boundary::config_values::json_value_to_python; use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::construct::new_record; @@ -64,31 +65,6 @@ fn option_scalar<'py>(py: Python<'py>, value: &Value) -> PyResult(py: Python<'py>, value: &Value) -> PyResult> { - match value { - Value::Array(items) => { - let values: Vec> = items - .iter() - .map(|item| option_item(py, item)) - .collect::>()?; - Ok(pyo3::types::PyList::new(py, values)?.into_any()) - } - Value::Object(entries) => { - let table: Bound<'py, PyDict> = PyDict::new(py); - for (key, item) in entries { - table.set_item(key, option_item(py, item)?)?; - } - Ok(table.into_any()) - } - Value::Null => Ok(py.None().into_bound(py)), - Value::Number(number) if number.as_i64().is_none() && number.as_u64().is_none() => { - number.as_f64().unwrap_or(f64::NAN).into_bound_py_any(py) - } - other => Ok(option_scalar(py, other)?.unwrap_or_else(|| py.None().into_bound(py))), - } -} - fn rule_options<'py>(py: Python<'py>, config: &RulesConfig) -> PyResult> { let result: Bound<'py, PyDict> = PyDict::new(py); for (code, options) in &config.rule_options { @@ -97,7 +73,7 @@ fn rule_options<'py>(py: Python<'py>, config: &RulesConfig) -> PyResult> = items .iter() - .map(|item| option_item(py, item)) + .map(|item| json_value_to_python(py, item)) .collect::>()?; values.set_item(name, PyTuple::new(py, tuple)?)?; } else if let Some(scalar) = option_scalar(py, value)? { diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 66bcebbc27..6edc268f3f 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -1,7 +1,9 @@ """Publication cases and isolated server credentials.""" -from dataclasses import dataclass -from typing import NamedTuple +from dataclasses import dataclass, field +from pathlib import Path +from subprocess import Popen +from typing import Literal, NamedTuple class SnapshotApi(NamedTuple): @@ -16,6 +18,14 @@ class ClaimedWorker(NamedTuple): bearer: str +class ScheduledWorker(NamedTuple): + api: SnapshotApi + project: Path + control_store: Path + process: Popen[bytes] + log: Path + + @dataclass(frozen=True) class SnapshotPublicationTestCase: description: str @@ -65,3 +75,16 @@ class LocationWorkTestCase: description: str database: str expected_connection: bool + + +@dataclass(frozen=True) +class WorkerExecutionTestCase: + description: str + model_sql: str + warehouse_query: str + expected_status: str + expected_rows: list[tuple[int]] + expected_node_count: int = 1 + files: dict[str, str] = field(default_factory=dict) + store_setup: str = "" + launcher: Literal["native", "uncooperative"] = "native" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index d10a50b37c..24c11ac8ff 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -6,12 +6,13 @@ import json import os import secrets +import signal import sqlite3 import subprocess import sys import time -from collections.abc import Iterator -from contextlib import contextmanager +from collections.abc import Callable, Iterator +from contextlib import AbstractContextManager, contextmanager, nullcontext, suppress from datetime import UTC, datetime, timedelta from io import StringIO from pathlib import Path @@ -21,10 +22,16 @@ from urllib.request import Request, urlopen from uuid import uuid4 +import duckdb +import pytest + from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( ClaimedWorker, + ScheduledWorker, SnapshotApi, + WorkerExecutionTestCase, ) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files class RejectedPublicationOutput(StringIO): @@ -210,6 +217,183 @@ def current_snapshot(api: SnapshotApi, target: str) -> dict[str, Any]: return result +def native_launcher(directory: Path, executable: str) -> AbstractContextManager[str]: + """Use the installed native launcher without a synthetic child.""" + del directory + return nullcontext(executable) + + +@contextmanager +def uncooperative_launcher(directory: Path, executable: str) -> Iterator[str]: + """Own a synthetic process that ignores TERM, including cleanup after a failed assertion.""" + del executable + script: Path = directory / "worker" + _ = script.write_text( + f"#!{sys.executable}\nimport os, signal, time\nfrom pathlib import Path\n" + "signal.signal(signal.SIGTERM, signal.SIG_IGN)\n" + "Path('worker.pid').write_text(str(os.getpid()))\n" + "time.sleep(90)\n" + ) + script.chmod(0o700) + try: + yield str(script) + finally: + with suppress(FileNotFoundError, ProcessLookupError): + os.kill(int((directory / "project/worker.pid").read_text()), signal.SIGKILL) + + +@contextmanager +def scheduled_worker( + directory: Path, monkeypatch: pytest.MonkeyPatch, test_case: WorkerExecutionTestCase +) -> Iterator[ScheduledWorker]: + """Declare, publish and schedule isolated DuckDB work without provisioning claims.""" + project: Path = directory / "project" + write_files( + project_dir=project, + files={ + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n' + '[connections.local]\ndatabase = "warehouse.duckdb"\n' + '[targets.development]\nschema = "main"\n' + ), + "models/orders.sql": ( + "MODEL (description 'Orders.', materialized table);\n" + test_case.model_sql + "\n" + ), + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, schedule\n" + 'annual_orders = job("annual_orders", select="orders", ' + 'description="Build orders.", ' + 'schedule=schedule(cron="0 0 1 1 *", timezone="UTC", ' + 'starts_at="2000-01-01T00:00:00Z"))\n' + ), + } + | test_case.files, + ) + executable: str = str(Path(sys.executable).with_name("sqb")) + launchers: dict[str, Callable[[Path, str], AbstractContextManager[str]]] = { + "native": native_launcher, + "uncooperative": uncooperative_launcher, + } + with ( + snapshot_api(directory / "control") as api, + launchers[test_case.launcher](directory, executable) as launch_executable, + ): + published: subprocess.CompletedProcess[str] = subprocess.run( + [ + executable, + "location", + "publish", + "--project-dir", + str(project), + "--location-id", + api.location_id, + "--target", + "development", + "--server", + api.url, + "--generation", + "1", + ], + env=os.environ | {"SQLBUILD_LOCATION_TOKEN": api.bearer}, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + assert published.returncode == 0, published.stderr + configuration: Path = directory / "launchers.json" + _ = configuration.write_text( + json.dumps( + { + "local": [ + { + "location_id": api.location_id, + "target": "development", + "project_dir": str(project), + "executable": launch_executable, + "server_url": api.url, + "max_workers": 1, + } + ] + } + ) + ) + control_store: Path = directory / "control/.sqlbuild/control.sqlite3" + with sqlite3.connect(control_store) as control: + _ = control.executescript(test_case.store_setup) + monkeypatch.setenv("SQLBUILD_SCHEDULER_TEST_STORE", f"sqlite://{control_store}") + log: Path = directory / "scheduler.log" + with location_process( + [ + executable, + "scheduler", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "--launchers", + str(configuration), + ], + log, + directory / "scheduler.stdout", + ) as process: + yield ScheduledWorker(api, project, control_store, process, log) + + +def worker_result(worker: ScheduledWorker, test_case: WorkerExecutionTestCase) -> dict[str, Any]: + """Run/node outcomes, Recall, claims and the physical warehouse agree.""" + api, project, control_store, process, log = worker + run: dict[str, Any] = wait_for_run(api, process, log) + assert run["node_counts_json"] == {test_case.expected_status: test_case.expected_node_count}, ( + log.read_text() + ) + assert run["job_key"] == "annual_orders" + assert "Worker launched for run" in log.read_text() + runs_url: str = f"{api.url}/api/locations/{api.location_id}/targets/development/runs" + headers: dict[str, str] = {"Authorization": f"Bearer {api.bearer}"} + with urlopen( + Request(f"{runs_url}/{run['run_id']}/nodes", headers=headers), timeout=5 + ) as response: + nodes: list[dict[str, Any]] = json.load(response) + assert nodes[0]["node_key"] == "model:orders" + assert nodes[0]["started_at"] is not None + assert nodes[0]["ended_at"] is not None + with sqlite3.connect(control_store) as control: + contexts: list[tuple[str | None]] = control.execute( + "SELECT external_context FROM recall_invocations WHERE command = 'location'" + ).fetchall() + assert contexts, log.read_text() + assert all( + context is not None and json.loads(context)["run_id"] == run["run_id"] + for (context,) in contexts + ) + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + with duckdb.connect(str(project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + return run + + +def wait_for_run( + api: SnapshotApi, + process: subprocess.Popen[bytes], + log: Path, + *, + pending: frozenset[str] = frozenset({"queued", "launched", "running", "unresponsive"}), +) -> dict[str, Any]: + """A terminal API projection must follow the scheduler's actual isolated warehouse execution.""" + request: Request = Request( + f"{api.url}/api/locations/{api.location_id}/targets/development/runs", + headers={"Authorization": f"Bearer {api.bearer}"}, + ) + deadline: float = time.monotonic() + 30 + current: dict[str, Any] = {"status": "queued"} + while current["status"] in pending: + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + with urlopen(request, timeout=2) as response: + current = next(iter(json.load(response)), current) + time.sleep(0.02) + return current + + def claimed_worker(directory: Path, api: SnapshotApi, publication: dict[str, Any]) -> ClaimedWorker: """Provision only the launch boundary; verification and lifecycle reporting use real HTTP.""" run_id: str = str(uuid4()) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index b357d3504a..9e8dad99f7 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -1,6 +1,12 @@ """Retained compiler baselines cross the native worker boundary before lifecycle writes.""" import json +import os +import signal +import sqlite3 +import subprocess +import sys +import time from pathlib import Path from typing import Any from urllib.request import Request, urlopen @@ -23,10 +29,15 @@ from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( ClaimedWorker, SnapshotWorkerTestCase, + WorkerExecutionTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( claimed_worker, + scheduled_worker, snapshot_api, + wait_for_log, + wait_for_run, + worker_result, ) from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files @@ -146,3 +157,267 @@ def test_given_a_retained_baseline_when_starting_a_worker_then_verification_and_ result: dict[str, Any] = json.load(response)[0] assert result["status"] == "success" assert result["node_counts_json"] == {"success": 1} + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "successful model writes actual rows", + "SELECT CAST(value AS BIGINT) AS amount FROM (VALUES ('23')) t(value)", + "SELECT amount FROM orders", + "success", + [(23,)], + ), + WorkerExecutionTestCase( + "warehouse error retains failed node evidence", + "SELECT CAST(value AS BIGINT) AS amount FROM (VALUES ('invalid')) t(value)", + "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'", + "failed", + [(0,)], + ), + WorkerExecutionTestCase( + "scalar function dependency is claimed and executed", + 'SELECT __udf("order_total")(23) AS amount', + "SELECT amount FROM orders", + "success", + [(46,)], + expected_node_count=2, + files={ + "functions/sql/order_total.sql": ( + "FUNCTION (description 'Order total.', " + "arguments (amount BIGINT), returns BIGINT);\nSELECT amount * 2\n" + ) + }, + ), + WorkerExecutionTestCase( + "table function dependency is claimed and executed", + 'SELECT amount FROM __table_fn("order_amounts")()', + "SELECT amount FROM orders", + "success", + [(31,)], + expected_node_count=2, + files={ + "functions/sql/order_amounts.sql": ( + "FUNCTION (description 'Order amounts.', returns table (amount BIGINT));\n" + "SELECT CAST(31 AS BIGINT) AS amount\n" + ) + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_scheduled_job_when_published_then_an_isolated_worker_reports_real_warehouse_results( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + run: dict[str, Any] = worker_result(worker, test_case) + assert run["status"] == test_case.expected_status, worker.log.read_text() + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "cancellation stops an active warehouse statement", + "SELECT sum(sin(value)) AS amount FROM range(10000000000) t(value)", + "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'", + "cancelled", + [(0,)], + ) + ], + ids=lambda case: case.description, +) +def test_given_an_executing_scheduled_worker_when_cancelled_then_the_statement_stops_before_claim_release( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + active: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, pending=frozenset({"queued", "launched"}) + ) + assert active["status"] == "running", worker.log.read_text() + deadline: float = time.monotonic() + 15 + with sqlite3.connect(worker.control_store) as control: + while not control.execute( + "SELECT 1 FROM recall_artifacts WHERE CAST(content AS TEXT) LIKE ?", + ("%range(10000000000)%",), + ).fetchone(): + assert worker.process.poll() is None, worker.log.read_text() + assert time.monotonic() < deadline, worker.log.read_text() + time.sleep(0.02) + cancelled: subprocess.CompletedProcess[str] = subprocess.run( + [ + str(Path(sys.executable).with_name("sqb")), + "admin", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "cancel-run", + active["run_id"], + "--event-id", + str(uuid4()), + "--reason", + "Cancel the active test statement.", + ], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + assert cancelled.returncode == 0, cancelled.stderr + run: dict[str, Any] = worker_result(worker, test_case) + assert run["status"] == test_case.expected_status, worker.log.read_text() + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "credential failure after a committed claim", + "SELECT 23 AS amount", + "", + "failed", + [], + store_setup=( + "CREATE TRIGGER reject_run_token BEFORE INSERT ON sqb_access_tokens " + "WHEN NEW.run_id IS NOT NULL " + "BEGIN SELECT RAISE(ABORT, 'synthetic run credential failure'); END;" + ), + ), + WorkerExecutionTestCase( + "launch publication failure after a committed claim", + "SELECT 23 AS amount", + "", + "failed", + [], + store_setup=( + "CREATE TRIGGER reject_launch BEFORE UPDATE OF status ON sqb_runs " + "WHEN NEW.status = 'launched' " + "BEGIN SELECT RAISE(ABORT, 'synthetic launch publication failure'); END;" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_committed_claim_when_launch_preparation_fails_then_unstarted_work_is_terminal_and_unreserved( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + assert worker.process.wait(timeout=15) != 0, worker.log.read_text() + request: Request = Request( + f"{worker.api.url}/api/locations/{worker.api.location_id}/targets/development/runs", + headers={"Authorization": f"Bearer {worker.api.bearer}"}, + ) + with urlopen(request, timeout=5) as response: + runs: list[dict[str, Any]] = json.load(response) + assert len(runs) == 1 + assert runs[0]["status"] == test_case.expected_status, worker.log.read_text() + assert runs[0]["node_counts_json"] == {"skipped": 1} + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + assert control.execute("SELECT count(*) FROM sqb_concurrency_slots").fetchone() == (0,) + assert control.execute("SELECT count(*) FROM sqb_run_liveness").fetchone() == (0,) + assert not (worker.project / "warehouse.duckdb").exists() + assert "Worker launched for run" not in worker.log.read_text() + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "failed terminal publication keeps the owner alive until recovery", + "SELECT 23 AS amount", + "", + "failed", + [], + store_setup=( + "CREATE TRIGGER reject_run_token BEFORE INSERT ON sqb_access_tokens " + "WHEN NEW.run_id IS NOT NULL " + "BEGIN SELECT RAISE(ABORT, 'synthetic run credential failure'); END;" + "CREATE TRIGGER reject_terminal BEFORE UPDATE OF status ON sqb_runs " + "WHEN NEW.status = 'failed' " + "BEGIN SELECT RAISE(ABORT, 'synthetic terminal publication failure'); END;" + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_a_stopped_worker_when_terminal_publication_fails_then_shutdown_retains_and_retries_its_claim( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + wait_for_log(worker.process, worker.log, "Retrying worker shutdown publication") + assert worker.process.poll() is None, worker.log.read_text() + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (1,) + assert control.execute("SELECT count(*) FROM sqb_run_liveness").fetchone() == (1,) + _ = control.execute("DROP TRIGGER reject_terminal") + assert worker.process.wait(timeout=10) != 0, worker.log.read_text() + request: Request = Request( + f"{worker.api.url}/api/locations/{worker.api.location_id}/targets/development/runs", + headers={"Authorization": f"Bearer {worker.api.bearer}"}, + ) + with urlopen(request, timeout=5) as response: + runs: list[dict[str, Any]] = json.load(response) + assert len(runs) == 1 + assert runs[0]["status"] == test_case.expected_status + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + assert control.execute("SELECT count(*) FROM sqb_run_liveness").fetchone() == (0,) + assert control.execute( + "SELECT count(*) FROM recall_events WHERE event_type='run_terminated'" + ).fetchone() == (1,) + + +@pytest.mark.skipif(os.name != "posix", reason="POSIX TERM/KILL escalation") +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "uncooperative child is reaped after a lease-preserving hard kill", + "SELECT 23 AS amount", + "", + "failed", + [], + launcher="uncooperative", + ) + ], + ids=lambda case: case.description, +) +def test_given_an_uncooperative_worker_when_scheduler_stops_then_the_fence_and_claims_survive_until_reaping( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + pidfile: Path = worker.project / "worker.pid" + deadline: float = time.monotonic() + 15 + while not pidfile.exists(): + assert worker.process.poll() is None, worker.log.read_text() + assert time.monotonic() < deadline, worker.log.read_text() + time.sleep(0.02) + pid: int = int(pidfile.read_text()) + worker.process.send_signal(signal.SIGTERM) + time.sleep(32) + assert worker.process.poll() is None, worker.log.read_text() + os.kill(pid, 0) + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (1,) + assert control.execute( + "SELECT julianday(expires_at) > julianday('now') FROM sqb_scheduler_lease" + ).fetchone() == (1,) + assert worker.process.wait(timeout=40) == 0, worker.log.read_text() + with pytest.raises(ProcessLookupError): + os.kill(pid, 0) + request: Request = Request( + f"{worker.api.url}/api/locations/{worker.api.location_id}/targets/development/runs", + headers={"Authorization": f"Bearer {worker.api.bearer}"}, + ) + with urlopen(request, timeout=5) as response: + runs: list[dict[str, Any]] = json.load(response) + assert len(runs) == 1 + assert runs[0]["status"] == test_case.expected_status, worker.log.read_text() + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + assert control.execute("SELECT count(*) FROM sqb_run_liveness").fetchone() == (0,) + assert control.execute( + "SELECT json_extract(payload, '$.change.change.exit_code') FROM recall_events " + "WHERE event_type='run_terminated'" + ).fetchone() == (137,) From 143c64811ef3f82c579a9ec31062fd5f72e236ef Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 13:51:38 +0000 Subject: [PATCH 35/59] feat: supervise Docker workers across scheduler restarts Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .github/workflows/verify.yml | 6 + crates/sqlbuild-cli/Cargo.toml | 2 +- .../src/native_commands/_helpers/execution.rs | 8 +- .../src/workers/_helpers/child.rs | 57 ++++ .../src/workers/_helpers/configuration.rs | 58 +++- .../src/workers/_helpers/docker.rs | 205 ++++++++++++++ .../src/workers/_helpers/invocation.rs | 47 ++++ .../sqlbuild-cli/src/workers/_helpers/mod.rs | 6 +- .../src/workers/_helpers/recovery.rs | 64 +++++ .../src/workers/_helpers/shutdown.rs | 49 +++- .../_helpers/{local.rs => supervisor.rs} | 161 ++++++----- crates/sqlbuild-cli/src/workers/constants.rs | 3 + crates/sqlbuild-cli/src/workers/main/mod.rs | 2 +- ...{open_local_workers.rs => open_workers.rs} | 11 +- crates/sqlbuild-cli/src/workers/mod.rs | 1 + crates/sqlbuild-cli/src/workers/models.rs | 68 ++++- .../tests/native_commands/worker/helpers.rs | 128 +++++++-- .../native_commands/worker/test_recovery.rs | 79 ++++++ .../native_commands/worker/test_types.rs | 7 + .../native_commands/worker/test_worker.rs | 2 + .../src/runs/main/list_run_claims.rs | 39 +++ .../src/runs/main/mod.rs | 1 + .../tests/runs/test_claims.rs | 32 +++ .../publication/_test_types.py | 26 +- .../publication/docker/__init__.py | 1 + .../publication/docker/helpers.py | 126 +++++++++ .../compiled_project/publication/helpers.py | 107 ++++++-- .../publication/test_docker_worker.py | 252 ++++++++++++++++++ .../publication/test_worker.py | 39 +-- 29 files changed, 1400 insertions(+), 187 deletions(-) create mode 100644 crates/sqlbuild-cli/src/workers/_helpers/child.rs create mode 100644 crates/sqlbuild-cli/src/workers/_helpers/docker.rs create mode 100644 crates/sqlbuild-cli/src/workers/_helpers/invocation.rs create mode 100644 crates/sqlbuild-cli/src/workers/_helpers/recovery.rs rename crates/sqlbuild-cli/src/workers/_helpers/{local.rs => supervisor.rs} (66%) create mode 100644 crates/sqlbuild-cli/src/workers/constants.rs rename crates/sqlbuild-cli/src/workers/main/{open_local_workers.rs => open_workers.rs} (62%) create mode 100644 crates/sqlbuild-cli/tests/native_commands/worker/test_recovery.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/list_run_claims.rs create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/__init__.py create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 3f51d3fbb9..73205dc120 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -283,7 +283,13 @@ jobs: run: make check-ci - name: Run native checks run: make rust-check + - name: Prepare Docker worker tests + run: | + docker info > /dev/null + docker pull alpine:3.23 - name: Run unit and integration tests + env: + SQLBUILD_REQUIRE_DOCKER_TESTS: "1" run: make test e2e_shards: diff --git a/crates/sqlbuild-cli/Cargo.toml b/crates/sqlbuild-cli/Cargo.toml index d5e621fd80..5568005174 100644 --- a/crates/sqlbuild-cli/Cargo.toml +++ b/crates/sqlbuild-cli/Cargo.toml @@ -19,6 +19,7 @@ path = "src/bin/sqlbuild.rs" blake3.workspace = true chrono.workspace = true getrandom.workspace = true +reqwest.workspace = true serde.workspace = true serde_json = { workspace = true, features = ["arbitrary_precision"] } sqlbuild-build-features.workspace = true @@ -40,7 +41,6 @@ ctrlc.workspace = true same-file.workspace = true [dev-dependencies] -reqwest.workspace = true rusqlite.workspace = true tempfile.workspace = true diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 41b1df3e3f..7de7f07935 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -48,8 +48,8 @@ use crate::native_commands::models::{ CommandOutput, InstalledPackage, LocationCompileRequest, LocationCompiler, LocationServe, LostStdout, NativeCommand, NativeInvocation, ProjectLocation, }; -use crate::workers::main::open_local_workers::open_local_workers; -use crate::workers::models::LocalWorkers; +use crate::workers::main::open_workers::open_workers; +use crate::workers::models::Workers; /// Request one publication without transferring interpreter ownership to a cancellable future. pub(crate) async fn compile_generation( @@ -314,8 +314,8 @@ pub(crate) async fn open_control_store( } /// Decode privileged CLI configuration before handing process ownership to the workers domain. -pub(crate) fn open_control_workers(namespace: &Namespace) -> Result { - open_local_workers( +pub(crate) fn open_control_workers(namespace: &Namespace) -> Result { + open_workers( text(namespace, "control_launchers").as_deref(), text(namespace, "control_database_env") .unwrap_or_else(|| "SQLBUILD_CONTROL_DATABASE_URL".into()), diff --git a/crates/sqlbuild-cli/src/workers/_helpers/child.rs b/crates/sqlbuild-cli/src/workers/_helpers/child.rs new file mode 100644 index 0000000000..a6ca868c82 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/_helpers/child.rs @@ -0,0 +1,57 @@ +//! Launchers share physical exit observation and termination, not durable lifecycle authority. + +use sqlbuild_orchestration::store::errors::ControlError; + +use crate::workers::_helpers::supervisor::{exit_code, terminate}; +use crate::workers::models::{WorkerChild, WorkerExit}; + +impl WorkerExit { + pub(in crate::workers) const fn code(self) -> Option { + match self { + Self::Code(code) => Some(code), + Self::Absent => None, + } + } +} + +impl WorkerChild { + pub(in crate::workers) async fn try_wait( + &mut self, + ) -> Result, ControlError> { + match self { + Self::Local(child) => Ok(child + .try_wait()? + .map(|exit| WorkerExit::Code(exit_code(exit)))), + Self::Docker(child) => child.try_wait().await, + } + } + + pub(in crate::workers) async fn wait(&mut self) -> Result { + if let Self::Local(child) = self { + return Ok(WorkerExit::Code(exit_code(child.wait().await?))); + } + loop { + if let Some(exit) = self.try_wait().await? { + return Ok(exit); + } + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + } + } + + pub(in crate::workers) async fn terminate(&mut self) -> Result<(), ControlError> { + match self { + Self::Local(child) => terminate(child), + Self::Docker(child) => child.signal("SIGTERM").await, + } + } + + pub(in crate::workers) async fn kill(&mut self) -> Result<(), ControlError> { + match self { + Self::Local(child) => { + child.start_kill()?; + Ok(()) + } + Self::Docker(child) => child.signal("SIGKILL").await, + } + } +} diff --git a/crates/sqlbuild-cli/src/workers/_helpers/configuration.rs b/crates/sqlbuild-cli/src/workers/_helpers/configuration.rs index bf486eeb40..7377d5d941 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/configuration.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/configuration.rs @@ -5,36 +5,78 @@ use std::collections::BTreeSet; use sqlbuild_orchestration::store::errors::ControlError; use uuid::Uuid; -use crate::workers::models::LocalLaunchers; +use crate::workers::models::{Launcher, LauncherScope, Launchers}; + +impl<'a> Launcher<'a> { + pub(in crate::workers) fn scope(self) -> LauncherScope<'a> { + match self { + Self::Local(location) => LauncherScope { + location_id: location.location_id, + target: &location.target, + server_url: &location.server_url, + max_workers: location.max_workers, + }, + Self::Docker(location) => LauncherScope { + location_id: location.location_id, + target: &location.target, + server_url: &location.server_url, + max_workers: location.max_workers, + }, + } + } +} + +impl Launchers { + pub(in crate::workers) fn locations(&self) -> impl Iterator> { + self.local + .iter() + .map(Launcher::Local) + .chain(self.docker.iter().map(Launcher::Docker)) + } +} pub(in crate::workers) fn read_configuration( path: Option<&str>, scheduler_enabled: bool, -) -> Result { - let configuration: LocalLaunchers = match path { +) -> Result { + let configuration: Launchers = match path { Some(path) => serde_json::from_slice(&std::fs::read(path)?) .map_err(|error| ControlError::Invalid(format!("launcher configuration: {error}")))?, - None => LocalLaunchers { + None => Launchers { local: Vec::new(), + docker: Vec::new(), concurrency_limits: Default::default(), }, }; - if !scheduler_enabled && !configuration.local.is_empty() { + if !scheduler_enabled && configuration.locations().next().is_some() { return Err(ControlError::Invalid( "launchers require the embedded scheduler".into(), )); } let mut scopes: BTreeSet<(Uuid, &str)> = BTreeSet::new(); for location in &configuration.local { - if location.target.trim().is_empty() - || !location.project_dir.is_absolute() + if !location.project_dir.is_absolute() || !location.project_dir.is_dir() || !location.executable.is_absolute() || !location.executable.is_file() - || !scopes.insert((location.location_id, location.target.as_str())) { return Err(ControlError::Invalid("local launchers require unique location/target scopes and existing absolute project/executable paths".into())); } } + for location in configuration.locations() { + let scope = location.scope(); + if scope.target.trim().is_empty() || !scopes.insert((scope.location_id, scope.target)) { + return Err(ControlError::Invalid( + "launchers require unique location/target scopes".into(), + )); + } + if let Launcher::Docker(location) = location + && (!location.socket.is_absolute() + || location.location_container.trim().is_empty() + || location.executable.trim().is_empty()) + { + return Err(ControlError::Invalid("Docker launchers require an absolute socket path, location container and executable".into())); + } + } Ok(configuration) } diff --git a/crates/sqlbuild-cli/src/workers/_helpers/docker.rs b/crates/sqlbuild-cli/src/workers/_helpers/docker.rs new file mode 100644 index 0000000000..7b0d5c9633 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/_helpers/docker.rs @@ -0,0 +1,205 @@ +//! The privileged role talks to a local Docker daemon; workers never receive its socket. + +use std::path::Path; + +use reqwest::{Method, StatusCode}; +use serde_json::{Value, json}; +use sqlbuild_orchestration::runs::models::RunClaim; +use sqlbuild_orchestration::store::errors::ControlError; + +use crate::workers::_helpers::invocation::{arguments, environment, excluded}; +use crate::workers::constants::IMAGE_ID_HEX_LENGTH; +use crate::workers::models::{DockerLauncher, DockerWorker, Launcher, WorkerExit}; + +impl DockerWorker { + pub(in crate::workers) fn prepare( + location: &DockerLauncher, + claim: &RunClaim, + ) -> Result { + let image_id = claim.request.image_id.as_deref().unwrap_or_default(); + match image_id.strip_prefix("sha256:") { + Some(digest) + if digest.len() == IMAGE_ID_HEX_LENGTH + && digest.as_bytes().iter().all(u8::is_ascii_hexdigit) => {} + _ => { + return Err(ControlError::Invalid( + "Docker workers require a snapshot pinned to a full sha256 image ID".into(), + )); + } + } + Ok(Self { + client: docker_client(&location.socket)?, + name: format!( + "sqlbuild-{}-{}", + claim.request.location_id, claim.request.run_id + ), + claim_id: claim.claim_id, + image_id: image_id.into(), + }) + } + + pub(in crate::workers) async fn launch( + &self, + location: &DockerLauncher, + claim: &RunClaim, + bearer: &str, + control_variable: &str, + ) -> Result<(), ControlError> { + let parent: Value = self + .request( + Method::GET, + &format!("containers/{}/json", location.location_container), + None, + ) + .await? + .error_for_status() + .map_err(docker_error)? + .json() + .await + .map_err(docker_error)?; + let project = parent["Config"]["WorkingDir"] + .as_str() + .filter(|path| path.starts_with('/')) + .ok_or_else(|| { + ControlError::Invalid( + "location container requires an absolute working directory".into(), + ) + })?; + let mut command = arguments(&Launcher::Docker(location).scope(), claim); + command.push(project.into()); + command.extend(["--image-id".into(), self.image_id.clone()]); + let mut variables: Vec = parent["Config"]["Env"] + .as_array() + .into_iter() + .flatten() + .filter_map(Value::as_str) + .filter(|entry| !excluded(entry.split('=').next().unwrap_or(entry), control_variable)) + .map(str::to_owned) + .collect(); + variables.extend( + environment(claim, bearer) + .into_iter() + .map(|(key, value)| format!("{key}={value}")), + ); + let body = json!({ + "Image": self.image_id, "Entrypoint": [location.executable], "Cmd": command, + "WorkingDir": project, "User": parent["Config"]["User"], "Env": variables, + "Labels": {"io.sqlbuild.claim-id": self.claim_id.to_string()}, + "HostConfig": { + "NetworkMode": format!("container:{}", location.location_container), + "VolumesFrom": [location.location_container], + "Memory": parent["HostConfig"]["Memory"], "MemorySwap": parent["HostConfig"]["MemorySwap"], + "NanoCpus": parent["HostConfig"]["NanoCpus"], "PidsLimit": parent["HostConfig"]["PidsLimit"], + "ReadonlyRootfs": parent["HostConfig"]["ReadonlyRootfs"], + }, + }); + self.request( + Method::POST, + &format!("containers/create?name={}", self.name), + Some(&body), + ) + .await? + .error_for_status() + .map_err(docker_error)?; + self.request( + Method::POST, + &format!("containers/{}/start", self.name), + None, + ) + .await? + .error_for_status() + .map_err(docker_error)?; + Ok(()) + } + + pub(in crate::workers) async fn try_wait(&self) -> Result, ControlError> { + let response = self + .request(Method::GET, &format!("containers/{}/json", self.name), None) + .await?; + if response.status() == StatusCode::NOT_FOUND { + return Ok(Some(WorkerExit::Absent)); + } + let container: Value = response + .error_for_status() + .map_err(docker_error)? + .json() + .await + .map_err(docker_error)?; + if container["Config"]["Labels"]["io.sqlbuild.claim-id"] != self.claim_id.to_string() + || container["Image"] != self.image_id + { + return Err(ControlError::Invalid( + "Docker container does not belong to the retained claim and image".into(), + )); + } + if container["State"]["Running"].as_bool() == Some(true) { + return Ok(None); + } + if container["State"]["Status"].as_str() == Some("created") { + return Ok(Some(WorkerExit::Absent)); + } + let code = container["State"]["ExitCode"] + .as_i64() + .ok_or_else(|| ControlError::Invalid("Docker container has no exit evidence".into()))?; + Ok(Some(WorkerExit::Code(code))) + } + + pub(in crate::workers) async fn signal(&self, signal: &str) -> Result<(), ControlError> { + if self.try_wait().await?.is_some() { + return Ok(()); + } + let response = self + .request( + Method::POST, + &format!("containers/{}/kill?signal={signal}", self.name), + None, + ) + .await?; + if !matches!( + response.status(), + StatusCode::NOT_FOUND | StatusCode::CONFLICT + ) { + response.error_for_status().map_err(docker_error)?; + } + Ok(()) + } + + async fn request( + &self, + method: Method, + path: &str, + body: Option<&Value>, + ) -> Result { + let request = self + .client + .request(method, format!("http://localhost/{path}")); + match body { + Some(body) => request.json(body), + None => request, + } + .send() + .await + .map_err(docker_error) + } +} + +fn docker_error(error: reqwest::Error) -> ControlError { + ControlError::Invalid(format!("Docker daemon request failed: {error}")) +} + +#[cfg(unix)] +fn docker_client(socket: &Path) -> Result { + reqwest::Client::builder() + .no_proxy() + .unix_socket(socket) + .timeout(std::time::Duration::from_secs(5)) + .build() + .map_err(docker_error) +} + +#[cfg(not(unix))] +fn docker_client(_socket: &Path) -> Result { + Err(ControlError::Invalid( + "Docker launchers require a Unix socket".into(), + )) +} diff --git a/crates/sqlbuild-cli/src/workers/_helpers/invocation.rs b/crates/sqlbuild-cli/src/workers/_helpers/invocation.rs new file mode 100644 index 0000000000..67f3054326 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/_helpers/invocation.rs @@ -0,0 +1,47 @@ +//! Both launchers hand off the same scoped invocation and exclude privileged credentials. + +use sqlbuild_core::command_output::constants::INVOCATION_CONTEXT_ENV; +use sqlbuild_orchestration::runs::models::RunClaim; + +use crate::workers::models::LauncherScope; + +pub(in crate::workers) fn arguments(scope: &LauncherScope<'_>, claim: &RunClaim) -> Vec { + [ + "location".into(), + "execute".into(), + "--server".into(), + scope.server_url.to_owned(), + "--location-id".into(), + scope.location_id.to_string(), + "--target".into(), + scope.target.to_owned(), + "--run-id".into(), + claim.request.run_id.to_string(), + "--claim-id".into(), + claim.claim_id.to_string(), + "--project-dir".into(), + ] + .into() +} + +pub(in crate::workers) fn excluded(name: &str, control_variable: &str) -> bool { + name == control_variable + || matches!( + name, + "SQLBUILD_CONTROL_DATABASE_URL" | "SQLBUILD_LOCATION_TOKEN" | "SQLBUILD_RUN_TOKEN" + ) + || name == INVOCATION_CONTEXT_ENV +} + +pub(in crate::workers) fn environment( + claim: &RunClaim, + bearer: &str, +) -> [(&'static str, String); 2] { + [ + ("SQLBUILD_RUN_TOKEN", bearer.into()), + (INVOCATION_CONTEXT_ENV, serde_json::json!({ + "run_id": claim.request.run_id, + "integration": {"name": "sqlbuild", "run_id": claim.request.run_id, "job_name": claim.request.job_key}, + }).to_string()), + ] +} diff --git a/crates/sqlbuild-cli/src/workers/_helpers/mod.rs b/crates/sqlbuild-cli/src/workers/_helpers/mod.rs index 16f45af3e7..b182827d2d 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/mod.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/mod.rs @@ -1,3 +1,7 @@ +mod child; pub(super) mod configuration; -mod local; +mod docker; +mod invocation; +mod recovery; mod shutdown; +mod supervisor; diff --git a/crates/sqlbuild-cli/src/workers/_helpers/recovery.rs b/crates/sqlbuild-cli/src/workers/_helpers/recovery.rs new file mode 100644 index 0000000000..1704f88f6b --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/_helpers/recovery.rs @@ -0,0 +1,64 @@ +//! A new leader resumes supervising pinned containers without relaunching their execution. + +use std::collections::BTreeSet; +use std::io::Write; + +use sqlbuild_orchestration::runs::main::list_run_claims::list_run_claims; +use sqlbuild_orchestration::scheduler::main::renew_lease::renew_lease; +use sqlbuild_orchestration::scheduler::models::SchedulerLease; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::models::ControlStore; + +use crate::workers::models::{ClaimedWorker, DockerWorker, WorkerChild, Workers}; + +impl Workers { + pub(in crate::workers) async fn recover( + &mut self, + store: &ControlStore, + lease: &SchedulerLease, + ) -> Result<(), ControlError> { + if self.recovered_epoch == Some(lease.epoch) { + return Ok(()); + } + let mut known: BTreeSet<_> = self + .active + .iter() + .map(|worker| worker.claim.claim_id) + .collect(); + for location in &self.configuration.docker { + let mut after = None; + loop { + renew_lease(store, lease).await?; + let claims = + list_run_claims(store, (location.location_id, &location.target), after).await?; + let Some(last) = claims.last() else { + break; + }; + after = Some(last.request.run_id); + for claim in claims { + if !known.insert(claim.claim_id) { + continue; + } + let child = match DockerWorker::prepare(location, &claim) { + Ok(child) => Some(WorkerChild::Docker(child)), + Err(ControlError::Invalid(_)) => None, + Err(error) => return Err(error), + }; + drop(writeln!( + std::io::stderr().lock(), + "Worker supervision recovered for run {}", + claim.request.run_id + )); + self.active.push(ClaimedWorker { + claim, + child, + stopping: None, + exit: None, + }); + } + } + } + self.recovered_epoch = Some(lease.epoch); + Ok(()) + } +} diff --git a/crates/sqlbuild-cli/src/workers/_helpers/shutdown.rs b/crates/sqlbuild-cli/src/workers/_helpers/shutdown.rs index 24ab13b5cf..9613633420 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/shutdown.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/shutdown.rs @@ -11,10 +11,10 @@ use sqlbuild_orchestration::scheduler::models::SchedulerLease; use sqlbuild_orchestration::store::errors::ControlError; use sqlbuild_orchestration::store::models::ControlStore; -use crate::workers::_helpers::local::{exit_code, publish, terminal, terminate}; -use crate::workers::models::{LocalWorker, LocalWorkers}; +use crate::workers::_helpers::supervisor::{publish, terminal}; +use crate::workers::models::{ClaimedWorker, WorkerChild, WorkerExit, Workers}; -impl LocalWorker { +impl ClaimedWorker { async fn reap( &mut self, store: &ControlStore, @@ -26,9 +26,11 @@ impl LocalWorker { if self.exit.is_some() { return Ok(()); } + let mut killed = false; if self.stopping.is_none() { - if terminate(child).is_err() { + if child.terminate().await.is_err() { child.kill().await?; + killed = true; } self.stopping = Some(Instant::now()); } @@ -40,10 +42,9 @@ impl LocalWorker { self.exit = Some(exit?); return Ok(()); } - _ = tokio::time::sleep_until(deadline.into()) => { + _ = tokio::time::sleep_until(deadline.into()), if !killed => { child.kill().await?; - self.exit = Some(child.wait().await?); - return Ok(()); + killed = true; } _ = renew.tick(), if lease.is_some() => { if let Some(lease) = lease @@ -66,9 +67,9 @@ impl LocalWorker { if !terminal(state.status) { let lease = lease.ok_or(ControlError::Fenced)?; let reason = if self.child.is_none() { - "local worker launch preparation failed before a child started" + "worker launch preparation failed before a child started" } else { - "scheduler stopped its local worker and reaped the child" + "scheduler confirmed its worker stopped or absent" }; publish( store, @@ -83,7 +84,7 @@ impl LocalWorker { RunOutcome::Failed }, reason: reason.into(), - exit_code: self.exit.map(exit_code), + exit_code: self.exit.and_then(WorkerExit::code), }, ), ) @@ -93,7 +94,7 @@ impl LocalWorker { } } -impl LocalWorkers { +impl Workers { pub(in crate::workers) async fn shutdown_workers( &mut self, store: &ControlStore, @@ -101,9 +102,31 @@ impl LocalWorkers { ) -> Result<(), ControlError> { let mut retrying = false; let mut renew = tokio::time::interval(Duration::from_secs(RENEW_INTERVAL_SECONDS)); + let mut renewed = Instant::now(); while !self.active.is_empty() { let mut failure = None; for index in (0..self.active.len()).rev() { + if renewed.elapsed() >= Duration::from_secs(RENEW_INTERVAL_SECONDS) + && let Some(lease) = lease + { + if let Err(error) = renew_lease(store, lease).await { + failure = Some(error); + } + renewed = Instant::now(); + } + if let Some(WorkerChild::Docker(child)) = &self.active[index].child { + match child.try_wait().await { + Ok(None) => { + self.active.swap_remove(index); + continue; + } + Ok(exit) => self.active[index].exit = exit, + Err(error) => { + failure.get_or_insert(error); + continue; + } + } + } match self.active[index].finish_shutdown(store, lease).await { Ok(()) => { self.active.swap_remove(index); @@ -121,7 +144,7 @@ impl LocalWorkers { if !retrying { drop(writeln!( std::io::stderr().lock(), - "Retrying worker shutdown publication; stopped workers and their claims remain owned" + "Retrying worker shutdown publication or physical observation; claims remain owned" )); retrying = true; } @@ -140,7 +163,7 @@ impl LocalWorkers { if retrying { drop(writeln!( std::io::stderr().lock(), - "Worker shutdown publication recovered; claims released" + "Worker shutdown recovered; stopped claims released and live containers remain recoverable" )); } Ok(()) diff --git a/crates/sqlbuild-cli/src/workers/_helpers/local.rs b/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs similarity index 66% rename from crates/sqlbuild-cli/src/workers/_helpers/local.rs rename to crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs index 49bbbe75af..b49de36293 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/local.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs @@ -1,11 +1,10 @@ -//! The privileged control role owns and reaps local children; workers receive claim-scoped HTTP. +//! One privileged owner supervises both launchers; workers receive only claim-scoped HTTP. use std::io::Write; use std::process::{ExitStatus, Stdio}; use std::time::{Duration, Instant}; use chrono::{TimeDelta, Utc}; -use sqlbuild_core::command_output::constants::INVOCATION_CONTEXT_ENV; use sqlbuild_orchestration::credentials::main::issue_token::issue_token; use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; use sqlbuild_orchestration::runs::main::change_run::change_run; @@ -14,25 +13,40 @@ use sqlbuild_orchestration::runs::main::observe_run::observe_run; use sqlbuild_orchestration::runs::models::{ RunAuthority, RunChange, RunChangeRequest, RunClaim, RunOutcome, RunStatus, }; +use sqlbuild_orchestration::scheduler::constants::RENEW_INTERVAL_SECONDS; +use sqlbuild_orchestration::scheduler::main::renew_lease::renew_lease; use sqlbuild_orchestration::scheduler::models::SchedulerLease; use sqlbuild_orchestration::store::errors::ControlError; use sqlbuild_orchestration::store::models::ControlStore; use tokio::process::{Child, Command}; use uuid::Uuid; -use crate::workers::models::{LocalLauncher, LocalWorker, LocalWorkers}; +use crate::workers::_helpers::invocation::{arguments, environment, excluded}; +use crate::workers::models::{ + ClaimedWorker, DockerWorker, Launcher, LocalLauncher, WorkerChild, WorkerExit, Workers, +}; -impl sqlbuild_orchestration::scheduler::types::RunLauncher for LocalWorkers { +impl sqlbuild_orchestration::scheduler::types::RunLauncher for Workers { async fn poll( &mut self, store: &ControlStore, lease: Option<&SchedulerLease>, ) -> Result<(), ControlError> { + if let Some(lease) = lease { + self.recover(store, lease).await?; + } + let mut renewed = Instant::now(); for index in (0..self.active.len()).rev() { + if renewed.elapsed() >= Duration::from_secs(RENEW_INTERVAL_SECONDS) + && let Some(lease) = lease + { + renew_lease(store, lease).await?; + renewed = Instant::now(); + } let worker = &mut self.active[index]; let mut state = observe_run(store, worker.claim.request.run_id).await?; if let Some(child) = worker.child.as_mut() { - worker.exit = worker.exit.or(child.try_wait()?); + worker.exit = worker.exit.or(child.try_wait().await?); } if worker.exit.is_none() && let Some(child) = worker.child.as_mut() @@ -48,7 +62,7 @@ impl sqlbuild_orchestration::scheduler::types::RunLauncher for LocalWorkers { if (state.cancel_requested_at.is_some() || not_started || silent) && worker.stopping.is_none() { - terminate(child)?; + child.terminate().await?; worker.stopping = Some(Instant::now()); } if worker @@ -56,7 +70,7 @@ impl sqlbuild_orchestration::scheduler::types::RunLauncher for LocalWorkers { .is_some_and(|at| at.elapsed() >= Duration::from_secs(60)) { child.kill().await?; - worker.exit = child.try_wait()?; + worker.exit = child.try_wait().await?; state = observe_run(store, worker.claim.request.run_id).await?; } } @@ -78,9 +92,8 @@ impl sqlbuild_orchestration::scheduler::types::RunLauncher for LocalWorkers { "local-exit", RunChange::Finished { outcome, - reason: "local worker exited; physical child has been reaped" - .into(), - exit_code: worker.exit.map(exit_code), + reason: "worker stopped; physical exit or absence confirmed".into(), + exit_code: worker.exit.and_then(WorkerExit::code), }, ), ) @@ -92,28 +105,53 @@ impl sqlbuild_orchestration::scheduler::types::RunLauncher for LocalWorkers { let Some(lease) = lease else { return Ok(()); }; - for location in &self.configuration.local { + self.launch_available(store, lease).await + } + + /// Never release claims until children have physically stopped, even if shutdown loses its fence. + async fn shutdown( + &mut self, + store: &ControlStore, + lease: Option<&SchedulerLease>, + ) -> Result<(), ControlError> { + self.shutdown_workers(store, lease).await + } +} + +impl Workers { + async fn launch_available( + &mut self, + store: &ControlStore, + lease: &SchedulerLease, + ) -> Result<(), ControlError> { + let mut renewed = Instant::now(); + for location in self.configuration.locations() { + let scope = location.scope(); let running = self .active .iter() .filter(|worker| { - worker.claim.request.location_id == location.location_id - && worker.claim.request.target == location.target + worker.claim.request.location_id == scope.location_id + && worker.claim.request.target == scope.target }) .count(); - for _ in running..location.max_workers.get() { + for _ in running..scope.max_workers.get() { + if renewed.elapsed() >= Duration::from_secs(RENEW_INTERVAL_SECONDS) { + renew_lease(store, lease).await?; + renewed = Instant::now(); + } let Some(claim) = claim_next_run( store, lease, &self.configuration.concurrency_limits, - Some((location.location_id, &location.target)), + Some((scope.location_id, scope.target)), ) .await? else { break; }; let index = self.active.len(); - self.active.push(LocalWorker { + self.active.push(ClaimedWorker { claim, child: None, stopping: None, @@ -121,11 +159,29 @@ impl sqlbuild_orchestration::scheduler::types::RunLauncher for LocalWorkers { }); let worker = &mut self.active[index]; let claim = &worker.claim; - if claim.request.image_id.is_some() { + if matches!(location, Launcher::Local(_)) && claim.request.image_id.is_some() { publish(store, lease, claim, ("local-image-rejected", RunChange::Finished { outcome: RunOutcome::Failed, reason: "image-pinned runs require a Docker launcher, not a local interpreter".into(), exit_code: None })).await?; self.active.swap_remove(index); continue; } + let container_id = if let Launcher::Docker(location) = location { + match DockerWorker::prepare(location, claim) { + Ok(child) => { + let name = child.name.clone(); + worker.child = Some(WorkerChild::Docker(child)); + Some(name) + } + Err(_) => { + publish(store, lease, claim, ("docker-image-rejected", RunChange::Finished { + outcome: RunOutcome::Failed, reason: "Docker worker requires a pinned image and supported launcher configuration".into(), exit_code: None, + })).await?; + self.active.swap_remove(index); + continue; + } + } + } else { + None + }; let context = AuditContext { actor: format!("scheduler/{}", lease.holder), granting_role: "scheduler".into(), @@ -146,17 +202,33 @@ impl sqlbuild_orchestration::scheduler::types::RunLauncher for LocalWorkers { store, lease, claim, - ("local-launched", RunChange::Launched { container_id: None }), + ("local-launched", RunChange::Launched { container_id }), ) .await?; - match spawn(location, claim, &credential.bearer, &self.control_variable) { - Ok(child) => { + let launched = match location { + Launcher::Local(location) => { + spawn(location, claim, &credential.bearer, &self.control_variable).map( + |child| { + worker.child = Some(WorkerChild::Local(child)); + }, + ) + } + Launcher::Docker(location) => { + if let Some(WorkerChild::Docker(child)) = &worker.child { + child + .launch(location, claim, &credential.bearer, &self.control_variable) + .await?; + } + Ok(()) + } + }; + match launched { + Ok(()) => { drop(writeln!( std::io::stderr().lock(), "Worker launched for run {}", claim.request.run_id )); - worker.child = Some(child); } Err(_) => { publish( @@ -180,15 +252,6 @@ impl sqlbuild_orchestration::scheduler::types::RunLauncher for LocalWorkers { } Ok(()) } - - /// Never release claims until children have physically stopped, even if shutdown loses its fence. - async fn shutdown( - &mut self, - store: &ControlStore, - lease: Option<&SchedulerLease>, - ) -> Result<(), ControlError> { - self.shutdown_workers(store, lease).await - } } fn spawn( @@ -198,39 +261,15 @@ fn spawn( control_variable: &str, ) -> Result { Ok(Command::new(&location.executable) - .args([ - "location", - "execute", - "--server", - &location.server_url, - "--location-id", - &location.location_id.to_string(), - "--target", - &location.target, - "--run-id", - &claim.request.run_id.to_string(), - "--claim-id", - &claim.claim_id.to_string(), - "--project-dir", - ]) + .args(arguments(&Launcher::Local(location).scope(), claim)) .arg(&location.project_dir) .current_dir(&location.project_dir) - .env_remove(control_variable) - .env_remove("SQLBUILD_CONTROL_DATABASE_URL") - .env_remove("SQLBUILD_LOCATION_TOKEN") - .env("SQLBUILD_RUN_TOKEN", bearer) - .env( - INVOCATION_CONTEXT_ENV, - serde_json::json!({ - "run_id": claim.request.run_id, - "integration": { - "name": "sqlbuild", - "run_id": claim.request.run_id, - "job_name": claim.request.job_key, - }, - }) - .to_string(), - ) + .env_clear() + .envs(std::env::vars_os().filter(|(key, _)| { + key.to_str() + .is_none_or(|key| !excluded(key, control_variable)) + })) + .envs(environment(claim, bearer)) .stdin(Stdio::null()) .stdout(Stdio::null()) .stderr(Stdio::inherit()) diff --git a/crates/sqlbuild-cli/src/workers/constants.rs b/crates/sqlbuild-cli/src/workers/constants.rs new file mode 100644 index 0000000000..881c48bea1 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/constants.rs @@ -0,0 +1,3 @@ +//! Immutable Docker image identities use the full SHA-256 hexadecimal digest. + +pub(super) const IMAGE_ID_HEX_LENGTH: usize = 64; diff --git a/crates/sqlbuild-cli/src/workers/main/mod.rs b/crates/sqlbuild-cli/src/workers/main/mod.rs index abf03f2f04..2c35ad2bd4 100644 --- a/crates/sqlbuild-cli/src/workers/main/mod.rs +++ b/crates/sqlbuild-cli/src/workers/main/mod.rs @@ -1 +1 @@ -pub(crate) mod open_local_workers; +pub(crate) mod open_workers; diff --git a/crates/sqlbuild-cli/src/workers/main/open_local_workers.rs b/crates/sqlbuild-cli/src/workers/main/open_workers.rs similarity index 62% rename from crates/sqlbuild-cli/src/workers/main/open_local_workers.rs rename to crates/sqlbuild-cli/src/workers/main/open_workers.rs index 33803a125e..544dae4fc5 100644 --- a/crates/sqlbuild-cli/src/workers/main/open_local_workers.rs +++ b/crates/sqlbuild-cli/src/workers/main/open_workers.rs @@ -1,18 +1,19 @@ -//! Open the installation's local process owner before accepting scheduler work. +//! Open the installation's process owner before accepting scheduler work. use sqlbuild_orchestration::store::errors::ControlError; use crate::workers::_helpers::configuration::read_configuration; -use crate::workers::models::LocalWorkers; +use crate::workers::models::Workers; -pub(crate) fn open_local_workers( +pub(crate) fn open_workers( configuration_path: Option<&str>, control_variable: String, scheduler_enabled: bool, -) -> Result { - Ok(LocalWorkers { +) -> Result { + Ok(Workers { configuration: read_configuration(configuration_path, scheduler_enabled)?, active: Vec::new(), control_variable, + recovered_epoch: None, }) } diff --git a/crates/sqlbuild-cli/src/workers/mod.rs b/crates/sqlbuild-cli/src/workers/mod.rs index ec01b51f5c..59bf71d77a 100644 --- a/crates/sqlbuild-cli/src/workers/mod.rs +++ b/crates/sqlbuild-cli/src/workers/mod.rs @@ -1,5 +1,6 @@ //! Native process owners launch claim-scoped workers without transferring control-store access. mod _helpers; +mod constants; pub(crate) mod main; pub(crate) mod models; diff --git a/crates/sqlbuild-cli/src/workers/models.rs b/crates/sqlbuild-cli/src/workers/models.rs index 6fb061ad4d..3cd00b537f 100644 --- a/crates/sqlbuild-cli/src/workers/models.rs +++ b/crates/sqlbuild-cli/src/workers/models.rs @@ -3,45 +3,97 @@ use std::collections::BTreeMap; use std::num::{NonZeroU32, NonZeroUsize}; use std::path::PathBuf; -use std::process::ExitStatus; use std::time::Instant; use sqlbuild_orchestration::runs::models::RunClaim; use tokio::process::Child; use uuid::Uuid; +/// Physical ownership and stop evidence are independent of the scheduler's durable run state. +#[derive(Debug)] +pub(super) enum WorkerChild { + Local(Child), + Docker(DockerWorker), +} + +#[derive(Clone, Copy, Debug)] +pub(super) enum WorkerExit { + Code(i64), + Absent, +} + #[derive(Debug, serde::Deserialize)] #[serde(deny_unknown_fields)] -pub(super) struct LocalLaunchers { +pub(super) struct Launchers { + #[serde(default)] pub(super) local: Vec, #[serde(default)] + pub(super) docker: Vec, + #[serde(default)] pub(super) concurrency_limits: BTreeMap, } +#[derive(Clone, Copy, Debug)] +pub(super) struct LauncherScope<'a> { + pub(super) location_id: Uuid, + pub(super) target: &'a str, + pub(super) server_url: &'a str, + pub(super) max_workers: NonZeroUsize, +} + /// No executable, project path or API address is supplied by an untrusted run request. #[derive(Debug, serde::Deserialize)] #[serde(deny_unknown_fields)] pub(super) struct LocalLauncher { pub(super) location_id: Uuid, pub(super) target: String, + pub(super) server_url: String, + pub(super) max_workers: NonZeroUsize, pub(super) project_dir: PathBuf, pub(super) executable: PathBuf, +} + +/// The installation chooses the daemon and location container; the snapshot pins the worker image. +#[derive(Debug, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct DockerLauncher { + pub(super) location_id: Uuid, + pub(super) target: String, pub(super) server_url: String, pub(super) max_workers: NonZeroUsize, + pub(super) socket: PathBuf, + pub(super) location_container: String, + pub(super) executable: String, +} + +#[derive(Clone, Copy, Debug)] +pub(super) enum Launcher<'a> { + Local(&'a LocalLauncher), + Docker(&'a DockerLauncher), +} + +/// Retain the deterministic identity before create/start, including uncertain HTTP responses. +#[derive(Debug)] +pub(super) struct DockerWorker { + pub(super) client: reqwest::Client, + pub(super) name: String, + pub(super) claim_id: Uuid, + pub(super) image_id: String, } #[derive(Debug)] -pub(super) struct LocalWorker { +pub(super) struct ClaimedWorker { pub(super) claim: RunClaim, - pub(super) child: Option, + pub(super) child: Option, pub(super) stopping: Option, - pub(super) exit: Option, + pub(super) exit: Option, } /// Physical ownership survives leadership loss; only the current fence changes durable lifecycle. #[derive(Debug)] -pub(crate) struct LocalWorkers { - pub(super) configuration: LocalLaunchers, - pub(super) active: Vec, +pub(crate) struct Workers { + pub(super) configuration: Launchers, + pub(super) active: Vec, pub(super) control_variable: String, + pub(super) recovered_epoch: Option, } diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs index 4340e30294..d9635b34d3 100644 --- a/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs +++ b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs @@ -1,12 +1,13 @@ //! Privileged test operators stay separate from the API-only worker session. use std::collections::BTreeMap; +use std::process::{Command, ExitStatus, Stdio}; use std::time::{Duration, SystemTime}; use sqlbuild_orchestration::credentials::main::create_location::create_location; use sqlbuild_orchestration::credentials::main::issue_token::issue_token; use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; -use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; +use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope, IssuedToken}; use sqlbuild_orchestration::runs::main::change_run::change_run; use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; use sqlbuild_orchestration::runs::models::{RunAuthority, RunChange, RunChangeRequest}; @@ -20,6 +21,7 @@ use sqlbuild_orchestration::worker::models::WorkerSession; use uuid::Uuid; use crate::admin_helpers::administer; +use crate::helpers::isolated_binary; use crate::test_types::TestControl; use crate::worker::test_types::WorkerControl; @@ -27,6 +29,30 @@ pub(crate) async fn start_claimed_worker( control: &TestControl, url: &str, ) -> (WorkerSession, WorkerControl) { + let (worker, baseline, token) = claim_worker_run(control, None).await; + change_run( + &worker.store, + RunAuthority::Scheduler(&worker.lease), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: worker.identity.0, + claim_id: Some(worker.identity.1), + change: RunChange::Launched { container_id: None }, + }, + ) + .await + .expect("launched worker"); + let client = connect_worker(url, &token.bearer, worker.identity).expect("worker connection"); + let session = start_worker(&client, &baseline) + .await + .expect("verified worker"); + (session, worker) +} + +pub(crate) async fn claim_worker_run( + control: &TestControl, + image_id: Option<&str>, +) -> (WorkerControl, SnapshotInput, IssuedToken) { let path = control.directory.path().join(".sqlbuild/control.sqlite3"); let store = open_store(path.to_str().expect("test database path")) .await @@ -47,7 +73,7 @@ pub(crate) async fn start_claimed_worker( git_revision: None, git_dirty: false, git_author: None, - image_id: None, + image_id: image_id.map(str::to_owned), environment_hash: "orders-environment".into(), sqb_version: env!("CARGO_PKG_VERSION").into(), nodes: BTreeMap::from([( @@ -124,30 +150,16 @@ pub(crate) async fn start_claimed_worker( ) .await .expect("run-scoped token"); - change_run( - &store, - RunAuthority::Scheduler(&lease), - &RunChangeRequest { - event_id: Uuid::now_v7(), - run_id, - claim_id: Some(identity.1), - change: RunChange::Launched { container_id: None }, - }, - ) - .await - .expect("launched worker"); - let client = connect_worker(url, &token.bearer, identity).expect("worker connection"); - let session = start_worker(&client, &baseline) - .await - .expect("verified worker"); ( - session, WorkerControl { store, context, + lease, token_id: token.identity.token_id, identity, }, + baseline, + token, ) } @@ -230,3 +242,81 @@ pub(crate) fn disconnect_worker(_worker: WorkerControl, control: &mut TestContro control.child.kill().expect("stop API process"); control.child.wait().expect("reap API process"); } + +pub(crate) fn start_recovery_scheduler( + control: &TestControl, + url: &str, + location_id: Uuid, +) -> TestControl { + let directory = tempfile::tempdir().expect("recovery scheduler logs"); + let configuration = directory.path().join("launchers.json"); + let launcher = serde_json::json!({"docker": [{ + "location_id": location_id, "target": "development", "server_url": url, + "max_workers": 1, "socket": directory.path().join("docker.sock"), + "location_container": "orders-location", "executable": "sqb" + }]}); + std::fs::write(&configuration, launcher.to_string()).expect("Docker scope configuration"); + let stderr = directory.path().join("scheduler.log"); + let child = Command::new(isolated_binary(directory.path())) + .args([ + "scheduler", + "--database-env", + "SQLBUILD_RECOVERY_TEST_STORE", + "--launchers", + ]) + .arg(configuration) + .current_dir(directory.path()) + .env("PATH", "") + .env( + "SQLBUILD_RECOVERY_TEST_STORE", + control.directory.path().join(".sqlbuild/control.sqlite3"), + ) + .stdout(Stdio::null()) + .stderr(std::fs::File::create(&stderr).expect("scheduler stderr")) + .spawn() + .expect("replacement scheduler"); + TestControl { + child, + directory, + stderr, + } +} + +pub(crate) async fn wait_for_worker_log(control: &mut TestControl, expected: &str) { + tokio::time::timeout(Duration::from_secs(5), async { + for _ in 0..250 { + let log = std::fs::read_to_string(&control.stderr).expect("worker supervision log"); + assert!( + control + .child + .try_wait() + .expect("scheduler status") + .is_none(), + "{log}" + ); + let Some(_) = log.find(expected) else { + tokio::time::sleep(Duration::from_millis(20)).await; + continue; + }; + return; + } + panic!("worker supervision exhausted readiness attempts"); + }) + .await + .expect("worker supervision phase completes"); +} + +pub(crate) async fn wait_for_worker_exit(control: &mut TestControl) -> ExitStatus { + tokio::time::timeout(Duration::from_secs(5), async { + for _ in 0..250 { + let Some(exit) = control.child.try_wait().expect("scheduler exit") else { + tokio::time::sleep(Duration::from_millis(20)).await; + continue; + }; + return exit; + } + panic!("worker supervision exhausted shutdown attempts"); + }) + .await + .expect("shutdown publishes retained terminal evidence") +} diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/test_recovery.rs b/crates/sqlbuild-cli/tests/native_commands/worker/test_recovery.rs new file mode 100644 index 0000000000..0cdf1611ca --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/worker/test_recovery.rs @@ -0,0 +1,79 @@ +//! A committed but unlaunchable claim must not poison every replacement scheduler. + +use std::time::Duration; + +use crate::helpers::{api_ready, start_control}; +use crate::worker::helpers::{ + claim_worker_run, start_recovery_scheduler, wait_for_worker_exit, wait_for_worker_log, +}; +use crate::worker::test_types::WorkerRecoveryTestCase; + +#[test] +fn given_an_unlaunchable_committed_claim_when_recovering_then_rejection_retries_keep_ownership() { + let test_cases = [ + WorkerRecoveryTestCase { + description: "snapshot without an image pin", + image_id: None, + expected_status: "failed", + }, + WorkerRecoveryTestCase { + description: "snapshot with a malformed image pin", + image_id: Some("sha256:invalid"), + expected_status: "failed", + }, + ]; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("recovery test runtime"); + for test_case in test_cases { + runtime.block_on(async { + let mut control = start_control(&["server", "--port", "0", "--no-scheduler"]); + let url = api_ready(&mut control).await; + let (worker, baseline, _) = claim_worker_run(&control, test_case.image_id).await; + let connection = + rusqlite::Connection::open(control.directory.path().join(".sqlbuild/control.sqlite3")) + .expect("observe durable ownership"); + connection + .busy_timeout(Duration::from_secs(5)) + .expect("database contention timeout"); + connection.execute_batch( + "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00Z'; + CREATE TRIGGER reject_recovery_publication BEFORE INSERT ON recall_events + WHEN NEW.event_type='run_terminated' BEGIN SELECT RAISE(ABORT, 'synthetic unavailable publication'); END;", + ).expect("expire the departed leader and interrupt terminal publication"); + let mut replacement = start_recovery_scheduler(&control, &url, baseline.location_id); + wait_for_worker_log(&mut replacement, "Retrying worker shutdown publication").await; + let held: (i64, i64) = connection.query_row( + "SELECT (SELECT COUNT(*) FROM sqb_write_claims), (SELECT COUNT(*) FROM sqb_run_liveness)", + [], |row| Ok((row.get(0)?, row.get(1)?)), + ).expect("retained claim while publication is unavailable"); + assert_eq!(held, (1, 1), "{}", test_case.description); + connection + .execute_batch("DROP TRIGGER reject_recovery_publication") + .expect("restore terminal publication"); + let exit = wait_for_worker_exit(&mut replacement).await; + assert!( + !exit.success(), + "the original publication failure remains visible" + ); + let finished: (String, i64, i64, i64) = connection.query_row( + "SELECT status, (SELECT COUNT(*) FROM sqb_write_claims), + (SELECT COUNT(*) FROM sqb_run_liveness), + (SELECT COUNT(*) FROM recall_events WHERE event_type='run_terminated') FROM sqb_runs WHERE run_id=?1", + [worker.identity.0.to_string()], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + ).expect("terminal fact and released durable claims"); + assert_eq!( + finished, + (test_case.expected_status.into(), 0, 0, 1), + "{}", + test_case.description + ); + assert!( + std::fs::read_to_string(&replacement.stderr) + .expect("shutdown recovery log") + .contains("Worker shutdown recovered") + ); + }); + } +} diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs b/crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs index 73fb9bf997..54d23d6116 100644 --- a/crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs +++ b/crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs @@ -6,9 +6,16 @@ pub(crate) struct WorkerHeartbeatTestCase { pub(crate) expected_cancelled: bool, } +pub(crate) struct WorkerRecoveryTestCase { + pub(crate) description: &'static str, + pub(crate) image_id: Option<&'static str>, + pub(crate) expected_status: &'static str, +} + pub(crate) struct WorkerControl { pub(crate) store: sqlbuild_orchestration::store::models::ControlStore, pub(crate) context: sqlbuild_orchestration::credentials::models::AuditContext, + pub(crate) lease: sqlbuild_orchestration::scheduler::models::SchedulerLease, pub(crate) token_id: uuid::Uuid, pub(crate) identity: (uuid::Uuid, uuid::Uuid), } diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/test_worker.rs b/crates/sqlbuild-cli/tests/native_commands/worker/test_worker.rs index 13b54de371..5bb5be2014 100644 --- a/crates/sqlbuild-cli/tests/native_commands/worker/test_worker.rs +++ b/crates/sqlbuild-cli/tests/native_commands/worker/test_worker.rs @@ -2,5 +2,7 @@ mod helpers; #[path = "test_heartbeat.rs"] mod test_heartbeat; +#[path = "test_recovery.rs"] +mod test_recovery; #[path = "test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-orchestration/src/runs/main/list_run_claims.rs b/crates/sqlbuild-orchestration/src/runs/main/list_run_claims.rs new file mode 100644 index 0000000000..216113dab9 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/list_run_claims.rs @@ -0,0 +1,39 @@ +//! Recover ownership from active claims, not stale lifecycle events or elapsed heartbeat guesses. + +use uuid::Uuid; + +use crate::runs::constants::MONITOR_BATCH_SIZE; +use crate::runs::models::RunClaim; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, decode_row::decode_row, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Keyset pages include claimed-but-unstarted work and never include a neighboring scope. +pub async fn list_run_claims( + store: &ControlStore, + (location_id, target): (Uuid, &str), + after: Option, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query( + transaction, + &format!( + "SELECT l.worker_instance AS claim_id, r.request_json AS request \ + FROM sqb_run_liveness l JOIN sqb_runs r ON r.run_id=l.run_id \ + WHERE r.location_id=$1 AND r.target=$2 AND ($3 IS NULL OR r.run_id>$3) \ + ORDER BY r.run_id LIMIT {MONITOR_BATCH_SIZE}" + ), + vec![ + Parameter::Id(location_id), + Parameter::Text(target.into()), + Parameter::OptionalId(after), + ], + ) + .await?; + let claims = rows + .into_iter() + .map(decode_row) + .collect::, _>>()?; + commit(transaction).await?; + Ok(claims) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/mod.rs b/crates/sqlbuild-orchestration/src/runs/main/mod.rs index a7989d799a..7609ff479d 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/mod.rs @@ -2,6 +2,7 @@ pub mod change_run; pub mod claim_next_run; pub mod heartbeat_run; pub(crate) mod job_write_set; +pub mod list_run_claims; pub mod list_run_nodes; pub mod list_runs; pub mod monitor_runs; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_claims.rs b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs index 43619dfc46..a34df737bc 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_claims.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs @@ -2,6 +2,7 @@ use std::collections::BTreeMap; use std::num::NonZeroU32; use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; +use sqlbuild_orchestration::runs::main::list_run_claims::list_run_claims; use sqlbuild_orchestration::runs::main::list_runs::list_runs; use sqlbuild_orchestration::runs::models::RunStatus; use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; @@ -87,6 +88,31 @@ fn given_higher_priority_work_for_other_scopes_when_a_local_launcher_claims_then "{}", test_case.description ); + assert_eq!( + list_run_claims(store, (location, "development"), None) + .await + .expect("recover queued claim"), + vec![claim.clone()], + "recovery includes a committed claim before launch" + ); + assert!( + list_run_claims(store, (location, "development"), Some(claim.request.run_id)) + .await + .expect("page after claimed run") + .is_empty() + ); + assert!( + list_run_claims(store, (other, "development"), None) + .await + .expect("unclaimed neighboring location") + .is_empty() + ); + assert!( + list_run_claims(store, (location, "secondary"), None) + .await + .expect("unclaimed neighboring target") + .is_empty() + ); assert!( claim_next_run( store, @@ -99,6 +125,12 @@ fn given_higher_priority_work_for_other_scopes_when_a_local_launcher_claims_then .is_none() ); finish(store, &lease, &claim).await; + assert!( + list_run_claims(store, (location, "development"), None) + .await + .expect("terminal claims are not recoverable") + .is_empty() + ); let next = claim_next_run(store, &lease, &BTreeMap::new(), None) .await .expect("unrestricted claim") diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 6edc268f3f..7b78e9f190 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -3,7 +3,7 @@ from dataclasses import dataclass, field from pathlib import Path from subprocess import Popen -from typing import Literal, NamedTuple +from typing import Any, Literal, NamedTuple class SnapshotApi(NamedTuple): @@ -24,6 +24,20 @@ class ScheduledWorker(NamedTuple): control_store: Path process: Popen[bytes] log: Path + docker: "DockerLocation | None" = None + + +class DockerLocation(NamedTuple): + image_id: str + container: str + socket: Path + + +class WorkerDeployment(NamedTuple): + kind: str + configuration: dict[str, Any] + publication_arguments: tuple[str, ...] = () + docker: DockerLocation | None = None @dataclass(frozen=True) @@ -86,5 +100,13 @@ class WorkerExecutionTestCase: expected_rows: list[tuple[int]] expected_node_count: int = 1 files: dict[str, str] = field(default_factory=dict) + files_after_publication: dict[str, str] = field(default_factory=dict) store_setup: str = "" - launcher: Literal["native", "uncooperative"] = "native" + launcher: Literal["native", "uncooperative", "docker"] = "native" + + +@dataclass(frozen=True) +class WorkerRestartTestCase(WorkerExecutionTestCase): + description: str + shutdown_signal: int = 15 + expected_scheduler_exit: int = 0 diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/__init__.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/__init__.py new file mode 100644 index 0000000000..8d7411a51a --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/__init__.py @@ -0,0 +1 @@ +"""Real container launcher verification.""" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py new file mode 100644 index 0000000000..58135a9dec --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py @@ -0,0 +1,126 @@ +"""A real Docker location with the installed runtime and an isolated synthetic warehouse.""" + +import json +import os +import shutil +import subprocess +import sys +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path +from uuid import uuid4 + +import pytest + +import sqlbuild +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + DockerLocation, + WorkerDeployment, +) + +DOCKER_SOCKET: Path = Path(os.environ.get("SQLBUILD_TEST_DOCKER_SOCKET", "/var/run/docker.sock")) +REQUIRES_DOCKER: pytest.MarkDecorator = pytest.mark.skipif( + os.environ.get("SQLBUILD_REQUIRE_DOCKER_TESTS") != "1" + and (sys.platform != "linux" or shutil.which("docker") is None or not DOCKER_SOCKET.exists()), + reason="Docker worker tests require a Linux Docker daemon", +) + + +def docker_command(socket: Path, *arguments: str) -> str: + """Keep daemon errors useful without dumping a container's credential-bearing environment.""" + result: subprocess.CompletedProcess[str] = subprocess.run( + ["docker", "--host", f"unix://{socket}", *arguments], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + timeout=20, + check=False, + ) + assert result.returncode == 0, result.stdout + return result.stdout.strip() + + +@contextmanager +def docker_deployment(directory: Path, executable: str) -> Iterator[WorkerDeployment]: + """The worker inherits only these runtime/project mounts, never the store or daemon socket.""" + socket: Path = DOCKER_SOCKET + project: Path = directory / "project" + image: str = os.environ.get("SQLBUILD_TEST_WORKER_IMAGE", "alpine:3.23") + image_id: str = docker_command(socket, "image", "inspect", "--format", "{{.Id}}", image) + name: str = f"sqlbuild-test-location-{uuid4()}" + python: Path = Path(sys.executable) + installations: list[Path] = [Path(sys.prefix), Path(sys.base_prefix)] + while python.is_symlink(): + python = python.parent / python.readlink() + installations.append(python.parent.parent) + mounts: list[str] = [] + for path in filter( + Path.exists, + dict.fromkeys( + [ + Path("/usr"), + Path("/lib"), + Path("/lib64"), + *installations, + Path(sqlbuild.__file__).parent.parent, + ] + ), + ): + mounts.extend(["--volume", f"{path}:{path}:ro"]) + mounts.extend(["--volume", f"{project}:{project}:rw"]) + container: str = docker_command( + socket, + "run", + "--detach", + "--name", + name, + "--network", + "host", + "--user", + f"{os.getuid()}:{os.getgid()}", + "--workdir", + str(project), + "--env", + "HOME=/tmp", + "--env", + "SQLBUILD_WORKER_TEST_VALUE=orders-context", + "--env", + "SQLBUILD_LOCATION_TOKEN=synthetic-location-credential", + "--env", + "SQLBUILD_CONTROL_DATABASE_URL=synthetic-control-store", + "--env", + "SQLBUILD_SCHEDULER_TEST_STORE=synthetic-configured-store", + *mounts, + image_id, + "/usr/bin/sleep", + "300", + ) + try: + yield WorkerDeployment( + "docker", + {"socket": str(socket), "location_container": container, "executable": executable}, + ("--image-id", image_id), + DockerLocation(image_id, container, socket), + ) + finally: + containers: list[str] = docker_command( + socket, "ps", "--all", "--quiet", "--filter", f"volume={project}" + ).split() + for owned in containers: + _ = (directory / f"docker-{owned}.log").write_text( + docker_command(socket, "logs", owned) + ) + _ = docker_command(socket, "rm", "--force", owned) + + +def worker_container(location: DockerLocation, location_id: str, run_id: str) -> dict[str, object]: + """Return only physical identity/exit evidence, not Docker's secret-bearing configuration.""" + name: str = f"sqlbuild-{location_id}-{run_id}" + result: dict[str, object] = json.loads( + docker_command(location.socket, "inspect", "--format", "{{json .State}}", name) + ) + assert ( + docker_command(location.socket, "inspect", "--format", "{{.Image}}", name) + == location.image_id + ) + return result diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index 24c11ac8ff..a91b471edf 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -29,8 +29,12 @@ ClaimedWorker, ScheduledWorker, SnapshotApi, + WorkerDeployment, WorkerExecutionTestCase, ) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.docker.helpers import ( + docker_deployment, +) from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files @@ -78,8 +82,10 @@ def wait_for_generation( return current -def wait_for_log(process: subprocess.Popen[bytes], log: Path, text: str) -> None: - deadline: float = time.monotonic() + 15 +def wait_for_log( + process: subprocess.Popen[bytes], log: Path, text: str, *, timeout: float = 15 +) -> None: + deadline: float = time.monotonic() + timeout while text not in log.read_text(): assert process.poll() is None, log.read_text() assert time.monotonic() < deadline, log.read_text() @@ -217,14 +223,17 @@ def current_snapshot(api: SnapshotApi, target: str) -> dict[str, Any]: return result -def native_launcher(directory: Path, executable: str) -> AbstractContextManager[str]: +def native_launcher(directory: Path, executable: str) -> AbstractContextManager[WorkerDeployment]: """Use the installed native launcher without a synthetic child.""" - del directory - return nullcontext(executable) + return nullcontext( + WorkerDeployment( + "local", {"project_dir": str(directory / "project"), "executable": executable} + ) + ) @contextmanager -def uncooperative_launcher(directory: Path, executable: str) -> Iterator[str]: +def uncooperative_launcher(directory: Path, executable: str) -> Iterator[WorkerDeployment]: """Own a synthetic process that ignores TERM, including cleanup after a failed assertion.""" del executable script: Path = directory / "worker" @@ -236,7 +245,9 @@ def uncooperative_launcher(directory: Path, executable: str) -> Iterator[str]: ) script.chmod(0o700) try: - yield str(script) + yield WorkerDeployment( + "local", {"project_dir": str(directory / "project"), "executable": str(script)} + ) finally: with suppress(FileNotFoundError, ProcessLookupError): os.kill(int((directory / "project/worker.pid").read_text()), signal.SIGKILL) @@ -270,13 +281,14 @@ def scheduled_worker( | test_case.files, ) executable: str = str(Path(sys.executable).with_name("sqb")) - launchers: dict[str, Callable[[Path, str], AbstractContextManager[str]]] = { + launchers: dict[str, Callable[[Path, str], AbstractContextManager[WorkerDeployment]]] = { "native": native_launcher, + "docker": docker_deployment, "uncooperative": uncooperative_launcher, } with ( snapshot_api(directory / "control") as api, - launchers[test_case.launcher](directory, executable) as launch_executable, + launchers[test_case.launcher](directory, executable) as deployment, ): published: subprocess.CompletedProcess[str] = subprocess.run( [ @@ -293,6 +305,7 @@ def scheduled_worker( api.url, "--generation", "1", + *deployment.publication_arguments, ], env=os.environ | {"SQLBUILD_LOCATION_TOKEN": api.bearer}, capture_output=True, @@ -301,23 +314,15 @@ def scheduled_worker( check=False, ) assert published.returncode == 0, published.stderr + write_files(project_dir=project, files=test_case.files_after_publication) configuration: Path = directory / "launchers.json" - _ = configuration.write_text( - json.dumps( - { - "local": [ - { - "location_id": api.location_id, - "target": "development", - "project_dir": str(project), - "executable": launch_executable, - "server_url": api.url, - "max_workers": 1, - } - ] - } - ) - ) + launcher: dict[str, Any] = { + "location_id": api.location_id, + "target": "development", + "server_url": api.url, + "max_workers": 1, + } | deployment.configuration + _ = configuration.write_text(json.dumps({deployment.kind: [launcher]})) control_store: Path = directory / "control/.sqlbuild/control.sqlite3" with sqlite3.connect(control_store) as control: _ = control.executescript(test_case.store_setup) @@ -335,18 +340,17 @@ def scheduled_worker( log, directory / "scheduler.stdout", ) as process: - yield ScheduledWorker(api, project, control_store, process, log) + yield ScheduledWorker(api, project, control_store, process, log, deployment.docker) def worker_result(worker: ScheduledWorker, test_case: WorkerExecutionTestCase) -> dict[str, Any]: """Run/node outcomes, Recall, claims and the physical warehouse agree.""" - api, project, control_store, process, log = worker + api, project, control_store, process, log, _ = worker run: dict[str, Any] = wait_for_run(api, process, log) assert run["node_counts_json"] == {test_case.expected_status: test_case.expected_node_count}, ( log.read_text() ) assert run["job_key"] == "annual_orders" - assert "Worker launched for run" in log.read_text() runs_url: str = f"{api.url}/api/locations/{api.location_id}/targets/development/runs" headers: dict[str, str] = {"Authorization": f"Bearer {api.bearer}"} with urlopen( @@ -357,6 +361,11 @@ def worker_result(worker: ScheduledWorker, test_case: WorkerExecutionTestCase) - assert nodes[0]["started_at"] is not None assert nodes[0]["ended_at"] is not None with sqlite3.connect(control_store) as control: + assert control.execute( + "SELECT count(*) FROM recall_events " + "WHERE invocation_id=? AND event_type='run_launched'", + (f"scheduler/{run['run_id']}",), + ).fetchone() == (1,) contexts: list[tuple[str | None]] = control.execute( "SELECT external_context FROM recall_invocations WHERE command = 'location'" ).fetchall() @@ -371,6 +380,48 @@ def worker_result(worker: ScheduledWorker, test_case: WorkerExecutionTestCase) - return run +def running_statement(worker: ScheduledWorker) -> dict[str, Any]: + """The worker has started the real long query, not merely entered its Python interpreter.""" + active: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, pending=frozenset({"queued", "launched"}) + ) + assert active["status"] == "running", worker.log.read_text() + deadline: float = time.monotonic() + 15 + with sqlite3.connect(worker.control_store) as control: + while not control.execute( + "SELECT 1 FROM recall_artifacts WHERE CAST(content AS TEXT) LIKE ?", + ("%range(10000000000)%",), + ).fetchone(): + assert worker.process.poll() is None, worker.log.read_text() + assert time.monotonic() < deadline, worker.log.read_text() + time.sleep(0.02) + return active + + +def cancel_running_statement(worker: ScheduledWorker) -> None: + """Wait for the real long-running query before exercising the administrative cancel command.""" + active: dict[str, Any] = running_statement(worker) + cancelled: subprocess.CompletedProcess[str] = subprocess.run( + [ + str(Path(sys.executable).with_name("sqb")), + "admin", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "cancel-run", + active["run_id"], + "--event-id", + str(uuid4()), + "--reason", + "Cancel the active test statement.", + ], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + assert cancelled.returncode == 0, cancelled.stderr + + def wait_for_run( api: SnapshotApi, process: subprocess.Popen[bytes], diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py new file mode 100644 index 0000000000..a09544ebf1 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py @@ -0,0 +1,252 @@ +"""A published schedule executes in a real pinned container, not a mocked launcher.""" + +import signal +import sqlite3 +import sys +import time +from pathlib import Path +from typing import Any + +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + ScheduledWorker, + WorkerExecutionTestCase, + WorkerRestartTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.docker.helpers import ( + REQUIRES_DOCKER, + docker_command, + worker_container, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + cancel_running_statement, + location_process, + running_statement, + scheduled_worker, + wait_for_log, + wait_for_run, + worker_result, +) + + +@REQUIRES_DOCKER +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "pinned container inherits project context but not privileged credentials", + "SELECT CAST(value AS BIGINT) AS amount FROM (VALUES ('37')) t(value)", + "SELECT amount FROM orders", + "success", + [(37,)], + launcher="docker", + files={ + "orchestration/orders.py": ( + "import os\nfrom sqlbuild.orchestration import job, schedule\n" + "if 'SQLBUILD_RUN_TOKEN' in os.environ:\n" + " assert os.getenv('SQLBUILD_WORKER_TEST_VALUE') == 'orders-context'\n" + " for name in ('SQLBUILD_LOCATION_TOKEN', 'SQLBUILD_CONTROL_DATABASE_URL', " + "'SQLBUILD_SCHEDULER_TEST_STORE'):\n" + " assert name not in os.environ, name\n" + 'annual_orders = job("annual_orders", select="orders", ' + 'description="Build orders.", ' + 'schedule=schedule(cron="0 0 1 1 *", timezone="UTC", ' + 'starts_at="2000-01-01T00:00:00Z"))\n' + ) + }, + ), + WorkerExecutionTestCase( + "container warehouse error retains failed node evidence", + "SELECT CAST(value AS BIGINT) AS amount FROM (VALUES ('invalid')) t(value)", + "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'", + "failed", + [(0,)], + launcher="docker", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_scheduled_docker_job_when_published_then_a_pinned_worker_executes_and_reports( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + run: dict[str, Any] = worker_result(worker, test_case) + assert run["status"] == test_case.expected_status, worker.log.read_text() + assert worker.docker is not None + deadline: float = time.monotonic() + 10 + state: dict[str, object] = worker_container( + worker.docker, worker.api.location_id, run["run_id"] + ) + while state["Running"]: + assert time.monotonic() < deadline, "Terminal worker did not physically exit" + time.sleep(0.02) + state = worker_container(worker.docker, worker.api.location_id, run["run_id"]) + assert state["ExitCode"] == {"success": 0, "failed": 1}[test_case.expected_status] + + +@REQUIRES_DOCKER +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "container refuses changed source before writing", + "SELECT 23 AS amount", + "", + "failed", + [], + launcher="docker", + files_after_publication={ + "models/orders.sql": "MODEL (description 'Orders.', materialized table);\nSELECT 41 AS amount\n" + }, + ) + ], + ids=lambda case: case.description, +) +def test_given_a_pinned_snapshot_when_container_source_drifts_then_no_warehouse_write_is_permitted( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + run: dict[str, Any] = wait_for_run(worker.api, worker.process, worker.log) + assert run["status"] == test_case.expected_status, worker.log.read_text() + assert run["node_counts_json"] == {"skipped": 1} + assert not (worker.project / "warehouse.duckdb").exists() + assert worker.docker is not None + name: str = f"sqlbuild-{worker.api.location_id}-{run['run_id']}" + assert "node content differs" in docker_command(worker.docker.socket, "logs", name) + assert ( + worker_container(worker.docker, worker.api.location_id, run["run_id"])["ExitCode"] == 1 + ) + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + + +@REQUIRES_DOCKER +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "Docker cancellation stops an active warehouse statement", + "SELECT sum(sin(value)) AS amount FROM range(10000000000) t(value)", + "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'", + "cancelled", + [(0,)], + launcher="docker", + ) + ], + ids=lambda case: case.description, +) +def test_given_an_executing_container_when_cancelled_then_the_statement_stops_before_claim_release( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + cancel_running_statement(worker) + run: dict[str, Any] = worker_result(worker, test_case) + assert run["status"] == test_case.expected_status, worker.log.read_text() + + +@REQUIRES_DOCKER +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "killed Docker worker preserves its physical exit code", + "SELECT sum(sin(value)) AS amount FROM range(10000000000) t(value)", + "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'", + "failed", + [(0,)], + launcher="docker", + ) + ], + ids=lambda case: case.description, +) +def test_given_an_executing_container_when_killed_then_the_monitor_records_exit_and_releases_claims( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + active: dict[str, Any] = running_statement(worker) + assert worker.docker is not None + name: str = f"sqlbuild-{worker.api.location_id}-{active['run_id']}" + _ = docker_command(worker.docker.socket, "kill", "--signal", "KILL", name) + run: dict[str, Any] = worker_result(worker, test_case) + assert run["status"] == test_case.expected_status, worker.log.read_text() + assert ( + worker_container(worker.docker, worker.api.location_id, run["run_id"])["ExitCode"] + == 137 + ) + with sqlite3.connect(worker.control_store) as control: + assert control.execute( + "SELECT json_extract(payload, '$.change.change.exit_code') FROM recall_events " + "WHERE event_type='run_terminated'" + ).fetchone() == (137,) + + +@REQUIRES_DOCKER +@pytest.mark.parametrize( + "test_case", + [ + WorkerRestartTestCase( + "graceful scheduler restart retains the executing container", + "SELECT sum(sin(value)) AS amount FROM range(10000000000) t(value)", + "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'", + "cancelled", + [(0,)], + launcher="docker", + shutdown_signal=signal.SIGTERM, + expected_scheduler_exit=0, + ), + WorkerRestartTestCase( + "abrupt scheduler death retains the executing container", + "SELECT sum(sin(value)) AS amount FROM range(10000000000) t(value)", + "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'", + "cancelled", + [(0,)], + launcher="docker", + shutdown_signal=signal.SIGKILL, + expected_scheduler_exit=-signal.SIGKILL, + ), + ], + ids=lambda case: case.description, +) +def test_given_an_executing_container_when_scheduler_restarts_then_the_same_worker_is_adopted_without_relaunch( + test_case: WorkerRestartTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + active: dict[str, Any] = running_statement(worker) + assert worker.docker is not None + before: dict[str, object] = worker_container( + worker.docker, worker.api.location_id, active["run_id"] + ) + worker.process.send_signal(test_case.shutdown_signal) + assert worker.process.wait(timeout=10) == test_case.expected_scheduler_exit, ( + worker.log.read_text() + ) + assert worker_container(worker.docker, worker.api.location_id, active["run_id"])["Running"] + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (1,) + log: Path = tmp_path / "replacement-scheduler.log" + with location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "scheduler", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "--launchers", + str(tmp_path / "launchers.json"), + ], + log, + tmp_path / "replacement-scheduler.stdout", + ) as process: + replacement: ScheduledWorker = worker._replace(process=process, log=log) + wait_for_log(process, log, "Worker supervision recovered for run", timeout=45) + assert ( + worker_container(worker.docker, worker.api.location_id, active["run_id"])["Pid"] + == before["Pid"] + ) + cancel_running_statement(replacement) + run: dict[str, Any] = worker_result(replacement, test_case) + assert run["status"] == test_case.expected_status, log.read_text() + with sqlite3.connect(worker.control_store) as control: + assert control.execute( + "SELECT count(*) FROM recall_events WHERE event_type='run_claimed'" + ).fetchone() == (1,) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index 9e8dad99f7..8658227d69 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -4,8 +4,6 @@ import os import signal import sqlite3 -import subprocess -import sys import time from pathlib import Path from typing import Any @@ -32,11 +30,11 @@ WorkerExecutionTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + cancel_running_statement, claimed_worker, scheduled_worker, snapshot_api, wait_for_log, - wait_for_run, worker_result, ) from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files @@ -224,7 +222,7 @@ def test_given_a_scheduled_job_when_published_then_an_isolated_worker_reports_re "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'", "cancelled", [(0,)], - ) + ), ], ids=lambda case: case.description, ) @@ -232,38 +230,7 @@ def test_given_an_executing_scheduled_worker_when_cancelled_then_the_statement_s test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: - active: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, pending=frozenset({"queued", "launched"}) - ) - assert active["status"] == "running", worker.log.read_text() - deadline: float = time.monotonic() + 15 - with sqlite3.connect(worker.control_store) as control: - while not control.execute( - "SELECT 1 FROM recall_artifacts WHERE CAST(content AS TEXT) LIKE ?", - ("%range(10000000000)%",), - ).fetchone(): - assert worker.process.poll() is None, worker.log.read_text() - assert time.monotonic() < deadline, worker.log.read_text() - time.sleep(0.02) - cancelled: subprocess.CompletedProcess[str] = subprocess.run( - [ - str(Path(sys.executable).with_name("sqb")), - "admin", - "--database-env", - "SQLBUILD_SCHEDULER_TEST_STORE", - "cancel-run", - active["run_id"], - "--event-id", - str(uuid4()), - "--reason", - "Cancel the active test statement.", - ], - capture_output=True, - text=True, - timeout=10, - check=False, - ) - assert cancelled.returncode == 0, cancelled.stderr + cancel_running_statement(worker) run: dict[str, Any] = worker_result(worker, test_case) assert run["status"] == test_case.expected_status, worker.log.read_text() From c310faafde0e16adb7037820b44988500fcda2a5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 15:48:02 +0000 Subject: [PATCH 36/59] feat: checkpoint backend groups and pinned reruns (WIP) Preserve the current backend implementation as a remote-review checkpoint. The pull request remains draft and this is not release-ready. Group admission and the four complexity fixes passed targeted real-stack verification. Pinned reruns pass the SQLite/PostgreSQL transaction regression; installed worker rerun coverage and final aggregate gates remain in progress. Six existing Fensu interface findings remain unresolved. Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../edge_cases/orchestration/nested_groups.py | 7 + .../orchestration/edge_cases/definitions.py | 35 +- .../sqlbuild-cli/src/argv/command_table.json | 73 ++ .../src/native_commands/_helpers/decision.rs | 30 +- .../src/native_commands/main/_run_admin.rs | 5 + .../src/native_commands/models.rs | 3 +- .../src/workers/_helpers/configuration.rs | 9 + .../src/workers/_helpers/supervisor.rs | 25 +- .../src/api/_helpers/snapshots.rs | 21 + .../src/api/main/serve_api.rs | 4 + .../sqlbuild-orchestration/src/api/models.rs | 1 + .../src/declarations/_helpers/documents.rs | 32 + .../src/declarations/_helpers/mod.rs | 1 + .../declarations/main/frozen_definitions.rs | 22 + .../declarations/main/group_dependencies.rs | 92 ++ .../src/declarations/main/mod.rs | 2 + .../declarations/main/publish_definitions.rs | 16 +- .../declarations/main/snapshot_definitions.rs | 47 +- .../src/declarations/models.rs | 9 + .../src/runs/_helpers/groups/admission.rs | 186 ++++ .../src/runs/_helpers/groups/capacity.rs | 121 +++ .../src/runs/_helpers/groups/context.rs | 86 ++ .../src/runs/_helpers/groups/lifecycle.rs | 206 +++++ .../src/runs/_helpers/groups/mod.rs | 5 + .../src/runs/_helpers/groups/selection.rs | 139 +++ .../runs/_helpers/{ => lifecycle}/changes.rs | 2 +- .../runs/_helpers/{ => lifecycle}/claims.rs | 57 +- .../runs/_helpers/{ => lifecycle}/events.rs | 2 +- .../src/runs/_helpers/lifecycle/mod.rs | 8 + .../runs/_helpers/{ => lifecycle}/nodes.rs | 2 +- .../runs/_helpers/{ => lifecycle}/output.rs | 2 +- .../_helpers/{ => lifecycle}/projection.rs | 2 +- .../{ => lifecycle}/publish_change.rs | 20 +- .../runs/_helpers/{ => lifecycle}/recall.rs | 2 +- .../src/runs/_helpers/mod.rs | 13 +- .../{job_request.rs => requests/job.rs} | 24 +- .../src/runs/_helpers/requests/mod.rs | 3 + .../{request.rs => requests/publication.rs} | 36 +- .../src/runs/_helpers/requests/rerun.rs | 68 ++ .../src/runs/main/advance_groups.rs | 51 ++ .../src/runs/main/change_run.rs | 6 +- .../src/runs/main/claim_next_run.rs | 71 +- .../src/runs/main/heartbeat_run.rs | 2 +- .../src/runs/main/job_write_set.rs | 8 +- .../src/runs/main/mod.rs | 2 + .../src/runs/main/monitor_runs.rs | 2 +- .../src/runs/main/publish_run_output.rs | 2 +- .../src/runs/main/publish_run_recall.rs | 4 +- .../src/runs/main/queue_run.rs | 65 +- .../src/runs/main/request_job_run.rs | 28 +- .../src/runs/main/request_rerun.rs | 54 ++ .../src/runs/main/request_run.rs | 2 +- .../src/runs/main/run_execution.rs | 3 + .../sqlbuild-orchestration/src/runs/models.rs | 79 ++ .../src/scheduler/_helpers/persistence.rs | 10 +- .../src/scheduler/main/evaluate_trigger.rs | 2 +- .../src/scheduler/models.rs | 1 + .../src/snapshots/_helpers/reading.rs | 19 +- .../src/snapshots/main/current_snapshot.rs | 30 +- .../snapshots/main/current_snapshot_record.rs | 23 + .../src/snapshots/main/mod.rs | 3 + .../snapshots/main/next_remote_generation.rs | 6 +- .../snapshots/main/read_snapshot_documents.rs | 36 + .../src/snapshots/main/read_snapshot_nodes.rs | 36 + .../tests/api/test_contract.rs | 38 + crates/sqlbuild-orchestration/tests/runs.rs | 8 + .../tests/runs/claims/helpers.rs | 72 ++ .../tests/runs/test_claims.rs | 93 ++ .../tests/runs/test_group_admission.rs | 803 ++++++++++++++++++ .../tests/runs/test_groups.rs | 423 +++++++++ .../tests/runs/test_reruns.rs | 301 +++++++ .../tests/runs/test_types.rs | 28 + .../src/bindings/_helpers/cli/dispatch.rs | 55 +- .../_helpers/compiled_project/snapshot.rs | 10 +- .../compiled_project/snapshot_declarations.rs | 143 +++- .../compiled_project/snapshot_execution.rs | 19 +- .../_helpers/location_command/worker.rs | 96 ++- .../native_host/orchestration_declarations.rs | 60 +- .../src/schema/_helpers/mod.rs | 1 - .../src/schema/_helpers/postgres_views.rs | 91 -- .../src/schema/_helpers/relational_views.rs | 235 +++-- .../src/schema/_helpers/rendering.rs | 16 + .../schema/main/control_view_statements.rs | 4 +- src/sqlbuild/orchestration.py | 5 +- .../main/compile_snapshot.py | 4 +- .../orchestration_declarations/models.py | 15 + .../compiler/compiled_project/_test_types.py | 1 + .../publication/_test_types.py | 10 + .../compiled_project/publication/helpers.py | 31 +- .../publication/test_worker.py | 295 +++++++ .../test_orchestration_snapshot.py | 76 +- 91 files changed, 4430 insertions(+), 466 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/nested_groups.py create mode 100644 crates/sqlbuild-orchestration/src/declarations/_helpers/documents.rs create mode 100644 crates/sqlbuild-orchestration/src/declarations/main/frozen_definitions.rs create mode 100644 crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/groups/admission.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/groups/capacity.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/groups/context.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/groups/lifecycle.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/groups/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/groups/selection.rs rename crates/sqlbuild-orchestration/src/runs/_helpers/{ => lifecycle}/changes.rs (98%) rename crates/sqlbuild-orchestration/src/runs/_helpers/{ => lifecycle}/claims.rs (60%) rename crates/sqlbuild-orchestration/src/runs/_helpers/{ => lifecycle}/events.rs (92%) create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs rename crates/sqlbuild-orchestration/src/runs/_helpers/{ => lifecycle}/nodes.rs (97%) rename crates/sqlbuild-orchestration/src/runs/_helpers/{ => lifecycle}/output.rs (97%) rename crates/sqlbuild-orchestration/src/runs/_helpers/{ => lifecycle}/projection.rs (98%) rename crates/sqlbuild-orchestration/src/runs/_helpers/{ => lifecycle}/publish_change.rs (73%) rename crates/sqlbuild-orchestration/src/runs/_helpers/{ => lifecycle}/recall.rs (97%) rename crates/sqlbuild-orchestration/src/runs/_helpers/{job_request.rs => requests/job.rs} (77%) create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/requests/mod.rs rename crates/sqlbuild-orchestration/src/runs/_helpers/{request.rs => requests/publication.rs} (62%) create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/requests/rerun.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/advance_groups.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/request_rerun.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot_record.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_documents.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_nodes.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/test_groups.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/test_reruns.rs delete mode 100644 crates/sqlbuild-recall/src/schema/_helpers/postgres_views.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/nested_groups.py b/benchmarks/duckshop/edge_cases/orchestration/nested_groups.py new file mode 100644 index 0000000000..35727b2092 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/nested_groups.py @@ -0,0 +1,7 @@ +"""A group can contain jobs, but cannot contain another group.""" + +from sqlbuild.orchestration import job, job_group + +orders = job("orders_job", description="Build orders.", select="orders") +daily = job_group("daily_orders", description="Build daily orders.", jobs=[orders]) +weekly = job_group("weekly_orders", description="Unsupported nested membership.", jobs=[daily]) diff --git a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py index 3fcca7b42e..af566e9dff 100644 --- a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py +++ b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py @@ -1,12 +1,12 @@ """Exercise alias deduplication and cyclic factory containers during native discovery.""" -from sqlbuild.orchestration import job, schedule +from sqlbuild.orchestration import job, job_group, schedule edge_orders = job( "edge_snapshot_orders", description="Refresh only the synthetic edge-case models.", select="tag:edge_case", - settings={"run_tests": False}, + settings={"run_tests": False, "vars": {"edge_delivery_batch": "1"}, "reload": True}, concurrency=2, schedule=schedule( cron="15 2 * * *", @@ -20,3 +20,34 @@ edge_alias = edge_orders edge_factory_results = [[edge_orders]] edge_factory_results.append(edge_factory_results) +edge_group = job_group( + "edge_snapshot_group", + description="Exercise group discovery through shared, cyclic declaration containers.", + jobs=[edge_orders], +) +edge_factory_results.append(edge_group) + +edge_inventory = job( + "edge_snapshot_inventory", + description="Exercise a child limit independent of its parent.", + select="tag:edge_case", + concurrency_groups=("edge_products_limit",), +) +edge_products = job( + "edge_snapshot_products", + description="Exercise the opposing parent and child acquisition order.", + select="tag:edge_case", + concurrency_groups=("edge_inventory_limit",), +) +edge_inventory_group = job_group( + "edge_inventory_group", + description="Complete inventory without reserving its child's product limit.", + jobs=[edge_inventory], + concurrency_groups=("edge_inventory_limit",), +) +edge_products_group = job_group( + "edge_products_group", + description="Wait for safe admission when both installation limits are one.", + jobs=[edge_products], + concurrency_groups=("edge_products_limit",), +) diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index 08bbcf87c6..ed0917d366 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -448,6 +448,74 @@ ] } }, + { + "name": "rerun-run", + "parser": { + "prog": "sqb admin rerun-run", + "description": "Retry a terminal run using its pinned snapshot and settings, with an atomic security audit.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": ["-h", "--help"], + "dest": "help", + "nargs": 0, + "default": {"suppress": true}, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "SOURCE_RUN_ID", + "required": true, + "help": "terminal source run UUID", + "group": 0 + }, + { + "kind": "store", + "options": ["--run-id"], + "dest": "admin_event_id", + "required": true, + "help": "new run UUID; reuse it with identical arguments after an uncertain result", + "group": 1 + }, + { + "kind": "store_true", + "options": ["--from-failure"], + "dest": "admin_from_failure", + "nargs": 0, + "default": false, + "help": "retry exactly the failed and skipped nodes, excluding successful nodes", + "group": 1 + }, + { + "kind": "store", + "options": ["--priority"], + "dest": "admin_priority", + "type": "int", + "help": "signed queue priority; higher values run first (default: 0)", + "group": 1 + }, + { + "kind": "store", + "options": ["--reason"], + "dest": "admin_reason", + "required": true, + "help": "nonblank reason recorded with the security audit", + "group": 1 + } + ] + } + }, { "name": "request-run", "parser": { @@ -800,6 +868,11 @@ "metavar": "request-run", "help": "queue a pinned published job with an atomic security audit" }, + { + "dest": "rerun-run", + "metavar": "rerun-run", + "help": "retry a terminal run or exactly its failed and skipped nodes" + }, { "dest": "cancel-run", "metavar": "cancel-run", diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index aae4693e44..c0e69b8143 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -8,7 +8,7 @@ use serde_json::json; use sqlbuild_orchestration::credentials::models::{CredentialScope, PrincipalKind}; use sqlbuild_orchestration::declarations::models::TriggerChange; use sqlbuild_orchestration::platform::models::{LocationWork, LocationWorkRequest}; -use sqlbuild_orchestration::runs::models::JobRunRequest; +use sqlbuild_orchestration::runs::models::{JobRunRequest, RerunRequest}; use sqlbuild_orchestration::store::errors::ControlError; use uuid::Uuid; @@ -89,8 +89,9 @@ pub(crate) fn control_admin(namespace: &Namespace) -> Result { - let identity_option = if action.as_deref() == Some("request-run") { + Some("change-trigger" | "cancel-run" | "request-run" | "rerun-run") => { + let identity_option = if matches!(action.as_deref(), Some("request-run" | "rerun-run")) + { "--run-id" } else { "--event-id" @@ -101,7 +102,22 @@ pub(crate) fn control_admin(namespace: &Namespace) -> Result value.parse::().map_err(|_| { + ControlError::Invalid("--priority must fit a signed 64-bit integer".into()) + })?, + _ => 0, + }; + if action.as_deref() == Some("rerun-run") { + return Ok(ControlAdmin::Rerun(RerunRequest { + run_id: event_id, + source_run_id: id, + from_failure: flag(namespace, "admin_from_failure"), + reason, + priority, + })); + } let target = text(namespace, "admin_target").unwrap_or_default(); let job_key = text(namespace, "admin_job").unwrap_or_default(); if target.trim().is_empty() || job_key.trim().is_empty() { @@ -109,12 +125,6 @@ pub(crate) fn control_admin(namespace: &Namespace) -> Result value.parse::().map_err(|_| { - ControlError::Invalid("--priority must fit a signed 64-bit integer".into()) - })?, - _ => 0, - }; return Ok(ControlAdmin::RequestJobRun(JobRunRequest { run_id: event_id, location_id: id, diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs index 7cfcd0074f..2580c2b194 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs @@ -12,6 +12,7 @@ use sqlbuild_orchestration::platform::main::location_work::location_work; use sqlbuild_orchestration::platform::main::request_location_work::request_location_work; use sqlbuild_orchestration::runs::main::change_run::change_run; use sqlbuild_orchestration::runs::main::request_job_run::request_job_run; +use sqlbuild_orchestration::runs::main::request_rerun::request_rerun; use sqlbuild_orchestration::runs::models::{RunAuthority, RunChange, RunChangeRequest}; use sqlbuild_orchestration::store::errors::ControlError; use uuid::Uuid; @@ -51,6 +52,10 @@ pub(crate) async fn run_admin(namespace: &Namespace) -> Result { + let applied = request_rerun(&store, &context, &request).await?; + Ok(json!({"run_id": request.run_id, "applied": applied})) + } ControlAdmin::RequestLocationWork(request) => { let applied = request_location_work(&store, &context, &request).await?; Ok(json!({"request_id": request.request_id, "applied": applied})) diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 619784b8bd..c8e6917938 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -7,7 +7,7 @@ use sqlbuild_cache::project_snapshot::models::{PathStamp, SnapshotRules}; use sqlbuild_orchestration::credentials::models::{CredentialScope, PrincipalKind}; use sqlbuild_orchestration::declarations::models::TriggerChange; use sqlbuild_orchestration::platform::models::LocationWorkRequest; -use sqlbuild_orchestration::runs::models::JobRunRequest; +use sqlbuild_orchestration::runs::models::{JobRunRequest, RerunRequest}; use uuid::Uuid; use crate::argv::models::Namespace; @@ -19,6 +19,7 @@ pub(crate) enum ControlAdmin { IssueToken(CredentialScope, DateTime), RevokeToken(Uuid), RequestJobRun(JobRunRequest), + Rerun(RerunRequest), RequestLocationWork(LocationWorkRequest), LocationWork(Uuid), CancelRun { diff --git a/crates/sqlbuild-cli/src/workers/_helpers/configuration.rs b/crates/sqlbuild-cli/src/workers/_helpers/configuration.rs index 7377d5d941..0027b59f29 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/configuration.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/configuration.rs @@ -33,6 +33,15 @@ impl Launchers { .map(Launcher::Local) .chain(self.docker.iter().map(Launcher::Docker)) } + + pub(in crate::workers) fn dispatch_scopes(&self) -> BTreeSet<(Uuid, String)> { + self.locations() + .map(|location| { + let scope = location.scope(); + (scope.location_id, scope.target.to_owned()) + }) + .collect() + } } pub(in crate::workers) fn read_configuration( diff --git a/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs b/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs index b49de36293..b46fe96bab 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs @@ -7,11 +7,12 @@ use std::time::{Duration, Instant}; use chrono::{TimeDelta, Utc}; use sqlbuild_orchestration::credentials::main::issue_token::issue_token; use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; +use sqlbuild_orchestration::runs::main::advance_groups::advance_groups; use sqlbuild_orchestration::runs::main::change_run::change_run; use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; use sqlbuild_orchestration::runs::main::observe_run::observe_run; use sqlbuild_orchestration::runs::models::{ - RunAuthority, RunChange, RunChangeRequest, RunClaim, RunOutcome, RunStatus, + RunAuthority, RunChange, RunChangeRequest, RunClaim, RunDispatch, RunOutcome, RunStatus, }; use sqlbuild_orchestration::scheduler::constants::RENEW_INTERVAL_SECONDS; use sqlbuild_orchestration::scheduler::main::renew_lease::renew_lease; @@ -33,6 +34,22 @@ impl sqlbuild_orchestration::scheduler::types::RunLauncher for Workers { lease: Option<&SchedulerLease>, ) -> Result<(), ControlError> { if let Some(lease) = lease { + let eligible = self.configuration.dispatch_scopes(); + for location in self.configuration.locations() { + let scope = location.scope(); + advance_groups( + store, + lease, + ( + &self.configuration.concurrency_limits, + RunDispatch { + scope: Some((scope.location_id, scope.target)), + eligible: Some(&eligible), + }, + ), + ) + .await?; + } self.recover(store, lease).await?; } let mut renewed = Instant::now(); @@ -125,6 +142,7 @@ impl Workers { lease: &SchedulerLease, ) -> Result<(), ControlError> { let mut renewed = Instant::now(); + let eligible = self.configuration.dispatch_scopes(); for location in self.configuration.locations() { let scope = location.scope(); let running = self @@ -144,7 +162,10 @@ impl Workers { store, lease, &self.configuration.concurrency_limits, - Some((scope.location_id, scope.target)), + RunDispatch { + scope: Some((scope.location_id, scope.target)), + eligible: Some(&eligible), + }, ) .await? else { diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/snapshots.rs b/crates/sqlbuild-orchestration/src/api/_helpers/snapshots.rs index e5719f11ce..0b6f56f2af 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/snapshots.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/snapshots.rs @@ -9,6 +9,7 @@ use crate::api::_helpers::authentication::require_location; use crate::api::errors::ApiError; use crate::credentials::models::TokenIdentity; use crate::snapshots::main::current_snapshot::current_snapshot; +use crate::snapshots::main::current_snapshot_record::current_snapshot_record; use crate::snapshots::main::publish_snapshot::publish_snapshot; use crate::snapshots::models::{CurrentSnapshot, SnapshotInput, SnapshotRecord}; use crate::store::models::ControlStore; @@ -54,3 +55,23 @@ pub(crate) async fn current( })?; Ok(Json(current)) } + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/snapshots/metadata", + params(("location_id" = Uuid, Path), ("target" = String, Path)), + responses((status = 200, body = SnapshotRecord), (status = 401, description = "Invalid credential"), + (status = 403, description = "Location not authorized"), (status = 404, description = "No successful snapshot")), + security(("bearer" = [])))] +pub(crate) async fn metadata( + State(store): State, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + let record = current_snapshot_record(&store, location_id, &target) + .await? + .ok_or_else(|| ApiError { + status: StatusCode::NOT_FOUND, + message: "no successful snapshot".into(), + })?; + Ok(Json(record)) +} diff --git a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs index a08c8a3130..37918480f4 100644 --- a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs +++ b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs @@ -37,6 +37,10 @@ pub async fn serve_api( "/api/locations/{location_id}/targets/{target}/snapshots", get(snapshots::current).post(snapshots::publish), ) + .route( + "/api/locations/{location_id}/targets/{target}/snapshots/metadata", + get(snapshots::metadata), + ) .route( "/api/locations/{location_id}/targets/{target}/runs", get(run_reads::recent), diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index b39ad9fd6c..06471e75a9 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -15,6 +15,7 @@ use crate::api::_helpers::{ location_work::result, snapshots::publish, snapshots::current, + snapshots::metadata, runs::execution, runs::report, runs::heartbeat, diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/documents.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/documents.rs new file mode 100644 index 0000000000..cf9e048bb0 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/documents.rs @@ -0,0 +1,32 @@ +//! Decode the same immutable declaration documents at publication and execution boundaries. + +use std::collections::BTreeMap; + +use serde::de::DeserializeOwned; +use serde_json::Value; + +use crate::declarations::models::Definitions; +use crate::store::errors::ControlError; + +fn document( + documents: &BTreeMap, + name: &str, +) -> Result, ControlError> { + documents.get(name).map_or_else( + || Ok(BTreeMap::new()), + |value| { + serde_json::from_value(value.clone()) + .map_err(|error| ControlError::Invalid(format!("{name}: {error}"))) + }, + ) +} + +pub(crate) fn decode_definitions( + documents: &BTreeMap, +) -> Result { + Ok(Definitions { + jobs: document(documents, "jobs")?, + groups: document(documents, "groups")?, + triggers: document(documents, "triggers")?, + }) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs index 40c4be79ad..fda8521da8 100644 --- a/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs @@ -1 +1,2 @@ +pub(crate) mod documents; pub(crate) mod trigger_state; diff --git a/crates/sqlbuild-orchestration/src/declarations/main/frozen_definitions.rs b/crates/sqlbuild-orchestration/src/declarations/main/frozen_definitions.rs new file mode 100644 index 0000000000..5b04c1b88f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/frozen_definitions.rs @@ -0,0 +1,22 @@ +//! Runs resolve immutable snapshot documents, never a later deployment's job projections. + +use uuid::Uuid; + +use crate::declarations::_helpers::documents::decode_definitions; +use crate::declarations::models::Definitions; +use crate::snapshots::main::read_snapshot_documents::read_snapshot_documents; +use crate::store::errors::ControlError; +use crate::store::models::ControlTransaction; + +pub(crate) async fn frozen_definitions( + transaction: ControlTransaction, + snapshot_id: Uuid, +) -> Result<(ControlTransaction, Definitions), ControlError> { + let (transaction, documents) = read_snapshot_documents( + transaction, + snapshot_id, + Some(&["jobs", "groups", "triggers"]), + ) + .await?; + Ok((transaction, decode_definitions(&documents)?)) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs b/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs new file mode 100644 index 0000000000..e896fba0a5 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs @@ -0,0 +1,92 @@ +//! Freeze job boundaries from snapshot lineage plus explicit waits, rejecting cycles. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::declarations::models::JobDefinition; +use crate::snapshots::models::SnapshotInput; +use crate::store::errors::ControlError; + +pub fn group_dependencies( + input: &SnapshotInput, + jobs: &BTreeMap, + (members, after): (&BTreeSet, &BTreeMap>), +) -> Result>, ControlError> { + if members.is_empty() + || members.iter().any(|name| !jobs.contains_key(name)) + || after + .iter() + .any(|(name, dependencies)| !members.contains(name) || !dependencies.is_subset(members)) + { + return Err(ControlError::Invalid( + "group needs known jobs and dependencies within its membership".into(), + )); + } + let edges = lineage_dependencies(input, jobs, (members, after)); + ensure_acyclic(&edges)?; + Ok(edges) +} + +fn lineage_dependencies( + input: &SnapshotInput, + jobs: &BTreeMap, + (members, after): (&BTreeSet, &BTreeMap>), +) -> BTreeMap> { + let mut owners: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new(); + for name in members { + for key in &jobs[name].selection { + owners.entry(key).or_default().insert(name); + } + } + let mut edges: BTreeMap> = BTreeMap::new(); + for name in members { + let mut dependencies = after.get(name).cloned().unwrap_or_default(); + let mut seen: BTreeSet<&str> = BTreeSet::new(); + let mut pending: Vec<&str> = jobs[name].selection.iter().map(String::as_str).collect(); + while let Some(key) = pending.pop() { + if !seen.insert(key) { + continue; + } + if let Some(node) = input.nodes.get(key) + && let Some(upstream) = node + .definition + .get("deps") + .and_then(serde_json::Value::as_array) + { + for upstream in upstream.iter().filter_map(serde_json::Value::as_str) { + if let Some(owners) = owners.get(upstream) { + dependencies.extend( + owners + .iter() + .filter(|owner| **owner != name) + .map(|owner| (*owner).to_owned()), + ); + } + pending.push(upstream); + } + } + } + edges.insert(name.clone(), dependencies); + } + edges +} + +fn ensure_acyclic(edges: &BTreeMap>) -> Result<(), ControlError> { + let mut remaining = edges.clone(); + while !remaining.is_empty() { + let ready: BTreeSet = remaining + .iter() + .filter(|(_, deps)| deps.is_empty()) + .map(|(name, _)| name.clone()) + .collect(); + if ready.is_empty() { + return Err(ControlError::Invalid( + "group dependencies contain a cycle".into(), + )); + } + remaining.retain(|name, deps| { + deps.retain(|dependency| !ready.contains(dependency)); + !ready.contains(name) + }); + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/mod.rs b/crates/sqlbuild-orchestration/src/declarations/main/mod.rs index a9078f70e6..d87701b504 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/mod.rs @@ -1,4 +1,6 @@ pub mod change_trigger; +pub(crate) mod frozen_definitions; +pub mod group_dependencies; pub mod list_trigger_events; pub mod list_triggers; pub(crate) mod publish_definitions; diff --git a/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs b/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs index 6110788d5e..72e6e9b2db 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs @@ -26,13 +26,23 @@ pub(crate) async fn publish_definitions( ) .await?; } - for (name, job) in definitions.jobs { + let jobs = definitions + .jobs + .into_iter() + .map(|(name, job)| (name, "job", job)) + .chain( + definitions + .groups + .into_iter() + .map(|(name, group)| (name, "group", group.job)), + ); + for (name, kind, job) in jobs { let definition = serde_json::to_value(&job).map_err(|error| ControlError::Invalid(error.to_string()))?; (transaction, _) = query(transaction, - "INSERT INTO sqb_jobs (location_id, target, job_key, snapshot_id, kind, definition_json, overlap_policy) VALUES ($1,$2,$3,$4,'job',$5,$6)", + "INSERT INTO sqb_jobs (location_id, target, job_key, snapshot_id, kind, definition_json, overlap_policy) VALUES ($1,$2,$3,$4,$7,$5,$6)", vec![Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone()), Parameter::Text(name.clone()), - Parameter::Id(snapshot.snapshot_id), Parameter::Json(definition), Parameter::Text(job.overlap.as_str().into())]).await?; + Parameter::Id(snapshot.snapshot_id), Parameter::Json(definition), Parameter::Text(job.overlap.as_str().into()), Parameter::Text(kind.into())]).await?; for node in job.selection { (transaction, _) = query(transaction, "INSERT INTO sqb_job_nodes (location_id, target, job_key, node_key) VALUES ($1,$2,$3,$4)", diff --git a/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs b/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs index 96987a439c..5966cd18b3 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs @@ -1,9 +1,9 @@ //! Validate resolved declarations before acquiring the snapshot publication lock. -use std::collections::BTreeMap; - -use serde::de::DeserializeOwned; +use std::collections::BTreeSet; +use crate::declarations::_helpers::documents::decode_definitions; +use crate::declarations::main::group_dependencies::group_dependencies; use crate::declarations::models::{Definitions, Schedule}; use crate::scheduler::constants::MAX_TICKS_PER_BATCH; use crate::scheduler::main::parse_cron::parse_cron; @@ -11,25 +11,9 @@ use crate::scheduler::models::CatchUp; use crate::snapshots::models::SnapshotInput; use crate::store::errors::ControlError; -fn document( - input: &SnapshotInput, - name: &str, -) -> Result, ControlError> { - input.documents.get(name).map_or_else( - || Ok(BTreeMap::new()), - |value| { - serde_json::from_value(value.clone()) - .map_err(|error| ControlError::Invalid(format!("{name}: {error}"))) - }, - ) -} - /// Names, references and resolved selections are checked again at the native trust boundary. pub fn snapshot_definitions(input: &SnapshotInput) -> Result { - let definitions = Definitions { - jobs: document(input, "jobs")?, - triggers: document(input, "triggers")?, - }; + let definitions = decode_definitions(&input.documents)?; for (name, job) in &definitions.jobs { if name.trim().is_empty() || job.description.trim().is_empty() @@ -44,10 +28,31 @@ pub fn snapshot_definitions(input: &SnapshotInput) -> Result = group.jobs.keys().cloned().collect(); + let dependencies = group_dependencies(input, &definitions.jobs, (&members, &group.jobs))?; + let selection: BTreeSet = group + .jobs + .keys() + .flat_map(|name| definitions.jobs[name].selection.iter().cloned()) + .collect(); + if name.trim().is_empty() + || definitions.jobs.contains_key(name) + || group.job.description.trim().is_empty() + || group.job.selection != selection + || group.jobs != dependencies + || !group.job.settings.is_empty() + { + return Err(ControlError::Invalid(format!( + "group {name} has invalid membership, selection or dependencies" + ))); + } + } for (name, trigger) in &definitions.triggers { if name.trim().is_empty() || trigger.description.trim().is_empty() - || !definitions.jobs.contains_key(&trigger.job_key) + || (!definitions.jobs.contains_key(&trigger.job_key) + && !definitions.groups.contains_key(&trigger.job_key)) { return Err(ControlError::Invalid(format!( "trigger {name} needs a description and a known job" diff --git a/crates/sqlbuild-orchestration/src/declarations/models.rs b/crates/sqlbuild-orchestration/src/declarations/models.rs index 4120b8c9b6..623d581f43 100644 --- a/crates/sqlbuild-orchestration/src/declarations/models.rs +++ b/crates/sqlbuild-orchestration/src/declarations/models.rs @@ -50,6 +50,14 @@ pub struct JobDefinition { pub concurrency_groups: BTreeSet, } +/// A virtual parent shares run controls, but only its physical children claim node writers. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct GroupDefinition { + #[serde(flatten)] + pub job: JobDefinition, + pub jobs: BTreeMap>, +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum Schedule { @@ -80,6 +88,7 @@ pub struct TriggerDefinition { #[derive(Clone, Debug, Default, PartialEq)] pub struct Definitions { pub jobs: BTreeMap, + pub groups: BTreeMap, pub triggers: BTreeMap, } diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/admission.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/admission.rs new file mode 100644 index 0000000000..f893687365 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/admission.rs @@ -0,0 +1,186 @@ +//! Reconstruct global parent allocations and remaining child demands in the reservation transaction. + +use std::collections::{BTreeMap, BTreeSet}; +use std::num::NonZeroU32; + +use uuid::Uuid; + +use crate::runs::_helpers::groups::capacity::{claim_resources, parent_resources}; +use crate::runs::_helpers::groups::selection::child_ready; +use crate::runs::models::{ + ConcurrencyResource, GroupCapacity, GroupRun, QueueCandidate, RunKind, RunStatus, +}; +use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::ControlTransaction; + +pub(crate) async fn group_capacity( + transaction: ControlTransaction, + limits: &BTreeMap, + eligible: Option<&BTreeSet<(Uuid, String)>>, +) -> Result<(ControlTransaction, GroupCapacity), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT r.request_json, r.queued_at, r.status, r.cancel_requested_at, r.node_counts_json, l.worker_instance FROM sqb_runs r \ + LEFT JOIN sqb_run_liveness l ON l.run_id=r.run_id \ + WHERE (r.kind='group' AND r.status IN ('queued','running')) \ + OR (r.status='queued' AND r.cancel_requested_at IS NULL AND l.run_id IS NULL) \ + ORDER BY r.priority DESC, r.queued_at, r.run_id", vec![]).await?; + let runs: Vec = rows.into_iter().map(decode_row).collect::>()?; + let parents: BTreeMap<_, _> = runs + .iter() + .filter(|run| run.request_json.kind == RunKind::Group) + .map(|run| (run.request_json.run_id, run)) + .collect(); + let (transaction, slots) = query(transaction, + "SELECT s.group_key, s.run_id FROM sqb_concurrency_slots s JOIN sqb_runs r ON r.run_id=s.run_id WHERE r.kind='group' AND r.status='running'", vec![]).await?; + let mut capacity = GroupCapacity { + available: limits + .iter() + .map(|(key, limit)| { + ( + ConcurrencyResource::Limit(key.clone()), + i64::from(limit.get()), + ) + }) + .collect(), + held: BTreeMap::new(), + needed: parents.keys().map(|id| (*id, BTreeSet::new())).collect(), + waiting: runs + .iter() + .filter(|run| { + run.state.status == RunStatus::Queued + && run.state.cancel_requested_at.is_none() + && eligible.is_none_or(|scopes| { + scopes.contains(&( + run.request_json.location_id, + run.request_json.target.clone(), + )) + }) + && run.request_json.parent_run_id.is_none_or(|id| { + parents.get(&id).is_some_and(|parent| { + parent.state.status == RunStatus::Running + && parent.state.cancel_requested_at.is_none() + }) + }) + }) + .map(|run| QueueCandidate { + request_json: run.request_json.clone(), + queued_at: run.queued_at, + }) + .collect(), + }; + for (id, parent) in &parents { + if parent.state.status == RunStatus::Running { + let held = capacity.held.entry(*id).or_default(); + held.extend( + parent_resources(&parent.request_json) + .into_iter() + .filter(|resource| matches!(resource, ConcurrencyResource::Overlap(..))), + ); + } + } + for slot in slots { + let id = serde_json::from_value(slot["run_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let key = slot["group_key"] + .as_str() + .ok_or_else(|| ControlError::Invalid("concurrency claim has no group".into()))?; + capacity + .held + .entry(id) + .or_default() + .insert(ConcurrencyResource::Limit(key.into())); + } + for child in runs + .iter() + .filter(|run| run.request_json.kind != RunKind::Group) + { + let Some(parent) = child + .request_json + .parent_run_id + .and_then(|id| parents.get(&id)) + else { + continue; + }; + if parent.state.cancel_requested_at.is_some() + || (parent.state.status == RunStatus::Queued + && !capacity + .waiting + .iter() + .any(|waiter| waiter.request_json.run_id == parent.request_json.run_id)) + { + continue; + } + let inherited = if parent.state.status == RunStatus::Running { + capacity.held[&parent.request_json.run_id].clone() + } else { + parent_resources(&parent.request_json) + }; + capacity + .needed + .entry(parent.request_json.run_id) + .or_default() + .extend(claim_resources(&child.request_json, Some(&inherited))); + } + for resource in capacity + .held + .values() + .flatten() + .chain(capacity.needed.values().flatten()) + { + if !capacity.available.contains_key(resource) { + match resource { + ConcurrencyResource::Overlap(..) => { + capacity.available.insert(resource.clone(), 1); + } + ConcurrencyResource::Limit(key) => { + return Err(ControlError::Invalid(format!( + "concurrency group {key} has no configured limit" + ))); + } + } + } + } + for resource in capacity.held.values().flatten() { + *capacity.available.entry(resource.clone()).or_insert(0) -= 1; + } + Ok((transaction, capacity)) +} + +/// Give earlier executable contenders first claim, without blocking children that drain parents. +pub(crate) async fn earlier_waiter_blocks( + mut transaction: ControlTransaction, + capacity: &GroupCapacity, + candidate: &QueueCandidate, +) -> Result<(ControlTransaction, bool), ControlError> { + if candidate.request_json.parent_run_id.is_some() { + return Ok((transaction, false)); + } + let footprint = parent_resources(&candidate.request_json); + for older in capacity.earlier_waiters(candidate) { + let inherited = older + .request_json + .parent_run_id + .and_then(|id| capacity.held.get(&id)); + let mut waiting = claim_resources(&older.request_json, inherited); + if candidate.request_json.kind == RunKind::Group { + waiting.extend( + capacity + .needed + .get(&older.request_json.run_id) + .into_iter() + .flatten() + .cloned(), + ); + } + if footprint.is_disjoint(&waiting) { + continue; + } + let ready; + (transaction, ready) = child_ready(transaction, &older.request_json).await?; + if ready { + return Ok((transaction, true)); + } + } + Ok((transaction, false)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/capacity.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/capacity.rs new file mode 100644 index 0000000000..23bc9fb8b2 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/capacity.rs @@ -0,0 +1,121 @@ +//! Completion-order safety for persistent parent claims and atomic, temporary child claims. + +use std::cmp::Reverse; +use std::collections::BTreeSet; + +use crate::runs::models::{ConcurrencyResource, GroupCapacity, QueueCandidate, RunRequest}; +use crate::scheduler::models::Overlap; + +pub(crate) fn parent_resources(request: &RunRequest) -> BTreeSet { + let mut resources: BTreeSet<_> = request + .concurrency_groups + .iter() + .cloned() + .map(ConcurrencyResource::Limit) + .collect(); + if let Some(key) = &request.job_key { + resources.insert(ConcurrencyResource::Overlap( + request.location_id, + request.target.clone(), + key.clone(), + )); + } + resources +} + +pub(crate) fn claim_resources( + request: &RunRequest, + inherited: Option<&BTreeSet>, +) -> BTreeSet { + parent_resources(request) + .into_iter() + .filter(|resource| match resource { + ConcurrencyResource::Limit(_) => inherited.is_none_or(|held| !held.contains(resource)), + ConcurrencyResource::Overlap(..) => request.overlap != Overlap::Allow, + }) + .collect() +} + +impl GroupCapacity { + /// Find a completion order without reserving children's independent limits. + pub(crate) fn safe(&self, candidate: &RunRequest) -> bool { + let mut work = self.available.clone(); + let mut held = self.held.clone(); + let resources = parent_resources(candidate); + for resource in &resources { + *work.entry(resource.clone()).or_insert(1) -= 1; + } + held.insert(candidate.run_id, resources); + while !held.is_empty() { + let before = held.len(); + held.retain(|id, resources| { + if self.needed[id] + .iter() + .any(|resource| work.get(resource).copied().unwrap_or(1) <= 0) + { + return true; + } + for resource in resources.iter() { + *work.entry(resource.clone()).or_insert(0) += 1; + } + false + }); + if held.len() == before { + return false; + } + } + true + } + + pub(crate) fn blocks_claim( + &self, + candidate: &QueueCandidate, + waiting: &BTreeSet, + ) -> bool { + let inherited = candidate + .request_json + .parent_run_id + .and_then(|id| self.held.get(&id)); + let mut consumed = claim_resources(&candidate.request_json, inherited); + consumed.extend( + parent_resources(&candidate.request_json) + .into_iter() + .filter(|resource| matches!(resource, ConcurrencyResource::Overlap(..))), + ); + !consumed.is_disjoint(waiting) + } + + /// Only temporary blockers drain ahead of younger work; persistent blockers must let owners finish. + pub(crate) fn record_waiter( + &self, + candidate: &RunRequest, + mut waiting: BTreeSet, + ) -> BTreeSet { + let inherited = candidate.parent_run_id.and_then(|id| self.held.get(&id)); + let resources = claim_resources(candidate, inherited); + if resources.iter().all(|resource| { + self.available + .get(resource) + .is_none_or(|available| *available > 0) + }) { + waiting.extend(resources); + } + waiting + } + + pub(crate) fn earlier_waiters<'a>( + &'a self, + candidate: &'a QueueCandidate, + ) -> impl Iterator { + let rank = |value: &QueueCandidate| { + ( + Reverse(value.request_json.priority), + value.queued_at, + value.request_json.run_id, + ) + }; + self.waiting + .iter() + .filter(move |older| rank(older) < rank(candidate)) + } +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/context.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/context.rs new file mode 100644 index 0000000000..be4f644f81 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/context.rs @@ -0,0 +1,86 @@ +//! Read group membership and failure policy from the parent's immutable publication. + +use std::collections::BTreeSet; + +use uuid::Uuid; + +use crate::declarations::main::frozen_definitions::frozen_definitions; +use crate::declarations::models::OnFailure; +use crate::runs::models::{GroupContext, GroupRun, RunStatus}; +use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; +use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn load_group( + transaction: ControlTransaction, + parent_id: Uuid, +) -> Result<(ControlTransaction, GroupContext), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT r.request_json, r.queued_at, r.status, r.cancel_requested_at, r.node_counts_json, l.worker_instance FROM sqb_runs r \ + LEFT JOIN sqb_run_liveness l ON l.run_id=r.run_id WHERE r.run_id=$1 AND r.kind='group'", + vec![Parameter::Id(parent_id)]).await?; + let parent: GroupRun = decode_row( + rows.into_iter() + .next() + .ok_or_else(|| ControlError::Invalid("group run is missing".into()))?, + )?; + let (transaction, rows) = query(transaction, + "SELECT r.request_json, r.queued_at, r.status, r.cancel_requested_at, r.node_counts_json, l.worker_instance FROM sqb_runs r \ + LEFT JOIN sqb_run_liveness l ON l.run_id=r.run_id WHERE r.parent_run_id=$1 ORDER BY r.run_id", + vec![Parameter::Id(parent_id)]).await?; + let children = rows.into_iter().map(decode_row).collect::>()?; + let (transaction, definitions) = + frozen_definitions(transaction, parent.request_json.snapshot_id).await?; + let (transaction, failures) = query( + transaction, + "SELECT n.node_key FROM sqb_run_nodes n JOIN sqb_runs r ON r.run_id=n.run_id \ + WHERE r.parent_run_id=$1 AND n.status='failed'", + vec![Parameter::Id(parent_id)], + ) + .await?; + let failures: BTreeSet = failures + .iter() + .filter_map(|row| row["node_key"].as_str().map(str::to_owned)) + .collect(); + let (transaction, nodes) = read_snapshot_nodes( + transaction, + parent.request_json.snapshot_id, + Some(&failures), + ) + .await?; + let critical_failed = nodes + .values() + .any(|node| node.definition["config"]["critical"].as_bool() == Some(true)); + Ok(( + transaction, + GroupContext { + parent, + children, + definitions, + critical_failed, + }, + )) +} + +impl GroupContext { + pub(crate) fn stopped(&self) -> bool { + self.critical_failed + || self.children.iter().any(|child| { + child.request_json.job_key.as_ref().is_some_and(|name| { + self.definitions.jobs[name].on_failure == OnFailure::StopGroup + && (child + .state + .node_counts_json + .get("failed") + .copied() + .unwrap_or_default() + > 0 + || matches!( + child.state.status, + RunStatus::Failed | RunStatus::PresumedFailed + )) + }) + }) + } +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/lifecycle.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/lifecycle.rs new file mode 100644 index 0000000000..9e7cdb8fe2 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/lifecycle.rs @@ -0,0 +1,206 @@ +//! A virtual parent advances only from durable child evidence under the scheduler's fence. + +use std::collections::{BTreeMap, BTreeSet}; +use std::num::NonZeroU32; + +use uuid::Uuid; + +use crate::runs::_helpers::groups::admission::{earlier_waiter_blocks, group_capacity}; +use crate::runs::_helpers::groups::context::load_group; +use crate::runs::_helpers::lifecycle::claims::reserve_claims; +use crate::runs::_helpers::lifecycle::publish_change::publish_change; +use crate::runs::models::{ + ChangingRun, GroupContext, NodeOutcome, QueueCandidate, RunChange, RunChangeRequest, + RunNodeRecord, RunOutcome, RunStatus, +}; +use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::{ControlTransaction, Parameter}; + +/// Callers have already checked leadership; the surrounding transaction fences the entire advance. +pub(crate) async fn group_change( + transaction: ControlTransaction, + run_id: Uuid, + change: RunChange, +) -> Result { + let (transaction, rows) = query(transaction, + "SELECT r.status, r.cancel_requested_at, r.node_counts_json, l.worker_instance FROM sqb_runs r \ + LEFT JOIN sqb_run_liveness l ON l.run_id=r.run_id WHERE r.run_id=$1", + vec![Parameter::Id(run_id)]).await?; + let state: ChangingRun = decode_row( + rows.into_iter() + .next() + .ok_or_else(|| ControlError::Invalid("group run is missing".into()))?, + )?; + let event_id = Uuid::new_v5( + &run_id, + &serde_json::to_vec(&change).map_err(|error| ControlError::Invalid(error.to_string()))?, + ); + publish_change( + transaction, + &RunChangeRequest { + event_id, + run_id, + claim_id: state.worker_instance, + change, + }, + &state, + ) + .await +} + +pub(crate) async fn advance_group( + transaction: ControlTransaction, + parent_id: Uuid, + limits: &BTreeMap, + eligible: Option<&BTreeSet<(Uuid, String)>>, +) -> Result { + let (mut transaction, group) = load_group(transaction, parent_id).await?; + let parent = &group.parent; + let children = &group.children; + if parent.state.status.is_terminal() { + return Ok(transaction); + } + let stopped = group.stopped(); + if parent.state.cancel_requested_at.is_some() || stopped { + for child in children { + if !child.state.status.is_terminal() && child.state.cancel_requested_at.is_none() { + transaction = group_change( + transaction, + child.request_json.run_id, + RunChange::CancelRequested { + actor: "system:group".into(), + reason: if stopped { + "group stopped after a child failure" + } else { + "parent group cancellation requested" + } + .into(), + }, + ) + .await?; + } + } + } else if parent.state.status == RunStatus::Queued { + let capacity; + (transaction, capacity) = group_capacity(transaction, limits, eligible).await?; + let blocked; + (transaction, blocked) = earlier_waiter_blocks( + transaction, + &capacity, + &QueueCandidate { + request_json: parent.request_json.clone(), + queued_at: parent.queued_at, + }, + ) + .await?; + if blocked || !capacity.safe(&parent.request_json) { + return Ok(transaction); + } + (transaction, _) = query(transaction, "SAVEPOINT group_claim", vec![]).await?; + let reserved; + (transaction, reserved) = reserve_claims(transaction, &parent.request_json, limits).await?; + if !reserved { + (transaction, _) = + query(transaction, "ROLLBACK TO SAVEPOINT group_claim", vec![]).await?; + } else { + transaction = group_change(transaction, parent_id, RunChange::Started).await?; + } + (transaction, _) = query(transaction, "RELEASE SAVEPOINT group_claim", vec![]).await?; + } + if !children.is_empty() + && children + .iter() + .all(|child| child.state.status.is_terminal()) + { + transaction = finish_group(transaction, &group, stopped).await?; + } + Ok(transaction) +} + +async fn finish_group( + transaction: ControlTransaction, + group: &GroupContext, + stopped: bool, +) -> Result { + let parent = &group.parent; + let parent_id = parent.request_json.run_id; + let (mut transaction, rows) = query(transaction, + "SELECT n.* FROM sqb_run_nodes n JOIN sqb_runs r ON r.run_id=n.run_id WHERE r.parent_run_id=$1 ORDER BY r.run_id, n.node_key", + vec![Parameter::Id(parent_id)]).await?; + let mut nodes: BTreeMap = BTreeMap::new(); + let rank = |status: &str| match status { + "failed" => 5, + "cancelled" => 4, + "skipped" => 3, + "warn" => 2, + "success" => 1, + _ => 0, + }; + for row in rows { + let node: RunNodeRecord = decode_row(row)?; + if nodes + .get(&node.node_key) + .is_none_or(|previous| rank(&node.status) > rank(&previous.status)) + { + nodes.insert(node.node_key.clone(), node); + } + } + let mut outcomes: BTreeSet = BTreeSet::new(); + for key in &parent.request_json.selection { + let node = nodes + .get(key) + .ok_or_else(|| ControlError::Invalid("group node has no child evidence".into()))?; + outcomes.insert(node.status.clone()); + let outcome: NodeOutcome = serde_json::from_value(serde_json::json!(node.status)) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + transaction = group_change( + transaction, + parent_id, + RunChange::NodeFinished { + node_key: key.clone(), + outcome, + rows_affected: node.rows_affected, + error_code: node.error_code.clone(), + skipped_because_of: node.skipped_because_of.clone(), + }, + ) + .await?; + } + let good = outcomes.contains("success") || outcomes.contains("warn"); + let bad = group + .children + .iter() + .any(|child| !matches!(child.state.status, RunStatus::Success | RunStatus::Warn)) + || outcomes + .iter() + .any(|status| !matches!(status.as_str(), "success" | "warn")); + let outcome = if parent.state.cancel_requested_at.is_some() { + RunOutcome::Cancelled + } else if stopped { + RunOutcome::Failed + } else if bad && good { + RunOutcome::Partial + } else if bad { + RunOutcome::Failed + } else if outcomes.contains("warn") + || group + .children + .iter() + .any(|child| child.state.status == RunStatus::Warn) + { + RunOutcome::Warn + } else { + RunOutcome::Success + }; + group_change( + transaction, + parent_id, + RunChange::Finished { + outcome, + reason: "all group children have durable terminal outcomes".into(), + exit_code: None, + }, + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/mod.rs new file mode 100644 index 0000000000..9f205ce914 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/mod.rs @@ -0,0 +1,5 @@ +pub(in crate::runs) mod admission; +pub(in crate::runs) mod capacity; +pub(in crate::runs) mod context; +pub(in crate::runs) mod lifecycle; +pub(in crate::runs) mod selection; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/selection.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/selection.rs new file mode 100644 index 0000000000..a42f48821c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/selection.rs @@ -0,0 +1,139 @@ +//! Failed nodes block only their lineage dependents unless an upstream job requests broader stops. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::declarations::models::OnFailure; +use crate::runs::_helpers::groups::context::load_group; +use crate::runs::models::{RunRequest, RunStatus}; +use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn child_ready( + transaction: ControlTransaction, + request: &RunRequest, +) -> Result<(ControlTransaction, bool), ControlError> { + let Some(parent_id) = request.parent_run_id else { + return Ok((transaction, true)); + }; + let (transaction, group) = load_group(transaction, parent_id).await?; + let name = request + .job_key + .as_ref() + .ok_or_else(|| ControlError::Invalid("group child has no job".into()))?; + let parent_name = group + .parent + .request_json + .job_key + .as_ref() + .ok_or_else(|| ControlError::Invalid("group has no declaration".into()))?; + let dependencies = &group.definitions.groups[parent_name].jobs[name]; + let ready = group.parent.state.status == RunStatus::Running + && group.parent.state.cancel_requested_at.is_none() + && !group.stopped() + && group.children.iter().all(|child| { + child + .request_json + .job_key + .as_ref() + .is_none_or(|name| !dependencies.contains(name)) + || child.state.status.is_terminal() + }); + Ok((transaction, ready)) +} + +pub(crate) async fn skipped_nodes( + transaction: ControlTransaction, + request: &RunRequest, +) -> Result<(ControlTransaction, BTreeMap), ControlError> { + let Some(parent_id) = request.parent_run_id else { + return Ok((transaction, BTreeMap::new())); + }; + let (transaction, group) = load_group(transaction, parent_id).await?; + let name = request + .job_key + .as_ref() + .ok_or_else(|| ControlError::Invalid("group child has no job".into()))?; + let parent_name = group + .parent + .request_json + .job_key + .as_ref() + .ok_or_else(|| ControlError::Invalid("group has no declaration".into()))?; + let edges = &group.definitions.groups[parent_name].jobs; + let mut ancestors: BTreeSet<&str> = BTreeSet::new(); + let mut pending: Vec<&str> = edges[name].iter().map(String::as_str).collect(); + while let Some(name) = pending.pop() { + if ancestors.insert(name) { + pending.extend(edges[name].iter().map(String::as_str)); + } + } + let (transaction, rows) = query(transaction, + "SELECT n.node_key, n.status, n.skipped_because_of FROM sqb_run_nodes n JOIN sqb_runs r ON r.run_id=n.run_id \ + WHERE r.parent_run_id=$1 AND r.run_id<>$2 AND n.status IN ('failed','skipped','cancelled')", + vec![Parameter::Id(parent_id), Parameter::Id(request.run_id)]).await?; + let failures: BTreeMap = rows + .into_iter() + .filter_map(|row| { + let key = row.get("node_key")?.as_str()?.to_owned(); + let cause = row + .get("skipped_because_of") + .and_then(serde_json::Value::as_str) + .unwrap_or(&key) + .to_owned(); + Some((key, cause)) + }) + .collect(); + let whole_job = group.children.iter().find(|child| { + child.request_json.job_key.as_ref().is_some_and(|name| { + ancestors.contains(name.as_str()) + && group.definitions.jobs[name].on_failure == OnFailure::SkipDownstreamJobs + }) && child.state.status.is_terminal() + && !matches!(child.state.status, RunStatus::Success | RunStatus::Warn) + }); + if let Some(child) = whole_job { + let cause = child + .request_json + .selection + .iter() + .find_map(|key| failures.get(key)) + .cloned() + .unwrap_or_else(|| format!("run:{}", child.request_json.run_id)); + return Ok(( + transaction, + request + .selection + .iter() + .map(|key| (key.clone(), cause.clone())) + .collect(), + )); + } + if failures.is_empty() { + return Ok((transaction, BTreeMap::new())); + } + let (transaction, nodes) = read_snapshot_nodes(transaction, request.snapshot_id, None).await?; + let mut skipped: BTreeMap = BTreeMap::new(); + for selected in &request.selection { + let mut seen: BTreeSet<&str> = BTreeSet::new(); + let mut pending = vec![selected.as_str()]; + while let Some(key) = pending.pop() { + if !seen.insert(key) { + continue; + } + if let Some(cause) = failures.get(key) { + skipped.insert(selected.clone(), cause.clone()); + break; + } + if let Some(node) = nodes.get(key) + && let Some(deps) = node + .definition + .get("deps") + .and_then(serde_json::Value::as_array) + { + pending.extend(deps.iter().filter_map(serde_json::Value::as_str)); + } + } + } + Ok((transaction, skipped)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/changes.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/changes.rs similarity index 98% rename from crates/sqlbuild-orchestration/src/runs/_helpers/changes.rs rename to crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/changes.rs index 6c9fcee455..8310532774 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/changes.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/changes.rs @@ -1,4 +1,4 @@ -//! Validate lifecycle authority before publishing any fact or releasing physical write claims. +//! Lifecycle authority checks precede fact publication and physical claim release. use crate::runs::models::{ ChangingRun, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/claims.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/claims.rs similarity index 60% rename from crates/sqlbuild-orchestration/src/runs/_helpers/claims.rs rename to crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/claims.rs index 8e69205b5f..0fcc9bebc1 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/claims.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/claims.rs @@ -1,9 +1,13 @@ -//! A rejected candidate rolls back its entire savepoint, including any partial write set. +//! Reserve write claims and concurrency slots inside the candidate's rollback savepoint. use std::collections::{BTreeMap, BTreeSet}; use std::num::NonZeroU32; -use crate::runs::models::RunRequest; +use serde_json::json; +use uuid::Uuid; + +use crate::runs::_helpers::lifecycle::events::append_run_event; +use crate::runs::models::{RunClaim, RunRequest}; use crate::scheduler::models::Overlap; use crate::store::errors::ControlError; use crate::store::main::query::query; @@ -21,8 +25,9 @@ pub(crate) async fn reserve_claims( let rows; (transaction, rows) = query( transaction, - "SELECT r.run_id FROM sqb_runs r JOIN sqb_run_liveness l ON l.run_id=r.run_id \ - WHERE r.location_id=$1 AND r.target=$2 AND r.job_key=$3 LIMIT 1", + "SELECT r.run_id FROM sqb_runs r LEFT JOIN sqb_run_liveness l ON l.run_id=r.run_id \ + WHERE r.location_id=$1 AND r.target=$2 AND r.job_key=$3 \ + AND (l.run_id IS NOT NULL OR (r.kind='group' AND r.status='running')) LIMIT 1", vec![ Parameter::Id(request.location_id), Parameter::Text(request.target.clone()), @@ -46,6 +51,18 @@ pub(crate) async fn reserve_claims( } } for group in &request.concurrency_groups { + if let Some(parent_id) = request.parent_run_id { + let inherited; + (transaction, inherited) = query( + transaction, + "SELECT slot FROM sqb_concurrency_slots WHERE group_key=$1 AND run_id=$2", + vec![Parameter::Text(group.clone()), Parameter::Id(parent_id)], + ) + .await?; + if !inherited.is_empty() { + continue; + } + } let limit = limits .get(group) .ok_or_else(|| { @@ -82,3 +99,35 @@ pub(crate) async fn reserve_claims( } Ok((transaction, true)) } + +/// Publish ownership only after every physical claim has been reserved in this transaction. +pub(crate) async fn publish_claim( + transaction: ControlTransaction, + request: RunRequest, +) -> Result<(ControlTransaction, RunClaim), ControlError> { + let claim = RunClaim { + claim_id: Uuid::new_v5(&request.run_id, b"run_claimed"), + request, + }; + let now = transaction.now; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_run_liveness (run_id, worker_instance, claimed_at, last_heartbeat_at) VALUES ($1,$2,$3,$3)", + vec![Parameter::Id(claim.request.run_id), Parameter::Id(claim.claim_id), Parameter::Time(now)]).await?; + let (transaction, cursor) = append_run_event( + transaction, + (claim.request.run_id, claim.claim_id), + "run_claimed", + json!({"claim": claim}), + ) + .await?; + let (transaction, _) = query( + transaction, + "UPDATE sqb_runs SET last_event_storage_id=$1 WHERE run_id=$2", + vec![ + Parameter::Integer(cursor), + Parameter::Id(claim.request.run_id), + ], + ) + .await?; + Ok((transaction, claim)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/events.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/events.rs similarity index 92% rename from crates/sqlbuild-orchestration/src/runs/_helpers/events.rs rename to crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/events.rs index cb7b38073c..a146a8e4a7 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/events.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/events.rs @@ -1,4 +1,4 @@ -//! All orchestration transitions share Recall's invocation sequence and transaction. +//! Lifecycle facts share Recall's invocation sequence and transaction. use serde_json::Value; use uuid::Uuid; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs new file mode 100644 index 0000000000..0ef9c931db --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs @@ -0,0 +1,8 @@ +pub(in crate::runs) mod changes; +pub(in crate::runs) mod claims; +pub(in crate::runs) mod events; +pub(in crate::runs) mod nodes; +pub(in crate::runs) mod output; +pub(in crate::runs) mod projection; +pub(in crate::runs) mod publish_change; +pub(in crate::runs) mod recall; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/nodes.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/nodes.rs similarity index 97% rename from crates/sqlbuild-orchestration/src/runs/_helpers/nodes.rs rename to crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/nodes.rs index 8ae44b1a94..6c609d9340 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/nodes.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/nodes.rs @@ -1,4 +1,4 @@ -//! Updating one node adjusts bounded status counters instead of rescanning every node per event. +//! Node lifecycle projections adjust bounded counters instead of rescanning nodes per event. use std::collections::BTreeMap; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/output.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/output.rs similarity index 97% rename from crates/sqlbuild-orchestration/src/runs/_helpers/output.rs rename to crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/output.rs index beb7a601ef..80f466411a 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/output.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/output.rs @@ -1,4 +1,4 @@ -//! Resolve output ownership from the physical claim and normalize Recall input before append. +//! Normalize Recall output under its physical claim's invocation envelope. use chrono::{DateTime, SecondsFormat}; use sqlbuild_recall::records::main::output_row::output_row; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/projection.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/projection.rs similarity index 98% rename from crates/sqlbuild-orchestration/src/runs/_helpers/projection.rs rename to crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/projection.rs index 9d0629b574..7f6ffb2de0 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/projection.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/projection.rs @@ -4,7 +4,7 @@ use std::collections::BTreeMap; use serde_json::json; -use crate::runs::_helpers::nodes::project_node; +use crate::runs::_helpers::lifecycle::nodes::project_node; use crate::runs::models::{ChangingRun, RunChange, RunChangeRequest, RunOutcome}; use crate::store::errors::ControlError; use crate::store::main::query::query; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/publish_change.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publish_change.rs similarity index 73% rename from crates/sqlbuild-orchestration/src/runs/_helpers/publish_change.rs rename to crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publish_change.rs index 18ba3f5835..68cc3edd32 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/publish_change.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publish_change.rs @@ -2,8 +2,8 @@ use serde_json::json; -use crate::runs::_helpers::events::append_run_event; -use crate::runs::_helpers::projection::project_change; +use crate::runs::_helpers::lifecycle::events::append_run_event; +use crate::runs::_helpers::lifecycle::projection::project_change; use crate::runs::models::{ChangingRun, RunChange, RunChangeRequest, RunOutcome}; use crate::store::errors::ControlError; use crate::store::main::query::query; @@ -22,9 +22,19 @@ pub(crate) async fn publish_change( ) .await?; let mut transaction = project_change(transaction, request, state).await?; - if matches!(request.change, RunChange::CancelRequested { .. }) - && state.worker_instance.is_none() - { + let mut cancel_unstarted = matches!(request.change, RunChange::CancelRequested { .. }) + && state.worker_instance.is_none(); + if cancel_unstarted { + let rows; + (transaction, rows) = query( + transaction, + "SELECT kind FROM sqb_runs WHERE run_id=$1", + vec![Parameter::Id(request.run_id)], + ) + .await?; + cancel_unstarted = rows.first().and_then(|row| row["kind"].as_str()) != Some("group"); + } + if cancel_unstarted { let terminal = RunChangeRequest { event_id: uuid::Uuid::new_v5(&request.event_id, b"cancelled-before-launch"), run_id: request.run_id, diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/recall.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/recall.rs similarity index 97% rename from crates/sqlbuild-orchestration/src/runs/_helpers/recall.rs rename to crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/recall.rs index 442b0ba521..ecd6740f79 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/recall.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/recall.rs @@ -1,4 +1,4 @@ -//! Global Recall keys cannot escape a worker's claim namespace. +//! Namespace Recall facts and validate artifact ownership before publication. use std::collections::BTreeMap; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs index a8a8e0a6f5..5d47d7cf23 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs @@ -1,10 +1,3 @@ -pub(crate) mod changes; -pub(crate) mod claims; -pub(crate) mod events; -pub(crate) mod job_request; -pub(crate) mod nodes; -pub(crate) mod output; -pub(crate) mod projection; -pub(crate) mod publish_change; -pub(crate) mod recall; -pub(crate) mod request; +pub(crate) mod groups; +pub(crate) mod lifecycle; +pub(crate) mod requests; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/job_request.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs similarity index 77% rename from crates/sqlbuild-orchestration/src/runs/_helpers/job_request.rs rename to crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs index 691436c7c5..edd8116dd6 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/job_request.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs @@ -1,4 +1,4 @@ -//! Freeze execution inputs from one published job under the caller's write transaction. +//! Freeze execution inputs from one published job or group in the caller's write transaction. use crate::credentials::models::AuditContext; use crate::declarations::models::JobDefinition; @@ -15,9 +15,9 @@ pub(crate) async fn frozen_job_request( ) -> Result<(ControlTransaction, RunRequest), ControlError> { let (transaction, jobs) = query( transaction, - "SELECT j.snapshot_id, j.definition_json, s.image_id FROM sqb_jobs j \ + "SELECT j.snapshot_id, j.definition_json, j.kind, s.image_id FROM sqb_jobs j \ JOIN sqb_current_snapshots c ON c.location_id=j.location_id AND c.target=j.target AND c.snapshot_id=j.snapshot_id \ - JOIN sqb_snapshots s ON s.snapshot_id=j.snapshot_id WHERE j.location_id=$1 AND j.target=$2 AND j.job_key=$3 AND j.kind='job'", + JOIN sqb_snapshots s ON s.snapshot_id=j.snapshot_id WHERE j.location_id=$1 AND j.target=$2 AND j.job_key=$3", vec![Parameter::Id(request.location_id), Parameter::Text(request.target.clone()), Parameter::Text(request.job_key.clone())], ).await?; let row = jobs.first().ok_or_else(|| { @@ -29,12 +29,10 @@ pub(crate) async fn frozen_job_request( .map_err(|error| ControlError::Invalid(error.to_string()))?; let image_id = serde_json::from_value(row["image_id"].clone()) .map_err(|error| ControlError::Invalid(error.to_string()))?; - let (transaction, write_set) = job_write_set( - transaction, - snapshot_id, - (request.location_id, &request.target, &request.job_key), - ) - .await?; + let kind: RunKind = serde_json::from_value(row["kind"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let (transaction, write_set) = + job_write_set(transaction, snapshot_id, &definition.selection).await?; Ok(( transaction, RunRequest { @@ -43,7 +41,7 @@ pub(crate) async fn frozen_job_request( target: request.target.clone(), snapshot_id, image_id, - kind: RunKind::Job, + kind, origin: RunOrigin::Manual, job_key: Some(request.job_key.clone()), parent_run_id: None, @@ -54,7 +52,11 @@ pub(crate) async fn frozen_job_request( requested_by: context.actor.clone(), priority: request.priority, selection: definition.selection, - write_set, + write_set: if kind == RunKind::Group { + Default::default() + } else { + write_set + }, settings: definition.settings, concurrency_groups: definition.concurrency_groups, overlap: definition.overlap, diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/mod.rs new file mode 100644 index 0000000000..010bce9e3a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/mod.rs @@ -0,0 +1,3 @@ +pub(in crate::runs) mod job; +pub(in crate::runs) mod publication; +pub(in crate::runs) mod rerun; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/request.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/publication.rs similarity index 62% rename from crates/sqlbuild-orchestration/src/runs/_helpers/request.rs rename to crates/sqlbuild-orchestration/src/runs/_helpers/requests/publication.rs index b4b727be05..5f6a1eae99 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/request.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/publication.rs @@ -1,15 +1,44 @@ -//! Validate and publish a frozen request inside the caller's audit transaction. +//! Validate and publish frozen requests inside the caller's audit transaction. use std::collections::BTreeSet; use serde_json::json; +use uuid::Uuid; +use crate::credentials::models::AuditContext; use crate::runs::main::queue_run::queue_run; use crate::runs::models::RunRequest; use crate::store::errors::ControlError; use crate::store::main::query::query; use crate::store::models::{ControlTransaction, Parameter}; +/// Check the original authored request before reading mutable publication pointers or outcomes. +pub(crate) async fn request_retry( + transaction: ControlTransaction, + context: &AuditContext, + (run_id, event_id, request): (Uuid, Uuid, &serde_json::Value), +) -> Result<(ControlTransaction, bool), ControlError> { + let (transaction, previous) = query( + transaction, + "SELECT a.actor, a.after_json FROM sqb_runs r LEFT JOIN sqb_security_audit a ON a.event_id=$1 WHERE r.run_id=$2", + vec![Parameter::Id(event_id), Parameter::Id(run_id)], + ).await?; + let Some(previous) = previous.first() else { + return Ok((transaction, false)); + }; + if previous.get("actor") != Some(&json!(context.actor)) + || previous + .get("after_json") + .and_then(|after| after.get("request")) + != Some(request) + { + return Err(ControlError::Invalid( + "run ID already has different request content".into(), + )); + } + Ok((transaction, true)) +} + pub(crate) async fn publish_request( transaction: ControlTransaction, request: &RunRequest, @@ -36,6 +65,11 @@ pub(crate) async fn publish_request( } return Ok((transaction, false)); } + if request.parent_run_id.is_some() { + return Err(ControlError::Invalid( + "group children are created atomically with their pinned parent".into(), + )); + } let (transaction, nodes) = query( transaction, "SELECT node_key FROM sqb_snapshot_nodes WHERE snapshot_id=$1 AND EXISTS \ diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/rerun.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/rerun.rs new file mode 100644 index 0000000000..cd1e9a7ecc --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/rerun.rs @@ -0,0 +1,68 @@ +//! Retry immutable execution inputs, narrowing only to durable failed and skipped outcomes. + +use std::collections::BTreeSet; + +use serde_json::from_value; + +use crate::credentials::models::AuditContext; +use crate::runs::models::{RerunRequest, RunOrigin, RunRequest, RunStatus}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn frozen_rerun( + transaction: ControlTransaction, + context: &AuditContext, + request: &RerunRequest, +) -> Result<(ControlTransaction, RunRequest), ControlError> { + let (mut transaction, rows) = query( + transaction, + "SELECT request_json, status FROM sqb_runs WHERE run_id=$1", + vec![Parameter::Id(request.source_run_id)], + ) + .await?; + let source = rows + .first() + .ok_or_else(|| ControlError::Invalid("source run was not found".into()))?; + let status: RunStatus = from_value(source["status"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if !status.is_terminal() { + return Err(ControlError::Invalid( + "rerun requires a terminal source run".into(), + )); + } + let mut frozen: RunRequest = from_value(source["request_json"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if request.from_failure { + let rows; + (transaction, rows) = query( + transaction, + "SELECT node_key FROM sqb_run_nodes WHERE run_id=$1 AND status IN ('failed','skipped')", + vec![Parameter::Id(request.source_run_id)], + ) + .await?; + let failed: BTreeSet<_> = rows + .iter() + .filter_map(|row| row["node_key"].as_str()) + .collect(); + frozen.selection.retain(|key| failed.contains(key.as_str())); + frozen + .write_set + .retain(|key| frozen.selection.contains(key)); + if frozen.selection.is_empty() { + return Err(ControlError::Invalid( + "source run has no failed or skipped nodes".into(), + )); + } + } + frozen.run_id = request.run_id; + frozen.origin = RunOrigin::Rerun; + frozen.parent_run_id = None; + frozen.root_run_id = request.run_id; + frozen.trigger_id = None; + frozen.tick_id = None; + frozen.rerun_of_run_id = Some(request.source_run_id); + frozen.requested_by = context.actor.clone(); + frozen.priority = request.priority; + Ok((transaction, frozen)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/advance_groups.rs b/crates/sqlbuild-orchestration/src/runs/main/advance_groups.rs new file mode 100644 index 0000000000..33b0e9320e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/advance_groups.rs @@ -0,0 +1,51 @@ +//! Resume virtual parents independently of launcher capacity or process restarts. + +use std::collections::BTreeMap; +use std::num::NonZeroU32; +use std::time::{Duration, Instant}; + +use crate::runs::_helpers::groups::lifecycle::advance_group; +use crate::runs::models::RunDispatch; +use crate::scheduler::constants::RENEW_INTERVAL_SECONDS; +use crate::scheduler::main::{check_lease::check_lease, renew_lease::renew_lease}; +use crate::scheduler::models::SchedulerLease; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn advance_groups<'a>( + store: &ControlStore, + lease: &SchedulerLease, + (limits, dispatch): (&BTreeMap, impl Into>), +) -> Result<(), ControlError> { + let dispatch = dispatch.into(); + let scope = dispatch.scope.ok_or_else(|| { + ControlError::Invalid("group advancement requires a dispatch scope".into()) + })?; + let mut after = None; + let mut renewed = Instant::now(); + loop { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query(transaction, + "SELECT run_id FROM sqb_runs WHERE kind='group' AND status IN ('queued','running') \ + AND location_id=$1 AND target=$2 AND ($3 IS NULL OR run_id>$3) ORDER BY run_id LIMIT 100", + vec![Parameter::Id(scope.0), Parameter::Text(scope.1.into()), Parameter::OptionalId(after)]).await?; + commit(transaction).await?; + if rows.is_empty() { + return Ok(()); + } + for row in rows { + if renewed.elapsed() >= Duration::from_secs(RENEW_INTERVAL_SECONDS) { + renew_lease(store, lease).await?; + renewed = Instant::now(); + } + let run_id: uuid::Uuid = serde_json::from_value(row["run_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let transaction = + check_lease(begin(store, TransactionMode::Write).await?, lease).await?; + let transaction = advance_group(transaction, run_id, limits, dispatch.eligible).await?; + commit(check_lease(transaction, lease).await?).await?; + after = Some(run_id); + } + } +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/change_run.rs b/crates/sqlbuild-orchestration/src/runs/main/change_run.rs index 012ff4dbe2..c43086efe7 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/change_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/change_run.rs @@ -3,8 +3,10 @@ use serde_json::json; use crate::credentials::main::append_audit::append_audit; -use crate::runs::_helpers::changes::{validate_authority, validate_change, validate_retry}; -use crate::runs::_helpers::publish_change::publish_change; +use crate::runs::_helpers::lifecycle::changes::{ + validate_authority, validate_change, validate_retry, +}; +use crate::runs::_helpers::lifecycle::publish_change::publish_change; use crate::runs::models::{ChangingRun, RunAuthority, RunChange, RunChangeRequest}; use crate::scheduler::main::check_lease::check_lease; use crate::store::errors::ControlError; diff --git a/crates/sqlbuild-orchestration/src/runs/main/claim_next_run.rs b/crates/sqlbuild-orchestration/src/runs/main/claim_next_run.rs index 9bf328e7ed..41b5686001 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/claim_next_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/claim_next_run.rs @@ -1,15 +1,14 @@ //! Claim priority-ordered work without allowing a blocked page to starve later candidates. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::num::NonZeroU32; -use serde_json::json; use sqlbuild_recall::schema::models::ColumnType; -use uuid::Uuid; -use crate::runs::_helpers::claims::reserve_claims; -use crate::runs::_helpers::events::append_run_event; -use crate::runs::models::{QueueCandidate, RunClaim}; +use crate::runs::_helpers::groups::admission::{earlier_waiter_blocks, group_capacity}; +use crate::runs::_helpers::groups::selection::child_ready; +use crate::runs::_helpers::lifecycle::claims::{publish_claim, reserve_claims}; +use crate::runs::models::{ConcurrencyResource, QueueCandidate, RunClaim, RunDispatch}; use crate::scheduler::main::check_lease::check_lease; use crate::scheduler::models::SchedulerLease; use crate::store::errors::ControlError; @@ -20,13 +19,20 @@ use crate::store::main::query::query; use crate::store::models::{Connection, ControlStore, Parameter, TransactionMode}; /// Limits come from one installation configuration, never from competing per-job declarations. -pub async fn claim_next_run( +pub async fn claim_next_run<'a>( store: &ControlStore, lease: &SchedulerLease, limits: &BTreeMap, - scope: Option<(Uuid, &str)>, + dispatch: impl Into>, ) -> Result, ControlError> { - let mut transaction = check_lease(begin(store, TransactionMode::Write).await?, lease).await?; + let dispatch = dispatch.into(); + let (mut transaction, capacity) = group_capacity( + check_lease(begin(store, TransactionMode::Write).await?, lease).await?, + limits, + dispatch.eligible, + ) + .await?; + let mut waiting: BTreeSet = BTreeSet::new(); let locking = if matches!(transaction.connection, Some(Connection::Postgres(_))) { " FOR UPDATE OF r SKIP LOCKED" } else { @@ -36,7 +42,7 @@ pub async fn claim_next_run( loop { let rows; (transaction, rows) = query(transaction, - &format!("SELECT r.request_json, r.queued_at FROM sqb_runs r WHERE r.status='queued' AND r.cancel_requested_at IS NULL \ + &format!("SELECT r.request_json, r.queued_at FROM sqb_runs r WHERE r.status='queued' AND r.kind<>'group' AND r.cancel_requested_at IS NULL \ AND NOT EXISTS (SELECT 1 FROM sqb_run_liveness l WHERE l.run_id=r.run_id) \ AND ($4 IS NULL OR (r.location_id=$4 AND r.target=$5)) \ AND ($1 IS NULL OR r.priority<$1 OR (r.priority=$1 AND (r.queued_at>$2 OR (r.queued_at=$2 AND r.run_id>$3)))) \ @@ -44,14 +50,27 @@ pub async fn claim_next_run( vec![after.as_ref().map_or(Parameter::Null(ColumnType::Integer), |value| Parameter::Integer(value.request_json.priority)), Parameter::OptionalTime(after.as_ref().map(|value| value.queued_at)), Parameter::OptionalId(after.as_ref().map(|value| value.request_json.run_id)), - Parameter::OptionalId(scope.map(|(location, _)| location)), - Parameter::OptionalText(scope.map(|(_, target)| target.to_owned()))]).await?; + Parameter::OptionalId(dispatch.scope.map(|(location, _)| location)), + Parameter::OptionalText(dispatch.scope.map(|(_, target)| target.to_owned()))]).await?; if rows.is_empty() { commit(check_lease(transaction, lease).await?).await?; return Ok(None); } for row in rows { let candidate: QueueCandidate = decode_row(row)?; + let ready; + (transaction, ready) = child_ready(transaction, &candidate.request_json).await?; + if !ready { + after = Some(candidate); + continue; + } + let blocked; + (transaction, blocked) = + earlier_waiter_blocks(transaction, &capacity, &candidate).await?; + if blocked || capacity.blocks_claim(&candidate, &waiting) { + after = Some(candidate); + continue; + } (transaction, _) = query(transaction, "SAVEPOINT candidate_claim", vec![]).await?; let reserved; (transaction, reserved) = @@ -61,34 +80,12 @@ pub async fn claim_next_run( query(transaction, "ROLLBACK TO SAVEPOINT candidate_claim", vec![]).await?; (transaction, _) = query(transaction, "RELEASE SAVEPOINT candidate_claim", vec![]).await?; + waiting = capacity.record_waiter(&candidate.request_json, waiting); after = Some(candidate); continue; } - let claim = RunClaim { - claim_id: Uuid::new_v5(&candidate.request_json.run_id, b"run_claimed"), - request: candidate.request_json, - }; - let now = transaction.now; - (transaction, _) = query(transaction, - "INSERT INTO sqb_run_liveness (run_id, worker_instance, claimed_at, last_heartbeat_at) VALUES ($1,$2,$3,$3)", - vec![Parameter::Id(claim.request.run_id), Parameter::Id(claim.claim_id), Parameter::Time(now)]).await?; - let cursor; - (transaction, cursor) = append_run_event( - transaction, - (claim.request.run_id, claim.claim_id), - "run_claimed", - json!({"claim": claim}), - ) - .await?; - (transaction, _) = query( - transaction, - "UPDATE sqb_runs SET last_event_storage_id=$1 WHERE run_id=$2", - vec![ - Parameter::Integer(cursor), - Parameter::Id(claim.request.run_id), - ], - ) - .await?; + let claim; + (transaction, claim) = publish_claim(transaction, candidate.request_json).await?; (transaction, _) = query(transaction, "RELEASE SAVEPOINT candidate_claim", vec![]).await?; commit(check_lease(transaction, lease).await?).await?; diff --git a/crates/sqlbuild-orchestration/src/runs/main/heartbeat_run.rs b/crates/sqlbuild-orchestration/src/runs/main/heartbeat_run.rs index a5399c6cdf..dc0b6db2be 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/heartbeat_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/heartbeat_run.rs @@ -3,7 +3,7 @@ use serde_json::json; use uuid::Uuid; -use crate::runs::_helpers::events::append_run_event; +use crate::runs::_helpers::lifecycle::events::append_run_event; use crate::runs::models::{RunChange, RunChangeRequest}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; diff --git a/crates/sqlbuild-orchestration/src/runs/main/job_write_set.rs b/crates/sqlbuild-orchestration/src/runs/main/job_write_set.rs index 101f419a95..91840568ee 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/job_write_set.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/job_write_set.rs @@ -11,12 +11,11 @@ use crate::store::models::{ControlTransaction, Parameter}; pub(crate) async fn job_write_set( transaction: ControlTransaction, snapshot_id: Uuid, - (location_id, target, job_key): (Uuid, &str, &str), + selection: &BTreeSet, ) -> Result<(ControlTransaction, BTreeSet), ControlError> { let (transaction, rows) = query(transaction, - "SELECT n.node_key FROM sqb_snapshot_nodes n JOIN sqb_job_nodes j ON j.node_key=n.node_key \ - WHERE n.snapshot_id=$1 AND j.location_id=$2 AND j.target=$3 AND j.job_key=$4 AND n.node_kind IN ('model','seed','udf','table_fn')", - vec![Parameter::Id(snapshot_id), Parameter::Id(location_id), Parameter::Text(target.into()), Parameter::Text(job_key.into())]).await?; + "SELECT node_key FROM sqb_snapshot_nodes WHERE snapshot_id=$1 AND node_kind IN ('model','seed','udf','table_fn')", + vec![Parameter::Id(snapshot_id)]).await?; Ok(( transaction, rows.into_iter() @@ -25,6 +24,7 @@ pub(crate) async fn job_write_set( .and_then(serde_json::Value::as_str) .map(str::to_owned) }) + .filter(|key| selection.contains(key)) .collect(), )) } diff --git a/crates/sqlbuild-orchestration/src/runs/main/mod.rs b/crates/sqlbuild-orchestration/src/runs/main/mod.rs index 7609ff479d..a69509d123 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/mod.rs @@ -1,3 +1,4 @@ +pub mod advance_groups; pub mod change_run; pub mod claim_next_run; pub mod heartbeat_run; @@ -11,5 +12,6 @@ pub mod publish_run_output; pub mod publish_run_recall; pub(crate) mod queue_run; pub mod request_job_run; +pub mod request_rerun; pub mod request_run; pub mod run_execution; diff --git a/crates/sqlbuild-orchestration/src/runs/main/monitor_runs.rs b/crates/sqlbuild-orchestration/src/runs/main/monitor_runs.rs index b23d4eb791..f6ea6f8420 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/monitor_runs.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/monitor_runs.rs @@ -3,7 +3,7 @@ use chrono::TimeDelta; use uuid::Uuid; -use crate::runs::_helpers::publish_change::publish_change; +use crate::runs::_helpers::lifecycle::publish_change::publish_change; use crate::runs::constants::{HEARTBEAT_TIMEOUT_SECONDS, MONITOR_BATCH_SIZE}; use crate::runs::models::{ChangingRun, RunChange, RunChangeRequest}; use crate::scheduler::main::check_lease::check_lease; diff --git a/crates/sqlbuild-orchestration/src/runs/main/publish_run_output.rs b/crates/sqlbuild-orchestration/src/runs/main/publish_run_output.rs index 8390019aab..17e4e0c7a3 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/publish_run_output.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/publish_run_output.rs @@ -3,7 +3,7 @@ use sqlbuild_recall::records::models::OutputChunk; use uuid::Uuid; -use crate::runs::_helpers::output::{claim_envelope, output_rows}; +use crate::runs::_helpers::lifecycle::output::{claim_envelope, output_rows}; use crate::runs::models::OutputPublication; use crate::store::errors::ControlError; use crate::store::main::append_recall::append_recall; diff --git a/crates/sqlbuild-orchestration/src/runs/main/publish_run_recall.rs b/crates/sqlbuild-orchestration/src/runs/main/publish_run_recall.rs index 9a70513ebd..30c14bb9d8 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/publish_run_recall.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/publish_run_recall.rs @@ -3,8 +3,8 @@ use sqlbuild_recall::store::models::InsertOutcome; use uuid::Uuid; -use crate::runs::_helpers::output::claim_envelope; -use crate::runs::_helpers::recall::{append_records, claim_record}; +use crate::runs::_helpers::lifecycle::output::claim_envelope; +use crate::runs::_helpers::lifecycle::recall::{append_records, claim_record}; use crate::runs::constants::{RECALL_BATCH_ROWS, RECALL_INVOCATION_ID_BYTES}; use crate::runs::models::RecallBatch; use crate::store::errors::ControlError; diff --git a/crates/sqlbuild-orchestration/src/runs/main/queue_run.rs b/crates/sqlbuild-orchestration/src/runs/main/queue_run.rs index 08a7101e43..b8cb2e9b89 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/queue_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/queue_run.rs @@ -4,13 +4,74 @@ use serde_json::json; use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; -use crate::runs::_helpers::events::append_run_event; -use crate::runs::models::RunRequest; +use crate::declarations::main::frozen_definitions::frozen_definitions; +use crate::runs::_helpers::lifecycle::events::append_run_event; +use crate::runs::main::job_write_set::job_write_set; +use crate::runs::models::{RunKind, RunOrigin, RunRequest}; use crate::store::errors::ControlError; use crate::store::main::query::query; use crate::store::models::{ControlTransaction, Parameter}; pub(crate) async fn queue_run( + transaction: ControlTransaction, + request: &RunRequest, +) -> Result { + let mut transaction = queue_one(transaction, request).await?; + if request.kind == RunKind::Group { + let definitions; + (transaction, definitions) = frozen_definitions(transaction, request.snapshot_id).await?; + let group = request + .job_key + .as_ref() + .and_then(|name| definitions.groups.get(name)) + .ok_or_else(|| { + ControlError::Invalid("group is absent from the pinned snapshot".into()) + })?; + let selection_matches = if request.origin == RunOrigin::Rerun { + request.selection.is_subset(&group.job.selection) + } else { + group.job.selection == request.selection + }; + if !selection_matches || !request.write_set.is_empty() { + return Err(ControlError::Invalid( + "group scope differs from its pinned membership".into(), + )); + } + let writers; + (transaction, writers) = + job_write_set(transaction, request.snapshot_id, &request.selection).await?; + for name in group.jobs.keys() { + let job = &definitions.jobs[name]; + let selection: std::collections::BTreeSet<_> = job + .selection + .intersection(&request.selection) + .cloned() + .collect(); + if selection.is_empty() { + continue; + } + let child = RunRequest { + run_id: Uuid::new_v5(&request.run_id, name.as_bytes()), + kind: RunKind::Job, + job_key: Some(name.clone()), + parent_run_id: Some(request.run_id), + rerun_of_run_id: request + .rerun_of_run_id + .map(|id| Uuid::new_v5(&id, name.as_bytes())), + write_set: selection.intersection(&writers).cloned().collect(), + selection, + settings: job.settings.clone(), + concurrency_groups: job.concurrency_groups.clone(), + overlap: job.overlap, + ..request.clone() + }; + transaction = queue_one(transaction, &child).await?; + } + } + Ok(transaction) +} + +async fn queue_one( mut transaction: ControlTransaction, request: &RunRequest, ) -> Result { diff --git a/crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs b/crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs index e768a67f4d..b54554cb29 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs @@ -5,14 +5,13 @@ use uuid::Uuid; use crate::credentials::main::append_audit::append_audit; use crate::credentials::models::AuditContext; -use crate::runs::_helpers::job_request::frozen_job_request; -use crate::runs::_helpers::request::publish_request; +use crate::runs::_helpers::requests::job::frozen_job_request; +use crate::runs::_helpers::requests::publication::{publish_request, request_retry}; use crate::runs::models::JobRunRequest; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; -use crate::store::main::query::query; -use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::models::{ControlStore, TransactionMode}; /// The context is a trusted control-store owner, not a role supplied by an HTTP caller. pub async fn request_job_run( @@ -32,22 +31,13 @@ pub async fn request_job_run( } let event_id = Uuid::new_v5(&request.run_id, b"manual-job-request"); let transaction = begin(store, TransactionMode::Write).await?; - let (transaction, previous) = query( + let (transaction, retry) = request_retry( transaction, - "SELECT a.actor, a.after_json FROM sqb_runs r LEFT JOIN sqb_security_audit a ON a.event_id=$1 WHERE r.run_id=$2", - vec![Parameter::Id(event_id), Parameter::Id(request.run_id)], - ).await?; - if let Some(previous) = previous.first() { - if previous.get("actor") != Some(&json!(context.actor)) - || previous - .get("after_json") - .and_then(|after| after.get("request")) - != Some(&json!(request)) - { - return Err(ControlError::Invalid( - "run ID already has different request content".into(), - )); - } + context, + (request.run_id, event_id, &json!(request)), + ) + .await?; + if retry { commit(transaction).await?; return Ok(false); } diff --git a/crates/sqlbuild-orchestration/src/runs/main/request_rerun.rs b/crates/sqlbuild-orchestration/src/runs/main/request_rerun.rs new file mode 100644 index 0000000000..f44d096434 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/request_rerun.rs @@ -0,0 +1,54 @@ +//! Publish a new pinned execution and its operator audit atomically, including retry identity. + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::AuditContext; +use crate::runs::_helpers::requests::publication::{publish_request, request_retry}; +use crate::runs::_helpers::requests::rerun::frozen_rerun; +use crate::runs::models::RerunRequest; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; + +/// This entry requires direct control-store ownership, just like a privileged manual job request. +pub async fn request_rerun( + store: &ControlStore, + context: &AuditContext, + request: &RerunRequest, +) -> Result { + if context.actor.trim().is_empty() + || context.granting_role.trim().is_empty() + || request.reason.trim().is_empty() + { + return Err(ControlError::Invalid( + "rerun needs an actor, granting role and reason".into(), + )); + } + let event_id = Uuid::new_v5(&request.run_id, b"rerun-request"); + let (transaction, retry) = request_retry( + begin(store, TransactionMode::Write).await?, + context, + (request.run_id, event_id, &json!(request)), + ) + .await?; + if retry { + commit(transaction).await?; + return Ok(false); + } + let (transaction, frozen) = frozen_rerun(transaction, context, request).await?; + let (transaction, _) = publish_request(transaction, &frozen).await?; + let transaction = append_audit( + transaction, + context, + (event_id, "run.rerun_requested"), + ( + json!({"location_id": frozen.location_id, "target": frozen.target, "job_key": frozen.job_key, "run_id": request.run_id}), + json!({"source_run_id": request.source_run_id}), + json!({"request": request, "frozen": frozen}), + ), + ).await?; + commit(transaction).await?; + Ok(true) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/request_run.rs b/crates/sqlbuild-orchestration/src/runs/main/request_run.rs index 71665e84da..f3041afbbc 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/request_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/request_run.rs @@ -1,6 +1,6 @@ //! Queue a frozen, already authorized request without recreating work on transport retries. -use crate::runs::_helpers::request::publish_request; +use crate::runs::_helpers::requests::publication::publish_request; use crate::runs::models::RunRequest; use crate::store::errors::ControlError; use crate::store::main::begin::begin; diff --git a/crates/sqlbuild-orchestration/src/runs/main/run_execution.rs b/crates/sqlbuild-orchestration/src/runs/main/run_execution.rs index 55c1da3519..f8e49fc1f2 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/run_execution.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/run_execution.rs @@ -4,6 +4,7 @@ use std::collections::BTreeMap; use uuid::Uuid; +use crate::runs::_helpers::groups::selection::skipped_nodes; use crate::runs::models::{ExecutionMetadata, RunClaim, RunExecution}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; @@ -45,6 +46,7 @@ pub async fn run_execution( Ok((fields["node_key"].clone(), fields["node_hash"].clone())) }) .collect::>()?; + let (transaction, skipped_nodes) = skipped_nodes(transaction, &metadata.request_json).await?; commit(transaction).await?; Ok(Some(RunExecution { claim: RunClaim { @@ -54,5 +56,6 @@ pub async fn run_execution( environment_hash: metadata.environment_hash, sqb_version: metadata.sqb_version, node_hashes, + skipped_nodes, })) } diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index 34444c32e8..c049379a61 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -63,6 +63,16 @@ pub enum RunStatus { PresumedFailed, } +impl RunStatus { + #[must_use] + pub const fn is_terminal(self) -> bool { + !matches!( + self, + Self::Queued | Self::Launched | Self::Running | Self::Unresponsive + ) + } +} + /// The authorized selection and pinned execution inputs are immutable once requested. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] pub struct RunRequest { @@ -100,6 +110,17 @@ pub struct JobRunRequest { pub priority: i64, } +/// A retry derives its execution scope from a terminal run, never from caller selectors. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RerunRequest { + pub run_id: Uuid, + pub source_run_id: Uuid, + pub from_failure: bool, + pub reason: String, + pub priority: i64, +} + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] pub struct RunRecord { pub run_id: Uuid, @@ -166,6 +187,9 @@ pub struct RunExecution { pub environment_hash: String, pub sqb_version: String, pub node_hashes: BTreeMap, + /// Derived from completed sibling facts and pinned lineage, never a mutable request rewrite. + #[serde(default)] + pub skipped_nodes: BTreeMap, } #[derive(Debug, Deserialize)] @@ -271,6 +295,28 @@ pub(crate) struct QueueCandidate { pub(crate) queued_at: DateTime, } +/// Fairness covers runnable launcher scopes, not queues the installation cannot dispatch. +#[derive(Clone, Copy, Debug)] +pub struct RunDispatch<'a> { + pub scope: Option<(Uuid, &'a str)>, + pub eligible: Option<&'a BTreeSet<(Uuid, String)>>, +} + +impl<'a> From> for RunDispatch<'a> { + fn from(scope: Option<(Uuid, &'a str)>) -> Self { + Self { + scope, + eligible: None, + } + } +} + +impl<'a> From<(Uuid, &'a str)> for RunDispatch<'a> { + fn from(scope: (Uuid, &'a str)) -> Self { + Some(scope).into() + } +} + #[derive(Clone, Debug, Deserialize)] pub(crate) struct ChangingRun { pub(crate) status: RunStatus, @@ -279,6 +325,39 @@ pub(crate) struct ChangingRun { pub(crate) node_counts_json: BTreeMap, } +/// Virtual parents and physical children use the same lifecycle evidence during group progress. +#[derive(Debug, Deserialize)] +pub(crate) struct GroupRun { + pub(crate) request_json: RunRequest, + pub(crate) queued_at: DateTime, + #[serde(flatten)] + pub(crate) state: ChangingRun, +} + +/// Parent overlap locks and installation-wide limits occupy different resource namespaces. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) enum ConcurrencyResource { + Limit(String), + Overlap(Uuid, String, String), +} + +/// A transaction-local safety calculation, never a second persisted allocation ledger. +#[derive(Debug)] +pub(crate) struct GroupCapacity { + pub(crate) available: BTreeMap, + pub(crate) held: BTreeMap>, + pub(crate) needed: BTreeMap>, + pub(crate) waiting: Vec, +} + +#[derive(Debug)] +pub(crate) struct GroupContext { + pub(crate) parent: GroupRun, + pub(crate) children: Vec, + pub(crate) definitions: crate::declarations::models::Definitions, + pub(crate) critical_failed: bool, +} + /// Differing retries retain conflict evidence while preserving the first output bytes. #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] pub struct OutputPublication { diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs index e84c4f73b9..3a69b46ea5 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs @@ -19,7 +19,7 @@ pub(crate) async fn persist_ticks( let (mut transaction, write_set) = job_write_set( transaction, trigger.snapshot_id, - (trigger.location_id, &trigger.target, &trigger.job_key), + &trigger.job_json.selection, ) .await?; for action in &plan.actions { @@ -44,7 +44,7 @@ pub(crate) async fn persist_ticks( target: trigger.target.clone(), snapshot_id: trigger.snapshot_id, image_id: trigger.image_id.clone(), - kind: RunKind::Job, + kind: trigger.run_kind, origin: RunOrigin::Schedule, job_key: Some(trigger.job_key.clone()), parent_run_id: None, @@ -55,7 +55,11 @@ pub(crate) async fn persist_ticks( requested_by: "system:scheduler".into(), priority: 0, selection: trigger.job_json.selection.clone(), - write_set: write_set.clone(), + write_set: if trigger.run_kind == RunKind::Group { + Default::default() + } else { + write_set.clone() + }, settings: trigger.job_json.settings.clone(), concurrency_groups: trigger.job_json.concurrency_groups.clone(), overlap: trigger.job_json.overlap, diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/evaluate_trigger.rs b/crates/sqlbuild-orchestration/src/scheduler/main/evaluate_trigger.rs index 1a2e98d3b7..7d065991ee 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/evaluate_trigger.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/evaluate_trigger.rs @@ -24,7 +24,7 @@ pub async fn evaluate_trigger( ) -> Result, ControlError> { let transaction = check_lease(begin(store, TransactionMode::Write).await?, lease).await?; let (transaction, rows) = query(transaction, - "SELECT t.trigger_id, t.location_id, t.target, t.job_key, t.snapshot_id, s.image_id, t.definition_json, j.definition_json AS job_json, \ + "SELECT t.trigger_id, t.location_id, t.target, t.job_key, t.snapshot_id, s.image_id, t.definition_json, j.definition_json AS job_json, j.kind AS run_kind, \ state.started_at, state.last_tick_for, state.status FROM sqb_triggers t \ JOIN sqb_trigger_state state ON state.trigger_id=t.trigger_id JOIN sqb_snapshots s ON s.snapshot_id=t.snapshot_id \ JOIN sqb_jobs j ON j.location_id=t.location_id AND j.target=t.target AND j.job_key=t.job_key \ diff --git a/crates/sqlbuild-orchestration/src/scheduler/models.rs b/crates/sqlbuild-orchestration/src/scheduler/models.rs index 9f4d421736..39594ed147 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/models.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/models.rs @@ -125,6 +125,7 @@ pub(crate) struct DueTrigger { pub(crate) image_id: Option, pub(crate) definition_json: TriggerDefinition, pub(crate) job_json: JobDefinition, + pub(crate) run_kind: crate::runs::models::RunKind, pub(crate) started_at: DateTime, pub(crate) last_tick_for: Option>, pub(crate) status: TriggerStatus, diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/reading.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/reading.rs index 91d5c5f6b0..a350003e29 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/_helpers/reading.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/reading.rs @@ -4,12 +4,29 @@ use std::collections::BTreeMap; use serde::de::DeserializeOwned; use serde_json::Value; +use uuid::Uuid; use crate::snapshots::_helpers::content::decompress; -use crate::snapshots::models::SnapshotNode; +use crate::snapshots::models::{SnapshotNode, SnapshotRecord}; use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::{ControlTransaction, Parameter}; use crate::store::types::Row; +pub(crate) async fn read_current_record( + transaction: ControlTransaction, + (location_id, target): (Uuid, &str), +) -> Result<(ControlTransaction, Option), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT s.snapshot_id, s.location_id, s.target, s.generation, s.manifest_digest, s.published_at \ + FROM sqb_current_snapshots c JOIN sqb_snapshots s ON s.snapshot_id=c.snapshot_id WHERE c.location_id=$1 AND c.target=$2", + vec![Parameter::Id(location_id), Parameter::Text(target.into())]).await?; + Ok(( + transaction, + rows.into_iter().next().map(decode_row).transpose()?, + )) +} + fn decode_body(value: &Value) -> Result { serde_json::from_slice(&decompress(value)?) .map_err(|error| ControlError::Invalid(error.to_string())) diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot.rs b/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot.rs index b668008c38..81889f713b 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot.rs @@ -2,8 +2,10 @@ use uuid::Uuid; -use crate::snapshots::_helpers::reading::{attach_sources, read_documents, read_nodes}; -use crate::snapshots::models::{CurrentSnapshot, SnapshotChange, SnapshotRecord}; +use crate::snapshots::_helpers::reading::{attach_sources, read_current_record}; +use crate::snapshots::main::read_snapshot_documents::read_snapshot_documents; +use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; +use crate::snapshots::models::{CurrentSnapshot, SnapshotChange}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; @@ -17,27 +19,21 @@ pub async fn current_snapshot( location_id: Uuid, target: &str, ) -> Result, ControlError> { - let transaction = begin(store, TransactionMode::Read).await?; - let (transaction, rows) = query(transaction, - "SELECT s.snapshot_id, s.location_id, s.target, s.generation, s.manifest_digest, s.published_at \ - FROM sqb_current_snapshots c JOIN sqb_snapshots s ON s.snapshot_id=c.snapshot_id WHERE c.location_id=$1 AND c.target=$2", - vec![Parameter::Id(location_id), Parameter::Text(target.into())]).await?; - let Some(row) = rows.into_iter().next() else { + let (transaction, record) = read_current_record( + begin(store, TransactionMode::Read).await?, + (location_id, target), + ) + .await?; + let Some(record) = record else { return Ok(None); }; - let record: SnapshotRecord = decode_row(row)?; - let (transaction, rows) = query(transaction, - "SELECT n.node_key, d.body FROM sqb_snapshot_nodes n JOIN sqb_node_definitions d ON d.node_hash=n.node_hash WHERE n.snapshot_id=$1 ORDER BY n.node_key", - vec![Parameter::Id(record.snapshot_id)]).await?; - let nodes = read_nodes(rows)?; + let (transaction, nodes) = read_snapshot_nodes(transaction, record.snapshot_id, None).await?; let (transaction, rows) = query(transaction, "SELECT s.node_key, s.kind, t.body FROM sqb_snapshot_sources s JOIN sqb_source_texts t ON t.text_hash=s.text_hash WHERE s.snapshot_id=$1", vec![Parameter::Id(record.snapshot_id)]).await?; let nodes = attach_sources(nodes, rows)?; - let (transaction, rows) = query(transaction, - "SELECT d.kind, b.body FROM sqb_snapshot_documents d JOIN sqb_documents b ON b.document_hash=d.document_hash WHERE d.snapshot_id=$1", - vec![Parameter::Id(record.snapshot_id)]).await?; - let documents = read_documents(rows)?; + let (transaction, documents) = + read_snapshot_documents(transaction, record.snapshot_id, None).await?; let (transaction, rows) = query(transaction, "SELECT change_kind, subject_key, before_hash, after_hash FROM sqb_snapshot_changes WHERE snapshot_id=$1 ORDER BY change_seq", vec![Parameter::Id(record.snapshot_id)]).await?; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot_record.rs b/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot_record.rs new file mode 100644 index 0000000000..58f44ee9a4 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot_record.rs @@ -0,0 +1,23 @@ +//! Reconcile publication identity without reading node definitions, source bodies or documents. + +use uuid::Uuid; + +use crate::snapshots::_helpers::reading::read_current_record; +use crate::snapshots::models::SnapshotRecord; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn current_snapshot_record( + store: &ControlStore, + location_id: Uuid, + target: &str, +) -> Result, ControlError> { + let (transaction, record) = read_current_record( + begin(store, TransactionMode::Read).await?, + (location_id, target), + ) + .await?; + commit(transaction).await?; + Ok(record) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs index 56f1b75755..6918426ec2 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs @@ -1,5 +1,8 @@ pub mod current_snapshot; +pub mod current_snapshot_record; pub mod next_remote_generation; pub mod publish_remote_snapshot; pub mod publish_snapshot; +pub(crate) mod read_snapshot_documents; +pub(crate) mod read_snapshot_nodes; pub mod verify_execution; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/next_remote_generation.rs b/crates/sqlbuild-orchestration/src/snapshots/main/next_remote_generation.rs index f92dabe4fa..3c4f974037 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/next_remote_generation.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/next_remote_generation.rs @@ -5,7 +5,7 @@ use uuid::Uuid; use crate::api::errors::ClientError; use crate::api::main::client_response::client_response; use crate::api::main::connect_client::connect_client; -use crate::snapshots::models::CurrentSnapshot; +use crate::snapshots::models::SnapshotRecord; pub async fn next_remote_generation( server_url: &str, @@ -21,10 +21,10 @@ pub async fn next_remote_generation( "targets", target, "snapshots", + "metadata", ])?; - match client_response::(connection.client.get(url)).await { + match client_response::(connection.client.get(url)).await { Ok(current) => current - .record .generation .checked_add(1) .ok_or(ClientError::Configuration( diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_documents.rs b/crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_documents.rs new file mode 100644 index 0000000000..45a03182f8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_documents.rs @@ -0,0 +1,36 @@ +//! Decode immutable documents within the caller's consistent snapshot transaction. + +use std::collections::BTreeMap; + +use serde_json::Value; +use uuid::Uuid; + +use crate::snapshots::_helpers::reading::read_documents; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn read_snapshot_documents( + transaction: ControlTransaction, + snapshot_id: Uuid, + kinds: Option<&[&str]>, +) -> Result<(ControlTransaction, BTreeMap), ControlError> { + if kinds.is_some_and(<[&str]>::is_empty) { + return Ok((transaction, BTreeMap::new())); + } + let mut parameters = vec![Parameter::Id(snapshot_id)]; + let filter = kinds.map_or(String::new(), |keys| { + parameters.extend(keys.iter().map(|key| Parameter::Text((*key).into()))); + format!( + " AND d.kind IN ({})", + (2..parameters.len() + 1) + .map(|index| format!("${index}")) + .collect::>() + .join(",") + ) + }); + let (transaction, rows) = query(transaction, + &format!("SELECT d.kind, b.body FROM sqb_snapshot_documents d JOIN sqb_documents b ON b.document_hash=d.document_hash WHERE d.snapshot_id=$1{filter}"), + parameters).await?; + Ok((transaction, read_documents(rows)?)) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_nodes.rs b/crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_nodes.rs new file mode 100644 index 0000000000..33240705d0 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_nodes.rs @@ -0,0 +1,36 @@ +//! Read retained node definitions without expanding their display-only source bodies. + +use std::collections::{BTreeMap, BTreeSet}; + +use uuid::Uuid; + +use crate::snapshots::_helpers::reading::read_nodes; +use crate::snapshots::models::SnapshotNode; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn read_snapshot_nodes( + transaction: ControlTransaction, + snapshot_id: Uuid, + selection: Option<&BTreeSet>, +) -> Result<(ControlTransaction, BTreeMap), ControlError> { + if selection.is_some_and(BTreeSet::is_empty) { + return Ok((transaction, BTreeMap::new())); + } + let mut parameters = vec![Parameter::Id(snapshot_id)]; + let filter = selection.map_or(String::new(), |keys| { + parameters.extend(keys.iter().cloned().map(Parameter::Text)); + format!( + " AND n.node_key IN ({})", + (2..parameters.len() + 1) + .map(|index| format!("${index}")) + .collect::>() + .join(",") + ) + }); + let (transaction, rows) = query(transaction, + &format!("SELECT n.node_key, d.body FROM sqb_snapshot_nodes n JOIN sqb_node_definitions d ON d.node_hash=n.node_hash WHERE n.snapshot_id=$1{filter} ORDER BY n.node_key"), + parameters).await?; + Ok((transaction, read_nodes(rows)?)) +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_contract.rs b/crates/sqlbuild-orchestration/tests/api/test_contract.rs index 9b4b904f29..be3b784298 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contract.rs @@ -16,6 +16,7 @@ use sqlbuild_orchestration::runs::models::{ RunAuthority, RunChange, RunChangeRequest, RunExecution, RunStatus, }; use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::snapshots::main::next_remote_generation::next_remote_generation; use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; use sqlbuild_orchestration::snapshots::main::verify_execution::verify_execution; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotRecord}; @@ -108,10 +109,47 @@ fn given_a_real_http_server_when_locations_and_workers_report_then_scopes_retrie assert!(openapi["components"]["schemas"]["SnapshotInput"].is_object()); assert!(openapi["components"]["schemas"]["RunChangeRequest"].is_object()); assert!(openapi["components"]["schemas"]["RunExecution"].is_object()); + assert_eq!( + next_remote_generation( + &api.url, + &location_token.bearer, + location_id, + "development" + ) + .await + .expect("first generation"), + 1 + ); let input = snapshot(location_id); let published = publish_remote_snapshot(&api.url, &location_token.bearer, &input) .await .expect("publish retained native data"); + let metadata: SnapshotRecord = client + .get(format!("{path}/metadata")) + .bearer_auth(&location_token.bearer) + .send() + .await + .expect("metadata response") + .json() + .await + .expect("metadata record without project bodies"); + assert_eq!(metadata, published); + assert_eq!( + next_remote_generation( + &api.url, + &location_token.bearer, + location_id, + "development" + ) + .await + .expect("reconcile published generation"), + 2 + ); + assert!(matches!( + next_remote_generation(&api.url, &location_token.bearer, other, "development") + .await, + Err(ClientError::Rejected(StatusCode::FORBIDDEN)) + )); let response = client .get(&path) .bearer_auth(&location_token.bearer) diff --git a/crates/sqlbuild-orchestration/tests/runs.rs b/crates/sqlbuild-orchestration/tests/runs.rs index b2b4e61852..df23471f89 100644 --- a/crates/sqlbuild-orchestration/tests/runs.rs +++ b/crates/sqlbuild-orchestration/tests/runs.rs @@ -1,3 +1,5 @@ +#[path = "runs/claims/helpers.rs"] +mod claim_helpers; #[path = "store/faults/helpers.rs"] mod fault_helpers; #[path = "store/faults/test_types.rs"] @@ -24,11 +26,17 @@ mod test_claims; mod test_coalesce; #[path = "runs/test_faults.rs"] mod test_faults; +#[path = "runs/test_group_admission.rs"] +mod test_group_admission; +#[path = "runs/test_groups.rs"] +mod test_groups; #[path = "runs/test_lifecycle.rs"] mod test_lifecycle; #[path = "runs/test_monitor.rs"] mod test_monitor; #[path = "runs/test_requests.rs"] mod test_requests; +#[path = "runs/test_reruns.rs"] +mod test_reruns; #[path = "runs/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs new file mode 100644 index 0000000000..4fe22bb22f --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs @@ -0,0 +1,72 @@ +//! Real slot accounting and successful completion through the public worker lifecycle. + +use std::collections::BTreeMap; + +use sqlbuild_orchestration::runs::main::change_run::change_run; +use sqlbuild_orchestration::runs::models::{ + NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunClaim, RunOutcome, +}; +use sqlbuild_orchestration::store::models::ControlStore; +use uuid::Uuid; + +pub(crate) async fn slot_counts(store: &ControlStore) -> BTreeMap { + let sql = "SELECT group_key, count(*) FROM sqb_concurrency_slots GROUP BY group_key"; + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + return rusqlite::Connection::open(path) + .expect("inspect SQLite") + .prepare(sql) + .expect("slot query") + .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) + .expect("SQLite slot counts") + .collect::>() + .expect("decode slot counts"); + }; + sqlx_core::query_as::query_as::<_, (String, i64)>(sql) + .fetch_all(pool) + .await + .expect("PostgreSQL slot counts") + .into_iter() + .collect() +} + +pub(crate) async fn succeed(store: &ControlStore, claim: &RunClaim) { + let mut changes = vec![RunChange::Started]; + changes.extend( + claim + .request + .selection + .iter() + .map(|node| RunChange::NodeFinished { + node_key: node.clone(), + outcome: NodeOutcome::Success, + rows_affected: Some(1), + error_code: None, + skipped_because_of: None, + }), + ); + changes.push(RunChange::Finished { + outcome: RunOutcome::Success, + reason: "synthetic child completed".into(), + exit_code: Some(0), + }); + for change in changes { + change_run( + store, + RunAuthority::Worker { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change, + }, + ) + .await + .expect("worker fact"); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_claims.rs b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs index a34df737bc..ad747388df 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_claims.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs @@ -11,6 +11,7 @@ use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; use uuid::Uuid; +use crate::claim_helpers::succeed; use crate::helpers::{queue, request, snapshot}; use crate::location_helpers::create_location; use crate::replica_helpers::second_store; @@ -439,3 +440,95 @@ fn given_more_than_one_page_of_blocked_runs_when_claiming_then_lower_priority_in }); } } + +#[test] +fn given_a_blocked_multi_limit_waiter_when_younger_work_arrives_then_shared_limits_drain_without_blocking_unrelated_work() + { + let test_cases = [ + GroupLimitTestCase { + description: "SQLite protects priority across atomic limit sets", + backend: Backend::Sqlite, + expected_unblocked_run: 4, + }, + GroupLimitTestCase { + description: "PostgreSQL protects priority across atomic limit sets", + backend: Backend::Postgres, + expected_unblocked_run: 4, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("snapshot"); + let scope = (location, published.snapshot_id); + for (id, priority, nodes, groups) in [ + (1, 100, &["inventory"][..], &["inventory_limit"][..]), + ( + 2, + 10, + &["orders"][..], + &["orders_limit", "inventory_limit"][..], + ), + (3, 1, &["customers"][..], &["orders_limit"][..]), + (4, 0, &["products"][..], &[][..]), + ] { + let mut input = request(scope, id, priority, nodes); + input.concurrency_groups = groups.iter().map(|key| (*key).into()).collect(); + queue(store, &input).await; + } + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::from([ + ("orders_limit".into(), NonZeroU32::new(1).expect("one")), + ("inventory_limit".into(), NonZeroU32::new(1).expect("one")), + ]); + let blocker = claim_next_run(store, &lease, &limits, None) + .await + .expect("claim") + .expect("highest priority"); + assert_eq!(blocker.request.run_id, Uuid::from_u128(1)); + let unrelated = claim_next_run(store, &lease, &limits, None) + .await + .expect("claim") + .expect("unrelated"); + assert_eq!( + unrelated.request.run_id, + Uuid::from_u128(test_case.expected_unblocked_run), + "{}", + test_case.description + ); + assert!( + claim_next_run(store, &lease, &limits, None) + .await + .expect("wait") + .is_none() + ); + succeed(store, &blocker).await; + let protected = claim_next_run(store, &lease, &limits, None) + .await + .expect("claim") + .expect("older waiter"); + assert_eq!(protected.request.run_id, Uuid::from_u128(2)); + succeed(store, &protected).await; + let younger = claim_next_run(store, &lease, &limits, None) + .await + .expect("claim") + .expect("younger retry"); + assert_eq!(younger.request.run_id, Uuid::from_u128(3)); + succeed(store, &younger).await; + succeed(store, &unrelated).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs b/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs new file mode 100644 index 0000000000..8dc2a9e113 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs @@ -0,0 +1,803 @@ +//! Real database admission preserves completion orders, capacities and fair retries across replicas. + +use std::collections::{BTreeMap, BTreeSet}; +use std::num::NonZeroU32; + +use serde_json::json; +use sqlbuild_orchestration::credentials::models::AuditContext; +use sqlbuild_orchestration::runs::main::{ + advance_groups::advance_groups, claim_next_run::claim_next_run, list_runs::list_runs, + request_job_run::request_job_run, +}; +use sqlbuild_orchestration::runs::models::{JobRunRequest, RunDispatch, RunStatus}; +use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use uuid::Uuid; + +use crate::claim_helpers::{slot_counts, succeed}; +use crate::helpers::{queue, request, snapshot}; +use crate::location_helpers::create_location; +use crate::replica_helpers::second_store; +use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store_test_types::Backend; +use crate::test_types::{ + AdmissionParent, GroupAdmissionTestCase, GroupQueueTestCase, MixedAdmissionTestCase, + RunLifecycleTestCase, +}; + +#[test] +fn given_mixed_physical_and_parent_waiters_when_admitting_then_priority_and_queue_order_win() { + let test_cases = [ + MixedAdmissionTestCase { + description: "higher-priority physical request precedes parent admission", + physical_priority: 10, + parent_priority: 1, + expected_selections: ["inventory", "orders"], + }, + MixedAdmissionTestCase { + description: "earlier physical request wins at equal priority", + physical_priority: 1, + parent_priority: 1, + expected_selections: ["inventory", "orders"], + }, + MixedAdmissionTestCase { + description: "higher-priority parent precedes physical request", + physical_priority: 1, + parent_priority: 10, + expected_selections: ["orders", "inventory"], + }, + ]; + for test_case in test_cases { + for backend in [Backend::Sqlite, Backend::Postgres] { + run(async { + let Some(fixture) = open_test_store(backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let actor = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let mut input = snapshot(location); + input.documents.insert( + "jobs".into(), + json!({"stage": {"description": "Build orders", "selection": ["orders"]}}), + ); + input.documents.insert("groups".into(), json!({"daily": {"description": "Group orders", "selection": ["orders"], "concurrency_groups": ["warehouse"], "jobs": {"stage": []}}})); + let published = publish_snapshot(store, &input).await.expect("publish"); + let mut physical = request( + (location, published.snapshot_id), + 900, + test_case.physical_priority, + &["inventory"], + ); + physical.concurrency_groups.insert("warehouse".into()); + queue(store, &physical).await; + request_job_run( + store, + &actor, + &JobRunRequest { + run_id: Uuid::from_u128(100), + location_id: location, + target: "development".into(), + job_key: "daily".into(), + reason: "Exercise mixed queue ordering".into(), + priority: test_case.parent_priority, + }, + ) + .await + .expect("queue group"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = + BTreeMap::from([("warehouse".into(), NonZeroU32::new(1).expect("one"))]); + let scope = (location, "development"); + for index in 0..test_case.expected_selections.len() { + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("advance"); + let claim = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim") + .expect(test_case.description); + assert_eq!( + claim.request.selection, + BTreeSet::from([test_case.expected_selections[index].into()]), + "{} {backend:?}", + test_case.description + ); + assert_eq!(slot_counts(store).await.get("warehouse"), Some(&1)); + succeed(store, &claim).await; + } + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("finish parent"); + assert!( + list_runs(store, scope, 10) + .await + .expect("runs") + .iter() + .all(|run| run.status == RunStatus::Success) + ); + assert!(slot_counts(store).await.is_empty()); + close_test_store(fixture).await; + }); + } + } +} + +#[test] +fn given_an_unserved_queued_group_when_dispatching_an_eligible_scope_then_unknown_child_limits_do_not_block_it() + { + let test_cases = [ + RunLifecycleTestCase { + description: "SQLite dispatch excludes unserved prospective demand", + backend: Backend::Sqlite, + expected_status: RunStatus::Success, + }, + RunLifecycleTestCase { + description: "PostgreSQL dispatch excludes unserved prospective demand", + backend: Backend::Postgres, + expected_status: RunStatus::Success, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let actor = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let unserved = create_location(store, "inventory", "test-admin") + .await + .expect("unserved location"); + let served = create_location(store, "orders", "test-admin") + .await + .expect("served location"); + for location in [unserved, served] { + let mut input = snapshot(location); + input.documents.insert("jobs".into(), json!({"stage": {"description": "Build orders", "selection": ["orders"], "concurrency_groups": if location == unserved {vec!["inventory_limit"]} else {vec![]}}})); + input.documents.insert("groups".into(), json!({"daily": {"description": "Group orders", "selection": ["orders"], "jobs": {"stage": []}}})); + publish_snapshot(store, &input).await.expect("publish"); + request_job_run( + store, + &actor, + &JobRunRequest { + run_id: location, + location_id: location, + target: "development".into(), + job_key: "daily".into(), + reason: "Exercise launcher eligibility".into(), + priority: 1, + }, + ) + .await + .expect("queue group"); + } + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::new(); + let eligible = BTreeSet::from([(served, "development".into())]); + let dispatch = RunDispatch { + scope: Some((served, "development")), + eligible: Some(&eligible), + }; + advance_groups(store, &lease, (&limits, dispatch)) + .await + .expect("eligible parent advances"); + let claim = claim_next_run(store, &lease, &limits, dispatch) + .await + .expect("eligible dispatch") + .expect("eligible child"); + assert_eq!(claim.request.location_id, served); + succeed(store, &claim).await; + advance_groups(store, &lease, (&limits, dispatch)) + .await + .expect("finish eligible parent"); + assert!( + list_runs(store, (served, "development"), 10) + .await + .expect("served runs") + .iter() + .all(|run| run.status == test_case.expected_status), + "{}", + test_case.description + ); + assert!( + list_runs(store, (unserved, "development"), 10) + .await + .expect("unserved runs") + .iter() + .all(|run| run.status == RunStatus::Queued) + ); + assert!(slot_counts(store).await.is_empty()); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_capacity_and_multistage_cycles_when_replicas_admit_and_resume_groups_then_every_group_finishes_within_limits() + { + let test_cases = [ + GroupAdmissionTestCase { + description: "capacity two permits both sides of a cycle", + limits: &[("orders_limit", 2), ("inventory_limit", 2)], + parents: &[ + AdmissionParent { + holds: &["orders_limit"], + children: &[&["inventory_limit"]], + }, + AdmissionParent { + holds: &["inventory_limit"], + children: &[&["orders_limit"]], + }, + ], + expected_running: 2, + reductions: &[&[("orders_limit", 1), ("inventory_limit", 1)]], + }, + GroupAdmissionTestCase { + description: "asymmetric capacity still has a safe completion order", + limits: &[("orders_limit", 2), ("inventory_limit", 1)], + parents: &[ + AdmissionParent { + holds: &["orders_limit"], + children: &[&["inventory_limit"]], + }, + AdmissionParent { + holds: &["inventory_limit"], + children: &[&["orders_limit"]], + }, + ], + expected_running: 2, + reductions: &[], + }, + GroupAdmissionTestCase { + description: "three-way cycle leaves one parent waiting", + limits: &[ + ("orders_limit", 1), + ("inventory_limit", 1), + ("products_limit", 1), + ], + parents: &[ + AdmissionParent { + holds: &["orders_limit"], + children: &[&["inventory_limit"]], + }, + AdmissionParent { + holds: &["inventory_limit"], + children: &[&["products_limit"]], + }, + AdmissionParent { + holds: &["products_limit"], + children: &[&["orders_limit"]], + }, + ], + expected_running: 2, + reductions: &[], + }, + GroupAdmissionTestCase { + description: "a free first stage does not conceal a later cycle", + limits: &[("orders_limit", 1), ("inventory_limit", 1)], + parents: &[ + AdmissionParent { + holds: &["orders_limit"], + children: &[&[], &["inventory_limit"]], + }, + AdmissionParent { + holds: &["inventory_limit"], + children: &[&[], &["orders_limit"]], + }, + ], + expected_running: 1, + reductions: &[], + }, + GroupAdmissionTestCase { + description: "a persistently blocked child lets its parent's peer drain shared limits", + limits: &[ + ("orders_limit", 1), + ("inventory_limit", 1), + ("products_limit", 1), + ], + parents: &[ + AdmissionParent { + holds: &["orders_limit"], + children: &[&["inventory_limit", "products_limit"]], + }, + AdmissionParent { + holds: &["inventory_limit"], + children: &[&["products_limit"]], + }, + ], + expected_running: 2, + reductions: &[], + }, + GroupAdmissionTestCase { + description: "multiple parent limits do not hoist independent child limits", + limits: &[ + ("orders_limit", 1), + ("inventory_limit", 1), + ("products_limit", 1), + ("support_limit", 1), + ], + parents: &[ + AdmissionParent { + holds: &["orders_limit", "products_limit"], + children: &[&["inventory_limit"]], + }, + AdmissionParent { + holds: &["inventory_limit"], + children: &[&["orders_limit", "products_limit"]], + }, + AdmissionParent { + holds: &["support_limit"], + children: &[&["orders_limit"]], + }, + ], + expected_running: 2, + reductions: &[], + }, + ]; + for test_case in test_cases { + for backend in [Backend::Sqlite, Backend::Postgres] { + run(async { + let Some(fixture) = open_test_store(backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let replica = second_store(store).await; + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let actor = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let mut input = snapshot(location); + let mut jobs = serde_json::Map::new(); + let mut groups = serde_json::Map::new(); + let mut child_count = 0; + for (index, parent) in test_case.parents.iter().enumerate() { + let mut members = serde_json::Map::new(); + let mut selection = BTreeSet::new(); + for (stage, needs) in parent.children.iter().enumerate() { + let name = format!("orders_job_{index}_{stage}"); + let node = + ["orders", "customers", "inventory", "products"][child_count % 4]; + child_count += 1; + selection.insert(node); + jobs.insert(name.clone(), json!({"description": "Build a synthetic stage", "selection": [node], "concurrency_groups": needs})); + let after: Vec = (0..stage) + .map(|earlier| format!("orders_job_{index}_{earlier}")) + .collect(); + members.insert(name, json!(after)); + } + groups.insert(format!("orders_group_{index}"), json!({"description": "Complete a synthetic group", "selection": selection, "concurrency_groups": parent.holds, "jobs": members})); + } + input.documents.insert("jobs".into(), json!(jobs)); + input.documents.insert("groups".into(), json!(groups)); + let published = publish_snapshot(store, &input) + .await + .expect("publish groups"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits: BTreeMap = test_case + .limits + .iter() + .map(|(key, count)| { + ( + (*key).into(), + NonZeroU32::new(*count).expect("positive capacity"), + ) + }) + .collect(); + let scope = (location, "development"); + for index in 0..test_case.parents.len() { + request_job_run( + store, + &actor, + &JobRunRequest { + // Higher-priority arrivals exercise safety, not only an older waiter's priority barrier. + run_id: Uuid::from_u128(1000 - index as u128), + location_id: location, + target: "development".into(), + job_key: format!("orders_group_{index}"), + reason: "Exercise capacity-aware admission".into(), + priority: index as i64 + 1, + }, + ) + .await + .expect("request group"); + let (first, second) = tokio::join!( + advance_groups(store, &lease, (&limits, scope)), + advance_groups(&replica, &lease, (&limits, scope)) + ); + first.expect("first scheduler admits higher-priority arrival"); + second.expect("replica cannot admit an unsafe arrival"); + } + for _ in 0..test_case.parents.len() { + let (first, second) = tokio::join!( + advance_groups(store, &lease, (&limits, scope)), + advance_groups(&replica, &lease, (&limits, scope)) + ); + first.expect("first scheduler advance"); + second.expect("second scheduler advance"); + } + let initial = list_runs(store, scope, 100).await.expect("initial runs"); + assert_eq!( + initial + .iter() + .filter(|run| run.status == RunStatus::Running) + .count(), + test_case.expected_running, + "{} {backend:?}", + test_case.description + ); + let before = slot_counts(store).await; + for reduction in test_case.reductions { + let reduced = reduction + .iter() + .map(|(key, count)| { + ( + (*key).into(), + NonZeroU32::new(*count).expect("positive reduction"), + ) + }) + .collect(); + assert!( + claim_next_run(&replica, &lease, &reduced, Some(scope)) + .await + .expect("unsafe parent state must not fail physical dispatch") + .is_none() + ); + assert_eq!( + slot_counts(store).await, + before, + "blocked work cannot mutate claims" + ); + let unrelated = + request((location, published.snapshot_id), 777, 1, &["products"]); + queue(store, &unrelated).await; + let claim = claim_next_run(&replica, &lease, &reduced, Some(scope)) + .await + .expect("unrelated recovery progress") + .expect("unrelated work still dispatches"); + assert_eq!(claim.request.run_id, unrelated.run_id); + succeed(store, &claim).await; + } + for iteration in 0..child_count { + let owner = [store, &replica][iteration % 2]; + for _ in 0..test_case.parents.len() { + advance_groups(owner, &lease, (&limits, scope)) + .await + .expect("resume group from durable state"); + } + let claim = claim_next_run(owner, &lease, &limits, Some(scope)) + .await + .expect("claim") + .expect(test_case.description); + // Re-evaluate while a physical child still holds its temporary claims. + advance_groups(&replica, &lease, (&limits, scope)) + .await + .expect("advance during running claim"); + let occupied = slot_counts(store).await; + assert!( + occupied + .iter() + .all(|(key, count)| *count <= i64::from(limits[key].get())), + "{} {occupied:?}", + test_case.description + ); + succeed(owner, &claim).await; + } + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("aggregate final children"); + let finished = list_runs(&replica, scope, 100) + .await + .expect("finished groups"); + assert_eq!( + finished.len(), + test_case.parents.len() + child_count + test_case.reductions.len() + ); + assert!( + finished.iter().all(|run| run.status == RunStatus::Success), + "{} {finished:?}", + test_case.description + ); + assert!( + slot_counts(store).await.is_empty(), + "terminal parents release their own slots" + ); + close_store(replica).await; + close_test_store(fixture).await; + }); + } + } +} + +#[test] +fn given_an_older_blocked_allow_group_when_an_independent_generation_is_queued_then_overlap_allow_still_runs() + { + let test_cases = [ + RunLifecycleTestCase { + description: "SQLite preserves overlap allow across generations", + backend: Backend::Sqlite, + expected_status: RunStatus::Running, + }, + RunLifecycleTestCase { + description: "PostgreSQL preserves overlap allow across generations", + backend: Backend::Postgres, + expected_status: RunStatus::Running, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let actor = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let mut input = snapshot(location); + input.documents.insert( + "jobs".into(), + json!({"stage": {"description": "Build a pinned stage", "selection": ["orders"]}}), + ); + input.documents.insert("groups".into(), json!({"daily": {"description": "Allow independent generations", "selection": ["orders"], "overlap": "allow", "concurrency_groups": ["orders_limit"], "jobs": {"stage": []}}})); + let first = publish_snapshot(store, &input) + .await + .expect("first snapshot"); + let older = JobRunRequest { + run_id: Uuid::from_u128(2000), + location_id: location, + target: "development".into(), + job_key: "daily".into(), + reason: "Older group".into(), + priority: 1, + }; + request_job_run(store, &actor, &older) + .await + .expect("older request"); + input.generation = 2; + input.documents.get_mut("jobs").expect("jobs")["stage"]["selection"] = + json!(["inventory"]); + input.documents.get_mut("groups").expect("groups")["daily"]["selection"] = + json!(["inventory"]); + input.documents.get_mut("groups").expect("groups")["daily"]["concurrency_groups"] = + json!(["inventory_limit"]); + publish_snapshot(store, &input) + .await + .expect("second snapshot"); + let newer = JobRunRequest { + run_id: Uuid::from_u128(1000), + reason: "Independent replacement generation".into(), + ..older.clone() + }; + request_job_run(store, &actor, &newer) + .await + .expect("newer request"); + let mut physical = request((location, first.snapshot_id), 3000, 10, &["products"]); + physical.concurrency_groups.insert("orders_limit".into()); + queue(store, &physical).await; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::from([ + ("orders_limit".into(), NonZeroU32::new(1).expect("one")), + ("inventory_limit".into(), NonZeroU32::new(1).expect("one")), + ]); + let scope = (location, "development"); + let blocker = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("physical claim") + .expect("higher priority blocker"); + assert_eq!(blocker.request.run_id, physical.run_id); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("independent group admission"); + let runs = list_runs(store, scope, 100).await.expect("runs"); + assert_eq!( + runs.iter() + .find(|run| run.run_id == older.run_id) + .expect("older") + .status, + RunStatus::Queued + ); + assert_eq!( + runs.iter() + .find(|run| run.run_id == newer.run_id) + .expect("newer") + .status, + test_case.expected_status, + "{}", + test_case.description + ); + let child = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("new child") + .expect("allow generation can progress"); + assert_eq!(child.request.parent_run_id, Some(newer.run_id)); + assert_eq!(child.request.selection, ["inventory".to_owned()].into()); + succeed(store, &child).await; + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("finish replacement parent"); + succeed(store, &blocker).await; + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("old parent retries after blocker drains"); + let child = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("old child") + .expect("older group makes progress"); + assert_eq!(child.request.parent_run_id, Some(older.run_id)); + succeed(store, &child).await; + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("complete all groups"); + assert!( + list_runs(store, scope, 100) + .await + .expect("finished") + .iter() + .all(|run| run.status == RunStatus::Success) + ); + assert!(slot_counts(store).await.is_empty()); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_a_large_healthy_project_with_blocked_groups_when_claiming_then_unrelated_work_has_a_bounded_write_lock() + { + let test_cases = [ + GroupQueueTestCase { + description: "SQLite scans a full blocked group page without expanding unrelated bodies", + backend: Backend::Sqlite, + group_count: 105, + node_count: 3000, + expected_max_claim_millis: 5000, + }, + GroupQueueTestCase { + description: "PostgreSQL scans a full blocked group page without expanding unrelated bodies", + backend: Backend::Postgres, + group_count: 105, + node_count: 3000, + expected_max_claim_millis: 5000, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let actor = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let mut input = snapshot(location); + let template = input.nodes["orders"].clone(); + for index in 0..test_case.node_count { + let mut node = template.clone(); + node.definition["description"] = + json!(format!("Orders {index} {}", "details ".repeat(128))); + input.nodes.insert(format!("orders_{index}"), node); + } + let jobs = (0..test_case.group_count).map(|index| (format!("job_{index}"), json!({"description": "Build orders", "selection": ["orders"], "concurrency_groups": ["warehouse"]}))).collect::>(); + let groups = (0..test_case.group_count).map(|index| (format!("group_{index}"), json!({"description": "Independent order group", "selection": ["orders"], "jobs": {format!("job_{index}"): []}}))).collect::>(); + input.documents.insert("jobs".into(), json!(jobs)); + input.documents.insert("groups".into(), json!(groups)); + input.documents.insert( + "columns".into(), + json!({"orders": "unused display metadata ".repeat(100000)}), + ); + let published = publish_snapshot(store, &input) + .await + .expect("publish large project"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::from([("warehouse".into(), NonZeroU32::new(1).expect("one"))]); + let scope = (location, "development"); + let mut physical = request((location, published.snapshot_id), 1, 10, &["products"]); + physical.concurrency_groups.insert("warehouse".into()); + queue(store, &physical).await; + let blocker = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim") + .expect("blocker"); + for index in 0..test_case.group_count { + request_job_run( + store, + &actor, + &JobRunRequest { + run_id: Uuid::from_u128(1000 + index as u128), + location_id: location, + target: "development".into(), + job_key: format!("group_{index}"), + reason: "Exercise blocked group queue".into(), + priority: 1, + }, + ) + .await + .expect("request group"); + } + let started = std::time::Instant::now(); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("admit independent parents"); + eprintln!( + "{:?}: {} parent admissions in {:?}", + test_case.backend, + test_case.group_count, + started.elapsed() + ); + let unrelated = request((location, published.snapshot_id), 2, -1, &["customers"]); + queue(store, &unrelated).await; + let started = std::time::Instant::now(); + let claim = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("scan blocked children") + .expect("unrelated work"); + let elapsed = started.elapsed(); + eprintln!( + "{:?}: {} groups / {} nodes, claim transaction {:?}", + test_case.backend, test_case.group_count, test_case.node_count, elapsed + ); + assert_eq!( + claim.request.run_id, unrelated.run_id, + "{}", + test_case.description + ); + assert!( + elapsed.as_millis() < test_case.expected_max_claim_millis, + "{}: {elapsed:?}", + test_case.description + ); + succeed(store, &claim).await; + succeed(store, &blocker).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_groups.rs b/crates/sqlbuild-orchestration/tests/runs/test_groups.rs new file mode 100644 index 0000000000..6a502d207d --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_groups.rs @@ -0,0 +1,423 @@ +//! Pinned groups coordinate real transactions, inherited slots, cancellation and replay. + +use std::collections::BTreeMap; +use std::num::NonZeroU32; + +use serde_json::json; +use sqlbuild_orchestration::credentials::models::AuditContext; +use sqlbuild_orchestration::runs::main::{ + advance_groups::advance_groups, change_run::change_run, claim_next_run::claim_next_run, + list_runs::list_runs, request_job_run::request_job_run, run_execution::run_execution, +}; +use sqlbuild_orchestration::runs::models::{ + JobRunRequest, NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, +}; +use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::store::main::{ + migrate_store::migrate_store, read_recall_events::read_recall_events, +}; +use uuid::Uuid; + +use crate::claim_helpers::{slot_counts, succeed}; +use crate::fault_helpers::{install_fault, remove_fault}; +use crate::helpers::snapshot; +use crate::location_helpers::create_location; +use crate::store_helpers::{close_test_store, open_test_store, run}; +use crate::store_test_types::Backend; +use crate::test_types::{RunLifecycleTestCase, TransactionFault}; + +#[test] +fn given_a_pinned_group_when_redeployed_and_cancelled_then_children_keep_scope_and_parent_waits_for_terminal_evidence() + { + let test_cases = [ + RunLifecycleTestCase { + description: "SQLite group cancellation and replay", + backend: Backend::Sqlite, + expected_status: RunStatus::Cancelled, + }, + RunLifecycleTestCase { + description: "PostgreSQL group cancellation and replay", + backend: Backend::Postgres, + expected_status: RunStatus::Cancelled, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let context = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let mut input = snapshot(location); + input.nodes.get_mut("orders").expect("orders").definition["deps"] = + json!(["customers"]); + input.documents.insert("jobs".into(), json!({ + "staging": {"description": "Stage customers", "selection": ["customers"], "concurrency_groups": ["warehouse"]}, + "marts": {"description": "Build orders", "selection": ["orders"], "concurrency_groups": ["warehouse"]} + })); + input.documents.insert("groups".into(), json!({"daily_orders": { + "description": "Stage then build orders", "selection": ["customers", "orders"], + "concurrency_groups": ["warehouse"], "jobs": {"staging": [], "marts": ["staging"]} + }})); + let published = publish_snapshot(store, &input) + .await + .expect("publish group"); + let request = JobRunRequest { + run_id: Uuid::now_v7(), + location_id: location, + target: "development".into(), + job_key: "daily_orders".into(), + reason: "Build the group".into(), + priority: 1, + }; + assert!( + request_job_run(store, &context, &request) + .await + .expect("request group") + ); + assert!( + !request_job_run(store, &context, &request) + .await + .expect("retry group") + ); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = + BTreeMap::from([("warehouse".into(), NonZeroU32::new(1).expect("one slot"))]); + let scope = (location, "development"); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("start virtual parent"); + let first = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim staging") + .expect("staging ready"); + assert_eq!(first.request.job_key.as_deref(), Some("staging")); + assert_eq!(first.request.parent_run_id, Some(request.run_id)); + assert_eq!(first.request.write_set, ["customers".to_owned()].into()); + assert!( + claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("dependency waits") + .is_none() + ); + let authority = RunAuthority::Worker { + run_id: first.request.run_id, + claim_id: first.claim_id, + }; + for change in [ + RunChange::Started, + RunChange::NodeFinished { + node_key: "customers".into(), + outcome: NodeOutcome::Success, + rows_affected: Some(11), + error_code: None, + skipped_because_of: None, + }, + RunChange::Finished { + outcome: RunOutcome::Success, + reason: "staging completed".into(), + exit_code: Some(0), + }, + ] { + change_run( + store, + authority, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: first.request.run_id, + claim_id: Some(first.claim_id), + change, + }, + ) + .await + .expect("staging fact"); + } + input.generation = 2; + input.documents.get_mut("jobs").expect("jobs")["marts"]["selection"] = + json!(["inventory"]); + input.documents.get_mut("groups").expect("groups")["daily_orders"]["selection"] = + json!(["customers", "inventory"]); + publish_snapshot(store, &input) + .await + .expect("redeploy different membership"); + let second = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim marts") + .expect("marts ready"); + assert_eq!(second.request.snapshot_id, published.snapshot_id); + assert_eq!(second.request.selection, ["orders".to_owned()].into()); + assert_eq!(second.request.write_set, second.request.selection); + assert!( + run_execution(store, (second.request.run_id, second.claim_id)) + .await + .expect("execution") + .expect("pinned") + .skipped_nodes + .is_empty() + ); + let authority = RunAuthority::Worker { + run_id: second.request.run_id, + claim_id: second.claim_id, + }; + change_run( + store, + authority, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: second.request.run_id, + claim_id: Some(second.claim_id), + change: RunChange::Started, + }, + ) + .await + .expect("start marts"); + change_run( + store, + RunAuthority::Operator(&context), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: request.run_id, + claim_id: None, + change: RunChange::CancelRequested { + actor: context.actor.clone(), + reason: "Cancel the group".into(), + }, + }, + ) + .await + .expect("cancel parent"); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("propagate cancellation"); + let runs = list_runs(store, scope, 100).await.expect("runs"); + assert_eq!( + runs.iter() + .find(|run| run.run_id == request.run_id) + .expect("parent") + .status, + RunStatus::Running, + "{}", + test_case.description + ); + assert_eq!( + runs.iter() + .find(|run| run.run_id == second.request.run_id) + .expect("active child") + .status, + RunStatus::Running + ); + let standalone = JobRunRequest { + run_id: Uuid::now_v7(), + job_key: "staging".into(), + ..request.clone() + }; + request_job_run(store, &context, &standalone) + .await + .expect("standalone request"); + assert!( + claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("parent retains its slot") + .is_none() + ); + change_run( + store, + authority, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: second.request.run_id, + claim_id: Some(second.claim_id), + change: RunChange::Finished { + outcome: RunOutcome::Cancelled, + reason: "worker confirmed its executor stopped".into(), + exit_code: Some(143), + }, + }, + ) + .await + .expect("physical child finished"); + let fault = TransactionFault { + description: "group terminal publication rolls back", + table: "recall_events", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected group publication failure')", + postgres_body: "RAISE EXCEPTION 'injected group publication failure'", + }; + install_fault(store, &fault).await; + assert!( + advance_groups(store, &lease, (&limits, scope)) + .await + .is_err() + ); + remove_fault(store, &fault).await; + for _ in 0..2 { + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("retry group completion"); + } + let runs = list_runs(store, scope, 100).await.expect("completed runs"); + assert_eq!( + runs.iter() + .find(|run| run.run_id == request.run_id) + .expect("parent") + .status, + test_case.expected_status + ); + let standalone_claim = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("slot released") + .expect("standalone staging ready"); + assert_eq!(standalone_claim.request.run_id, standalone.run_id); + assert_eq!(standalone_claim.request.parent_run_id, None); + let events = read_recall_events(store, 0, 1000).await.expect("facts"); + assert_eq!( + events + .iter() + .filter(|event| event["event_type"] == "run_terminated" + && event["invocation_id"] == format!("scheduler/{}", request.run_id)) + .count(), + 1 + ); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_opposing_parent_and_child_limits_when_advancing_groups_then_both_groups_finish_without_exceeding_limits() + { + let test_cases = [ + RunLifecycleTestCase { + description: "SQLite opposing group reservations", + backend: Backend::Sqlite, + expected_status: RunStatus::Running, + }, + RunLifecycleTestCase { + description: "PostgreSQL opposing group reservations", + backend: Backend::Postgres, + expected_status: RunStatus::Running, + }, + ]; + let mut completed = Vec::new(); + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let context = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let mut input = snapshot(location); + input.documents.insert("jobs".into(), json!({ + "orders_job": {"description": "Build orders", "selection": ["orders"], "concurrency_groups": ["inventory_limit"]}, + "inventory_job": {"description": "Build inventory", "selection": ["customers"], "concurrency_groups": ["orders_limit"]} + })); + input.documents.insert("groups".into(), json!({ + "orders_group": {"description": "Orders group", "selection": ["orders"], "concurrency_groups": ["orders_limit"], "jobs": {"orders_job": []}}, + "inventory_group": {"description": "Inventory group", "selection": ["customers"], "concurrency_groups": ["inventory_limit"], "jobs": {"inventory_job": []}} + })); + publish_snapshot(store, &input) + .await + .expect("publish groups"); + for job_key in ["orders_group", "inventory_group"] { + request_job_run( + store, + &context, + &JobRunRequest { + run_id: Uuid::now_v7(), + location_id: location, + target: "development".into(), + job_key: job_key.into(), + reason: "Build independent groups".into(), + priority: 1, + }, + ) + .await + .expect("request group"); + } + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::from([ + ("orders_limit".into(), NonZeroU32::new(1).expect("one slot")), + ( + "inventory_limit".into(), + NonZeroU32::new(1).expect("one slot"), + ), + ]); + let scope = (location, "development"); + for _ in 0..2 { + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("advance groups"); + } + let runs = list_runs(store, scope, 100).await.expect("runs"); + assert_eq!(runs.len(), 4, "{}", test_case.description); + assert_eq!( + runs.iter() + .filter(|run| run.status == test_case.expected_status) + .count(), + 1, + "{}", + test_case.description + ); + for _ in 0..2 { + let child = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim child") + .expect("a group must make progress"); + let slots = slot_counts(store).await; + assert_eq!( + slots.len(), + 2, + "child-only slot is acquired only for its worker" + ); + assert!(slots.values().all(|occupied| *occupied == 1), "{slots:?}"); + assert!( + claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("held child") + .is_none() + ); + succeed(store, &child).await; + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("complete parent and retry waiting group"); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("retry independent of run-id ordering"); + } + let runs = list_runs(store, scope, 100).await.expect("completed runs"); + assert!( + runs.iter().all(|run| run.status == RunStatus::Success), + "{runs:?}" + ); + completed.push(test_case.description); + close_test_store(fixture).await; + }); + } + assert_eq!(completed.len(), 2, "both backends must run"); +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs new file mode 100644 index 0000000000..a17b54cee9 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs @@ -0,0 +1,301 @@ +//! Terminal retries retain their snapshot and independent limits without repeating successful work. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::json; +use sqlbuild_orchestration::credentials::models::AuditContext; +use sqlbuild_orchestration::runs::main::{ + advance_groups::advance_groups, change_run::change_run, claim_next_run::claim_next_run, + list_runs::list_runs, request_job_run::request_job_run, request_rerun::request_rerun, +}; +use sqlbuild_orchestration::runs::models::{ + JobRunRequest, NodeOutcome, RerunRequest, RunAuthority, RunChange, RunChangeRequest, RunOrigin, + RunOutcome, RunStatus, +}; +use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use uuid::Uuid; + +use crate::claim_helpers::{slot_counts, succeed}; +use crate::fault_helpers::{install_fault, remove_fault}; +use crate::helpers::snapshot; +use crate::location_helpers::create_location; +use crate::store_helpers::{close_test_store, open_test_store, run}; +use crate::store_test_types::Backend; +use crate::test_types::{RunLifecycleTestCase, TransactionFault}; + +#[test] +fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_nodes_retry_atomically() + { + let test_cases = [ + RunLifecycleTestCase { + description: "SQLite pinned group rerun", + backend: Backend::Sqlite, + expected_status: RunStatus::Success, + }, + RunLifecycleTestCase { + description: "PostgreSQL pinned group rerun", + backend: Backend::Postgres, + expected_status: RunStatus::Success, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let context = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let mut input = snapshot(location); + input + .nodes + .get_mut("inventory") + .expect("inventory") + .definition["deps"] = json!(["orders"]); + input.documents.insert("jobs".into(), json!({ + "staging": {"description": "Stage orders", "selection": ["orders", "customers"], "concurrency_groups": ["warehouse"], "settings": {"vars": {"batch": 23}}}, + "marts": {"description": "Build inventory", "selection": ["inventory"], "concurrency_groups": ["inventory_limit"]}, + "completed": {"description": "Build products", "selection": ["products"]} + })); + input.documents.insert("groups".into(), json!({"daily_orders": { + "description": "Build daily orders", "selection": ["customers", "orders", "inventory", "products"], + "concurrency_groups": ["warehouse"], "jobs": {"staging": [], "marts": ["staging"], "completed": []} + }})); + let published = publish_snapshot(store, &input).await.expect("publish"); + let source_id = Uuid::now_v7(); + request_job_run( + store, + &context, + &JobRunRequest { + run_id: source_id, + location_id: location, + target: "development".into(), + job_key: "daily_orders".into(), + reason: "Build orders".into(), + priority: 1, + }, + ) + .await + .expect("request group"); + let retry = RerunRequest { + run_id: Uuid::now_v7(), + source_run_id: source_id, + from_failure: true, + reason: "Retry failed orders".into(), + priority: 7, + }; + assert!( + request_rerun(store, &context, &retry).await.is_err(), + "a live source cannot be rerun" + ); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::from([ + ("warehouse".into(), std::num::NonZeroU32::new(1).unwrap()), + ( + "inventory_limit".into(), + std::num::NonZeroU32::new(1).unwrap(), + ), + ]); + let scope = (location, "development"); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("admit source"); + for _ in 0..3 { + let claim = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim source") + .expect("ready child"); + let mut changes = vec![RunChange::Started]; + for node in &claim.request.selection { + let outcome = match node.as_str() { + "products" => NodeOutcome::Success, + "orders" => NodeOutcome::Failed, + "customers" => NodeOutcome::Cancelled, + _ => NodeOutcome::Skipped, + }; + changes.push(RunChange::NodeFinished { + node_key: node.clone(), + outcome, + rows_affected: None, + error_code: None, + skipped_because_of: (outcome == NodeOutcome::Skipped) + .then(|| "orders".into()), + }); + } + changes.push(RunChange::Finished { + outcome: if claim.request.selection.contains("products") { + RunOutcome::Success + } else { + RunOutcome::Failed + }, + reason: "synthetic source outcome".into(), + exit_code: Some(1), + }); + for change in changes { + change_run( + store, + RunAuthority::Worker { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change, + }, + ) + .await + .expect("source worker fact"); + } + } + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("finish source"); + assert_eq!( + list_runs(store, scope, 20) + .await + .unwrap() + .iter() + .find(|run| run.run_id == source_id) + .unwrap() + .status, + RunStatus::Partial + ); + input.generation = 2; + input.image_id = Some("sha256:synthetic-orders-image-2".into()); + input.documents.get_mut("jobs").unwrap()["staging"]["settings"] = + json!({"vars": {"batch": 91}}); + publish_snapshot(store, &input).await.expect("redeploy"); + let fault = TransactionFault { + description: "rerun audit failure", + table: "sqb_security_audit", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected audit failure')", + postgres_body: "RAISE EXCEPTION 'injected audit failure'", + }; + install_fault(store, &fault).await; + assert!(request_rerun(store, &context, &retry).await.is_err()); + remove_fault(store, &fault).await; + assert_eq!( + list_runs(store, scope, 20).await.unwrap().len(), + 4, + "audit failure rolls back parent and all children" + ); + let (first, again) = tokio::join!( + request_rerun(store, &context, &retry), + request_rerun(store, &context, &retry) + ); + assert_ne!(first.expect("rerun"), again.expect("replica retry")); + let changed = RerunRequest { + from_failure: false, + ..retry.clone() + }; + assert!(request_rerun(store, &context, &changed).await.is_err()); + let other = AuditContext { + actor: "other-operator".into(), + ..context.clone() + }; + assert!(request_rerun(store, &other, &retry).await.is_err()); + for request in [ + retry.clone(), + RerunRequest { + run_id: Uuid::now_v7(), + from_failure: false, + ..retry.clone() + }, + ] { + if !request.from_failure { + assert!(request_rerun(store, &context, &request).await.unwrap()); + } + let expected: BTreeSet = if request.from_failure { + ["orders", "inventory"].map(str::to_owned).into() + } else { + ["orders", "inventory", "customers", "products"] + .map(str::to_owned) + .into() + }; + let runs = list_runs(store, scope, 20).await.unwrap(); + assert_eq!( + runs.iter() + .find(|run| run.run_id == request.run_id) + .unwrap() + .selection_json, + expected + ); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("admit rerun"); + let mut selected = BTreeSet::new(); + for _ in 0..if request.from_failure { 2 } else { 3 } { + let claim = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim rerun") + .expect("ready retry child"); + assert_eq!(claim.request.snapshot_id, published.snapshot_id); + assert_eq!( + claim.request.image_id, + Some("sha256:synthetic-orders-image".into()) + ); + assert_eq!(claim.request.origin, RunOrigin::Rerun); + assert_eq!(claim.request.priority, 7); + if claim.request.job_key.as_deref() == Some("staging") { + assert_eq!( + claim.request.settings, + BTreeMap::from([("vars".into(), json!({"batch": 23}))]) + ); + } + selected.extend(claim.request.selection.iter().cloned()); + succeed(store, &claim).await; + } + assert_eq!(selected, expected); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("finish rerun"); + assert_eq!( + list_runs(store, scope, 20) + .await + .unwrap() + .iter() + .find(|run| run.run_id == request.run_id) + .unwrap() + .status, + test_case.expected_status, + "{}", + test_case.description + ); + assert!(slot_counts(store).await.is_empty()); + assert!( + !request_rerun(store, &context, &request) + .await + .expect("completed transport retry") + ); + let successful = RerunRequest { + run_id: Uuid::now_v7(), + source_run_id: request.run_id, + from_failure: true, + ..request.clone() + }; + assert!( + request_rerun(store, &context, &successful).await.is_err(), + "successful work has nothing to retry from failure" + ); + } + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_types.rs b/crates/sqlbuild-orchestration/tests/runs/test_types.rs index 0a21b7f2ef..9ee50b5f8c 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_types.rs @@ -62,3 +62,31 @@ pub(crate) struct MonitorBacklogTestCase { pub(crate) run_count: usize, pub(crate) expected_batches: [usize; 3], } + +pub(crate) struct AdmissionParent { + pub(crate) holds: &'static [&'static str], + pub(crate) children: &'static [&'static [&'static str]], +} + +pub(crate) struct GroupAdmissionTestCase { + pub(crate) description: &'static str, + pub(crate) limits: &'static [(&'static str, u32)], + pub(crate) parents: &'static [AdmissionParent], + pub(crate) expected_running: usize, + pub(crate) reductions: &'static [&'static [(&'static str, u32)]], +} + +pub(crate) struct GroupQueueTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) group_count: usize, + pub(crate) node_count: usize, + pub(crate) expected_max_claim_millis: u128, +} + +pub(crate) struct MixedAdmissionTestCase { + pub(crate) description: &'static str, + pub(crate) physical_priority: i64, + pub(crate) parent_priority: i64, + pub(crate) expected_selections: [&'static str; 2], +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index 17c4dea5e1..7c0ebc870f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -76,7 +76,7 @@ const RAW_QUERY_ATTRIBUTES: [&str; 4] = [ ]; /// One invocation being dispatched: its namespace and the values derived from it once. -struct Invocation<'py> { +pub(crate) struct Invocation<'py> { args: Bound<'py, PyAny>, project_dir: Bound<'py, PyAny>, select: Bound<'py, PyTuple>, @@ -84,6 +84,27 @@ struct Invocation<'py> { } impl<'py> Invocation<'py> { + pub(crate) fn new(args: &Bound<'py, PyAny>) -> PyResult { + let raw_project_dir = native_field(args, "project_dir")?; + let project_dir = if raw_project_dir.is_none() { + raw_project_dir + } else { + path_class(args.py())? + .call1((raw_project_dir,))? + .call_method0("resolve")? + }; + let (select, selector_files) = selector_inputs(args)?; + if native_field(args, "select_file")?.is_truthy()? { + withhold_launch_record(args.py())?; + } + Ok(Self { + args: args.clone(), + project_dir, + select, + selector_files, + }) + } + fn py(&self) -> Python<'py> { self.args.py() } @@ -97,7 +118,7 @@ impl<'py> Invocation<'py> { } /// The value of one handler argument or request field. - fn value(&self, source: Source) -> PyResult> { + pub(crate) fn value(&self, source: Source) -> PyResult> { let py: Python<'py> = self.py(); match source { Source::Attr(name) => self.attr(name), @@ -425,24 +446,7 @@ fn routed_command<'py>( recall: &InvocationRecall, ) -> PyResult> { let py: Python<'py> = args.py(); - let raw_project_dir: Bound<'py, PyAny> = native_field(args, "project_dir")?; - let project_dir: Bound<'py, PyAny> = if raw_project_dir.is_none() { - raw_project_dir - } else { - path_class(py)? - .call1((raw_project_dir,))? - .call_method0("resolve")? - }; - let (select, selector_files) = selector_inputs(args)?; - if native_field(args, "select_file")?.is_truthy()? { - withhold_launch_record(py)?; - } - let invocation = Invocation { - args: args.clone(), - project_dir, - select, - selector_files, - }; + let invocation = Invocation::new(args)?; match command_dispatch(&command_facts(args)?) { Dispatch::Nothing => 0.into_bound_py_any(py), Dispatch::Fail(failure) => Err(user_error(py, failure)), @@ -473,6 +477,12 @@ pub(crate) fn dispatch_command<'py>( handlers: Option>, recall: &InvocationRecall, ) -> PyResult> { + within(&warehouse_scope(args)?, |_| { + routed_command(args, handlers.as_ref(), recall) + }) +} + +pub(crate) fn warehouse_scope<'py>(args: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = args.py(); let warehouse: Bound<'py, PyAny> = native_field(args, "warehouse")?; if !warehouse.is_none() { @@ -491,7 +501,7 @@ pub(crate) fn dispatch_command<'py>( ) .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))?; } - let scope: Bound<'py, PyAny> = Bound::new( + Ok(Bound::new( py, command_warehouse_scope( &native_field(args, "command")?, @@ -499,8 +509,7 @@ pub(crate) fn dispatch_command<'py>( Some(&native_field(args, "vars")?), )?, )? - .into_any(); - within(&scope, |_| routed_command(args, handlers.as_ref(), recall)) + .into_any()) } /// The parsed values that select a route, read from the namespace. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs index a2f44bf10a..0d95fcf560 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs @@ -15,13 +15,13 @@ use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_re use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode, SnapshotRecord}; use crate::bindings::_helpers::compiled_project::snapshot_declarations::{ - AuthoredJob, compile_declarations, discover_declarations, + AuthoredDeclaration, compile_declarations, discover_declarations, }; use crate::bindings::_helpers::compiled_project::snapshot_models::model_nodes; use crate::bindings::_helpers::compiled_project::snapshot_resources::resource_nodes; use crate::bindings::_helpers::compiled_project::snapshot_worker::NativeOrchestrationWorker; use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; -use crate::bindings::_helpers::native_host::orchestration_declarations::validate_jobs; +use crate::bindings::_helpers::native_host::orchestration_declarations::collect_declarations; use crate::bindings::_helpers::native_host::orchestration_environment::environment_hash; use crate::bindings::_helpers::records::fields::field; @@ -126,11 +126,11 @@ pub(crate) fn build_orchestration_snapshot( None => discover_declarations(py, &project_dir)?, }; if !declarations.is_empty() { - validate_jobs(py, &declarations)?; + let declarations = collect_declarations(py, declarations)?; let authored = declarations .iter() - .map(|declaration| declaration.extract::()) - .collect::>>()?; + .map(|declaration| declaration.extract::()) + .collect::>>()?; let graph = project.lineage_graph(py)?; input = compile_declarations(input, graph.bind(py).get().graph(), authored)?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index 54ad885b83..d5922477fc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -14,9 +14,10 @@ use sqlbuild_core::constants::PYTHON_INIT_MODULE_STEM; use sqlbuild_discovery::constants::{ORCHESTRATION_DIRECTORY, PYTHON_FILE_SUFFIX}; use sqlbuild_discovery::tree::main::rglob::rglob; use sqlbuild_discovery::tree::models::ProjectTree; +use sqlbuild_orchestration::declarations::main::group_dependencies::group_dependencies; use sqlbuild_orchestration::declarations::main::snapshot_definitions::snapshot_definitions; use sqlbuild_orchestration::declarations::models::{ - JobDefinition, OnFailure, Schedule, TriggerDefinition, + GroupDefinition, JobDefinition, OnFailure, Schedule, TriggerDefinition, }; use sqlbuild_orchestration::scheduler::models::Overlap; use sqlbuild_orchestration::snapshots::models::SnapshotInput; @@ -83,23 +84,79 @@ pub(crate) struct AuthoredJob { targets: Vec, } +#[derive(FromPyObject, PartialEq, Eq, PartialOrd, Ord)] +struct AuthoredJobReference { + name: String, +} + +#[derive(FromPyObject)] +pub(crate) struct AuthoredGroup { + name: String, + description: String, + jobs: Vec, + after: BTreeMap>, + overlap: String, + concurrency_groups: Vec, + schedule: Option, + targets: Vec, +} + +#[derive(FromPyObject)] +pub(crate) enum AuthoredDeclaration { + Job(AuthoredJob), + Group(AuthoredGroup), +} + +impl AuthoredSchedule { + fn compile(self, name: &str, description: &str) -> PyResult { + let catch_up = if self.max_catch_up.is_null() { + Value::String(self.catch_up) + } else { + serde_json::json!({self.catch_up: {"max_runs": self.max_catch_up}}) + }; + Ok(TriggerDefinition { + description: description.into(), + job_key: name.into(), + schedule: Schedule::Cron { expression: self.cron, timezone: self.timezone }, + starts_at: serde_json::from_value(self.starts_at) + .map_err(|error| PyValueError::new_err(format!("starts_at: {error}")))?, + default_status: serde_json::from_value(Value::String(self.default_status)) + .map_err(|error| PyValueError::new_err(format!("default_status: {error}")))?, + catch_up: serde_json::from_value(catch_up).map_err(|error| PyValueError::new_err(format!( + "catch_up: {error}; use 'latest' without a bound or 'all' with a positive max_catch_up" + )))?, + }) + } +} + pub(crate) fn compile_declarations( mut input: SnapshotInput, graph: &ProjectGraph, - declarations: Vec, + declarations: Vec, ) -> PyResult { if declarations.is_empty() { return Ok(input); } - if input.documents.contains_key("jobs") || input.documents.contains_key("triggers") { + if ["jobs", "groups", "triggers"] + .iter() + .any(|key| input.documents.contains_key(*key)) + { return Err(PyValueError::new_err( "authored declarations cannot overwrite snapshot job or trigger documents", )); } let mut jobs: BTreeMap = BTreeMap::new(); + let mut authored_groups: Vec = Vec::new(); + let mut authored_jobs: Vec = Vec::new(); + for declaration in declarations { + match declaration { + AuthoredDeclaration::Job(job) => authored_jobs.push(job), + AuthoredDeclaration::Group(group) => authored_groups.push(group), + } + } let mut triggers: BTreeMap = BTreeMap::new(); let mut names: BTreeSet = BTreeSet::new(); - for job in declarations { + for job in authored_jobs { if !names.insert(job.name.clone()) { return Err(PyValueError::new_err(format!("duplicate job {}", job.name))); } @@ -134,31 +191,9 @@ pub(crate) fn compile_declarations( .map(|(kind, name)| format!("{kind}:{name}")) .collect(); if let Some(schedule) = job.schedule { - let catch_up = if schedule.max_catch_up.is_null() { - Value::String(schedule.catch_up) - } else { - serde_json::json!({schedule.catch_up: {"max_runs": schedule.max_catch_up}}) - }; triggers.insert( job.name.clone(), - TriggerDefinition { - description: job.description.clone(), - job_key: job.name.clone(), - schedule: Schedule::Cron { - expression: schedule.cron, - timezone: schedule.timezone, - }, - starts_at: serde_json::from_value(schedule.starts_at) - .map_err(|error| PyValueError::new_err(format!("starts_at: {error}")))?, - default_status: serde_json::from_value(Value::String(schedule.default_status)) - .map_err(|error| { - PyValueError::new_err(format!("default_status: {error}")) - })?, - catch_up: serde_json::from_value(catch_up) - .map_err(|error| PyValueError::new_err(format!( - "catch_up: {error}; use 'latest' without a bound or 'all' with a positive max_catch_up" - )))?, - }, + schedule.compile(&job.name, &job.description)?, ); } jobs.insert( @@ -175,8 +210,62 @@ pub(crate) fn compile_declarations( }, ); } + let mut groups: BTreeMap = BTreeMap::new(); + for group in authored_groups { + if !names.insert(group.name.clone()) { + return Err(PyValueError::new_err(format!( + "duplicate job or group {}", + group.name + ))); + } + if !group.targets.is_empty() && !group.targets.contains(&input.target) { + continue; + } + let members: BTreeSet = group.jobs.iter().map(|job| job.name.clone()).collect(); + if members.len() != group.jobs.len() { + return Err(PyValueError::new_err("group members must be distinct")); + } + let mut after: BTreeMap> = BTreeMap::new(); + for (job, dependencies) in group.after { + after.insert( + job.name, + dependencies + .into_iter() + .map(|dependency| dependency.name) + .collect(), + ); + } + let dependencies = group_dependencies(&input, &jobs, (&members, &after)) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + let selection: BTreeSet = members + .iter() + .flat_map(|name| jobs[name].selection.iter().cloned()) + .collect(); + if let Some(schedule) = group.schedule { + triggers.insert( + group.name.clone(), + schedule.compile(&group.name, &group.description)?, + ); + } + groups.insert( + group.name, + GroupDefinition { + job: JobDefinition { + description: group.description, + selection, + settings: BTreeMap::new(), + overlap: serde_json::from_value(Value::String(group.overlap)) + .map_err(|error| PyValueError::new_err(error.to_string()))?, + on_failure: OnFailure::SkipDependents, + concurrency_groups: group.concurrency_groups.into_iter().collect(), + }, + jobs: dependencies, + }, + ); + } for (key, document) in [ ("jobs", serde_json::to_value(jobs)), + ("groups", serde_json::to_value(groups)), ("triggers", serde_json::to_value(triggers)), ] { input.documents.insert( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs index 57d0f43e90..a7aa2f894c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs @@ -23,11 +23,24 @@ pub(crate) struct WorkerBuild { impl WorkerBuild { pub(crate) fn new(worker: &Bound<'_, NativeOrchestrationWorker>) -> PyResult> { - Ok(Arc::new(Self { + let state = Arc::new(Self { execution: worker.get().execution(worker.py())?, worker: worker.clone().unbind(), completed: Mutex::default(), - })) + }); + for key in state.execution.skipped_nodes.keys() { + state.complete(worker.py(), key, (NodeOutcome::Skipped, None, None))?; + } + Ok(state) + } + + pub(crate) fn has_work(&self) -> bool { + self.execution + .claim + .request + .selection + .iter() + .any(|key| !self.execution.skipped_nodes.contains_key(key)) } pub(crate) fn check_cancelled(&self, py: Python<'_>) -> PyResult<()> { @@ -103,7 +116,7 @@ impl WorkerBuild { outcome, rows_affected, error_code, - skipped_because_of: None, + skipped_because_of: self.execution.skipped_nodes.get(key).cloned(), }, )?; self.completed diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs index 472717ab03..52da11300d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -5,72 +5,68 @@ use std::time::Duration; use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::types::PyString; +use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; +use sqlbuild_cli::dispatch::main::command_dispatch::command_dispatch; +use sqlbuild_cli::dispatch::models::Dispatch; use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_orchestration::runs::models::RunOutcome; -use crate::bindings::_helpers::boundary::config_values::json_value_to_python; -use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with, within}; use crate::bindings::_helpers::build_command::command::run_build; +use crate::bindings::_helpers::cli::argv::parse_cli_invocation; +use crate::bindings::_helpers::cli::dispatch::{Invocation, command_facts, warehouse_scope}; use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSnapshot; use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; use crate::bindings::_helpers::compiled_project::snapshot_worker::NativeOrchestrationWorker; use crate::bindings::_helpers::native_host::host::{entry, write_console}; use crate::bindings::_helpers::recall::session::InvocationRecall; -use crate::bindings::_helpers::records::construct::new_record; -use crate::bindings::_helpers::records::ids::RecordId; +use crate::bindings::_helpers::records::fields::{field, replaced}; fn build_request<'py>( py: Python<'py>, discovered: Bound<'py, PyAny>, arguments: &KeywordArguments<'py>, worker: &Bound<'py, NativeOrchestrationWorker>, -) -> PyResult> { +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { let execution = worker.get().execution(py)?; let frozen = execution.claim.request; - for name in frozen.settings.keys() { - if !matches!( - name.as_str(), - "full_refresh" - | "fail_fast" - | "concurrency" - | "cli_vars" - | "run_tests" - | "run_audits" - | "load_sources" - | "reload_sources" - | "include_python" - | "allow_snapshot_full_refresh" - | "allow_snapshot_schema_change" - | "allow_table_type_downgrade" - | "allow_retention_decrease" - | "allow_missing_migration_origin" - | "max_microbatches" - | "test_timeout" - ) { - return Err(PyValueError::new_err(format!( - "unsupported worker build setting {name}" - ))); - } - } - let settings = PyDict::new(py); - for (name, value) in &frozen.settings { - settings.set_item(name, json_value_to_python(py, value)?)?; - } - settings.set_item("project_dir", arguments.value("project_dir")?)?; - settings.set_item("selected_target", frozen.target)?; - let mut selectors: Vec = Vec::new(); + let mut argv: Vec> = std::iter::once("build".to_owned()) + .chain(job_settings_arguments(&frozen.settings).map_err(PyValueError::new_err)?) + .map(|argument| PyString::new(py, &argument)) + .collect(); + argv.extend([ + PyString::new(py, "--project-dir"), + arguments.value("project_dir")?.str()?, + PyString::new(py, "--target"), + PyString::new(py, &frozen.target), + PyString::new(py, "--no-color"), + PyString::new(py, "--json"), + ]); for key in &frozen.selection { + if execution.skipped_nodes.contains_key(key) { + continue; + } let (_, name) = key .split_once(':') .ok_or_else(|| PyValueError::new_err("frozen run contains an invalid node key"))?; - selectors.push(name.to_owned()); + argv.extend([PyString::new(py, "--select"), PyString::new(py, name)]); } - settings.set_item("select", selectors)?; - settings.set_item("discovered_inputs", discovered)?; - settings.set_item("no_color", true)?; - settings.set_item("json_output", true)?; - new_record(py, RecordId::BuildCommandRequest, (), Some(&settings)) + let parsed = parse_cli_invocation(py, argv, false)?; + let args = field(&parsed, "args")?; + if args.is_none() { + return Err(PyValueError::new_err("invalid frozen build settings")); + } + let Dispatch::Run(route, None) = command_dispatch(&command_facts(&args)?) else { + return Err(PyValueError::new_err( + "frozen settings must dispatch a build", + )); + }; + let request = Invocation::new(&args)?.value(route.arguments[0].source)?; + Ok(( + replaced(&request, &[("discovered_inputs", discovered)])?, + warehouse_scope(&args)?, + )) } /// The heartbeat is native; only driver cancellation crosses the executor host boundary. @@ -130,13 +126,21 @@ pub(super) fn execute_worker<'py>( Some(Arc::clone(&worker)); } let result = (|| { - let request = build_request(py, discovered, arguments, &owner)?; + if !worker.has_work() { + write_console( + py, + "stderr", + "Run inputs verified. All selected nodes are blocked by upstream failures.\n", + )?; + return Ok(0); + } + let (request, scope) = build_request(py, discovered, arguments, &owner)?; write_console( py, "stderr", "Run inputs verified. Executing frozen selection...\n", )?; - run_cancellable(py, &worker, &request) + within(&scope, |_| run_cancellable(py, &worker, &request)) })(); let code = result.as_ref().copied().unwrap_or(1); let outcome = if recall.is_some() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs index 76ee58e258..d758b406a0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs @@ -4,7 +4,7 @@ use std::collections::HashSet; use std::path::Path; use pyo3::exceptions::{PyTypeError, PyValueError}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; use sqlbuild_host::entries::models::UserCodeHost; @@ -31,6 +31,7 @@ pub(crate) fn module_jobs<'py>( options.set_item("project_dir", project_dir)?; let module = entry(py, UserCodeHost::LoadOrchestrationModule)?.call((), Some(&options))?; let job_class = py.import("sqlbuild.orchestration")?.getattr("Job")?; + let group_class = py.import("sqlbuild.orchestration")?.getattr("JobGroup")?; let attributes = module.getattr("__dict__")?.cast_into::()?; let mut pending: Vec> = attributes.values().iter().collect(); let mut seen: HashSet = HashSet::new(); @@ -39,7 +40,7 @@ pub(crate) fn module_jobs<'py>( if !seen.insert(value.as_ptr() as usize) { continue; } - if value.is_instance(&job_class)? { + if value.is_instance(&job_class)? || value.is_instance(&group_class)? { jobs.push(value); } else if value.is_instance_of::() || value.is_instance_of::() { pending.extend(value.try_iter()?.collect::>>()?); @@ -48,17 +49,56 @@ pub(crate) fn module_jobs<'py>( Ok(jobs) } -pub(crate) fn validate_jobs( - py: Python<'_>, - declarations: &[Bound<'_, PyAny>], -) -> Result<(), PyErr> { +pub(crate) fn collect_declarations<'py>( + py: Python<'py>, + declarations: Vec>, +) -> PyResult>> { let job = py.import("sqlbuild.orchestration")?.getattr("Job")?; - for declaration in declarations { - if !declaration.is_instance(&job)? { + let group = py.import("sqlbuild.orchestration")?.getattr("JobGroup")?; + let mut roots: HashSet = HashSet::new(); + for declaration in &declarations { + if !roots.insert(declaration.as_ptr() as usize) { + return Err(PyValueError::new_err(format!( + "duplicate job {}", + declaration.getattr("name")?.extract::()? + ))); + } + } + let mut pending = declarations; + let mut seen: HashSet = HashSet::new(); + let mut collected: Vec> = Vec::new(); + while let Some(declaration) = pending.pop() { + if !seen.insert(declaration.as_ptr() as usize) { + continue; + } + if declaration.is_instance(&group)? { + for member in declaration.getattr("jobs")?.try_iter()? { + let member = member?; + if !member.is_instance(&job)? { + return Err(PyTypeError::new_err("group members must be Job objects")); + } + pending.push(member); + } + for (member, dependencies) in declaration.getattr("after")?.cast::()?.iter() { + if !member.is_instance(&job)? { + return Err(PyTypeError::new_err( + "group dependencies must be Job objects", + )); + } + for dependency in dependencies.try_iter()? { + if !dependency?.is_instance(&job)? { + return Err(PyTypeError::new_err( + "group dependencies must be Job objects", + )); + } + } + } + } else if !declaration.is_instance(&job)? { return Err(PyTypeError::new_err( - "snapshot declarations must be sqlbuild.orchestration.Job objects", + "snapshot declarations must be sqlbuild.orchestration.Job or JobGroup objects", )); } + collected.push(declaration); } - Ok(()) + Ok(collected) } diff --git a/crates/sqlbuild-recall/src/schema/_helpers/mod.rs b/crates/sqlbuild-recall/src/schema/_helpers/mod.rs index e40b47cb19..6282151cb0 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/mod.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/mod.rs @@ -1,5 +1,4 @@ pub(crate) mod clickhouse_views; -pub(crate) mod postgres_views; pub(crate) mod relational_tables; pub(crate) mod relational_views; pub(crate) mod rendering; diff --git a/crates/sqlbuild-recall/src/schema/_helpers/postgres_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/postgres_views.rs deleted file mode 100644 index ece90e302a..0000000000 --- a/crates/sqlbuild-recall/src/schema/_helpers/postgres_views.rs +++ /dev/null @@ -1,91 +0,0 @@ -//! PostgreSQL expressions for the shared relational Recall view definitions. - -use crate::schema::_helpers::relational_views::relational_view_bodies; - -fn json_fields(mut body: String) -> String { - for (field, cast) in [ - ("audit_name", ""), - ("audit_definition_name", ""), - ("target_name", ""), - ("target_database", ""), - ("target_schema", ""), - ("attached_target_kind", ""), - ("attached_target_name", ""), - ("attached_column_name", ""), - ("severity", ""), - ("violation_count", "::BIGINT"), - ("measured_value", "::DOUBLE PRECISION"), - ("evaluation_mode", ""), - ("execution_error", ""), - ("evidence_count", "::BIGINT"), - ("evidence_truncated", "::BOOLEAN::INTEGER"), - ("sample_digest", ""), - ("error_message", ""), - ("signal_number", "::BIGINT"), - ("cost_status", ""), - ] { - body = body.replace( - &format!("json_extract(payload, '$.{field}')"), - &format!("(payload ->> '{field}'){cast}"), - ); - } - for (field, cast) in [ - ("name", ""), - ("job_name", ""), - ("run_id", ""), - ("retry_number", "::BIGINT"), - ] { - body = body.replace( - &format!("json_extract(context.external_context, '$.integration.{field}')"), - &format!("(context.external_context #>> '{{integration,{field}}}'){cast}"), - ); - } - body.replace( - "json_valid(context.external_context)", - "context.external_context IS NOT NULL", - ) - .replace( - "json_array_length(context.selection)", - "jsonb_array_length(context.selection)", - ) -} - -/// Keep display bytes until readers decode them: PostgreSQL text rejects NUL and malformed UTF-8. -fn postgres_body(body: String) -> String { - let mut body = json_fields(body) - .replace("strftime('%Y-%m-%dT%H:%M:%fZ', 'now')", "CURRENT_TIMESTAMP") - .replace("MAX(artifact.digest IS NOT NULL)", "MAX(CAST(artifact.digest IS NOT NULL AS INTEGER))") - .replace("artifact.digest IS NOT NULL AS available", "CAST(artifact.digest IS NOT NULL AS INTEGER) AS available") - .replace("MAX(\n COALESCE(received.max_sequence", "GREATEST(\n COALESCE(received.max_sequence") - .replace("MAX(COALESCE(received.queue_drops", "GREATEST(COALESCE(received.queue_drops") - .replace("COALESCE(context.recall_filter_active, 0)", "COALESCE(CAST(context.recall_filter_active AS INTEGER), 0)") - .replace("COALESCE(delivery.flush_complete, 0)", "COALESCE(CAST(delivery.flush_complete AS INTEGER), 0)") - .replace("CAST(data AS TEXT)", "data") - .replace("CAST(artifact.content AS TEXT)", "artifact.content") - .replace(")\nGROUP BY", ") AS source_rows\nGROUP BY") - .replace(")\n GROUP BY", ") AS source_rows\n GROUP BY") - .replace(")\nWHERE position", ") AS source_rows\nWHERE position") - .replace("ROUND(\n (unixepoch(last_heartbeat_at, 'subsec') - unixepoch(started_at, 'subsec')) * 1000.0, 3\n )", - "CAST(ROUND(CAST(EXTRACT(EPOCH FROM (last_heartbeat_at - started_at)) * 1000 AS NUMERIC), 3) AS DOUBLE PRECISION)"); - for expression in [ - "byte_count", - "CASE WHEN stream = 'stdout' THEN byte_count ELSE 0 END", - "CASE WHEN stream = 'stderr' THEN byte_count ELSE 0 END", - "dropped_before", - "COALESCE(omitted_sequences, 1)", - "COALESCE(omitted_bytes, 0)", - ] { - body = body.replace( - &format!("SUM({expression})"), - &format!("CAST(SUM({expression}) AS BIGINT)"), - ); - } - body -} - -pub(crate) fn postgres_view_bodies() -> Vec<(&'static str, String)> { - relational_view_bodies(true) - .into_iter() - .map(|(name, body)| (name, postgres_body(body))) - .collect() -} diff --git a/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs index c2bffe4987..321bdb28a8 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs @@ -1,11 +1,15 @@ -//! Shared relational view definitions, with explicit dialect expressions for binary transcripts. +//! Shared relational views emit dialect differences at each expression's definition. +use crate::schema::_helpers::rendering::{json_field, relational_cast}; use crate::schema::constants::OUTPUT_TAIL_RECORDS; -/// The current time as stored timestamps spell it. -const NOW: &str = "strftime('%Y-%m-%dT%H:%M:%fZ', 'now')"; +const EVENTS: &str = "SELECT * FROM recall_events"; +const DIAGNOSTICS: &str = "SELECT * FROM recall_diagnostics"; +const CONFLICTS: &str = "SELECT * FROM recall_conflicts"; -const STATEMENTS: &str = "WITH facts AS ( +fn statements(postgres: bool) -> String { + let available = relational_cast(postgres, ("artifact.digest IS NOT NULL", "INTEGER")); + format!("WITH facts AS ( SELECT * FROM recall_events WHERE statement_id IS NOT NULL AND event_type LIKE 'statement!_%' ESCAPE '!' @@ -41,7 +45,7 @@ terminals AS ( ), captures AS ( SELECT firsts.statement_id, ( - SELECT MAX(artifact.digest IS NOT NULL) + SELECT MAX({available}) FROM recall_artifact_refs AS ref LEFT JOIN recall_artifacts AS artifact ON artifact.digest = ref.digest WHERE ref.invocation_id = firsts.invocation_id @@ -102,9 +106,13 @@ LEFT JOIN captures ON captures.statement_id = firsts.statement_id LEFT JOIN resources ON resources.invocation_id = firsts.invocation_id AND resources.resource_id = firsts.resource_id -WHERE firsts.position = 1"; +WHERE firsts.position = 1") +} -const AUDIT_RESULTS: &str = "SELECT +fn audit_results(postgres: bool) -> String { + let field = |path, cast| json_field(postgres, ("payload", path), cast); + format!( + "SELECT invocation_id, invocation_started_at, event_id, @@ -112,40 +120,63 @@ const AUDIT_RESULTS: &str = "SELECT run_id, resource_id, resource_attempt_id, - json_extract(payload, '$.audit_name') AS audit_name, - json_extract(payload, '$.audit_definition_name') AS audit_definition_name, - CASE WHEN json_extract(payload, '$.target_name') IS NULL THEN NULL ELSE - COALESCE(json_extract(payload, '$.target_database') || '.', '') - || COALESCE(json_extract(payload, '$.target_schema') || '.', '') - || json_extract(payload, '$.target_name') + {audit_name} AS audit_name, + {audit_definition_name} AS audit_definition_name, + CASE WHEN {target_name} IS NULL THEN NULL ELSE + COALESCE({target_database} || '.', '') + || COALESCE({target_schema} || '.', '') + || {target_name} END AS target_relation, - json_extract(payload, '$.attached_target_kind') AS attached_target_kind, - json_extract(payload, '$.attached_target_name') AS attached_target_name, - json_extract(payload, '$.attached_column_name') AS \"column\", + {attached_target_kind} AS attached_target_kind, + {attached_target_name} AS attached_target_name, + {attached_column_name} AS \"column\", outcome, - json_extract(payload, '$.severity') AS severity, - json_extract(payload, '$.violation_count') AS violation_count, - json_extract(payload, '$.measured_value') AS measured_value, - json_extract(payload, '$.evaluation_mode') AS evaluation_mode, - json_extract(payload, '$.execution_error') AS execution_error, + {severity} AS severity, + {violation_count} AS violation_count, + {measured_value} AS measured_value, + {evaluation_mode} AS evaluation_mode, + {execution_error} AS execution_error, sql_digest, - json_extract(payload, '$.evidence_count') AS evidence_count, - json_extract(payload, '$.evidence_truncated') AS evidence_truncated, - json_extract(payload, '$.sample_digest') AS sample_digest + {evidence_count} AS evidence_count, + {evidence_truncated} AS evidence_truncated, + {sample_digest} AS sample_digest FROM recall_events -WHERE event_type = 'audit_completed'"; +WHERE event_type = 'audit_completed'", + audit_name = field("audit_name", ""), + audit_definition_name = field("audit_definition_name", ""), + target_name = field("target_name", ""), + target_database = field("target_database", ""), + target_schema = field("target_schema", ""), + attached_target_kind = field("attached_target_kind", ""), + attached_target_name = field("attached_target_name", ""), + attached_column_name = field("attached_column_name", ""), + severity = field("severity", ""), + violation_count = field("violation_count", "::BIGINT"), + measured_value = field("measured_value", "::DOUBLE PRECISION"), + evaluation_mode = field("evaluation_mode", ""), + execution_error = field("execution_error", ""), + evidence_count = field("evidence_count", "::BIGINT"), + evidence_truncated = field("evidence_truncated", "::BOOLEAN::INTEGER"), + sample_digest = field("sample_digest", "") + ) +} -const OUTPUT_INVOCATIONS: &str = "WITH received AS ( +fn output_invocations(postgres: bool) -> String { + let sum = |expression| relational_cast(postgres, (&format!("SUM({expression})"), "BIGINT")); + let maximum = if postgres { "GREATEST" } else { "MAX" }; + let filter = relational_cast(postgres, ("context.recall_filter_active", "INTEGER")); + let flush = relational_cast(postgres, ("delivery.flush_complete", "INTEGER")); + format!("WITH received AS ( SELECT invocation_id, MIN(invocation_started_at) AS invocation_started_at, COUNT(*) AS chunks, - SUM(byte_count) AS bytes, - SUM(CASE WHEN stream = 'stdout' THEN byte_count ELSE 0 END) AS stdout_bytes, - SUM(CASE WHEN stream = 'stderr' THEN byte_count ELSE 0 END) AS stderr_bytes, + {bytes} AS bytes, + {stdout_bytes} AS stdout_bytes, + {stderr_bytes} AS stderr_bytes, MAX(sequence) AS max_sequence, - SUM(dropped_before) AS queue_drops, - SUM(COALESCE(omitted_sequences, 1)) AS spans, - SUM(COALESCE(omitted_bytes, 0)) AS truncated_bytes + {drops} AS queue_drops, + {spans} AS spans, + {truncated} AS truncated_bytes FROM recall_output GROUP BY invocation_id ), @@ -166,15 +197,15 @@ counted AS ( COALESCE(received.stderr_bytes, 0) AS stderr_bytes, COALESCE(received.truncated_bytes, 0) AS truncated_bytes, received.max_sequence, - MAX( + {maximum}( COALESCE(received.max_sequence + 1 - received.spans, 0), COALESCE(delivery.accepted, 0) - COALESCE(received.chunks, 0) ) AS missing_sequences, - MAX(COALESCE(received.queue_drops, 0), COALESCE(delivery.dropped, 0)) AS reported_drops, + {maximum}(COALESCE(received.queue_drops, 0), COALESCE(delivery.dropped, 0)) AS reported_drops, COALESCE(delivery.filtered, 0) AS filtered, COALESCE(delivery.failed, 0) AS failed, - COALESCE(context.recall_filter_active, 0) AS filter_active, - COALESCE(delivery.flush_complete, 0) AS flush_complete, + COALESCE({filter}, 0) AS filter_active, + COALESCE({flush}, 0) AS flush_complete, delivery.invocation_id IS NOT NULL AS delivery_reported, COALESCE(delivery.delivered, 0) AS delivered FROM ids @@ -210,13 +241,21 @@ SELECT AND chunks >= delivered THEN 1 ELSE 0 END AS complete -FROM counted"; - -const EVENTS: &str = "SELECT * FROM recall_events"; - -const DIAGNOSTICS: &str = "SELECT * FROM recall_diagnostics"; +FROM counted", + bytes = sum("byte_count"), stdout_bytes = sum("CASE WHEN stream = 'stdout' THEN byte_count ELSE 0 END"), + stderr_bytes = sum("CASE WHEN stream = 'stderr' THEN byte_count ELSE 0 END"), drops = sum("dropped_before"), + spans = sum("COALESCE(omitted_sequences, 1)"), truncated = sum("COALESCE(omitted_bytes, 0)")) +} -const ARTIFACTS: &str = "SELECT +fn artifacts(postgres: bool) -> String { + let available = relational_cast(postgres, ("artifact.digest IS NOT NULL", "INTEGER")); + let text = if postgres { + "artifact.content" + } else { + "CAST(artifact.content AS TEXT)" + }; + format!( + "SELECT refs.invocation_id, refs.invocation_started_at, refs.digest, @@ -227,15 +266,15 @@ const ARTIFACTS: &str = "SELECT artifact.kind, artifact.media_type, artifact.size, - artifact.digest IS NOT NULL AS available, + {available} AS available, CASE WHEN artifact.digest IS NULL THEN 'expired_or_not_captured' END AS unavailable_reason, CASE WHEN artifact.media_type LIKE 'text/%' OR artifact.media_type = 'application/json' - THEN CAST(artifact.content AS TEXT) END AS text, + THEN {text} END AS text, artifact.content FROM recall_artifact_refs AS refs -LEFT JOIN recall_artifacts AS artifact ON artifact.digest = refs.digest"; - -const CONFLICTS: &str = "SELECT * FROM recall_conflicts"; +LEFT JOIN recall_artifacts AS artifact ON artifact.digest = refs.digest" + ) +} fn output_transcripts(postgres: bool) -> String { let columns: Vec = [ @@ -261,7 +300,27 @@ fn output_transcripts(postgres: bool) -> String { ) } -fn invocations() -> String { +fn invocations(postgres: bool) -> String { + let now = if postgres { + "CURRENT_TIMESTAMP" + } else { + "strftime('%Y-%m-%dT%H:%M:%fZ', 'now')" + }; + let selection_count = if postgres { + "jsonb_array_length(context.selection)" + } else { + "json_array_length(context.selection)" + }; + let valid_context = if postgres { + "context.external_context IS NOT NULL" + } else { + "json_valid(context.external_context)" + }; + let elapsed = if postgres { + "CAST(ROUND(CAST(EXTRACT(EPOCH FROM (last_heartbeat_at - started_at)) * 1000 AS NUMERIC), 3) AS DOUBLE PRECISION)" + } else { + "ROUND((unixepoch(last_heartbeat_at, 'subsec') - unixepoch(started_at, 'subsec')) * 1000.0, 3)" + }; format!( "WITH ids AS ( SELECT invocation_id, MIN(invocation_started_at) AS invocation_started_at @@ -270,7 +329,7 @@ fn invocations() -> String { UNION ALL SELECT invocation_id, invocation_started_at FROM recall_heartbeats UNION ALL SELECT invocation_id, invocation_started_at FROM recall_events WHERE event_type IN ('invocation_started', 'invocation_completed', 'invocation_failed') - ) + ) AS source_rows GROUP BY invocation_id ), started AS ( @@ -281,9 +340,9 @@ started AS ( ), terminals AS ( SELECT invocation_id, event_type, occurred_at, exit_code, duration_ms, error_type, error_code, - json_extract(payload, '$.error_message') AS error_message, - json_extract(payload, '$.signal_number') AS signal_number, - json_extract(payload, '$.cost_status') AS cost_status, + {error_message} AS error_message, + {signal_number} AS signal_number, + {cost_status} AS cost_status, ROW_NUMBER() OVER ( PARTITION BY invocation_id ORDER BY invocation_sequence DESC, occurred_at DESC, event_id DESC @@ -308,7 +367,7 @@ projected AS ( AND terminals.signal_number IS NOT NULL THEN 'cancelled' WHEN terminals.event_type = 'invocation_failed' THEN 'failed' WHEN beats.stale_at IS NULL THEN 'unknown' - WHEN beats.stale_at >= {NOW} THEN 'running' + WHEN beats.stale_at >= {now} THEN 'running' ELSE 'abandoned' END AS status, terminals.event_type AS terminal_event, @@ -328,7 +387,7 @@ projected AS ( context.adapter, context.project_name, context.selection, - COALESCE(context.selection_count, json_array_length(context.selection)) + COALESCE(context.selection_count, {selection_count}) AS selection_count, context.selection_truncated, context.selector_files, @@ -339,17 +398,17 @@ projected AS ( context.host_token, context.pid, context.external_context, - CASE WHEN json_valid(context.external_context) - THEN json_extract(context.external_context, '$.integration.name') END + CASE WHEN {valid_context} + THEN {integration} END AS integration, - CASE WHEN json_valid(context.external_context) - THEN json_extract(context.external_context, '$.integration.job_name') END + CASE WHEN {valid_context} + THEN {integration_job} END AS integration_job, - CASE WHEN json_valid(context.external_context) - THEN json_extract(context.external_context, '$.integration.run_id') END + CASE WHEN {valid_context} + THEN {integration_run_id} END AS integration_run_id, - CASE WHEN json_valid(context.external_context) - THEN json_extract(context.external_context, '$.integration.retry_number') END + CASE WHEN {valid_context} + THEN {integration_retry} END AS integration_retry, context.recall_filter_active, beats.last_heartbeat_at @@ -371,9 +430,7 @@ SELECT END AS ended_at, CASE WHEN terminal_event IS NOT NULL THEN terminal_duration_ms - WHEN status = 'abandoned' THEN ROUND( - (unixepoch(last_heartbeat_at, 'subsec') - unixepoch(started_at, 'subsec')) * 1000.0, 3 - ) + WHEN status = 'abandoned' THEN {elapsed} END AS duration_ms, CASE WHEN terminal_event IS NOT NULL THEN 0 @@ -409,18 +466,46 @@ SELECT integration_retry, recall_filter_active, last_heartbeat_at -FROM projected" +FROM projected", + error_message = json_field(postgres, ("payload", "error_message"), ""), + signal_number = json_field(postgres, ("payload", "signal_number"), "::BIGINT"), + cost_status = json_field(postgres, ("payload", "cost_status"), ""), + integration = json_field( + postgres, + ("context.external_context", "integration.name"), + "" + ), + integration_job = json_field( + postgres, + ("context.external_context", "integration.job_name"), + "" + ), + integration_run_id = json_field( + postgres, + ("context.external_context", "integration.run_id"), + "" + ), + integration_retry = json_field( + postgres, + ("context.external_context", "integration.retry_number"), + "::BIGINT" + ) ) } -fn output_tail() -> String { +fn output_tail(postgres: bool) -> String { + let text = if postgres { + "data" + } else { + "CAST(data AS TEXT)" + }; format!( "SELECT invocation_id, invocation_started_at, sequence, stream, occurred_at, byte_count, - CAST(data AS TEXT) AS text + {text} AS text FROM ( SELECT *, ROW_NUMBER() OVER (PARTITION BY invocation_id ORDER BY sequence DESC) AS position FROM recall_output -) +) AS source_rows WHERE position <= {OUTPUT_TAIL_RECORDS}" ) } @@ -428,15 +513,15 @@ WHERE position <= {OUTPUT_TAIL_RECORDS}" /// Each supported view's name and relational body, in creation order. pub(crate) fn relational_view_bodies(postgres: bool) -> Vec<(&'static str, String)> { vec![ - ("invocations", invocations()), - ("statements", STATEMENTS.to_owned()), - ("audit_results", AUDIT_RESULTS.to_owned()), - ("output_invocations", OUTPUT_INVOCATIONS.to_owned()), - ("output_tail", output_tail()), + ("invocations", invocations(postgres)), + ("statements", statements(postgres)), + ("audit_results", audit_results(postgres)), + ("output_invocations", output_invocations(postgres)), + ("output_tail", output_tail(postgres)), ("output_transcripts", output_transcripts(postgres)), ("events", EVENTS.to_owned()), ("diagnostics", DIAGNOSTICS.to_owned()), - ("artifacts", ARTIFACTS.to_owned()), + ("artifacts", artifacts(postgres)), ("conflicts", CONFLICTS.to_owned()), ] } diff --git a/crates/sqlbuild-recall/src/schema/_helpers/rendering.rs b/crates/sqlbuild-recall/src/schema/_helpers/rendering.rs index d7f3eb635b..f7355f0442 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/rendering.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/rendering.rs @@ -68,6 +68,22 @@ pub(crate) fn clickhouse_ttl(spec: &TableSpec, options: &DdlOptions) -> String { format!("{} + INTERVAL {days} DAY", spec.expiry_column) } +pub(crate) fn json_field(postgres: bool, (column, path): (&str, &str), cast: &str) -> String { + if postgres { + format!("({column} #>> '{{{}}}'){cast}", path.replace('.', ",")) + } else { + format!("json_extract({column}, '$.{path}')") + } +} + +pub(crate) fn relational_cast(postgres: bool, (expression, kind): (&str, &str)) -> String { + if postgres { + format!("CAST({expression} AS {kind})") + } else { + expression.into() + } +} + /// The engine clause, replicated with one shard and every replica on a cluster. pub(crate) fn clickhouse_engine(spec: &TableSpec, options: &DdlOptions) -> String { let version: &str = spec.clickhouse_version.unwrap_or_default(); diff --git a/crates/sqlbuild-recall/src/schema/main/control_view_statements.rs b/crates/sqlbuild-recall/src/schema/main/control_view_statements.rs index 826dd58587..1645d1ab19 100644 --- a/crates/sqlbuild-recall/src/schema/main/control_view_statements.rs +++ b/crates/sqlbuild-recall/src/schema/main/control_view_statements.rs @@ -2,7 +2,7 @@ use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; -use crate::schema::_helpers::postgres_views::postgres_view_bodies; +use crate::schema::_helpers::relational_views::relational_view_bodies; use crate::schema::main::view_statements::view_statements; use crate::schema::models::{DdlOptions, Dialect}; @@ -10,7 +10,7 @@ use crate::schema::models::{DdlOptions, Dialect}; #[must_use] pub fn control_view_statements(postgres: bool) -> Vec { if postgres { - postgres_view_bodies() + relational_view_bodies(true) .into_iter() .map(|(name, body)| { format!( diff --git a/src/sqlbuild/orchestration.py b/src/sqlbuild/orchestration.py index 94bb6c3ace..d305deef89 100644 --- a/src/sqlbuild/orchestration.py +++ b/src/sqlbuild/orchestration.py @@ -1,8 +1,9 @@ """Public authoring and snapshot compilation API for orchestration declarations.""" from sqlbuild.orchestration_declarations.main.compile_snapshot import compile_snapshot -from sqlbuild.orchestration_declarations.models import Job, Schedule +from sqlbuild.orchestration_declarations.models import Job, JobGroup, Schedule from sqlbuild.orchestration_declarations.models import Job as job +from sqlbuild.orchestration_declarations.models import JobGroup as job_group from sqlbuild.orchestration_declarations.models import Schedule as schedule -__all__ = ("Job", "Schedule", "compile_snapshot", "job", "schedule") +__all__ = ("Job", "JobGroup", "Schedule", "compile_snapshot", "job", "job_group", "schedule") diff --git a/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py b/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py index 3cba43a00c..cc10a5eb4a 100644 --- a/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py +++ b/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py @@ -7,7 +7,7 @@ from sqlbuild._native import NativeOrchestrationSnapshot, build_orchestration_snapshot from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.orchestration_declarations.models import Job +from sqlbuild.orchestration_declarations.models import Job, JobGroup def compile_snapshot( @@ -15,7 +15,7 @@ def compile_snapshot( project: CompiledProjectView, metadata_json: str, project_dir: Path, - declarations: Sequence[Job] | None = None, + declarations: Sequence[Job | JobGroup] | None = None, ) -> NativeOrchestrationSnapshot: """Compile authoring records and retained project facts into a data-only snapshot.""" return build_orchestration_snapshot( diff --git a/src/sqlbuild/orchestration_declarations/models.py b/src/sqlbuild/orchestration_declarations/models.py index fa8ff9efc1..e113ce4aa4 100644 --- a/src/sqlbuild/orchestration_declarations/models.py +++ b/src/sqlbuild/orchestration_declarations/models.py @@ -2,6 +2,7 @@ from __future__ import annotations +from collections.abc import Sequence from dataclasses import dataclass, field from typing import Literal @@ -35,3 +36,17 @@ class Job: concurrency_groups: tuple[str, ...] = field(default=(), kw_only=True) schedule: Schedule | None = field(default=None, kw_only=True) targets: tuple[str, ...] = field(default=(), kw_only=True) + + +@dataclass(frozen=True, eq=False) +class JobGroup: + """Jobs ordered by compiled lineage and optional explicit dependencies.""" + + name: str + description: str = field(kw_only=True) + jobs: Sequence[Job] = field(kw_only=True) + after: dict[Job, Sequence[Job]] = field(default_factory=dict, kw_only=True) + overlap: Literal["coalesce", "queue", "allow"] = field(default="coalesce", kw_only=True) + concurrency_groups: tuple[str, ...] = field(default=(), kw_only=True) + schedule: Schedule | None = field(default=None, kw_only=True) + targets: tuple[str, ...] = field(default=(), kw_only=True) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index 53f479aa3c..293ae24452 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -252,6 +252,7 @@ class SnapshotDiscoveryTestCase: files: dict[str, str] expected_jobs: tuple[str, ...] declarations: tuple[()] | None = None + expected_groups: dict[str, dict[str, list[str]]] = field(default_factory=dict) @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 7b78e9f190..135c64aaf6 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -102,9 +102,19 @@ class WorkerExecutionTestCase: files: dict[str, str] = field(default_factory=dict) files_after_publication: dict[str, str] = field(default_factory=dict) store_setup: str = "" + concurrency_limits: dict[str, int] = field(default_factory=dict) launcher: Literal["native", "uncooperative", "docker"] = "native" +@dataclass(frozen=True) +class GroupExecutionTestCase(WorkerExecutionTestCase): + description: str + on_failure: Literal["skip_dependents", "skip_downstream_jobs", "stop_group"] = "skip_dependents" + expected_child_statuses: dict[str, tuple[str, ...]] = field(default_factory=dict) + expected_totals: tuple[str, str | None] = ("skipped", "model:broken_orders") + expected_marts_started: bool = True + + @dataclass(frozen=True) class WorkerRestartTestCase(WorkerExecutionTestCase): description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index a91b471edf..1b98f71fb1 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -322,7 +322,14 @@ def scheduled_worker( "server_url": api.url, "max_workers": 1, } | deployment.configuration - _ = configuration.write_text(json.dumps({deployment.kind: [launcher]})) + _ = configuration.write_text( + json.dumps( + { + deployment.kind: [launcher], + "concurrency_limits": test_case.concurrency_limits, + } + ) + ) control_store: Path = directory / "control/.sqlbuild/control.sqlite3" with sqlite3.connect(control_store) as control: _ = control.executescript(test_case.store_setup) @@ -380,10 +387,14 @@ def worker_result(worker: ScheduledWorker, test_case: WorkerExecutionTestCase) - return run -def running_statement(worker: ScheduledWorker) -> dict[str, Any]: +def running_statement(worker: ScheduledWorker, *, job_key: str | None = None) -> dict[str, Any]: """The worker has started the real long query, not merely entered its Python interpreter.""" active: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, pending=frozenset({"queued", "launched"}) + worker.api, + worker.process, + worker.log, + pending=frozenset({"queued", "launched"}), + job_key=job_key, ) assert active["status"] == "running", worker.log.read_text() deadline: float = time.monotonic() + 15 @@ -398,9 +409,11 @@ def running_statement(worker: ScheduledWorker) -> dict[str, Any]: return active -def cancel_running_statement(worker: ScheduledWorker) -> None: +def cancel_running_statement( + worker: ScheduledWorker, *, job_key: str | None = None, run_id: str | None = None +) -> None: """Wait for the real long-running query before exercising the administrative cancel command.""" - active: dict[str, Any] = running_statement(worker) + active: dict[str, Any] = running_statement(worker, job_key=job_key) cancelled: subprocess.CompletedProcess[str] = subprocess.run( [ str(Path(sys.executable).with_name("sqb")), @@ -408,7 +421,7 @@ def cancel_running_statement(worker: ScheduledWorker) -> None: "--database-env", "SQLBUILD_SCHEDULER_TEST_STORE", "cancel-run", - active["run_id"], + run_id or active["run_id"], "--event-id", str(uuid4()), "--reason", @@ -428,6 +441,7 @@ def wait_for_run( log: Path, *, pending: frozenset[str] = frozenset({"queued", "launched", "running", "unresponsive"}), + job_key: str | None = None, ) -> dict[str, Any]: """A terminal API projection must follow the scheduler's actual isolated warehouse execution.""" request: Request = Request( @@ -440,7 +454,10 @@ def wait_for_run( assert process.poll() is None, log.read_text() assert time.monotonic() < deadline, log.read_text() with urlopen(request, timeout=2) as response: - current = next(iter(json.load(response)), current) + runs: list[dict[str, Any]] = json.load(response) + by_job: dict[str | None, dict[str, Any]] = {run["job_key"]: run for run in runs} + by_job[None] = next(iter(runs), current) + current = by_job.get(job_key, current) time.sleep(0.02) return current diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index 8658227d69..a95849fa86 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -5,11 +5,13 @@ import signal import sqlite3 import time +from dataclasses import replace from pathlib import Path from typing import Any from urllib.request import Request, urlopen from uuid import uuid4 +import duckdb import pytest from sqlbuild._native import ( @@ -26,6 +28,7 @@ ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( ClaimedWorker, + GroupExecutionTestCase, SnapshotWorkerTestCase, WorkerExecutionTestCase, ) @@ -35,11 +38,220 @@ scheduled_worker, snapshot_api, wait_for_log, + wait_for_run, worker_result, ) from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files +@pytest.mark.parametrize( + "test_case", + [ + GroupExecutionTestCase( + "group execution preserves independent downstream work", + "SELECT CAST(value AS INTEGER) AS amount FROM (VALUES ('invalid')) AS source(value)", + "SELECT amount FROM orders", + "partial", + [(11,)], + expected_child_statuses={"staging": ("partial",), "marts": ("partial",)}, + ), + GroupExecutionTestCase( + "successful jobs honor compiled boundaries", + "SELECT 17 AS amount", + "SELECT amount FROM totals", + "success", + [(17,)], + expected_child_statuses={"staging": ("success",), "marts": ("success",)}, + expected_totals=("success", None), + ), + GroupExecutionTestCase( + "whole downstream job is blocked without building an empty selection", + "SELECT CAST(value AS INTEGER) AS amount FROM (VALUES ('invalid')) AS source(value)", + "SELECT count(*) FROM information_schema.tables WHERE table_name IN ('orders','totals')", + "partial", + [(0,)], + on_failure="skip_downstream_jobs", + expected_child_statuses={"staging": ("partial",), "marts": ("failed",)}, + ), + GroupExecutionTestCase( + "job failure stops the group", + "SELECT CAST(value AS INTEGER) AS amount FROM (VALUES ('invalid')) AS source(value)", + "SELECT count(*) FROM information_schema.tables WHERE table_name IN ('orders','totals')", + "failed", + [(0,)], + on_failure="stop_group", + expected_child_statuses={"staging": ("partial", "cancelled"), "marts": ("cancelled",)}, + expected_totals=("cancelled", None), + expected_marts_started=False, + ), + GroupExecutionTestCase( + "critical node stops the group under the default policy", + "SELECT CAST(value AS INTEGER) AS amount FROM (VALUES ('invalid')) AS source(value)", + "SELECT count(*) FROM information_schema.tables WHERE table_name IN ('orders','totals')", + "failed", + [(0,)], + expected_child_statuses={"staging": ("partial", "cancelled"), "marts": ("cancelled",)}, + expected_totals=("cancelled", None), + expected_marts_started=False, + files={ + "models/broken_orders.sql": ( + "MODEL (description 'Critical orders.', materialized table, critical true);\n" + "SELECT CAST(value AS INTEGER) AS amount FROM (VALUES ('invalid')) AS source(value)\n" + ), + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_scheduled_group_when_workers_execute_then_dependencies_and_failure_policy_hold( + test_case: GroupExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project_dir: Path = tmp_path / "project" + project_dir.mkdir() + with duckdb.connect(str(project_dir / "warehouse.duckdb")) as warehouse: + warehouse.execute("CREATE TABLE broken_orders AS SELECT 99 AS amount") + test_case = replace( + test_case, + files={ + "models/orders.sql": ( + "MODEL (description 'Independent orders.', materialized table);\n" + 'SELECT amount FROM __ref("stg_orders")\n' + ), + "models/stg_orders.sql": ( + "MODEL (description 'Staged orders.', materialized table);\nSELECT 11 AS amount\n" + ), + "models/broken_orders.sql": ( + "MODEL (description 'Source values.', materialized table);\n" + test_case.model_sql + ), + "models/totals.sql": ( + "MODEL (description 'Dependent totals.', materialized table);\n" + 'SELECT amount FROM __ref("broken_orders")\n' + ), + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, job_group, schedule\n" + 'staging = job("staging", description="Stage orders.", ' + f'select="stg_orders broken_orders", on_failure="{test_case.on_failure}")\n' + 'marts = job("marts", description="Build independent marts.", select="orders totals")\n' + 'annual_orders = job_group("annual_orders", description="Stage then build marts.", ' + "jobs=[marts, staging], " + 'schedule=schedule(cron="0 0 1 1 *", timezone="UTC", ' + 'starts_at="2000-01-01T00:00:00Z"))\n' + ), + } + | test_case.files, + ) + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + parent: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_orders" + ) + assert parent["status"] == test_case.expected_status, worker.log.read_text() + request: Request = Request( + f"{worker.api.url}/api/locations/{worker.api.location_id}/targets/development/runs", + headers={"Authorization": f"Bearer {worker.api.bearer}"}, + ) + with urlopen(request, timeout=5) as response: + runs: dict[str, Any] = {run["job_key"]: run for run in json.load(response)} + assert set(runs) == {"annual_orders", *test_case.expected_child_statuses} + assert all( + runs[name]["status"] in statuses + for name, statuses in test_case.expected_child_statuses.items() + ) + assert (runs["marts"]["started_at"] is not None) == test_case.expected_marts_started + assert runs["staging"]["ended_at"] <= ( + runs["marts"]["started_at"] or runs["staging"]["ended_at"] + ) + with sqlite3.connect(worker.control_store) as control: + assert ( + control.execute( + "SELECT parent_run_id, root_run_id FROM sqb_runs WHERE kind='job' ORDER BY job_key" + ).fetchall() + == [(runs["annual_orders"]["run_id"],) * 2] * 2 + ) + assert ( + control.execute( + "SELECT status, skipped_because_of FROM sqb_run_nodes WHERE run_id=? AND node_key='model:totals'", + (runs["marts"]["run_id"],), + ).fetchone() + == test_case.expected_totals + ) + assert control.execute( + "SELECT status FROM sqb_run_nodes WHERE run_id=? AND node_key='model:broken_orders'", + (runs["staging"]["run_id"],), + ).fetchone() == ({"success": "success"}.get(test_case.expected_status, "failed"),) + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + assert control.execute("SELECT count(*) FROM sqb_run_liveness").fetchone() == (0,) + with duckdb.connect(str(project_dir / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + description="opposing parent and independent child limits", + model_sql="SELECT 11 AS amount", + warehouse_query="SELECT amount FROM orders UNION ALL SELECT amount FROM inventory ORDER BY amount", + expected_status="success", + expected_rows=[(11,), (23,)], + concurrency_limits={"orders_limit": 1, "inventory_limit": 1}, + files={ + "models/inventory.sql": ( + "MODEL (description 'Inventory.', materialized table);\nSELECT 23 AS amount\n" + ), + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, job_group, schedule\n" + "orders = job('orders_job', description='Build orders.', select='orders', " + "concurrency_groups=('inventory_limit',))\n" + "inventory = job('inventory_job', description='Build inventory.', select='inventory', " + "concurrency_groups=('orders_limit',))\n" + "annual_orders = job_group('annual_orders', description='Orders group.', jobs=[orders], " + "concurrency_groups=('orders_limit',), " + "schedule=schedule(cron='0 0 1 1 *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" + "annual_inventory = job_group('annual_inventory', description='Inventory group.', " + "jobs=[inventory], concurrency_groups=('inventory_limit',), " + "schedule=schedule(cron='0 0 1 1 *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" + ), + }, + store_setup=( + "CREATE TABLE test_slot_claims (run_id TEXT, group_key TEXT, occupied INTEGER);\n" + "CREATE TRIGGER test_observe_slots AFTER INSERT ON sqb_concurrency_slots BEGIN " + "INSERT INTO test_slot_claims SELECT NEW.run_id, NEW.group_key, count(*) " + "FROM sqb_concurrency_slots WHERE group_key=NEW.group_key; END;\n" + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_opposing_group_limits_when_scheduled_then_both_real_workers_finish_without_hoisting_limits( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + orders: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_orders" + ) + inventory: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_inventory" + ) + assert orders["status"] == inventory["status"] == "success", worker.log.read_text() + with sqlite3.connect(worker.control_store) as control: + assert control.execute( + "SELECT r.job_key, c.group_key, max(c.occupied) FROM test_slot_claims c " + "JOIN sqb_runs r ON r.run_id=c.run_id GROUP BY r.job_key, c.group_key ORDER BY r.job_key" + ).fetchall() == [ + ("annual_inventory", "inventory_limit", 1), + ("annual_orders", "orders_limit", 1), + ("inventory_job", "orders_limit", 1), + ("orders_job", "inventory_limit", 1), + ] + assert control.execute( + "SELECT count(*) FROM sqb_runs WHERE status='success'" + ).fetchone() == (4,) + assert control.execute("SELECT count(*) FROM sqb_concurrency_slots").fetchone() == (0,) + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + + @pytest.mark.parametrize( "test_case", [ @@ -167,6 +379,27 @@ def test_given_a_retained_baseline_when_starting_a_worker_then_verification_and_ "success", [(23,)], ), + WorkerExecutionTestCase( + "authored variables and reload use ordinary build argument conversion", + "SELECT @@batch AS amount", + "SELECT amount FROM orders", + "success", + [(23,)], + files={ + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n[vars]\nbatch = "3"\n' + '[connections.local]\ndatabase = "warehouse.duckdb"\n' + '[targets.development]\nschema = "main"\n' + ), + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, schedule\n" + "annual_orders = job('annual_orders', select='orders', description='Build orders.', " + "settings={'vars': {'batch': 23}, 'reload': True, 'run_tests': False}, " + "schedule=schedule(cron='0 0 1 1 *', timezone='UTC', " + "starts_at='2000-01-01T00:00:00Z'))\n" + ), + }, + ), WorkerExecutionTestCase( "warehouse error retains failed node evidence", "SELECT CAST(value AS BIGINT) AS amount FROM (VALUES ('invalid')) t(value)", @@ -235,6 +468,68 @@ def test_given_an_executing_scheduled_worker_when_cancelled_then_the_statement_s assert run["status"] == test_case.expected_status, worker.log.read_text() +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "parent cancellation interrupts its active child and cancels its waiting child", + "SELECT sum(sin(value)) AS amount FROM range(10000000000) t(value)", + "SELECT count(*) FROM duckdb_tables() WHERE table_name IN ('orders','totals')", + "cancelled", + [(0,)], + files={ + "models/totals.sql": ( + "MODEL (description 'Downstream totals.', materialized table);\n" + 'SELECT amount FROM __ref("orders")\n' + ), + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, job_group, schedule\n" + "orders = job('orders', description='Long query.', select='orders')\n" + "totals = job('totals', description='Waiting query.', select='totals')\n" + "annual_orders = job_group('annual_orders', description='Cancellable group.', " + "jobs=[totals, orders], schedule=schedule(cron='0 0 1 1 *', timezone='UTC', " + "starts_at='2000-01-01T00:00:00Z'))\n" + ), + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_an_executing_group_when_parent_cancelled_then_all_children_stop_before_parent_completion( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + parent: dict[str, Any] = wait_for_run( + worker.api, + worker.process, + worker.log, + pending=frozenset({"queued"}), + job_key="annual_orders", + ) + assert parent["status"] == "running", worker.log.read_text() + cancel_running_statement(worker, job_key="orders", run_id=parent["run_id"]) + completed: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_orders" + ) + assert completed["status"] == test_case.expected_status, worker.log.read_text() + with sqlite3.connect(worker.control_store) as control: + assert control.execute( + "SELECT job_key, status FROM sqb_runs ORDER BY job_key" + ).fetchall() == [ + ("annual_orders", "cancelled"), + ("orders", "cancelled"), + ("totals", "cancelled"), + ] + assert control.execute( + "SELECT count(*) FROM sqb_runs WHERE parent_run_id=? AND ended_at>?", + (parent["run_id"], completed["ended_at"]), + ).fetchone() == (0,) + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + assert control.execute("SELECT count(*) FROM sqb_run_liveness").fetchone() == (0,) + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 03b75e763a..f274c11f7e 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -554,6 +554,29 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject }, expected_jobs=("daily", "staging"), ), + SnapshotDiscoveryTestCase( + description="group-only references retain lineage and explicit waits", + files={ + "orchestration/jobs.py": ( + "from sqlbuild.orchestration import job, job_group\n" + "def factory():\n" + " staging = job('staging', description='Stage orders.', select='stg_orders')\n" + " marts = job('marts', description='Build orders.', select='orders')\n" + " inventory = job('inventory', description='Repeat staging.', select='stg_orders')\n" + " return job_group('daily', description='Ordered jobs.', " + "jobs=[marts, inventory, staging], after={inventory: [staging]})\n" + "daily = factory()\n" + ), + }, + expected_jobs=("inventory", "marts", "staging"), + expected_groups={ + "daily": { + "inventory": ["staging"], + "marts": ["inventory", "staging"], + "staging": [], + } + }, + ), SnapshotDiscoveryTestCase( description="explicit empty declarations do not execute project files", files={"orchestration/jobs.py": "raise RuntimeError('must not import')\n"}, @@ -570,7 +593,16 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject "MODEL (", "MODEL (tags ['edge_case'],", 1 ), }, - expected_jobs=("edge_snapshot_orders",), + expected_jobs=( + "edge_snapshot_inventory", + "edge_snapshot_orders", + "edge_snapshot_products", + ), + expected_groups={ + "edge_snapshot_group": {"edge_snapshot_orders": []}, + "edge_inventory_group": {"edge_snapshot_inventory": []}, + "edge_products_group": {"edge_snapshot_products": []}, + }, ), SnapshotDiscoveryTestCase( description="a project without declarations has no job document", @@ -598,6 +630,10 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects ).to_json() ) assert tuple(publication["documents"].get("jobs", {})) == test_case.expected_jobs + assert { + name: definition["jobs"] + for name, definition in publication["documents"].get("groups", {}).items() + } == test_case.expected_groups for definition in publication["documents"].get("jobs", {}).values(): assert set(definition["selection"]) <= publication["nodes"].keys() @@ -605,6 +641,44 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects @pytest.mark.parametrize( "test_case", [ + SnapshotDiscoveryRejectionTestCase( + description="group members are jobs, not nested groups", + files={ + "orchestration/jobs.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/nested_groups.py" + ).read_text(), + }, + expected_error=TypeError, + expected_message="group members must be Job objects", + ), + SnapshotDiscoveryRejectionTestCase( + description="explicit group cycles cannot publish", + files={ + "orchestration/jobs.py": ( + "from sqlbuild.orchestration import job, job_group\n" + "staging = job('staging', description='Stage orders.', select='stg_orders')\n" + "marts = job('marts', description='Build orders.', select='orders')\n" + "daily = job_group('daily', description='Cyclic jobs.', " + "jobs=[marts, staging], after={staging: [marts]})\n" + ), + }, + expected_error=ValueError, + expected_message="group dependencies contain a cycle", + ), + SnapshotDiscoveryRejectionTestCase( + description="group waits cannot refer to nonmember jobs", + files={ + "orchestration/jobs.py": ( + "from sqlbuild.orchestration import job, job_group\n" + "staging = job('staging', description='Stage orders.', select='stg_orders')\n" + "marts = job('marts', description='Build orders.', select='orders')\n" + "daily = job_group('daily', description='Unknown upstream.', " + "jobs=[marts], after={marts: [staging]})\n" + ), + }, + expected_error=ValueError, + expected_message="group needs known jobs and dependencies within its membership", + ), SnapshotDiscoveryRejectionTestCase( description="distinct objects cannot share a job name", files={ From e4948a54deb5e79db706ac800195906f71a26b63 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:13:01 +0000 Subject: [PATCH 37/59] fix: preserve failure-only reruns and transitive job ordering Keep successful and cancelled nodes out of failure-only reruns, retain waits through omitted completed jobs, and prove the complete local and Docker worker paths. The backend remains WIP. SQLite/PostgreSQL API, run and Recall regressions pass; the publication suite passes 42 cases. The exact static target still reports six known unused-public-entry findings. Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../orchestration/rerun_from_failure.sql | 8 ++ .../sqlbuild-cli/src/argv/command_table.json | 1 + .../argv/tests/test_parse_cli_arguments.rs | 29 ++++ .../src/runs/_helpers/groups/selection.rs | 27 ++-- .../src/runs/_helpers/requests/job.rs | 57 ++++++++ .../src/runs/main/queue_run.rs | 55 +------ .../tests/runs/test_reruns.rs | 134 +++++++++++------- .../tests/runs/test_types.rs | 12 +- .../compiled_project/publication/helpers.py | 5 +- .../publication/test_rerun.py | 124 ++++++++++++++++ 10 files changed, 336 insertions(+), 116 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/rerun_from_failure.sql create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rerun.py diff --git a/benchmarks/duckshop/edge_cases/orchestration/rerun_from_failure.sql b/benchmarks/duckshop/edge_cases/orchestration/rerun_from_failure.sql new file mode 100644 index 0000000000..8304e49843 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/rerun_from_failure.sql @@ -0,0 +1,8 @@ +MODEL (description 'Retry a failed cast after warehouse input changes.', materialized table); +SELECT CASE + WHEN EXISTS ( + SELECT 1 FROM information_schema.tables WHERE table_name = 'retry_ready' + ) THEN 17 + ELSE CAST(value AS INTEGER) +END AS amount +FROM (VALUES ('invalid')) AS source(value) diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index ed0917d366..fea155edee 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -493,6 +493,7 @@ "options": ["--from-failure"], "dest": "admin_from_failure", "nargs": 0, + "const": true, "default": false, "help": "retry exactly the failed and skipped nodes, excluding successful nodes", "group": 1 diff --git a/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs b/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs index ab5d022216..62ba60f4db 100644 --- a/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs +++ b/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs @@ -49,6 +49,35 @@ fn given_accepted_arguments_when_parsing_then_attributes_hold_the_parsed_values( attribute: "contract_command", expected_value: Value::Str("diff".to_owned()), }, + ParsedValueTestCase { + description: "an explicit failed-node rerun never becomes a full rerun", + argv: &[ + "admin", + "rerun-run", + "00000000-0000-0000-0000-000000000001", + "--run-id", + "00000000-0000-0000-0000-000000000002", + "--reason", + "Retry failed orders", + "--from-failure", + ], + attribute: "admin_from_failure", + expected_value: Value::Bool(true), + }, + ParsedValueTestCase { + description: "a full rerun is the default without failure narrowing", + argv: &[ + "admin", + "rerun-run", + "00000000-0000-0000-0000-000000000001", + "--run-id", + "00000000-0000-0000-0000-000000000002", + "--reason", + "Retry all orders", + ], + attribute: "admin_from_failure", + expected_value: Value::Bool(false), + }, ]; for test_case in test_cases { diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/selection.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/selection.rs index a42f48821c..4f2c66310c 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/selection.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/selection.rs @@ -28,7 +28,7 @@ pub(crate) async fn child_ready( .job_key .as_ref() .ok_or_else(|| ControlError::Invalid("group has no declaration".into()))?; - let dependencies = &group.definitions.groups[parent_name].jobs[name]; + let dependencies = upstream_jobs(&group.definitions.groups[parent_name].jobs, name); let ready = group.parent.state.status == RunStatus::Running && group.parent.state.cancel_requested_at.is_none() && !group.stopped() @@ -37,7 +37,7 @@ pub(crate) async fn child_ready( .request_json .job_key .as_ref() - .is_none_or(|name| !dependencies.contains(name)) + .is_none_or(|name| !dependencies.contains(name.as_str())) || child.state.status.is_terminal() }); Ok((transaction, ready)) @@ -62,13 +62,7 @@ pub(crate) async fn skipped_nodes( .as_ref() .ok_or_else(|| ControlError::Invalid("group has no declaration".into()))?; let edges = &group.definitions.groups[parent_name].jobs; - let mut ancestors: BTreeSet<&str> = BTreeSet::new(); - let mut pending: Vec<&str> = edges[name].iter().map(String::as_str).collect(); - while let Some(name) = pending.pop() { - if ancestors.insert(name) { - pending.extend(edges[name].iter().map(String::as_str)); - } - } + let ancestors = upstream_jobs(edges, name); let (transaction, rows) = query(transaction, "SELECT n.node_key, n.status, n.skipped_because_of FROM sqb_run_nodes n JOIN sqb_runs r ON r.run_id=n.run_id \ WHERE r.parent_run_id=$1 AND r.run_id<>$2 AND n.status IN ('failed','skipped','cancelled')", @@ -137,3 +131,18 @@ pub(crate) async fn skipped_nodes( } Ok((transaction, skipped)) } + +/// Completed jobs omitted from a retry do not sever explicit ordering between retained jobs. +fn upstream_jobs<'a>( + edges: &'a BTreeMap>, + name: &str, +) -> BTreeSet<&'a str> { + let mut ancestors: BTreeSet<&str> = BTreeSet::new(); + let mut pending: Vec<&str> = edges[name].iter().map(String::as_str).collect(); + while let Some(name) = pending.pop() { + if ancestors.insert(name) { + pending.extend(edges[name].iter().map(String::as_str)); + } + } + ancestors +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs index edd8116dd6..edbfae8f23 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs @@ -1,6 +1,11 @@ //! Freeze execution inputs from one published job or group in the caller's write transaction. +use std::collections::BTreeSet; + +use uuid::Uuid; + use crate::credentials::models::AuditContext; +use crate::declarations::main::frozen_definitions::frozen_definitions; use crate::declarations::models::JobDefinition; use crate::runs::main::job_write_set::job_write_set; use crate::runs::models::{JobRunRequest, RunKind, RunOrigin, RunRequest}; @@ -63,3 +68,55 @@ pub(crate) async fn frozen_job_request( }, )) } + +pub(crate) async fn frozen_group_children( + transaction: ControlTransaction, + request: &RunRequest, +) -> Result<(ControlTransaction, Vec), ControlError> { + let (transaction, definitions) = frozen_definitions(transaction, request.snapshot_id).await?; + let group = request + .job_key + .as_ref() + .and_then(|name| definitions.groups.get(name)) + .ok_or_else(|| ControlError::Invalid("group is absent from the pinned snapshot".into()))?; + let selection_matches = if request.origin == RunOrigin::Rerun { + request.selection.is_subset(&group.job.selection) + } else { + group.job.selection == request.selection + }; + if !selection_matches || !request.write_set.is_empty() { + return Err(ControlError::Invalid( + "group scope differs from its pinned membership".into(), + )); + } + let (transaction, writers) = + job_write_set(transaction, request.snapshot_id, &request.selection).await?; + let mut children: Vec = Vec::new(); + for name in group.jobs.keys() { + let job = &definitions.jobs[name]; + let selection: BTreeSet<_> = job + .selection + .intersection(&request.selection) + .cloned() + .collect(); + if selection.is_empty() { + continue; + } + children.push(RunRequest { + run_id: Uuid::new_v5(&request.run_id, name.as_bytes()), + kind: RunKind::Job, + job_key: Some(name.clone()), + parent_run_id: Some(request.run_id), + rerun_of_run_id: request + .rerun_of_run_id + .map(|id| Uuid::new_v5(&id, name.as_bytes())), + write_set: selection.intersection(&writers).cloned().collect(), + selection, + settings: job.settings.clone(), + concurrency_groups: job.concurrency_groups.clone(), + overlap: job.overlap, + ..request.clone() + }); + } + Ok((transaction, children)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/queue_run.rs b/crates/sqlbuild-orchestration/src/runs/main/queue_run.rs index b8cb2e9b89..fb4d86c590 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/queue_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/queue_run.rs @@ -4,10 +4,9 @@ use serde_json::json; use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; -use crate::declarations::main::frozen_definitions::frozen_definitions; use crate::runs::_helpers::lifecycle::events::append_run_event; -use crate::runs::main::job_write_set::job_write_set; -use crate::runs::models::{RunKind, RunOrigin, RunRequest}; +use crate::runs::_helpers::requests::job::frozen_group_children; +use crate::runs::models::{RunKind, RunRequest}; use crate::store::errors::ControlError; use crate::store::main::query::query; use crate::store::models::{ControlTransaction, Parameter}; @@ -18,53 +17,9 @@ pub(crate) async fn queue_run( ) -> Result { let mut transaction = queue_one(transaction, request).await?; if request.kind == RunKind::Group { - let definitions; - (transaction, definitions) = frozen_definitions(transaction, request.snapshot_id).await?; - let group = request - .job_key - .as_ref() - .and_then(|name| definitions.groups.get(name)) - .ok_or_else(|| { - ControlError::Invalid("group is absent from the pinned snapshot".into()) - })?; - let selection_matches = if request.origin == RunOrigin::Rerun { - request.selection.is_subset(&group.job.selection) - } else { - group.job.selection == request.selection - }; - if !selection_matches || !request.write_set.is_empty() { - return Err(ControlError::Invalid( - "group scope differs from its pinned membership".into(), - )); - } - let writers; - (transaction, writers) = - job_write_set(transaction, request.snapshot_id, &request.selection).await?; - for name in group.jobs.keys() { - let job = &definitions.jobs[name]; - let selection: std::collections::BTreeSet<_> = job - .selection - .intersection(&request.selection) - .cloned() - .collect(); - if selection.is_empty() { - continue; - } - let child = RunRequest { - run_id: Uuid::new_v5(&request.run_id, name.as_bytes()), - kind: RunKind::Job, - job_key: Some(name.clone()), - parent_run_id: Some(request.run_id), - rerun_of_run_id: request - .rerun_of_run_id - .map(|id| Uuid::new_v5(&id, name.as_bytes())), - write_set: selection.intersection(&writers).cloned().collect(), - selection, - settings: job.settings.clone(), - concurrency_groups: job.concurrency_groups.clone(), - overlap: job.overlap, - ..request.clone() - }; + let children; + (transaction, children) = frozen_group_children(transaction, request).await?; + for child in children { transaction = queue_one(transaction, &child).await?; } } diff --git a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs index a17b54cee9..0729862d1f 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs @@ -23,7 +23,7 @@ use crate::helpers::snapshot; use crate::location_helpers::create_location; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; -use crate::test_types::{RunLifecycleTestCase, TransactionFault}; +use crate::test_types::{RerunExecutionTestCase, RunLifecycleTestCase, TransactionFault}; #[test] fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_nodes_retry_atomically() @@ -57,11 +57,6 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ ip_address: None, }; let mut input = snapshot(location); - input - .nodes - .get_mut("inventory") - .expect("inventory") - .definition["deps"] = json!(["orders"]); input.documents.insert("jobs".into(), json!({ "staging": {"description": "Stage orders", "selection": ["orders", "customers"], "concurrency_groups": ["warehouse"], "settings": {"vars": {"batch": 23}}}, "marts": {"description": "Build inventory", "selection": ["inventory"], "concurrency_groups": ["inventory_limit"]}, @@ -69,7 +64,7 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ })); input.documents.insert("groups".into(), json!({"daily_orders": { "description": "Build daily orders", "selection": ["customers", "orders", "inventory", "products"], - "concurrency_groups": ["warehouse"], "jobs": {"staging": [], "marts": ["staging"], "completed": []} + "concurrency_groups": ["warehouse"], "jobs": {"staging": [], "marts": ["completed"], "completed": ["staging"]} }})); let published = publish_snapshot(store, &input).await.expect("publish"); let source_id = Uuid::now_v7(); @@ -88,7 +83,7 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ .await .expect("request group"); let retry = RerunRequest { - run_id: Uuid::now_v7(), + run_id: Uuid::from_u128(1), source_run_id: source_id, from_failure: true, reason: "Retry failed orders".into(), @@ -103,16 +98,30 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ .expect("elect") .expect("leader"); let limits = BTreeMap::from([ - ("warehouse".into(), std::num::NonZeroU32::new(1).unwrap()), + ( + "warehouse".into(), + std::num::NonZeroU32::new(1).expect("positive capacity"), + ), ( "inventory_limit".into(), - std::num::NonZeroU32::new(1).unwrap(), + std::num::NonZeroU32::new(1).expect("positive capacity"), ), ]); let scope = (location, "development"); advance_groups(store, &lease, (&limits, scope)) .await .expect("admit source"); + let node_outcomes = BTreeMap::from([ + ("products", NodeOutcome::Success), + ("orders", NodeOutcome::Failed), + ("customers", NodeOutcome::Cancelled), + ("inventory", NodeOutcome::Skipped), + ]); + let job_outcomes = BTreeMap::from([ + ("staging", RunOutcome::Failed), + ("completed", RunOutcome::Success), + ("marts", RunOutcome::Failed), + ]); for _ in 0..3 { let claim = claim_next_run(store, &lease, &limits, Some(scope)) .await @@ -120,12 +129,7 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ .expect("ready child"); let mut changes = vec![RunChange::Started]; for node in &claim.request.selection { - let outcome = match node.as_str() { - "products" => NodeOutcome::Success, - "orders" => NodeOutcome::Failed, - "customers" => NodeOutcome::Cancelled, - _ => NodeOutcome::Skipped, - }; + let outcome = node_outcomes[node.as_str()]; changes.push(RunChange::NodeFinished { node_key: node.clone(), outcome, @@ -136,11 +140,7 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ }); } changes.push(RunChange::Finished { - outcome: if claim.request.selection.contains("products") { - RunOutcome::Success - } else { - RunOutcome::Failed - }, + outcome: job_outcomes[claim.request.job_key.as_deref().expect("source job")], reason: "synthetic source outcome".into(), exit_code: Some(1), }); @@ -168,16 +168,16 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ assert_eq!( list_runs(store, scope, 20) .await - .unwrap() + .expect("list completed source") .iter() .find(|run| run.run_id == source_id) - .unwrap() + .expect("source exists") .status, RunStatus::Partial ); input.generation = 2; input.image_id = Some("sha256:synthetic-orders-image-2".into()); - input.documents.get_mut("jobs").unwrap()["staging"]["settings"] = + input.documents.get_mut("jobs").expect("job declarations")["staging"]["settings"] = json!({"vars": {"batch": 91}}); publish_snapshot(store, &input).await.expect("redeploy"); let fault = TransactionFault { @@ -192,7 +192,10 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ assert!(request_rerun(store, &context, &retry).await.is_err()); remove_fault(store, &fault).await; assert_eq!( - list_runs(store, scope, 20).await.unwrap().len(), + list_runs(store, scope, 20) + .await + .expect("list rolled-back runs") + .len(), 4, "audit failure rolls back parent and all children" ); @@ -211,41 +214,67 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ ..context.clone() }; assert!(request_rerun(store, &other, &retry).await.is_err()); - for request in [ - retry.clone(), - RerunRequest { - run_id: Uuid::now_v7(), - from_failure: false, - ..retry.clone() + let settings = BTreeMap::from([ + ( + "staging", + BTreeMap::from([("vars".into(), json!({"batch": 23}))]), + ), + ("completed", BTreeMap::new()), + ("marts", BTreeMap::new()), + ]); + for execution in [ + RerunExecutionTestCase { + description: "failure retry omits successful and cancelled nodes", + request: retry.clone(), + expected_applied: false, + expected_selection: ["orders", "inventory"].map(str::to_owned).into(), + expected_jobs: &["staging", "marts"], }, - ] { - if !request.from_failure { - assert!(request_rerun(store, &context, &request).await.unwrap()); - } - let expected: BTreeSet = if request.from_failure { - ["orders", "inventory"].map(str::to_owned).into() - } else { - ["orders", "inventory", "customers", "products"] + RerunExecutionTestCase { + description: "full retry retains all original nodes", + request: RerunRequest { + run_id: Uuid::now_v7(), + from_failure: false, + ..retry.clone() + }, + expected_applied: true, + expected_selection: ["orders", "inventory", "customers", "products"] .map(str::to_owned) - .into() - }; - let runs = list_runs(store, scope, 20).await.unwrap(); + .into(), + expected_jobs: &["staging", "completed", "marts"], + }, + ] { + let request = execution.request; + assert_eq!( + request_rerun(store, &context, &request) + .await + .expect("publish rerun"), + execution.expected_applied, + "{}", + execution.description + ); + let runs = list_runs(store, scope, 20).await.expect("list reruns"); assert_eq!( runs.iter() .find(|run| run.run_id == request.run_id) - .unwrap() + .expect("rerun exists") .selection_json, - expected + execution.expected_selection ); advance_groups(store, &lease, (&limits, scope)) .await .expect("admit rerun"); - let mut selected = BTreeSet::new(); - for _ in 0..if request.from_failure { 2 } else { 3 } { + let mut selected: BTreeSet = BTreeSet::new(); + for expected_job in execution.expected_jobs { let claim = claim_next_run(store, &lease, &limits, Some(scope)) .await .expect("claim rerun") .expect("ready retry child"); + assert_eq!( + claim.request.job_key.as_deref(), + Some(*expected_job), + "a completed intermediate job must not erase its ancestor's wait boundary" + ); assert_eq!(claim.request.snapshot_id, published.snapshot_id); assert_eq!( claim.request.image_id, @@ -253,26 +282,21 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ ); assert_eq!(claim.request.origin, RunOrigin::Rerun); assert_eq!(claim.request.priority, 7); - if claim.request.job_key.as_deref() == Some("staging") { - assert_eq!( - claim.request.settings, - BTreeMap::from([("vars".into(), json!({"batch": 23}))]) - ); - } + assert_eq!(claim.request.settings, settings[expected_job]); selected.extend(claim.request.selection.iter().cloned()); succeed(store, &claim).await; } - assert_eq!(selected, expected); + assert_eq!(selected, execution.expected_selection); advance_groups(store, &lease, (&limits, scope)) .await .expect("finish rerun"); assert_eq!( list_runs(store, scope, 20) .await - .unwrap() + .expect("list finished reruns") .iter() .find(|run| run.run_id == request.run_id) - .unwrap() + .expect("finished rerun exists") .status, test_case.expected_status, "{}", diff --git a/crates/sqlbuild-orchestration/tests/runs/test_types.rs b/crates/sqlbuild-orchestration/tests/runs/test_types.rs index 9ee50b5f8c..0646241008 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_types.rs @@ -1,6 +1,8 @@ //! Expected queue outcomes for each real control-store backend. -use sqlbuild_orchestration::runs::models::RunStatus; +use std::collections::BTreeSet; + +use sqlbuild_orchestration::runs::models::{RerunRequest, RunStatus}; pub(crate) use crate::fault_types::TransactionFault; use crate::store_test_types::Backend; @@ -50,6 +52,14 @@ pub(crate) struct RunLifecycleTestCase { pub(crate) expected_status: RunStatus, } +pub(crate) struct RerunExecutionTestCase { + pub(crate) description: &'static str, + pub(crate) request: RerunRequest, + pub(crate) expected_applied: bool, + pub(crate) expected_selection: BTreeSet, + pub(crate) expected_jobs: &'static [&'static str], +} + pub(crate) struct RunMonitorTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index 1b98f71fb1..f0717f1000 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -442,6 +442,7 @@ def wait_for_run( *, pending: frozenset[str] = frozenset({"queued", "launched", "running", "unresponsive"}), job_key: str | None = None, + run_id: str | None = None, ) -> dict[str, Any]: """A terminal API projection must follow the scheduler's actual isolated warehouse execution.""" request: Request = Request( @@ -457,7 +458,9 @@ def wait_for_run( runs: list[dict[str, Any]] = json.load(response) by_job: dict[str | None, dict[str, Any]] = {run["job_key"]: run for run in runs} by_job[None] = next(iter(runs), current) - current = by_job.get(job_key, current) + by_id: dict[str | None, dict[str, Any]] = {run["run_id"]: run for run in runs} + by_id[None] = by_job.get(job_key, current) + current = by_id.get(run_id, current) time.sleep(0.02) return current diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rerun.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rerun.py new file mode 100644 index 0000000000..f7edf07364 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rerun.py @@ -0,0 +1,124 @@ +"""Pinned group retries rebuild failed and skipped nodes without repeating successful writes.""" + +import json +import sqlite3 +import subprocess +import sys +from dataclasses import replace +from pathlib import Path +from typing import Any +from uuid import uuid4 + +import duckdb +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + WorkerExecutionTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + scheduled_worker, + wait_for_run, +) + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + description="local group retries only failed and skipped nodes", + model_sql="SELECT 11 AS amount", + warehouse_query="SELECT amount FROM orders UNION ALL SELECT amount FROM totals ORDER BY amount", + expected_status="success", + expected_rows=[(17,), (99,)], + ), + WorkerExecutionTestCase( + description="Docker group retries only failed and skipped nodes", + model_sql="SELECT 11 AS amount", + warehouse_query="SELECT amount FROM orders UNION ALL SELECT amount FROM totals ORDER BY amount", + expected_status="success", + expected_rows=[(17,), (99,)], + launcher="docker", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_partial_group_when_retrying_from_failure_then_successful_nodes_are_not_rewritten( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + test_case = replace( + test_case, + files={ + "models/broken_orders.sql": Path( + "benchmarks/duckshop/edge_cases/orchestration/rerun_from_failure.sql" + ).read_text(), + "models/totals.sql": ( + "MODEL (description 'Downstream totals.', materialized table);\n" + 'SELECT amount FROM __ref("broken_orders")\n' + ), + "models/inventory.sql": ( + "MODEL (description 'Independent inventory.', materialized table);\n" + "SELECT 29 AS amount\n" + ), + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, job_group, schedule\n" + "staging = job('staging', description='Stage orders.', select='orders broken_orders')\n" + "marts = job('marts', description='Build totals.', select='totals')\n" + "inventory = job('inventory', description='Build inventory.', select='inventory')\n" + "annual_orders = job_group('annual_orders', description='Retry boundaries.', " + "jobs=[inventory, marts, staging], schedule=schedule(cron='0 0 1 1 *', " + "timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" + ), + }, + ) + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + source: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_orders" + ) + assert source["status"] == "partial", worker.log.read_text() + with duckdb.connect(str(worker.project / "warehouse.duckdb")) as warehouse: + warehouse.execute("CREATE TABLE retry_ready AS SELECT 1 AS id") + warehouse.execute("UPDATE orders SET amount=99") + warehouse.execute("UPDATE inventory SET amount=71") + run_id: str = str(uuid4()) + command: list[str] = [ + str(Path(sys.executable).with_name("sqb")), + "admin", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "rerun-run", + source["run_id"], + "--run-id", + run_id, + "--from-failure", + "--reason", + "Retry failed and skipped nodes after fixing warehouse input.", + ] + for expected_applied in [True, False]: + requested: subprocess.CompletedProcess[str] = subprocess.run( + command, capture_output=True, text=True, timeout=10, check=False + ) + assert requested.returncode == 0, requested.stderr + assert json.loads(requested.stdout) == {"run_id": run_id, "applied": expected_applied} + completed: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, run_id=run_id + ) + assert completed["status"] == test_case.expected_status, worker.log.read_text() + assert completed["snapshot_id"] == source["snapshot_id"] + assert completed["selection_json"] == ["model:broken_orders", "model:totals"] + assert completed["node_counts_json"] == {"success": 2} + with sqlite3.connect(worker.control_store) as control: + children: list[tuple[str, str]] = control.execute( + "SELECT job_key, selection_json FROM sqb_runs WHERE parent_run_id=? ORDER BY job_key", + (run_id,), + ).fetchall() + assert [(name, json.loads(selection)) for name, selection in children] == [ + ("marts", ["model:totals"]), + ("staging", ["model:broken_orders"]), + ] + assert control.execute( + "SELECT count(*) FROM sqb_security_audit WHERE action='run.rerun_requested'" + ).fetchone() == (1,) + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + assert warehouse.sql("SELECT amount FROM inventory").fetchall() == [(71,)] From 6c07dab58e224160f6161387fb3fbb3c10457f6c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:13:14 +0000 Subject: [PATCH 38/59] test: complete scheduled-function DuckShop coverage Exercise the inherited seed and scalar function together, attach the required not-null audit and assert concrete SQL unit-test results. The failing DuckShop compile checks now pass without relaxing any Rules or changing the core benchmark checksum. Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../edge_cases/rpt_edge_scheduled_function.sql | 11 ++++++++++- .../test_rpt_edge_scheduled_function.sql | 15 +++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_scheduled_function.sql diff --git a/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_scheduled_function.sql b/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_scheduled_function.sql index 5e33820e53..f86364144c 100644 --- a/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_scheduled_function.sql +++ b/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_scheduled_function.sql @@ -2,6 +2,15 @@ MODEL ( description 'A scheduled model whose implicit scalar-function writer must be claimed.', tags [edge_case], materialized table, + columns ( + quantity (type INTEGER, nullable false, audits [not_null]), + ), ); -SELECT __udf("edge_next_quantity")(CAST(23 AS INTEGER)) AS quantity +WITH final AS ( + SELECT __udf("edge_next_quantity")(q.quantity) AS quantity + FROM __seed("edge_quantity_steps") AS q +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_scheduled_function.sql b/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_scheduled_function.sql new file mode 100644 index 0000000000..8db1011dad --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_scheduled_function.sql @@ -0,0 +1,15 @@ +TEST (name "rpt_edge_scheduled_function__advances_the_inherited_function_quantity"); + +WITH __seed__edge_quantity_steps AS ( + SELECT v.quantity + FROM VALUES + (-2), + (23) AS v(quantity) +), + +__expected__rpt_edge_scheduled_function AS ( + SELECT v.quantity + FROM VALUES + (-1), + (24) AS v(quantity) +) From 2303d6cc587fb6509bc893d3700ba9cfbdbd34e4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:58:25 +0000 Subject: [PATCH 39/59] feat: recover Linux local workers (WIP) Retain inherited claim locks through scheduler death and verify physical absence before releasing claims. Acquire ownership before fenced launch publication so a delayed stale scheduler cannot spawn after recovery. WIP: local restart recovery is Linux-only. The backend remains incomplete and the static gate is red on six existing test/API-boundary findings; no checks are bypassed or relaxed. Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- Cargo.lock | 11 ++ Cargo.toml | 1 + .../orchestration/local_worker_recovery.sql | 3 + crates/sqlbuild-cli/Cargo.toml | 3 + .../src/workers/_helpers/child.rs | 6 + .../src/workers/_helpers/local.rs | 128 ++++++++++++ .../sqlbuild-cli/src/workers/_helpers/mod.rs | 2 + .../src/workers/_helpers/recovery.rs | 28 ++- .../src/workers/_helpers/supervisor.rs | 43 ++-- crates/sqlbuild-cli/src/workers/constants.rs | 4 + crates/sqlbuild-cli/src/workers/models.rs | 11 ++ .../publication/test_local_recovery.py | 184 ++++++++++++++++++ 12 files changed, 402 insertions(+), 22 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/local_worker_recovery.sql create mode 100644 crates/sqlbuild-cli/src/workers/_helpers/local.rs create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py diff --git a/Cargo.lock b/Cargo.lock index dbef530346..d3c2ae6c05 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -348,6 +348,16 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2550f75b8cfac212855f6b1885455df8eaee8fe8e246b647d69146142e016084" +[[package]] +name = "command-fds" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b60b5124979fccd9addd89d8b97a1d6eebb4950694520c75ddd722535ea443f" +dependencies = [ + "nix 0.31.3", + "thiserror 2.0.21", +] + [[package]] name = "command-group" version = "5.0.1" @@ -2493,6 +2503,7 @@ version = "0.186.2" dependencies = [ "blake3", "chrono", + "command-fds", "ctrlc", "getrandom 0.4.3", "reqwest", diff --git a/Cargo.toml b/Cargo.toml index 61061e23b9..fd65e8af7b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -99,6 +99,7 @@ syn-v3-macro-features = { package = "syn", version = "3", default-features = fal thiserror = "2" # Windows only: `sqb` ignores Ctrl+C while the Python CLI it started handles it. ctrlc = "3" +command-fds = "0.3" quote = "1" rayon = "1" # Static reading of project Python source; the parser fensu uses. diff --git a/benchmarks/duckshop/edge_cases/orchestration/local_worker_recovery.sql b/benchmarks/duckshop/edge_cases/orchestration/local_worker_recovery.sql new file mode 100644 index 0000000000..fe752e60d2 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/local_worker_recovery.sql @@ -0,0 +1,3 @@ +-- A long-running statement for scheduler crash, adoption and cancellation checks. +-- The supervising test cancels this query; it is not part of benchmark measurements. +SELECT sum(sin(value)) AS amount FROM range(10000000000) t(value) diff --git a/crates/sqlbuild-cli/Cargo.toml b/crates/sqlbuild-cli/Cargo.toml index 5568005174..cac9dd214d 100644 --- a/crates/sqlbuild-cli/Cargo.toml +++ b/crates/sqlbuild-cli/Cargo.toml @@ -36,6 +36,9 @@ uuid.workspace = true [target.'cfg(unix)'.dependencies] rustix.workspace = true +[target.'cfg(target_os = "linux")'.dependencies] +command-fds.workspace = true + [target.'cfg(windows)'.dependencies] ctrlc.workspace = true same-file.workspace = true diff --git a/crates/sqlbuild-cli/src/workers/_helpers/child.rs b/crates/sqlbuild-cli/src/workers/_helpers/child.rs index a6ca868c82..75d44e336f 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/child.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/child.rs @@ -22,6 +22,8 @@ impl WorkerChild { Self::Local(child) => Ok(child .try_wait()? .map(|exit| WorkerExit::Code(exit_code(exit)))), + #[cfg(target_os = "linux")] + Self::RecoveredLocal(child) => child.try_wait(), Self::Docker(child) => child.try_wait().await, } } @@ -41,6 +43,8 @@ impl WorkerChild { pub(in crate::workers) async fn terminate(&mut self) -> Result<(), ControlError> { match self { Self::Local(child) => terminate(child), + #[cfg(target_os = "linux")] + Self::RecoveredLocal(child) => child.signal(rustix::process::Signal::TERM), Self::Docker(child) => child.signal("SIGTERM").await, } } @@ -51,6 +55,8 @@ impl WorkerChild { child.start_kill()?; Ok(()) } + #[cfg(target_os = "linux")] + Self::RecoveredLocal(child) => child.signal(rustix::process::Signal::KILL), Self::Docker(child) => child.signal("SIGKILL").await, } } diff --git a/crates/sqlbuild-cli/src/workers/_helpers/local.rs b/crates/sqlbuild-cli/src/workers/_helpers/local.rs new file mode 100644 index 0000000000..82a1c4fba2 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/_helpers/local.rs @@ -0,0 +1,128 @@ +//! Linux dev workers inherit a claim lock before exec; recovery never guesses from a stale PID. + +use std::fs::OpenOptions; +use std::os::unix::fs::MetadataExt; + +use command_fds::{CommandFdExt, FdMapping}; +use rustix::fs::{FlockOperation, flock}; +use rustix::process::{Pid, PidfdFlags, Signal, pidfd_open, pidfd_send_signal}; +use sqlbuild_core::command_output::constants::INVOCATION_CONTEXT_ENV; +use sqlbuild_orchestration::runs::models::RunClaim; +use sqlbuild_orchestration::store::errors::ControlError; +use tokio::process::Command; + +use crate::workers::constants::LOCAL_CLAIM_FD; +use crate::workers::models::{LocalLauncher, LocalWorker, WorkerExit}; + +impl LocalWorker { + pub(in crate::workers) fn open( + location: &LocalLauncher, + claim: &RunClaim, + ) -> Result { + let directory = location.project_dir.join(".sqlbuild/workers"); + std::fs::create_dir_all(&directory)?; + let lock = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(directory.join(format!("{}.lock", claim.claim_id)))?; + Ok(Self { + lock, + run_id: claim.request.run_id, + stopped: false, + }) + } + + /// Keep the acquired lock until terminal publication, so a delayed spawn cannot start afterward. + pub(in crate::workers) fn try_wait(&mut self) -> Result, ControlError> { + if !self.stopped { + match flock(&self.lock, FlockOperation::NonBlockingLockExclusive) { + Ok(()) => self.stopped = true, + Err(rustix::io::Errno::WOULDBLOCK) => return Ok(None), + Err(error) => return Err(std::io::Error::from(error).into()), + } + } + Ok(Some(WorkerExit::Absent)) + } + + /// Only a kernel handle whose current descriptor and invocation match this claim can be signalled. + pub(in crate::workers) fn signal(&self, signal: Signal) -> Result<(), ControlError> { + let expected = self.lock.metadata()?; + for entry in std::fs::read_dir("/proc")? { + let entry = entry?; + let name = entry.file_name(); + let Some(name) = name.to_str() else { + continue; + }; + let raw = match name.parse::() { + Ok(raw) => raw, + Err(_) => continue, + }; + let Some(pid) = Pid::from_raw(raw) else { + continue; + }; + let descriptor = entry.path().join(format!("fd/{LOCAL_CLAIM_FD}")); + if !same_file(&descriptor, &expected) { + continue; + } + let handle = match pidfd_open(pid, PidfdFlags::empty()) { + Ok(handle) => handle, + Err(rustix::io::Errno::SRCH) => continue, + Err(error) => return Err(std::io::Error::from(error).into()), + }; + if !same_file(&descriptor, &expected) || !self.owns_invocation(&entry.path())? { + continue; + } + match pidfd_send_signal(handle, signal) { + Ok(()) | Err(rustix::io::Errno::SRCH) => {} + Err(error) => return Err(std::io::Error::from(error).into()), + } + } + Ok(()) + } + + fn owns_invocation(&self, process: &std::path::Path) -> Result { + let environment = match std::fs::read(process.join("environ")) { + Ok(environment) => environment, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error.into()), + }; + let prefix = format!("{INVOCATION_CONTEXT_ENV}="); + for variable in environment.split(|byte| *byte == 0) { + if let Some(context) = variable.strip_prefix(prefix.as_bytes()) { + let context: serde_json::Value = + serde_json::from_slice(context).map_err(|error| { + ControlError::Invalid(format!( + "invalid local worker invocation context: {error}" + )) + })?; + return Ok(context["run_id"].as_str() == Some(&self.run_id.to_string())); + } + } + Ok(false) + } +} + +fn same_file(path: &std::path::Path, expected: &std::fs::Metadata) -> bool { + std::fs::metadata(path) + .is_ok_and(|actual| actual.dev() == expected.dev() && actual.ino() == expected.ino()) +} + +/// The child inherits the lock even before Python starts or a worker handshake can succeed. +pub(in crate::workers) fn inherit_claim( + command: &mut Command, + location: &LocalLauncher, + claim: &RunClaim, +) -> Result<(), ControlError> { + let owner = LocalWorker::open(location, claim)?; + flock(&owner.lock, FlockOperation::NonBlockingLockExclusive).map_err(std::io::Error::from)?; + command + .as_std_mut() + .fd_mappings(vec![FdMapping { + parent_fd: owner.lock.into(), + child_fd: LOCAL_CLAIM_FD, + }]) + .map_err(|error| ControlError::Invalid(format!("local claim descriptor: {error}")))?; + Ok(()) +} diff --git a/crates/sqlbuild-cli/src/workers/_helpers/mod.rs b/crates/sqlbuild-cli/src/workers/_helpers/mod.rs index b182827d2d..a0ead33080 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/mod.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/mod.rs @@ -2,6 +2,8 @@ mod child; pub(super) mod configuration; mod docker; mod invocation; +#[cfg(target_os = "linux")] +mod local; mod recovery; mod shutdown; mod supervisor; diff --git a/crates/sqlbuild-cli/src/workers/_helpers/recovery.rs b/crates/sqlbuild-cli/src/workers/_helpers/recovery.rs index 1704f88f6b..c54d87b613 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/recovery.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/recovery.rs @@ -9,7 +9,7 @@ use sqlbuild_orchestration::scheduler::models::SchedulerLease; use sqlbuild_orchestration::store::errors::ControlError; use sqlbuild_orchestration::store::models::ControlStore; -use crate::workers::models::{ClaimedWorker, DockerWorker, WorkerChild, Workers}; +use crate::workers::models::{ClaimedWorker, DockerWorker, Launcher, WorkerChild, Workers}; impl Workers { pub(in crate::workers) async fn recover( @@ -25,12 +25,17 @@ impl Workers { .iter() .map(|worker| worker.claim.claim_id) .collect(); - for location in &self.configuration.docker { + for location in self.configuration.locations() { + #[cfg(not(target_os = "linux"))] + if matches!(location, Launcher::Local(_)) { + continue; + } + let scope = location.scope(); let mut after = None; loop { renew_lease(store, lease).await?; let claims = - list_run_claims(store, (location.location_id, &location.target), after).await?; + list_run_claims(store, (scope.location_id, scope.target), after).await?; let Some(last) = claims.last() else { break; }; @@ -39,10 +44,19 @@ impl Workers { if !known.insert(claim.claim_id) { continue; } - let child = match DockerWorker::prepare(location, &claim) { - Ok(child) => Some(WorkerChild::Docker(child)), - Err(ControlError::Invalid(_)) => None, - Err(error) => return Err(error), + let child = match location { + #[cfg(target_os = "linux")] + Launcher::Local(location) => Some(WorkerChild::RecoveredLocal( + crate::workers::models::LocalWorker::open(location, &claim)?, + )), + #[cfg(not(target_os = "linux"))] + Launcher::Local(_) => continue, + Launcher::Docker(location) => match DockerWorker::prepare(location, &claim) + { + Ok(child) => Some(WorkerChild::Docker(child)), + Err(ControlError::Invalid(_)) => None, + Err(error) => return Err(error), + }, }; drop(writeln!( std::io::stderr().lock(), diff --git a/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs b/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs index b46fe96bab..b9aa97e091 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs @@ -219,6 +219,15 @@ impl Workers { Utc::now() + TimeDelta::days(7), ) .await?; + let local_command = match location { + Launcher::Local(location) => Some(prepare_spawn( + location, + claim, + &credential.bearer, + &self.control_variable, + )?), + Launcher::Docker(_) => None, + }; publish( store, lease, @@ -226,16 +235,17 @@ impl Workers { ("local-launched", RunChange::Launched { container_id }), ) .await?; - let launched = match location { - Launcher::Local(location) => { - spawn(location, claim, &credential.bearer, &self.control_variable).map( - |child| { - worker.child = Some(WorkerChild::Local(child)); - }, - ) - } - Launcher::Docker(location) => { - if let Some(WorkerChild::Docker(child)) = &worker.child { + let launched = match local_command { + Some(mut command) => command + .spawn() + .map(|child| { + worker.child = Some(WorkerChild::Local(child)); + }) + .map_err(ControlError::from), + None => { + if let (Launcher::Docker(location), Some(WorkerChild::Docker(child))) = + (location, &worker.child) + { child .launch(location, claim, &credential.bearer, &self.control_variable) .await?; @@ -275,13 +285,14 @@ impl Workers { } } -fn spawn( +fn prepare_spawn( location: &LocalLauncher, claim: &RunClaim, bearer: &str, control_variable: &str, -) -> Result { - Ok(Command::new(&location.executable) +) -> Result { + let mut command = Command::new(&location.executable); + command .args(arguments(&Launcher::Local(location).scope(), claim)) .arg(&location.project_dir) .current_dir(&location.project_dir) @@ -294,8 +305,10 @@ fn spawn( .stdin(Stdio::null()) .stdout(Stdio::null()) .stderr(Stdio::inherit()) - .kill_on_drop(true) - .spawn()?) + .kill_on_drop(true); + #[cfg(target_os = "linux")] + crate::workers::_helpers::local::inherit_claim(&mut command, location, claim)?; + Ok(command) } pub(in crate::workers) async fn publish( diff --git a/crates/sqlbuild-cli/src/workers/constants.rs b/crates/sqlbuild-cli/src/workers/constants.rs index 881c48bea1..06a889dbfb 100644 --- a/crates/sqlbuild-cli/src/workers/constants.rs +++ b/crates/sqlbuild-cli/src/workers/constants.rs @@ -1,3 +1,7 @@ //! Immutable Docker image identities use the full SHA-256 hexadecimal digest. pub(super) const IMAGE_ID_HEX_LENGTH: usize = 64; + +/// Keep inherited claim ownership away from standard IO and the runtime's usual low descriptors. +#[cfg(target_os = "linux")] +pub(super) const LOCAL_CLAIM_FD: i32 = 198; diff --git a/crates/sqlbuild-cli/src/workers/models.rs b/crates/sqlbuild-cli/src/workers/models.rs index 3cd00b537f..415247a7f7 100644 --- a/crates/sqlbuild-cli/src/workers/models.rs +++ b/crates/sqlbuild-cli/src/workers/models.rs @@ -13,9 +13,20 @@ use uuid::Uuid; #[derive(Debug)] pub(super) enum WorkerChild { Local(Child), + #[cfg(target_os = "linux")] + RecoveredLocal(LocalWorker), Docker(DockerWorker), } +/// The inherited lock proves physical ownership without relying on a remembered process ID. +#[cfg(target_os = "linux")] +#[derive(Debug)] +pub(super) struct LocalWorker { + pub(super) lock: std::fs::File, + pub(super) run_id: Uuid, + pub(super) stopped: bool, +} + #[derive(Clone, Copy, Debug)] pub(super) enum WorkerExit { Code(i64), diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py new file mode 100644 index 0000000000..938a1b34b3 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py @@ -0,0 +1,184 @@ +"""A replacement Linux dev scheduler retains physical ownership of surviving local workers.""" + +import os +import signal +import sqlite3 +import sys +import time +from contextlib import suppress +from pathlib import Path +from types import ModuleType +from typing import Any + +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + ScheduledWorker, + WorkerExecutionTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + cancel_running_statement, + location_process, + running_statement, + scheduled_worker, + wait_for_interpreter, + wait_for_log, + wait_for_run, + worker_result, +) + +fcntl: ModuleType = pytest.importorskip("fcntl") + + +@pytest.mark.skipif(sys.platform != "linux", reason="Linux inherited-lock and pidfd recovery") +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "scheduler SIGKILL does not orphan a running local warehouse statement", + Path( + "benchmarks/duckshop/edge_cases/orchestration/local_worker_recovery.sql" + ).read_text(), + "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'", + "cancelled", + [(0,)], + ) + ], + ids=lambda case: case.description, +) +def test_given_a_running_local_worker_when_scheduler_dies_then_restart_adopts_and_cancels_without_relaunch( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + active: dict[str, Any] = running_statement(worker) + with sqlite3.connect(worker.control_store) as control: + claim_id: str = control.execute( + "SELECT worker_instance FROM sqb_run_liveness WHERE run_id=?", (active["run_id"],) + ).fetchone()[0] + lock: Path = worker.project / ".sqlbuild/workers" / f"{claim_id}.lock" + with lock.open("rb") as ownership: + with pytest.raises(BlockingIOError): + fcntl.flock(ownership, fcntl.LOCK_EX | fcntl.LOCK_NB) + worker.process.kill() + assert worker.process.wait(timeout=10) == -signal.SIGKILL + with pytest.raises(BlockingIOError): + fcntl.flock(ownership, fcntl.LOCK_EX | fcntl.LOCK_NB) + log: Path = tmp_path / "replacement-scheduler.log" + with location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "scheduler", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "--launchers", + str(tmp_path / "launchers.json"), + ], + log, + tmp_path / "replacement-scheduler.stdout", + ) as process: + replacement: ScheduledWorker = worker._replace(process=process, log=log) + wait_for_log(process, log, "Worker supervision recovered for run", timeout=45) + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == ( + 1, + ) + cancel_running_statement(replacement) + run: dict[str, Any] = worker_result(replacement, test_case) + assert run["status"] == test_case.expected_status, log.read_text() + with sqlite3.connect(worker.control_store) as control: + assert control.execute( + "SELECT count(*) FROM recall_events WHERE event_type='run_claimed'" + ).fetchone() == (1,) + deadline: float = time.monotonic() + 10 + while True: + try: + fcntl.flock(ownership, fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + assert time.monotonic() < deadline, log.read_text() + time.sleep(0.02) + + +@pytest.mark.skipif(sys.platform != "linux", reason="Linux inherited-lock and pidfd recovery") +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "pre-handshake child survives restart and failed terminal publication retains its claim", + "SELECT 23 AS amount", + "", + "failed", + [], + launcher="uncooperative", + store_setup=( + "CREATE TRIGGER reject_terminal BEFORE UPDATE OF status ON sqb_runs " + "WHEN NEW.status='failed' " + "BEGIN SELECT RAISE(ABORT, 'synthetic terminal publication failure'); END;" + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_an_unstarted_orphan_when_recovery_publication_fails_then_claims_wait_for_physical_exit_and_commit( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + pid: int = wait_for_interpreter(worker.process, worker.project / "worker.pid", worker.log) + active: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, pending=frozenset({"queued"}) + ) + assert active["status"] == "launched" + handle: int = os.pidfd_open(pid) + try: + worker.process.kill() + assert worker.process.wait(timeout=10) == -signal.SIGKILL + log: Path = tmp_path / "replacement-scheduler.log" + with location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "scheduler", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "--launchers", + str(tmp_path / "launchers.json"), + ], + log, + tmp_path / "replacement-scheduler.stdout", + ) as process: + wait_for_log(process, log, "Worker supervision recovered for run", timeout=45) + signal.pidfd_send_signal(handle, 0) + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == ( + 1, + ) + assert control.execute("SELECT status FROM sqb_runs").fetchone() == ( + "launched", + ) + signal.pidfd_send_signal(handle, signal.SIGKILL) + wait_for_log(process, log, "Retrying worker shutdown publication") + assert process.poll() is None, log.read_text() + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == ( + 1, + ) + _ = control.execute("DROP TRIGGER reject_terminal") + assert process.wait(timeout=10) != 0, log.read_text() + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT status FROM sqb_runs").fetchone() == ( + test_case.expected_status, + ) + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == ( + 0, + ) + assert control.execute( + "SELECT count(*) FROM recall_events WHERE event_type='run_terminated'" + ).fetchone() == (1,) + assert control.execute( + "SELECT json_extract(payload, '$.change.change.exit_code') FROM recall_events " + "WHERE event_type='run_terminated'" + ).fetchone() == (None,) + assert not (worker.project / "warehouse.duckdb").exists() + finally: + with suppress(ProcessLookupError): + signal.pidfd_send_signal(handle, signal.SIGKILL) + os.close(handle) From b601a6ea8384397c7e42a8f2e18fe45c68a94f5c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 17:26:09 +0000 Subject: [PATCH 40/59] refactor: keep scheduler internals owner-local Remove test-only production convenience exports and retain the real SQLite/PostgreSQL contracts in owner-local test harnesses. Exercise CLI worker lease setup through the public scheduler loop. Shared fixtures are declared once under their owners. Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../tests/native_commands/worker/helpers.rs | 57 +++++++++++++++++-- .../native_commands/worker/test_types.rs | 7 +++ crates/sqlbuild-orchestration/src/api/mod.rs | 4 ++ .../src/credentials/mod.rs | 4 ++ .../src/platform/mod.rs | 4 ++ .../src/runs/main/mod.rs | 1 - .../src/runs/main/request_run.rs | 16 ------ crates/sqlbuild-orchestration/src/runs/mod.rs | 4 ++ .../scheduler/_helpers/drain_due_triggers.rs | 2 +- .../{acquire_lease.rs => _acquire_lease.rs} | 2 +- ...aluate_trigger.rs => _evaluate_trigger.rs} | 2 +- ...scheduler_cycle.rs => _scheduler_cycle.rs} | 4 +- .../src/scheduler/main/mod.rs | 6 +- .../src/scheduler/main/run_scheduler.rs | 2 +- .../src/scheduler/mod.rs | 2 +- .../src/scheduler/tests.rs | 2 + .../src/snapshots/mod.rs | 4 ++ .../src/store/main/mod.rs | 2 - .../src/store/main/publish_recall.rs | 25 -------- .../src/store/main/read_recall_events.rs | 28 --------- .../sqlbuild-orchestration/src/store/mod.rs | 4 ++ crates/sqlbuild-orchestration/tests/api.rs | 28 --------- .../tests/api/helpers.rs | 7 +-- .../tests/api/output/helpers.rs | 10 ++-- .../tests/api/recall/helpers.rs | 12 ++-- .../tests/api/test_contract.rs | 48 ++++++++-------- .../tests/api/test_contracts.rs | 22 +++++++ .../tests/api/test_location_health.rs | 28 +++++---- .../tests/api/test_location_work.rs | 38 ++++++------- .../tests/api/test_reads.rs | 49 ++++++++-------- .../tests/api/test_types.rs | 7 +-- .../tests/api/test_worker.rs | 46 +++++++-------- .../tests/api/work/helpers.rs | 2 +- .../tests/credentials.rs | 20 ------- .../tests/credentials/helpers.rs | 2 +- .../tests/credentials/locations/helpers.rs | 8 +-- .../tests/credentials/test_contract.rs | 36 ++++++------ .../tests/credentials/test_contracts.rs | 10 ++++ .../tests/credentials/test_types.rs | 4 +- .../sqlbuild-orchestration/tests/platform.rs | 14 ----- .../tests/platform/helpers.rs | 12 ++-- .../tests/platform/test_contracts.rs | 8 +++ .../tests/platform/test_types.rs | 4 +- .../tests/platform/test_watchdog.rs | 22 +++---- crates/sqlbuild-orchestration/tests/runs.rs | 42 -------------- .../tests/runs/claims/helpers.rs | 6 +- .../tests/runs/helpers.rs | 23 ++++++-- .../tests/runs/monitor/helpers.rs | 14 ++--- .../tests/runs/termination/helpers.rs | 10 ++-- .../tests/runs/test_claims.rs | 40 +++++++------ .../tests/runs/test_coalesce.rs | 25 ++++---- .../tests/runs/test_contracts.rs | 28 +++++++++ .../tests/runs/test_faults.rs | 34 +++++------ .../tests/runs/test_group_admission.rs | 28 ++++----- .../tests/runs/test_groups.rs | 31 +++++----- .../tests/runs/test_lifecycle.rs | 34 +++++------ .../tests/runs/test_monitor.rs | 47 ++++++++------- .../tests/runs/test_requests.rs | 30 +++++----- .../tests/runs/test_reruns.rs | 30 +++++----- .../tests/runs/test_types.rs | 6 +- .../sqlbuild-orchestration/tests/scheduler.rs | 22 ------- .../tests/scheduler/helpers.rs | 20 ++++--- .../tests/scheduler/test_contract.rs | 8 +-- .../tests/scheduler/test_contracts.rs | 10 ++++ .../tests/scheduler/test_types.rs | 2 +- .../tests/scheduler/ticks/helpers.rs | 38 ++++++------- .../tests/scheduler/ticks/test_contract.rs | 8 +-- .../tests/scheduler/ticks/test_loop.rs | 37 ++++++------ .../tests/scheduler/ticks/test_triggers.rs | 34 +++++------ .../tests/scheduler/ticks/test_types.rs | 18 +++--- .../sqlbuild-orchestration/tests/snapshots.rs | 14 ----- .../tests/snapshots/helpers.rs | 24 ++++---- .../tests/snapshots/test_contract.rs | 8 +-- .../tests/snapshots/test_contracts.rs | 8 +++ .../tests/snapshots/test_types.rs | 2 +- crates/sqlbuild-orchestration/tests/store.rs | 16 ------ .../tests/store/faults/helpers.rs | 4 +- .../tests/store/helpers.rs | 6 +- .../tests/store/leases/helpers.rs | 2 +- .../tests/store/recall/helpers.rs | 51 +++++++++++++++-- .../tests/store/recall/test_contract.rs | 10 ++-- .../tests/store/recall/test_types.rs | 2 +- .../tests/store/recall/test_views.rs | 12 ++-- .../tests/store/replicas/helpers.rs | 4 +- .../tests/store/test_contracts.rs | 22 +++++++ .../tests/store/test_types.rs | 2 +- 86 files changed, 714 insertions(+), 717 deletions(-) delete mode 100644 crates/sqlbuild-orchestration/src/runs/main/request_run.rs rename crates/sqlbuild-orchestration/src/scheduler/main/{acquire_lease.rs => _acquire_lease.rs} (97%) rename crates/sqlbuild-orchestration/src/scheduler/main/{evaluate_trigger.rs => _evaluate_trigger.rs} (98%) rename crates/sqlbuild-orchestration/src/scheduler/main/{scheduler_cycle.rs => _scheduler_cycle.rs} (96%) delete mode 100644 crates/sqlbuild-orchestration/src/store/main/publish_recall.rs delete mode 100644 crates/sqlbuild-orchestration/src/store/main/read_recall_events.rs delete mode 100644 crates/sqlbuild-orchestration/tests/api.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/test_contracts.rs delete mode 100644 crates/sqlbuild-orchestration/tests/credentials.rs create mode 100644 crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs delete mode 100644 crates/sqlbuild-orchestration/tests/platform.rs create mode 100644 crates/sqlbuild-orchestration/tests/platform/test_contracts.rs delete mode 100644 crates/sqlbuild-orchestration/tests/runs.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/test_contracts.rs delete mode 100644 crates/sqlbuild-orchestration/tests/scheduler.rs create mode 100644 crates/sqlbuild-orchestration/tests/scheduler/test_contracts.rs delete mode 100644 crates/sqlbuild-orchestration/tests/snapshots.rs create mode 100644 crates/sqlbuild-orchestration/tests/snapshots/test_contracts.rs delete mode 100644 crates/sqlbuild-orchestration/tests/store.rs create mode 100644 crates/sqlbuild-orchestration/tests/store/test_contracts.rs diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs index d9635b34d3..5a5a7fc4a0 100644 --- a/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs +++ b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs @@ -11,10 +11,14 @@ use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope, use sqlbuild_orchestration::runs::main::change_run::change_run; use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; use sqlbuild_orchestration::runs::models::{RunAuthority, RunChange, RunChangeRequest}; -use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler; +use sqlbuild_orchestration::scheduler::models::{SchedulerLease, SchedulerState}; +use sqlbuild_orchestration::scheduler::types::RunLauncher; use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; +use sqlbuild_orchestration::store::errors::ControlError; use sqlbuild_orchestration::store::main::open_store::open_store; +use sqlbuild_orchestration::store::models::ControlStore; use sqlbuild_orchestration::worker::main::connect_worker::connect_worker; use sqlbuild_orchestration::worker::main::start_worker::start_worker; use sqlbuild_orchestration::worker::models::WorkerSession; @@ -23,7 +27,33 @@ use uuid::Uuid; use crate::admin_helpers::administer; use crate::helpers::isolated_binary; use crate::test_types::TestControl; -use crate::worker::test_types::WorkerControl; +use crate::worker::test_types::{LeaseCapture, WorkerControl}; + +impl RunLauncher for LeaseCapture { + async fn poll( + &mut self, + _store: &ControlStore, + lease: Option<&SchedulerLease>, + ) -> Result<(), ControlError> { + if let Some(lease) = lease { + self.lease + .take() + .expect("first scheduler lease") + .send(lease.clone()) + .expect("lease receiver"); + self.shutdown.send(true).expect("scheduler shutdown"); + } + Ok(()) + } + + async fn shutdown( + &mut self, + _store: &ControlStore, + _lease: Option<&SchedulerLease>, + ) -> Result<(), ControlError> { + Ok(()) + } +} pub(crate) async fn start_claimed_worker( control: &TestControl, @@ -120,10 +150,25 @@ pub(crate) async fn claim_worker_run( [&run], |row| row.get(0)).expect("trusted manual request audit"); assert_eq!(count, 1); drop(connection); - let lease = acquire_lease(&store, Uuid::now_v7()) - .await - .expect("acquire lease") - .expect("leader"); + let (lease_sender, lease_receiver) = tokio::sync::oneshot::channel(); + let (shutdown, signal) = tokio::sync::watch::channel(false); + let (_changes, changes) = tokio::sync::watch::channel(()); + tokio::time::timeout( + Duration::from_secs(5), + run_scheduler( + &store, + SchedulerState::new(Uuid::now_v7()), + (signal, changes), + LeaseCapture { + lease: Some(lease_sender), + shutdown, + }, + ), + ) + .await + .expect("scheduler acquires leadership promptly") + .expect("scheduler shutdown"); + let lease = lease_receiver.await.expect("public scheduler leadership"); let claim = claim_next_run( &store, &lease, diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs b/crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs index 54d23d6116..81ecbc260f 100644 --- a/crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs +++ b/crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs @@ -1,5 +1,12 @@ use crate::test_types::TestControl; +pub(crate) struct LeaseCapture { + pub(crate) lease: Option< + tokio::sync::oneshot::Sender, + >, + pub(crate) shutdown: tokio::sync::watch::Sender, +} + pub(crate) struct WorkerHeartbeatTestCase { pub(crate) description: &'static str, pub(crate) interruption: fn(WorkerControl, &mut TestControl), diff --git a/crates/sqlbuild-orchestration/src/api/mod.rs b/crates/sqlbuild-orchestration/src/api/mod.rs index 951756c642..cb776dbca6 100644 --- a/crates/sqlbuild-orchestration/src/api/mod.rs +++ b/crates/sqlbuild-orchestration/src/api/mod.rs @@ -5,3 +5,7 @@ pub(crate) mod constants; pub mod errors; pub mod main; pub mod models; + +#[cfg(test)] +#[path = "../../tests/api/test_contracts.rs"] +mod tests; diff --git a/crates/sqlbuild-orchestration/src/credentials/mod.rs b/crates/sqlbuild-orchestration/src/credentials/mod.rs index a3ce0d1f39..acde203166 100644 --- a/crates/sqlbuild-orchestration/src/credentials/mod.rs +++ b/crates/sqlbuild-orchestration/src/credentials/mod.rs @@ -4,3 +4,7 @@ pub(crate) mod _helpers; pub(crate) mod constants; pub mod main; pub mod models; + +#[cfg(test)] +#[path = "../../tests/credentials/test_contracts.rs"] +pub(crate) mod tests; diff --git a/crates/sqlbuild-orchestration/src/platform/mod.rs b/crates/sqlbuild-orchestration/src/platform/mod.rs index 5359000d16..32d92d3cc2 100644 --- a/crates/sqlbuild-orchestration/src/platform/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/mod.rs @@ -1,3 +1,7 @@ pub(crate) mod _helpers; pub mod main; pub mod models; + +#[cfg(test)] +#[path = "../../tests/platform/test_contracts.rs"] +mod tests; diff --git a/crates/sqlbuild-orchestration/src/runs/main/mod.rs b/crates/sqlbuild-orchestration/src/runs/main/mod.rs index a69509d123..6359a19779 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/mod.rs @@ -13,5 +13,4 @@ pub mod publish_run_recall; pub(crate) mod queue_run; pub mod request_job_run; pub mod request_rerun; -pub mod request_run; pub mod run_execution; diff --git a/crates/sqlbuild-orchestration/src/runs/main/request_run.rs b/crates/sqlbuild-orchestration/src/runs/main/request_run.rs deleted file mode 100644 index f3041afbbc..0000000000 --- a/crates/sqlbuild-orchestration/src/runs/main/request_run.rs +++ /dev/null @@ -1,16 +0,0 @@ -//! Queue a frozen, already authorized request without recreating work on transport retries. - -use crate::runs::_helpers::requests::publication::publish_request; -use crate::runs::models::RunRequest; -use crate::store::errors::ControlError; -use crate::store::main::begin::begin; -use crate::store::main::commit::commit; -use crate::store::models::{ControlStore, TransactionMode}; - -/// The API must compute and authorize this selection; callers cannot supply their own write scope. -pub async fn request_run(store: &ControlStore, request: &RunRequest) -> Result { - let transaction = begin(store, TransactionMode::Write).await?; - let (transaction, applied) = publish_request(transaction, request).await?; - commit(transaction).await?; - Ok(applied) -} diff --git a/crates/sqlbuild-orchestration/src/runs/mod.rs b/crates/sqlbuild-orchestration/src/runs/mod.rs index f147865646..529a59766a 100644 --- a/crates/sqlbuild-orchestration/src/runs/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/mod.rs @@ -2,3 +2,7 @@ pub(crate) mod _helpers; pub(crate) mod constants; pub mod main; pub mod models; + +#[cfg(test)] +#[path = "../../tests/runs/test_contracts.rs"] +pub(crate) mod tests; diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs index ab6a639022..f2edb08a87 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs @@ -2,7 +2,7 @@ use uuid::Uuid; -use crate::scheduler::main::evaluate_trigger::evaluate_trigger; +use crate::scheduler::main::_evaluate_trigger::evaluate_trigger; use crate::scheduler::main::renew_lease::renew_lease; use crate::scheduler::models::{SchedulerCycle, SchedulerLease}; use crate::store::errors::ControlError; diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/acquire_lease.rs b/crates/sqlbuild-orchestration/src/scheduler/main/_acquire_lease.rs similarity index 97% rename from crates/sqlbuild-orchestration/src/scheduler/main/acquire_lease.rs rename to crates/sqlbuild-orchestration/src/scheduler/main/_acquire_lease.rs index 1c1bf2ec6a..20b4286e1b 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/acquire_lease.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/_acquire_lease.rs @@ -12,7 +12,7 @@ use crate::store::main::query::query; use crate::store::models::{ControlStore, Parameter, TransactionMode}; /// An existing live lease is never stolen, including by a duplicate caller with the same holder. -pub async fn acquire_lease( +pub(crate) async fn acquire_lease( store: &ControlStore, holder: Uuid, ) -> Result, ControlError> { diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/evaluate_trigger.rs b/crates/sqlbuild-orchestration/src/scheduler/main/_evaluate_trigger.rs similarity index 98% rename from crates/sqlbuild-orchestration/src/scheduler/main/evaluate_trigger.rs rename to crates/sqlbuild-orchestration/src/scheduler/main/_evaluate_trigger.rs index 7d065991ee..2646e237ee 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/evaluate_trigger.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/_evaluate_trigger.rs @@ -17,7 +17,7 @@ use crate::store::main::query::query; use crate::store::models::{ControlStore, Parameter, TransactionMode}; /// A removed or paused trigger is inert; a lost fence rolls back every write, including Recall. -pub async fn evaluate_trigger( +pub(crate) async fn evaluate_trigger( store: &ControlStore, lease: &SchedulerLease, trigger_id: Uuid, diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/scheduler_cycle.rs b/crates/sqlbuild-orchestration/src/scheduler/main/_scheduler_cycle.rs similarity index 96% rename from crates/sqlbuild-orchestration/src/scheduler/main/scheduler_cycle.rs rename to crates/sqlbuild-orchestration/src/scheduler/main/_scheduler_cycle.rs index 8db0fea9b5..1d94e6944a 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/scheduler_cycle.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/_scheduler_cycle.rs @@ -8,7 +8,7 @@ use crate::runs::constants::MONITOR_BATCH_SIZE; use crate::runs::main::monitor_runs::monitor_runs; use crate::scheduler::_helpers::drain_due_triggers::drain_due_triggers; use crate::scheduler::constants::RENEW_INTERVAL_SECONDS; -use crate::scheduler::main::acquire_lease::acquire_lease; +use crate::scheduler::main::_acquire_lease::acquire_lease; use crate::scheduler::main::renew_lease::renew_lease; use crate::scheduler::models::{SchedulerCycle, SchedulerState}; use crate::store::errors::ControlError; @@ -17,7 +17,7 @@ use crate::store::main::commit::commit; use crate::store::main::query::query; use crate::store::models::{ControlStore, TransactionMode}; -pub async fn scheduler_cycle( +pub(crate) async fn scheduler_cycle( store: &ControlStore, mut state: SchedulerState, ) -> Result<(SchedulerState, SchedulerCycle), ControlError> { diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/mod.rs b/crates/sqlbuild-orchestration/src/scheduler/main/mod.rs index 60225c3875..f91c3d1180 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/mod.rs @@ -1,12 +1,12 @@ +pub(in crate::scheduler) mod _acquire_lease; +pub(in crate::scheduler) mod _evaluate_trigger; pub(in crate::scheduler) mod _next_cron_time; pub(in crate::scheduler) mod _plan_ticks; pub(in crate::scheduler) mod _previous_cron_time; -pub mod acquire_lease; +pub(in crate::scheduler) mod _scheduler_cycle; pub(crate) mod check_lease; -pub mod evaluate_trigger; pub mod list_ticks; pub(crate) mod next_schedule_time; pub mod parse_cron; pub mod renew_lease; pub mod run_scheduler; -pub mod scheduler_cycle; diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs b/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs index b4fd01d5e5..71954a8f71 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs @@ -2,7 +2,7 @@ use tokio::sync::watch; -use crate::scheduler::main::scheduler_cycle::scheduler_cycle; +use crate::scheduler::main::_scheduler_cycle::scheduler_cycle; use crate::scheduler::models::SchedulerState; use crate::scheduler::types::RunLauncher; use crate::store::errors::ControlError; diff --git a/crates/sqlbuild-orchestration/src/scheduler/mod.rs b/crates/sqlbuild-orchestration/src/scheduler/mod.rs index cf840ff555..f5b66b8aca 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/mod.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/mod.rs @@ -7,4 +7,4 @@ pub mod models; pub mod types; #[cfg(test)] -mod tests; +pub(crate) mod tests; diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests.rs b/crates/sqlbuild-orchestration/src/scheduler/tests.rs index e0731c81e8..927f476ce9 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/tests.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/tests.rs @@ -1,3 +1,5 @@ +#[path = "../../tests/scheduler/test_contracts.rs"] +pub(crate) mod contracts; #[path = "tests/helpers.rs"] mod helpers; #[path = "tests/test_backlog.rs"] diff --git a/crates/sqlbuild-orchestration/src/snapshots/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/mod.rs index 5359000d16..04ae56c307 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/mod.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/mod.rs @@ -1,3 +1,7 @@ pub(crate) mod _helpers; pub mod main; pub mod models; + +#[cfg(test)] +#[path = "../../tests/snapshots/test_contracts.rs"] +mod tests; diff --git a/crates/sqlbuild-orchestration/src/store/main/mod.rs b/crates/sqlbuild-orchestration/src/store/main/mod.rs index 715e39afaf..d8119ad010 100644 --- a/crates/sqlbuild-orchestration/src/store/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/store/main/mod.rs @@ -6,8 +6,6 @@ pub(crate) mod decode_row; pub mod list_recall_invocations; pub mod migrate_store; pub mod open_store; -pub mod publish_recall; pub(crate) mod query; -pub mod read_recall_events; pub mod read_recall_view; pub(crate) mod refresh_clock; diff --git a/crates/sqlbuild-orchestration/src/store/main/publish_recall.rs b/crates/sqlbuild-orchestration/src/store/main/publish_recall.rs deleted file mode 100644 index bbe77db9a8..0000000000 --- a/crates/sqlbuild-orchestration/src/store/main/publish_recall.rs +++ /dev/null @@ -1,25 +0,0 @@ -//! Worker publication is atomic; scheduler writes use the same append inside their tick transaction. - -use sqlbuild_recall::records::models::RecallRow; - -use crate::store::errors::ControlError; -use crate::store::main::append_recall::append_recall; -use crate::store::main::begin::begin; -use crate::store::main::commit::commit; -use crate::store::models::{ControlStore, RecallAppendOutcome, TransactionMode}; - -/// Preserve the first copy of each key and record differing retries as Recall conflicts. -pub async fn publish_recall( - store: &ControlStore, - rows: &[RecallRow], -) -> Result, ControlError> { - let mut transaction = begin(store, TransactionMode::Write).await?; - let mut outcomes = Vec::with_capacity(rows.len()); - for row in rows { - let outcome; - (transaction, outcome) = append_recall(transaction, row).await?; - outcomes.push(outcome); - } - commit(transaction).await?; - Ok(outcomes) -} diff --git a/crates/sqlbuild-orchestration/src/store/main/read_recall_events.rs b/crates/sqlbuild-orchestration/src/store/main/read_recall_events.rs deleted file mode 100644 index ec9a4f533d..0000000000 --- a/crates/sqlbuild-orchestration/src/store/main/read_recall_events.rs +++ /dev/null @@ -1,28 +0,0 @@ -//! Stable cursor paging follows serialized committed appends, not producer timestamps. - -use crate::store::errors::ControlError; -use crate::store::main::begin::begin; -use crate::store::main::commit::commit; -use crate::store::main::query::query; -use crate::store::models::{ControlStore, Parameter, TransactionMode}; -use crate::store::types::Row; - -/// Page events in storage order; equal producer timestamps and late arrivals remain visible. -pub async fn read_recall_events( - store: &ControlStore, - after: i64, - limit: u32, -) -> Result, ControlError> { - let transaction = begin(store, TransactionMode::Read).await?; - let (transaction, rows) = query( - transaction, - "SELECT * FROM recall_events WHERE storage_id > $1 ORDER BY storage_id LIMIT $2", - vec![ - Parameter::Integer(after), - Parameter::Integer(i64::from(limit.min(1000))), - ], - ) - .await?; - commit(transaction).await?; - Ok(rows) -} diff --git a/crates/sqlbuild-orchestration/src/store/mod.rs b/crates/sqlbuild-orchestration/src/store/mod.rs index d92242c092..1228df7ae6 100644 --- a/crates/sqlbuild-orchestration/src/store/mod.rs +++ b/crates/sqlbuild-orchestration/src/store/mod.rs @@ -3,3 +3,7 @@ pub mod errors; pub mod main; pub mod models; pub(crate) mod types; + +#[cfg(test)] +#[path = "../../tests/store/test_contracts.rs"] +pub(crate) mod tests; diff --git a/crates/sqlbuild-orchestration/tests/api.rs b/crates/sqlbuild-orchestration/tests/api.rs deleted file mode 100644 index 0b7016321c..0000000000 --- a/crates/sqlbuild-orchestration/tests/api.rs +++ /dev/null @@ -1,28 +0,0 @@ -#[path = "api/helpers.rs"] -mod helpers; -#[path = "credentials/locations/helpers.rs"] -mod location_helpers; -#[path = "api/output/helpers.rs"] -mod output_helpers; -#[path = "api/recall/helpers.rs"] -mod recall_helpers; -#[path = "runs/helpers.rs"] -mod run_helpers; -#[path = "store/helpers.rs"] -mod store_helpers; -#[path = "store/test_types.rs"] -mod store_test_types; -#[path = "api/test_contract.rs"] -mod test_contract; -#[path = "api/test_location_health.rs"] -mod test_location_health; -#[path = "api/test_location_work.rs"] -mod test_location_work; -#[path = "api/test_reads.rs"] -mod test_reads; -#[path = "api/test_types.rs"] -mod test_types; -#[path = "api/test_worker.rs"] -mod test_worker; -#[path = "api/work/helpers.rs"] -mod work_helpers; diff --git a/crates/sqlbuild-orchestration/tests/api/helpers.rs b/crates/sqlbuild-orchestration/tests/api/helpers.rs index bb65bdcf07..9743c15603 100644 --- a/crates/sqlbuild-orchestration/tests/api/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/helpers.rs @@ -1,9 +1,8 @@ use std::time::Duration; -use sqlbuild_orchestration::api::main::serve_api::serve_api; -use sqlbuild_orchestration::store::models::ControlStore; - -use crate::test_types::TestApi; +use crate::api::main::serve_api::serve_api; +use crate::api::tests::test_types::TestApi; +use crate::store::models::ControlStore; pub(crate) async fn start_api(store: &ControlStore) -> TestApi { let listener = tokio::net::TcpListener::bind("127.0.0.1:0") diff --git a/crates/sqlbuild-orchestration/tests/api/output/helpers.rs b/crates/sqlbuild-orchestration/tests/api/output/helpers.rs index 230312a314..6d9f455567 100644 --- a/crates/sqlbuild-orchestration/tests/api/output/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/output/helpers.rs @@ -1,11 +1,11 @@ //! Worker output contract helpers inspect durable bytes on both databases. +use crate::runs::models::OutputPublication; +use crate::store::main::read_recall_view::read_recall_view; +use crate::store::models::ControlStore; +use crate::worker::errors::WorkerError; +use crate::worker::models::WorkerClient; use serde_json::json; -use sqlbuild_orchestration::runs::models::OutputPublication; -use sqlbuild_orchestration::store::main::read_recall_view::read_recall_view; -use sqlbuild_orchestration::store::models::ControlStore; -use sqlbuild_orchestration::worker::errors::WorkerError; -use sqlbuild_orchestration::worker::models::WorkerClient; use sqlbuild_recall::records::models::{OmittedOutput, OutputChunk, OutputStream}; use uuid::Uuid; diff --git a/crates/sqlbuild-orchestration/tests/api/recall/helpers.rs b/crates/sqlbuild-orchestration/tests/api/recall/helpers.rs index edc555b357..d6bf078a67 100644 --- a/crates/sqlbuild-orchestration/tests/api/recall/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/recall/helpers.rs @@ -2,12 +2,12 @@ use std::time::Duration; +use crate::runs::models::{RecallBatch, RecallRecord}; +use crate::store::main::read_recall_view::read_recall_view; +use crate::store::models::ControlStore; +use crate::worker::errors::WorkerError; +use crate::worker::models::WorkerClient; use serde_json::json; -use sqlbuild_orchestration::runs::models::{RecallBatch, RecallRecord}; -use sqlbuild_orchestration::store::main::read_recall_view::read_recall_view; -use sqlbuild_orchestration::store::models::ControlStore; -use sqlbuild_orchestration::worker::errors::WorkerError; -use sqlbuild_orchestration::worker::models::WorkerClient; use sqlbuild_recall::records::main::artifact_rows::artifact_rows; use sqlbuild_recall::records::main::event_object_row::event_object_row; use sqlbuild_recall::records::main::sha256_hex::sha256_hex; @@ -20,7 +20,7 @@ use sqlbuild_recall::writer::main::start_recall_session::start_recall_session; use sqlbuild_recall::writer::models::SessionConfig; use uuid::Uuid; -use crate::output_helpers::stored_bytes; +use crate::api::tests::output_helpers::stored_bytes; fn record(row: RecallRow) -> RecallRecord { RecallRecord { diff --git a/crates/sqlbuild-orchestration/tests/api/test_contract.rs b/crates/sqlbuild-orchestration/tests/api/test_contract.rs index be3b784298..4b311bbb1b 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contract.rs @@ -1,34 +1,32 @@ use std::collections::BTreeMap; use std::time::Duration; +use crate::api::errors::ClientError; +use crate::credentials::main::issue_token::issue_token; +use crate::credentials::main::revoke_token::revoke_token; +use crate::credentials::models::{AuditContext, CredentialScope}; +use crate::runs::main::change_run::change_run; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::list_runs::list_runs; +use crate::runs::models::{RunAuthority, RunChange, RunChangeRequest, RunExecution, RunStatus}; +use crate::runs::tests::helpers::request_run; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::next_remote_generation::next_remote_generation; +use crate::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; +use crate::snapshots::main::verify_execution::verify_execution; +use crate::snapshots::models::{SnapshotInput, SnapshotRecord}; +use crate::store::main::migrate_store::migrate_store; use chrono::Utc; use reqwest::StatusCode; use serde_json::json; -use sqlbuild_orchestration::api::errors::ClientError; -use sqlbuild_orchestration::credentials::main::issue_token::issue_token; -use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; -use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; -use sqlbuild_orchestration::runs::main::change_run::change_run; -use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; -use sqlbuild_orchestration::runs::main::list_runs::list_runs; -use sqlbuild_orchestration::runs::main::request_run::request_run; -use sqlbuild_orchestration::runs::models::{ - RunAuthority, RunChange, RunChangeRequest, RunExecution, RunStatus, -}; -use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::snapshots::main::next_remote_generation::next_remote_generation; -use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; -use sqlbuild_orchestration::snapshots::main::verify_execution::verify_execution; -use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotRecord}; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use uuid::Uuid; -use crate::helpers::{start_api, stop_api}; -use crate::location_helpers::create_location; -use crate::run_helpers::{queue, request, snapshot}; -use crate::store_helpers::{close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::test_types::ApiContractTestCase; +use crate::api::tests::helpers::{start_api, stop_api}; +use crate::api::tests::test_types::ApiContractTestCase; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; #[test] fn given_a_real_http_server_when_locations_and_workers_report_then_scopes_retries_and_terminal_expiry_are_enforced() @@ -436,7 +434,7 @@ fn given_a_real_http_server_when_locations_and_workers_report_then_scopes_retrie ); } } - let mut forged = json!(crate::output_helpers::chunk( + let mut forged = json!(crate::api::tests::output_helpers::chunk( 0, sqlbuild_recall::records::models::OutputStream::Stdout, b"forged" @@ -502,7 +500,7 @@ fn given_a_real_http_server_when_locations_and_workers_report_then_scopes_retrie let terminal = RunChangeRequest { event_id: Uuid::now_v7(), change: RunChange::Finished { - outcome: sqlbuild_orchestration::runs::models::RunOutcome::Cancelled, + outcome: crate::runs::models::RunOutcome::Cancelled, reason: "worker acknowledged cancellation".into(), exit_code: Some(143), }, diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs new file mode 100644 index 0000000000..d141d28f37 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -0,0 +1,22 @@ +//! HTTP contracts compiled within the API owner. + +#[path = "helpers.rs"] +mod helpers; +#[path = "output/helpers.rs"] +mod output_helpers; +#[path = "recall/helpers.rs"] +mod recall_helpers; +#[path = "test_contract.rs"] +mod test_contract; +#[path = "test_location_health.rs"] +mod test_location_health; +#[path = "test_location_work.rs"] +mod test_location_work; +#[path = "test_reads.rs"] +mod test_reads; +#[path = "test_types.rs"] +mod test_types; +#[path = "test_worker.rs"] +mod test_worker; +#[path = "work/helpers.rs"] +mod work_helpers; diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_health.rs b/crates/sqlbuild-orchestration/tests/api/test_location_health.rs index 38928875b7..002eed46e0 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_location_health.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_location_health.rs @@ -1,23 +1,21 @@ +use crate::credentials::main::issue_token::issue_token; +use crate::credentials::models::{AuditContext, CredentialScope}; +use crate::platform::main::report_remote_location_health::report_remote_location_health; +use crate::platform::models::{LocationHealth, LocationHealthObservation, LocationHealthReport}; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::helpers::read_recall_events; use chrono::Utc; use reqwest::StatusCode; use serde_json::json; -use sqlbuild_orchestration::credentials::main::issue_token::issue_token; -use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; -use sqlbuild_orchestration::platform::main::report_remote_location_health::report_remote_location_health; -use sqlbuild_orchestration::platform::models::{ - LocationHealth, LocationHealthObservation, LocationHealthReport, -}; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; -use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; use uuid::Uuid; -use crate::helpers::{start_api, stop_api}; -use crate::location_helpers::create_location; -use crate::run_helpers::snapshot; -use crate::store_helpers::{close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::test_types::LocationHealthTestCase; +use crate::api::tests::helpers::{start_api, stop_api}; +use crate::api::tests::test_types::LocationHealthTestCase; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::snapshot; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; #[test] fn given_scoped_location_observations_when_retried_or_read_then_history_and_authority_are_preserved() diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs index 52b883742f..03d21e27b5 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs @@ -1,27 +1,27 @@ -use chrono::{Duration, Utc}; -use reqwest::StatusCode; -use sqlbuild_orchestration::api::errors::ClientError; -use sqlbuild_orchestration::credentials::main::issue_token::issue_token; -use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; -use sqlbuild_orchestration::platform::main::connect_location_work::connect_location_work; -use sqlbuild_orchestration::platform::main::location_work::location_work; -use sqlbuild_orchestration::platform::main::report_location_health::report_location_health; -use sqlbuild_orchestration::platform::main::request_location_work::request_location_work; -use sqlbuild_orchestration::platform::models::{ +use crate::api::errors::ClientError; +use crate::credentials::main::issue_token::issue_token; +use crate::credentials::models::{AuditContext, CredentialScope}; +use crate::platform::main::connect_location_work::connect_location_work; +use crate::platform::main::location_work::location_work; +use crate::platform::main::report_location_health::report_location_health; +use crate::platform::main::request_location_work::request_location_work; +use crate::platform::models::{ LocationHealth, LocationHealthReport, LocationWork, LocationWorkOutcome, LocationWorkPoll, LocationWorkRecord, LocationWorkRequest, LocationWorkResult, }; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use chrono::{Duration, Utc}; +use reqwest::StatusCode; use uuid::Uuid; -use crate::helpers::{start_api, stop_api}; -use crate::location_helpers::create_location; -use crate::run_helpers::snapshot; -use crate::store_helpers::{close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::test_types::LocationWorkTestCase; -use crate::work_helpers::execute; +use crate::api::tests::helpers::{start_api, stop_api}; +use crate::api::tests::test_types::LocationWorkTestCase; +use crate::api::tests::work_helpers::execute; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::snapshot; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; #[test] fn given_outbound_work_when_claimed_retried_or_replaced_then_scope_and_immutable_results_are_enforced() diff --git a/crates/sqlbuild-orchestration/tests/api/test_reads.rs b/crates/sqlbuild-orchestration/tests/api/test_reads.rs index 598dbd7496..1a3b7a06b9 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_reads.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_reads.rs @@ -3,38 +3,37 @@ use std::collections::BTreeMap; use std::time::Duration; +use crate::credentials::main::create_principal::create_principal; +use crate::credentials::main::issue_token::issue_token; +use crate::credentials::main::revoke_token::revoke_token; +use crate::credentials::models::{AuditContext, CredentialScope, PrincipalKind}; +use crate::declarations::main::change_trigger::change_trigger; +use crate::declarations::main::list_triggers::list_triggers; +use crate::declarations::models::{TriggerChange, TriggerChangeRequest}; +use crate::runs::main::change_run::change_run; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::models::{RunAuthority, RunChange, RunChangeRequest, RunOutcome}; +use crate::runs::tests::helpers::request_run; +use crate::scheduler::tests::contracts::helpers::{acquire_lease, evaluate_trigger}; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::helpers::publish_recall; +use crate::worker::main::connect_worker::connect_worker; use chrono::Utc; use reqwest::StatusCode; use serde_json::json; -use sqlbuild_orchestration::credentials::main::create_principal::create_principal; -use sqlbuild_orchestration::credentials::main::issue_token::issue_token; -use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; -use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope, PrincipalKind}; -use sqlbuild_orchestration::declarations::main::change_trigger::change_trigger; -use sqlbuild_orchestration::declarations::main::list_triggers::list_triggers; -use sqlbuild_orchestration::declarations::models::{TriggerChange, TriggerChangeRequest}; -use sqlbuild_orchestration::runs::main::change_run::change_run; -use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; -use sqlbuild_orchestration::runs::main::request_run::request_run; -use sqlbuild_orchestration::runs::models::{RunAuthority, RunChange, RunChangeRequest, RunOutcome}; -use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::scheduler::main::evaluate_trigger::evaluate_trigger; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; -use sqlbuild_orchestration::store::main::publish_recall::publish_recall; -use sqlbuild_orchestration::worker::main::connect_worker::connect_worker; use sqlbuild_recall::records::main::output_row::output_row; use sqlbuild_recall::records::models::{OutputStream, RowEnvelope}; use uuid::Uuid; -use crate::helpers::{read_json, start_api, stop_api}; -use crate::location_helpers::create_location; -use crate::output_helpers::chunk; -use crate::recall_helpers::recall_contract; -use crate::run_helpers::{request, snapshot}; -use crate::store_helpers::{close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::test_types::ApiReadTestCase; +use crate::api::tests::helpers::{read_json, start_api, stop_api}; +use crate::api::tests::output_helpers::chunk; +use crate::api::tests::recall_helpers::recall_contract; +use crate::api::tests::test_types::ApiReadTestCase; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::{request, snapshot}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; #[test] fn given_published_locations_and_scheduled_work_when_read_through_http_then_resources_and_histories_remain_scoped() diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index cf088cb040..133e475a53 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -1,7 +1,6 @@ -use sqlbuild_orchestration::runs::models::RunStatus; -use sqlbuild_orchestration::store::errors::ControlError; - -use crate::store_test_types::Backend; +use crate::runs::models::RunStatus; +use crate::store::errors::ControlError; +use crate::store::tests::store_test_types::Backend; pub(crate) struct ApiContractTestCase { pub(crate) description: &'static str, diff --git a/crates/sqlbuild-orchestration/tests/api/test_worker.rs b/crates/sqlbuild-orchestration/tests/api/test_worker.rs index 60d5932881..e9d9e1aa80 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_worker.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_worker.rs @@ -1,36 +1,36 @@ use std::collections::BTreeMap; +use crate::credentials::main::issue_token::issue_token; +use crate::credentials::models::{AuditContext, CredentialScope}; +use crate::runs::main::change_run::change_run; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::list_runs::list_runs; +use crate::runs::models::{ + NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, +}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::worker::errors::WorkerError; +use crate::worker::main::connect_worker::connect_worker; +use crate::worker::main::start_worker::start_worker; use axum::Router; use axum::response::Redirect; use axum::routing::get; use chrono::Utc; -use sqlbuild_orchestration::credentials::main::issue_token::issue_token; -use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; -use sqlbuild_orchestration::runs::main::change_run::change_run; -use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; -use sqlbuild_orchestration::runs::main::list_runs::list_runs; -use sqlbuild_orchestration::runs::models::{ - NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, -}; -use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::errors::ControlError; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; -use sqlbuild_orchestration::worker::errors::WorkerError; -use sqlbuild_orchestration::worker::main::connect_worker::connect_worker; -use sqlbuild_orchestration::worker::main::start_worker::start_worker; use uuid::Uuid; -use crate::helpers::{start_api, stop_api}; -use crate::location_helpers::create_location; -use crate::output_helpers::{chunk, output_contract}; -use crate::recall_helpers::recall_contract; -use crate::run_helpers::{queue, request, snapshot}; -use crate::store_helpers::{close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::test_types::{ +use crate::api::tests::helpers::{start_api, stop_api}; +use crate::api::tests::output_helpers::{chunk, output_contract}; +use crate::api::tests::recall_helpers::recall_contract; +use crate::api::tests::test_types::{ ApiContractTestCase, TestApi, WorkerDeadlineTestCase, WorkerRedirectTestCase, }; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; #[test] fn given_a_claimed_run_when_a_worker_connects_then_verification_precedes_start_and_reports_are_retryable() diff --git a/crates/sqlbuild-orchestration/tests/api/work/helpers.rs b/crates/sqlbuild-orchestration/tests/api/work/helpers.rs index 32a7a3cbd3..2debfcd9b6 100644 --- a/crates/sqlbuild-orchestration/tests/api/work/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/work/helpers.rs @@ -1,6 +1,6 @@ //! Fault injection changes only disposable test stores, never product time or lease semantics. -use sqlbuild_orchestration::store::models::ControlStore; +use crate::store::models::ControlStore; pub(crate) async fn execute(store: &ControlStore, sql: &str) { let ControlStore::Postgres(pool) = store else { diff --git a/crates/sqlbuild-orchestration/tests/credentials.rs b/crates/sqlbuild-orchestration/tests/credentials.rs deleted file mode 100644 index b31553bc0d..0000000000 --- a/crates/sqlbuild-orchestration/tests/credentials.rs +++ /dev/null @@ -1,20 +0,0 @@ -#[path = "store/faults/helpers.rs"] -mod fault_helpers; -#[path = "store/faults/test_types.rs"] -mod fault_types; -#[path = "credentials/helpers.rs"] -mod helpers; -#[path = "store/replicas/helpers.rs"] -mod replica_helpers; -#[path = "runs/helpers.rs"] -mod run_helpers; -#[path = "store/helpers.rs"] -mod store_helpers; -#[path = "store/test_types.rs"] -mod store_test_types; -#[path = "runs/termination/helpers.rs"] -mod terminal_helpers; -#[path = "credentials/test_contract.rs"] -mod test_contract; -#[path = "credentials/test_types.rs"] -mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/credentials/helpers.rs b/crates/sqlbuild-orchestration/tests/credentials/helpers.rs index f86c76460c..4c28e90101 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/helpers.rs @@ -1,4 +1,4 @@ -use sqlbuild_orchestration::store::models::ControlStore; +use crate::store::models::ControlStore; pub(crate) async fn location_state(store: &ControlStore) -> (i64, i64) { let sql = "SELECT (SELECT COUNT(*) FROM sqb_locations), \ diff --git a/crates/sqlbuild-orchestration/tests/credentials/locations/helpers.rs b/crates/sqlbuild-orchestration/tests/credentials/locations/helpers.rs index c6edcc2874..9a48e51758 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/locations/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/locations/helpers.rs @@ -1,8 +1,8 @@ //! Shared location fixtures use the same audited identity operation as operator bootstrap. -use sqlbuild_orchestration::credentials::models::AuditContext; -use sqlbuild_orchestration::store::errors::ControlError; -use sqlbuild_orchestration::store::models::ControlStore; +use crate::credentials::models::AuditContext; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; use uuid::Uuid; pub(crate) async fn create_location( @@ -10,7 +10,7 @@ pub(crate) async fn create_location( name: &str, actor: &str, ) -> Result { - sqlbuild_orchestration::credentials::main::create_location::create_location( + crate::credentials::main::create_location::create_location( store, name, &AuditContext { diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs index 67526d5c57..d26aa25639 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs @@ -1,26 +1,26 @@ use std::collections::BTreeMap; +use crate::credentials::main::authenticate_token::authenticate_token; +use crate::credentials::main::create_location::create_location; +use crate::credentials::main::create_principal::create_principal; +use crate::credentials::main::issue_token::issue_token; +use crate::credentials::main::revoke_token::revoke_token; +use crate::credentials::models::{AuditContext, CredentialScope, PrincipalKind}; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; use chrono::{Duration, Utc}; -use sqlbuild_orchestration::credentials::main::authenticate_token::authenticate_token; -use sqlbuild_orchestration::credentials::main::create_location::create_location; -use sqlbuild_orchestration::credentials::main::create_principal::create_principal; -use sqlbuild_orchestration::credentials::main::issue_token::issue_token; -use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; -use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope, PrincipalKind}; -use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; -use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use uuid::Uuid; -use crate::fault_helpers::{install_fault, remove_fault}; -use crate::helpers::{location_state, stored_hashes}; -use crate::replica_helpers::second_store; -use crate::run_helpers::{queue, request, snapshot}; -use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::terminal_helpers::finish; -use crate::test_types::{CredentialTestCase, TransactionFault}; +use crate::credentials::tests::helpers::{location_state, stored_hashes}; +use crate::credentials::tests::test_types::{CredentialTestCase, TransactionFault}; +use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::runs::tests::terminal_helpers::finish; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; #[test] fn given_scoped_tokens_when_revoked_or_the_run_ends_then_all_replicas_reject_them_and_audit_failure_rolls_back() diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs b/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs new file mode 100644 index 0000000000..9e6edc1c94 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs @@ -0,0 +1,10 @@ +//! Credential contracts compiled within the credential owner. + +#[path = "helpers.rs"] +mod helpers; +#[path = "locations/helpers.rs"] +pub(crate) mod location_helpers; +#[path = "test_contract.rs"] +mod test_contract; +#[path = "test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_types.rs b/crates/sqlbuild-orchestration/tests/credentials/test_types.rs index 077ee03ce9..c2efe4bf2a 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_types.rs @@ -1,5 +1,5 @@ -pub(crate) use crate::fault_types::TransactionFault; -use crate::store_test_types::Backend; +pub(crate) use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::store_test_types::Backend; pub(crate) struct CredentialTestCase { pub(crate) description: &'static str, diff --git a/crates/sqlbuild-orchestration/tests/platform.rs b/crates/sqlbuild-orchestration/tests/platform.rs deleted file mode 100644 index 3243ae357c..0000000000 --- a/crates/sqlbuild-orchestration/tests/platform.rs +++ /dev/null @@ -1,14 +0,0 @@ -#[path = "platform/helpers.rs"] -mod helpers; -#[path = "credentials/locations/helpers.rs"] -mod location_helpers; -#[path = "store/replicas/helpers.rs"] -mod replica_helpers; -#[path = "store/helpers.rs"] -mod store_helpers; -#[path = "store/test_types.rs"] -mod store_test_types; -#[path = "platform/test_types.rs"] -mod test_types; -#[path = "platform/test_watchdog.rs"] -mod test_watchdog; diff --git a/crates/sqlbuild-orchestration/tests/platform/helpers.rs b/crates/sqlbuild-orchestration/tests/platform/helpers.rs index 310870ecfb..a00d9adf24 100644 --- a/crates/sqlbuild-orchestration/tests/platform/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/platform/helpers.rs @@ -1,17 +1,17 @@ use std::collections::{BTreeMap, BTreeSet}; use std::time::Duration; +use crate::platform::models::PlatformEvent; +use crate::snapshots::models::{SnapshotInput, SnapshotNode}; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; +use crate::store::tests::helpers::read_recall_events; use chrono::{TimeDelta, Utc}; use serde_json::json; -use sqlbuild_orchestration::platform::models::PlatformEvent; -use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; -use sqlbuild_orchestration::store::errors::ControlError; -use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; -use sqlbuild_orchestration::store::models::ControlStore; use tokio::task::JoinHandle; use uuid::Uuid; -use crate::test_types::Observation; +use crate::platform::tests::test_types::Observation; pub(crate) fn snapshot(location_id: Uuid) -> SnapshotInput { let future = Utc::now() + TimeDelta::days(10); diff --git a/crates/sqlbuild-orchestration/tests/platform/test_contracts.rs b/crates/sqlbuild-orchestration/tests/platform/test_contracts.rs new file mode 100644 index 0000000000..1a231c8e58 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/platform/test_contracts.rs @@ -0,0 +1,8 @@ +//! Platform contracts compiled within the platform owner. + +#[path = "helpers.rs"] +mod helpers; +#[path = "test_types.rs"] +mod test_types; +#[path = "test_watchdog.rs"] +mod test_watchdog; diff --git a/crates/sqlbuild-orchestration/tests/platform/test_types.rs b/crates/sqlbuild-orchestration/tests/platform/test_types.rs index 418d6a8cb3..01cf50f419 100644 --- a/crates/sqlbuild-orchestration/tests/platform/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/platform/test_types.rs @@ -1,8 +1,8 @@ +use crate::platform::models::PlatformEvent; use chrono::{DateTime, Utc}; use serde::Deserialize; -use sqlbuild_orchestration::platform::models::PlatformEvent; -use crate::store_test_types::Backend; +use crate::store::tests::store_test_types::Backend; pub(crate) struct WatchdogTestCase { pub(crate) description: &'static str, diff --git a/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs b/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs index e119342ea7..0d03b0e495 100644 --- a/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs +++ b/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs @@ -1,19 +1,19 @@ use std::collections::BTreeSet; use std::time::Duration; -use sqlbuild_orchestration::platform::main::run_watchdog::run_watchdog; -use sqlbuild_orchestration::platform::models::PlatformEvent; -use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use crate::platform::main::run_watchdog::run_watchdog; +use crate::platform::models::PlatformEvent; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; use uuid::Uuid; -use crate::helpers::{snapshot, wait_for_scans}; -use crate::location_helpers::create_location; -use crate::replica_helpers::second_store; -use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::test_types::WatchdogTestCase; +use crate::credentials::tests::location_helpers::create_location; +use crate::platform::tests::helpers::{snapshot, wait_for_scans}; +use crate::platform::tests::test_types::WatchdogTestCase; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; #[test] fn given_two_server_replicas_when_scheduling_is_stalled_then_health_facts_are_durable_deduplicated_and_independent_of_leadership() diff --git a/crates/sqlbuild-orchestration/tests/runs.rs b/crates/sqlbuild-orchestration/tests/runs.rs deleted file mode 100644 index df23471f89..0000000000 --- a/crates/sqlbuild-orchestration/tests/runs.rs +++ /dev/null @@ -1,42 +0,0 @@ -#[path = "runs/claims/helpers.rs"] -mod claim_helpers; -#[path = "store/faults/helpers.rs"] -mod fault_helpers; -#[path = "store/faults/test_types.rs"] -mod fault_types; -#[path = "runs/helpers.rs"] -mod helpers; -#[path = "store/leases/helpers.rs"] -mod lease_helpers; -#[path = "credentials/locations/helpers.rs"] -mod location_helpers; -#[path = "runs/monitor/helpers.rs"] -mod monitor_helpers; -#[path = "store/replicas/helpers.rs"] -mod replica_helpers; -#[path = "store/helpers.rs"] -mod store_helpers; -#[path = "store/test_types.rs"] -mod store_test_types; -#[path = "runs/termination/helpers.rs"] -mod terminal_helpers; -#[path = "runs/test_claims.rs"] -mod test_claims; -#[path = "runs/test_coalesce.rs"] -mod test_coalesce; -#[path = "runs/test_faults.rs"] -mod test_faults; -#[path = "runs/test_group_admission.rs"] -mod test_group_admission; -#[path = "runs/test_groups.rs"] -mod test_groups; -#[path = "runs/test_lifecycle.rs"] -mod test_lifecycle; -#[path = "runs/test_monitor.rs"] -mod test_monitor; -#[path = "runs/test_requests.rs"] -mod test_requests; -#[path = "runs/test_reruns.rs"] -mod test_reruns; -#[path = "runs/test_types.rs"] -mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs index 4fe22bb22f..6b2c0e2b1d 100644 --- a/crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs @@ -2,11 +2,11 @@ use std::collections::BTreeMap; -use sqlbuild_orchestration::runs::main::change_run::change_run; -use sqlbuild_orchestration::runs::models::{ +use crate::runs::main::change_run::change_run; +use crate::runs::models::{ NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunClaim, RunOutcome, }; -use sqlbuild_orchestration::store::models::ControlStore; +use crate::store::models::ControlStore; use uuid::Uuid; pub(crate) async fn slot_counts(store: &ControlStore) -> BTreeMap { diff --git a/crates/sqlbuild-orchestration/tests/runs/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/helpers.rs index 586e14c21e..c86ac7fe47 100644 --- a/crates/sqlbuild-orchestration/tests/runs/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/runs/helpers.rs @@ -2,14 +2,27 @@ use std::collections::{BTreeMap, BTreeSet}; +use crate::runs::_helpers::requests::publication::publish_request; +use crate::runs::models::{RunKind, RunOrigin, RunRequest}; +use crate::scheduler::models::Overlap; +use crate::snapshots::models::{SnapshotInput, SnapshotNode}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{ControlStore, TransactionMode}; use serde_json::json; -use sqlbuild_orchestration::runs::main::request_run::request_run; -use sqlbuild_orchestration::runs::models::{RunKind, RunOrigin, RunRequest}; -use sqlbuild_orchestration::scheduler::models::Overlap; -use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; -use sqlbuild_orchestration::store::models::ControlStore; use uuid::Uuid; +pub(crate) async fn request_run( + store: &ControlStore, + request: &RunRequest, +) -> Result { + let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, applied) = publish_request(transaction, request).await?; + commit(transaction).await?; + Ok(applied) +} + pub(crate) fn snapshot(location_id: Uuid) -> SnapshotInput { SnapshotInput { location_id, diff --git a/crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs index ec318744e7..c69347e480 100644 --- a/crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs @@ -2,17 +2,15 @@ use std::collections::BTreeMap; +use crate::runs::main::change_run::change_run; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::models::{RunAuthority, RunChange, RunChangeRequest, RunClaim, RunRequest}; +use crate::scheduler::models::SchedulerLease; +use crate::store::models::ControlStore; use chrono::{TimeDelta, Utc}; -use sqlbuild_orchestration::runs::main::change_run::change_run; -use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; -use sqlbuild_orchestration::runs::models::{ - RunAuthority, RunChange, RunChangeRequest, RunClaim, RunRequest, -}; -use sqlbuild_orchestration::scheduler::models::SchedulerLease; -use sqlbuild_orchestration::store::models::ControlStore; use uuid::Uuid; -use crate::helpers::queue; +use crate::runs::tests::helpers::queue; pub(crate) async fn start_run( store: &ControlStore, diff --git a/crates/sqlbuild-orchestration/tests/runs/termination/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/termination/helpers.rs index 9d6c347e0b..5b31b83c2f 100644 --- a/crates/sqlbuild-orchestration/tests/runs/termination/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/runs/termination/helpers.rs @@ -1,11 +1,9 @@ //! Terminal facts exercise claim release through the public run lifecycle. -use sqlbuild_orchestration::runs::main::change_run::change_run; -use sqlbuild_orchestration::runs::models::{ - RunAuthority, RunChange, RunChangeRequest, RunClaim, RunOutcome, -}; -use sqlbuild_orchestration::scheduler::models::SchedulerLease; -use sqlbuild_orchestration::store::models::ControlStore; +use crate::runs::main::change_run::change_run; +use crate::runs::models::{RunAuthority, RunChange, RunChangeRequest, RunClaim, RunOutcome}; +use crate::scheduler::models::SchedulerLease; +use crate::store::models::ControlStore; use uuid::Uuid; pub(crate) async fn finish(store: &ControlStore, lease: &SchedulerLease, claim: &RunClaim) { diff --git a/crates/sqlbuild-orchestration/tests/runs/test_claims.rs b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs index ad747388df..d8c4151143 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_claims.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs @@ -1,26 +1,26 @@ use std::collections::BTreeMap; use std::num::NonZeroU32; -use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; -use sqlbuild_orchestration::runs::main::list_run_claims::list_run_claims; -use sqlbuild_orchestration::runs::main::list_runs::list_runs; -use sqlbuild_orchestration::runs::models::RunStatus; -use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; -use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::list_run_claims::list_run_claims; +use crate::runs::main::list_runs::list_runs; +use crate::runs::models::RunStatus; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::helpers::read_recall_events; use uuid::Uuid; -use crate::claim_helpers::succeed; -use crate::helpers::{queue, request, snapshot}; -use crate::location_helpers::create_location; -use crate::replica_helpers::second_store; -use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::terminal_helpers::finish; -use crate::test_types::{ +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::claim_helpers::succeed; +use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::runs::tests::terminal_helpers::finish; +use crate::runs::tests::test_types::{ BacklogClaimTestCase, GroupLimitTestCase, PriorityClaimsTestCase, ScopedClaimTestCase, }; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; #[test] fn given_higher_priority_work_for_other_scopes_when_a_local_launcher_claims_then_only_its_location_and_target_are_reserved() @@ -413,11 +413,9 @@ fn given_more_than_one_page_of_blocked_runs_when_claiming_then_lower_priority_in for id in batch { queue(store, &request(scope, *id, 0, &["orders"])).await; } - lease = sqlbuild_orchestration::scheduler::main::renew_lease::renew_lease( - store, &lease, - ) - .await - .expect("keep leader live while preparing backlog"); + lease = crate::scheduler::main::renew_lease::renew_lease(store, &lease) + .await + .expect("keep leader live while preparing backlog"); } queue(store, &request(scope, 105, -1, &["customers"])).await; let started = std::time::Instant::now(); diff --git a/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs b/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs index 3149b775d9..b9eb5090c4 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs @@ -1,21 +1,20 @@ use std::collections::BTreeMap; +use crate::declarations::main::list_triggers::list_triggers; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::list_runs::list_runs; +use crate::scheduler::models::TickDecision; +use crate::scheduler::tests::contracts::helpers::{acquire_lease, evaluate_trigger}; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; use serde_json::json; -use sqlbuild_orchestration::declarations::main::list_triggers::list_triggers; -use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; -use sqlbuild_orchestration::runs::main::list_runs::list_runs; -use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::scheduler::main::evaluate_trigger::evaluate_trigger; -use sqlbuild_orchestration::scheduler::models::TickDecision; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use uuid::Uuid; -use crate::helpers::snapshot; -use crate::location_helpers::create_location; -use crate::store_helpers::{close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::test_types::CoalescedRunTestCase; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::snapshot; +use crate::runs::tests::test_types::CoalescedRunTestCase; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; #[test] fn given_coalescing_triggers_when_a_previous_run_has_been_claimed_then_new_ticks_wait_in_a_new_run_instead_of_covering_executing_work() diff --git a/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs b/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs new file mode 100644 index 0000000000..530eb54c35 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs @@ -0,0 +1,28 @@ +#[path = "claims/helpers.rs"] +mod claim_helpers; +#[path = "helpers.rs"] +pub(crate) mod helpers; +#[path = "monitor/helpers.rs"] +mod monitor_helpers; +#[path = "termination/helpers.rs"] +pub(crate) mod terminal_helpers; +#[path = "test_claims.rs"] +mod test_claims; +#[path = "test_coalesce.rs"] +mod test_coalesce; +#[path = "test_faults.rs"] +mod test_faults; +#[path = "test_group_admission.rs"] +mod test_group_admission; +#[path = "test_groups.rs"] +mod test_groups; +#[path = "test_lifecycle.rs"] +mod test_lifecycle; +#[path = "test_monitor.rs"] +mod test_monitor; +#[path = "test_requests.rs"] +mod test_requests; +#[path = "test_reruns.rs"] +mod test_reruns; +#[path = "test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_faults.rs b/crates/sqlbuild-orchestration/tests/runs/test_faults.rs index 00d20c38c3..f784d6485b 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_faults.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_faults.rs @@ -1,25 +1,25 @@ use std::collections::BTreeMap; use std::num::NonZeroU32; -use sqlbuild_orchestration::runs::main::change_run::change_run; -use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; -use sqlbuild_orchestration::runs::main::heartbeat_run::heartbeat_run; -use sqlbuild_orchestration::runs::main::list_runs::list_runs; -use sqlbuild_orchestration::runs::models::{RunAuthority, RunChange, RunChangeRequest}; -use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::errors::ControlError; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; -use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; +use crate::runs::main::change_run::change_run; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::heartbeat_run::heartbeat_run; +use crate::runs::main::list_runs::list_runs; +use crate::runs::models::{RunAuthority, RunChange, RunChangeRequest}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::helpers::read_recall_events; use uuid::Uuid; -use crate::fault_helpers::{install_fault, remove_fault}; -use crate::helpers::{queue, request, snapshot}; -use crate::lease_helpers::expire_lease; -use crate::location_helpers::create_location; -use crate::store_helpers::{close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::test_types::{ClaimFailureTestCase, TransactionFault}; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::runs::tests::test_types::{ClaimFailureTestCase, TransactionFault}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::lease_helpers::expire_lease; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; #[test] fn given_failures_during_claim_publication_when_retried_then_claims_slots_liveness_facts_and_projection_commit_together() diff --git a/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs b/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs index 8dc2a9e113..a57ce0a896 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs @@ -3,28 +3,28 @@ use std::collections::{BTreeMap, BTreeSet}; use std::num::NonZeroU32; -use serde_json::json; -use sqlbuild_orchestration::credentials::models::AuditContext; -use sqlbuild_orchestration::runs::main::{ +use crate::credentials::models::AuditContext; +use crate::runs::main::{ advance_groups::advance_groups, claim_next_run::claim_next_run, list_runs::list_runs, request_job_run::request_job_run, }; -use sqlbuild_orchestration::runs::models::{JobRunRequest, RunDispatch, RunStatus}; -use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use crate::runs::models::{JobRunRequest, RunDispatch, RunStatus}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use serde_json::json; use uuid::Uuid; -use crate::claim_helpers::{slot_counts, succeed}; -use crate::helpers::{queue, request, snapshot}; -use crate::location_helpers::create_location; -use crate::replica_helpers::second_store; -use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::test_types::{ +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::claim_helpers::{slot_counts, succeed}; +use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::runs::tests::test_types::{ AdmissionParent, GroupAdmissionTestCase, GroupQueueTestCase, MixedAdmissionTestCase, RunLifecycleTestCase, }; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; #[test] fn given_mixed_physical_and_parent_waiters_when_admitting_then_priority_and_queue_order_win() { diff --git a/crates/sqlbuild-orchestration/tests/runs/test_groups.rs b/crates/sqlbuild-orchestration/tests/runs/test_groups.rs index 6a502d207d..b2fe494aef 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_groups.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_groups.rs @@ -3,29 +3,28 @@ use std::collections::BTreeMap; use std::num::NonZeroU32; -use serde_json::json; -use sqlbuild_orchestration::credentials::models::AuditContext; -use sqlbuild_orchestration::runs::main::{ +use crate::credentials::models::AuditContext; +use crate::runs::main::{ advance_groups::advance_groups, change_run::change_run, claim_next_run::claim_next_run, list_runs::list_runs, request_job_run::request_job_run, run_execution::run_execution, }; -use sqlbuild_orchestration::runs::models::{ +use crate::runs::models::{ JobRunRequest, NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, }; -use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::{ - migrate_store::migrate_store, read_recall_events::read_recall_events, -}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::helpers::read_recall_events; +use serde_json::json; use uuid::Uuid; -use crate::claim_helpers::{slot_counts, succeed}; -use crate::fault_helpers::{install_fault, remove_fault}; -use crate::helpers::snapshot; -use crate::location_helpers::create_location; -use crate::store_helpers::{close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::test_types::{RunLifecycleTestCase, TransactionFault}; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::claim_helpers::{slot_counts, succeed}; +use crate::runs::tests::helpers::snapshot; +use crate::runs::tests::test_types::{RunLifecycleTestCase, TransactionFault}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; #[test] fn given_a_pinned_group_when_redeployed_and_cancelled_then_children_keep_scope_and_parent_waits_for_terminal_evidence() diff --git a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs index 5c23383e12..2f98d185eb 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs @@ -1,26 +1,26 @@ use std::collections::BTreeMap; -use sqlbuild_orchestration::credentials::models::AuditContext; -use sqlbuild_orchestration::runs::main::change_run::change_run; -use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; -use sqlbuild_orchestration::runs::main::heartbeat_run::heartbeat_run; -use sqlbuild_orchestration::runs::main::list_run_nodes::list_run_nodes; -use sqlbuild_orchestration::runs::main::list_runs::list_runs; -use sqlbuild_orchestration::runs::models::{ +use crate::credentials::models::AuditContext; +use crate::runs::main::change_run::change_run; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::heartbeat_run::heartbeat_run; +use crate::runs::main::list_run_nodes::list_run_nodes; +use crate::runs::main::list_runs::list_runs; +use crate::runs::models::{ NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, }; -use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; -use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::helpers::read_recall_events; use uuid::Uuid; -use crate::fault_helpers::{install_fault, remove_fault}; -use crate::helpers::{queue, request, snapshot}; -use crate::location_helpers::create_location; -use crate::store_helpers::{close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::test_types::{RunLifecycleTestCase, TransactionFault}; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::runs::tests::test_types::{RunLifecycleTestCase, TransactionFault}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; #[test] fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_facts_outcomes_and_claims_are_atomic() diff --git a/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs b/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs index 228dd090a2..62fe549bc8 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs @@ -3,32 +3,31 @@ use std::collections::BTreeMap; use std::time::Duration; -use sqlbuild_orchestration::runs::main::change_run::change_run; -use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; -use sqlbuild_orchestration::runs::main::heartbeat_run::heartbeat_run; -use sqlbuild_orchestration::runs::main::list_runs::list_runs; -use sqlbuild_orchestration::runs::main::monitor_runs::monitor_runs; -use sqlbuild_orchestration::runs::models::{ - RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, -}; -use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::scheduler::main::renew_lease::renew_lease; -use sqlbuild_orchestration::scheduler::main::scheduler_cycle::scheduler_cycle; -use sqlbuild_orchestration::scheduler::models::SchedulerState; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::errors::ControlError; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; -use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; +use crate::runs::main::change_run::change_run; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::heartbeat_run::heartbeat_run; +use crate::runs::main::list_runs::list_runs; +use crate::runs::main::monitor_runs::monitor_runs; +use crate::runs::models::{RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus}; +use crate::scheduler::main::renew_lease::renew_lease; +use crate::scheduler::models::SchedulerState; +use crate::scheduler::tests::contracts::helpers::{acquire_lease, scheduler_cycle}; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::helpers::read_recall_events; use uuid::Uuid; -use crate::fault_helpers::{install_fault, remove_fault}; -use crate::helpers::{queue, request, snapshot}; -use crate::lease_helpers::expire_lease; -use crate::location_helpers::create_location; -use crate::monitor_helpers::{age_heartbeat, start_run}; -use crate::store_helpers::{close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::test_types::{MonitorBacklogTestCase, RunMonitorTestCase, TransactionFault}; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::runs::tests::monitor_helpers::{age_heartbeat, start_run}; +use crate::runs::tests::test_types::{ + MonitorBacklogTestCase, RunMonitorTestCase, TransactionFault, +}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::lease_helpers::expire_lease; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; #[test] fn given_silent_and_live_workers_when_the_scheduler_monitors_then_fenced_atomic_facts_retain_claims_and_allow_recovery() diff --git a/crates/sqlbuild-orchestration/tests/runs/test_requests.rs b/crates/sqlbuild-orchestration/tests/runs/test_requests.rs index 226cb57a5b..5ad011d62f 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_requests.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_requests.rs @@ -2,24 +2,24 @@ use std::collections::BTreeMap; +use crate::credentials::models::AuditContext; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::list_runs::list_runs; +use crate::runs::main::request_job_run::request_job_run; +use crate::runs::models::{JobRunRequest, RunStatus}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::helpers::read_recall_events; use serde_json::json; -use sqlbuild_orchestration::credentials::models::AuditContext; -use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; -use sqlbuild_orchestration::runs::main::list_runs::list_runs; -use sqlbuild_orchestration::runs::main::request_job_run::request_job_run; -use sqlbuild_orchestration::runs::models::{JobRunRequest, RunStatus}; -use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; -use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; use uuid::Uuid; -use crate::fault_helpers::{install_fault, remove_fault}; -use crate::helpers::snapshot; -use crate::location_helpers::create_location; -use crate::store_helpers::{close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::test_types::{RunLifecycleTestCase, TransactionFault}; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::snapshot; +use crate::runs::tests::test_types::{RunLifecycleTestCase, TransactionFault}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; #[test] fn given_a_published_job_when_requested_and_retried_after_redeploy_then_audit_and_frozen_execution_commit_once() diff --git a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs index 0729862d1f..4c50700634 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs @@ -2,28 +2,30 @@ use std::collections::{BTreeMap, BTreeSet}; -use serde_json::json; -use sqlbuild_orchestration::credentials::models::AuditContext; -use sqlbuild_orchestration::runs::main::{ +use crate::credentials::models::AuditContext; +use crate::runs::main::{ advance_groups::advance_groups, change_run::change_run, claim_next_run::claim_next_run, list_runs::list_runs, request_job_run::request_job_run, request_rerun::request_rerun, }; -use sqlbuild_orchestration::runs::models::{ +use crate::runs::models::{ JobRunRequest, NodeOutcome, RerunRequest, RunAuthority, RunChange, RunChangeRequest, RunOrigin, RunOutcome, RunStatus, }; -use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use serde_json::json; use uuid::Uuid; -use crate::claim_helpers::{slot_counts, succeed}; -use crate::fault_helpers::{install_fault, remove_fault}; -use crate::helpers::snapshot; -use crate::location_helpers::create_location; -use crate::store_helpers::{close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::test_types::{RerunExecutionTestCase, RunLifecycleTestCase, TransactionFault}; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::claim_helpers::{slot_counts, succeed}; +use crate::runs::tests::helpers::snapshot; +use crate::runs::tests::test_types::{ + RerunExecutionTestCase, RunLifecycleTestCase, TransactionFault, +}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; #[test] fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_nodes_retry_atomically() diff --git a/crates/sqlbuild-orchestration/tests/runs/test_types.rs b/crates/sqlbuild-orchestration/tests/runs/test_types.rs index 0646241008..fc108dd8c5 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_types.rs @@ -2,10 +2,10 @@ use std::collections::BTreeSet; -use sqlbuild_orchestration::runs::models::{RerunRequest, RunStatus}; +use crate::runs::models::{RerunRequest, RunStatus}; -pub(crate) use crate::fault_types::TransactionFault; -use crate::store_test_types::Backend; +pub(crate) use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::store_test_types::Backend; pub(crate) struct ScopedClaimTestCase { pub(crate) description: &'static str, diff --git a/crates/sqlbuild-orchestration/tests/scheduler.rs b/crates/sqlbuild-orchestration/tests/scheduler.rs deleted file mode 100644 index 774474ceef..0000000000 --- a/crates/sqlbuild-orchestration/tests/scheduler.rs +++ /dev/null @@ -1,22 +0,0 @@ -#[path = "store/faults/helpers.rs"] -mod fault_helpers; -#[path = "store/faults/test_types.rs"] -mod fault_types; -#[path = "scheduler/helpers.rs"] -mod helpers; -#[path = "store/leases/helpers.rs"] -mod lease_helpers; -#[path = "credentials/locations/helpers.rs"] -mod location_helpers; -#[path = "store/replicas/helpers.rs"] -mod replica_helpers; -#[path = "store/helpers.rs"] -mod store_helpers; -#[path = "store/test_types.rs"] -mod store_test_types; -#[path = "scheduler/test_contract.rs"] -mod test_contract; -#[path = "scheduler/test_types.rs"] -mod test_types; -#[path = "scheduler/ticks/test_ticks.rs"] -mod ticks; diff --git a/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs index 8cac2b9924..a1e8ee33dc 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs @@ -1,15 +1,17 @@ -//! Contention and expiry exercise public APIs against the real databases, not fake clocks. +//! Contention and expiry exercise scheduler ownership against real databases, not fake clocks. -use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::scheduler::main::renew_lease::renew_lease; -use sqlbuild_orchestration::store::errors::ControlError; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +pub(crate) use crate::scheduler::main::_acquire_lease::acquire_lease; +pub(crate) use crate::scheduler::main::_evaluate_trigger::evaluate_trigger; +pub(crate) use crate::scheduler::main::_scheduler_cycle::scheduler_cycle; +use crate::scheduler::main::renew_lease::renew_lease; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; use uuid::Uuid; -use crate::lease_helpers::expire_lease; -use crate::replica_helpers::second_store; -use crate::store_helpers::{close_store, close_test_store, open_test_store}; -use crate::store_test_types::Backend; +use crate::store::tests::lease_helpers::expire_lease; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store}; +use crate::store::tests::store_test_types::Backend; pub(crate) async fn leadership_contract(backend: Backend) -> Option { let test_store = open_test_store(backend).await?; diff --git a/crates/sqlbuild-orchestration/tests/scheduler/test_contract.rs b/crates/sqlbuild-orchestration/tests/scheduler/test_contract.rs index 084f5fbb10..13bdfd4250 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/test_contract.rs @@ -1,7 +1,7 @@ -use crate::helpers::leadership_contract; -use crate::store_helpers::run; -use crate::store_test_types::Backend; -use crate::test_types::LeadershipTestCase; +use crate::scheduler::tests::contracts::helpers::leadership_contract; +use crate::scheduler::tests::contracts::test_types::LeadershipTestCase; +use crate::store::tests::store_helpers::run; +use crate::store::tests::store_test_types::Backend; #[test] fn given_two_schedulers_when_leadership_expires_then_only_the_new_epoch_can_renew() { diff --git a/crates/sqlbuild-orchestration/tests/scheduler/test_contracts.rs b/crates/sqlbuild-orchestration/tests/scheduler/test_contracts.rs new file mode 100644 index 0000000000..57d98cb7cf --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/test_contracts.rs @@ -0,0 +1,10 @@ +//! Real-store scheduler contracts compiled within the scheduler owner. + +#[path = "helpers.rs"] +pub(crate) mod helpers; +#[path = "test_contract.rs"] +mod test_contract; +#[path = "test_types.rs"] +mod test_types; +#[path = "ticks/test_ticks.rs"] +mod ticks; diff --git a/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs b/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs index c7db9ff190..fd5543086b 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs @@ -1,6 +1,6 @@ //! Leadership scenarios run independently on both supported engines. -use crate::store_test_types::Backend; +use crate::store::tests::store_test_types::Backend; #[derive(Debug)] pub(crate) struct LeadershipTestCase { diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs index 7d4bac4e82..223bc36025 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs @@ -2,28 +2,28 @@ use std::collections::BTreeMap; +use crate::declarations::main::list_triggers::list_triggers; +use crate::runs::main::list_runs::list_runs; +use crate::scheduler::main::_acquire_lease::acquire_lease; +use crate::scheduler::main::_evaluate_trigger::evaluate_trigger; +use crate::scheduler::main::list_ticks::list_ticks; +use crate::scheduler::models::TickOutcome; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::models::{SnapshotInput, SnapshotNode}; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::models::ControlStore; use serde_json::{Value, json}; -use sqlbuild_orchestration::declarations::main::list_triggers::list_triggers; -use sqlbuild_orchestration::runs::main::list_runs::list_runs; -use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::scheduler::main::evaluate_trigger::evaluate_trigger; -use sqlbuild_orchestration::scheduler::main::list_ticks::list_ticks; -use sqlbuild_orchestration::scheduler::models::TickOutcome; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; -use sqlbuild_orchestration::store::errors::ControlError; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; -use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; -use sqlbuild_orchestration::store::models::ControlStore; use uuid::Uuid; -use crate::fault_helpers::{install_fault, remove_fault}; -use crate::lease_helpers::expire_lease; -use crate::location_helpers::create_location; -use crate::replica_helpers::second_store; -use crate::store_helpers::{close_store, close_test_store, open_test_store}; -use crate::store_test_types::Backend; -use crate::ticks::test_types::TransactionFault; +use crate::credentials::tests::location_helpers::create_location; +use crate::scheduler::tests::contracts::ticks::test_types::TransactionFault; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::helpers::read_recall_events; +use crate::store::tests::lease_helpers::expire_lease; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store}; +use crate::store::tests::store_test_types::Backend; pub(crate) fn snapshot(location_id: Uuid) -> SnapshotInput { SnapshotInput { diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_contract.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_contract.rs index af4492a29d..91d523b69f 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_contract.rs @@ -1,9 +1,9 @@ //! Crash points must not separate a tick from its queued work and durable cursor. -use crate::store_helpers::run; -use crate::store_test_types::Backend; -use crate::ticks::helpers::transaction_contract; -use crate::ticks::test_types::TickTransactionTestCase; +use crate::scheduler::tests::contracts::ticks::helpers::transaction_contract; +use crate::scheduler::tests::contracts::ticks::test_types::TickTransactionTestCase; +use crate::store::tests::store_helpers::run; +use crate::store::tests::store_test_types::Backend; #[test] fn given_two_schedulers_and_faults_between_writes_when_restarting_then_ticks_runs_recall_and_cursor_commit_together() diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs index e62bf51595..382d0f17bf 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs @@ -2,20 +2,22 @@ use std::time::Duration; -use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler; -use sqlbuild_orchestration::scheduler::main::scheduler_cycle::scheduler_cycle; -use sqlbuild_orchestration::scheduler::models::SchedulerState; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use crate::scheduler::main::_scheduler_cycle::scheduler_cycle; +use crate::scheduler::main::run_scheduler::run_scheduler; +use crate::scheduler::models::SchedulerState; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; use uuid::Uuid; -use crate::lease_helpers::expire_lease; -use crate::location_helpers::create_location; -use crate::replica_helpers::second_store; -use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::ticks::helpers::snapshot; -use crate::ticks::test_types::{IdleLauncher, TickTransactionTestCase}; +use crate::credentials::tests::location_helpers::create_location; +use crate::scheduler::tests::contracts::ticks::helpers::snapshot; +use crate::scheduler::tests::contracts::ticks::test_types::{ + IdleLauncher, TickTransactionTestCase, +}; +use crate::store::tests::lease_helpers::expire_lease; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; #[test] fn given_embedded_and_standalone_schedulers_when_taking_over_then_only_one_drains_due_work_and_idle_shutdown_is_prompt() @@ -75,13 +77,10 @@ fn given_embedded_and_standalone_schedulers_when_taking_over_then_only_one_drain .1 .leader ); - let runs = sqlbuild_orchestration::runs::main::list_runs::list_runs( - store, - (location, "development"), - 100, - ) - .await - .expect("queued run"); + let runs = + crate::runs::main::list_runs::list_runs(store, (location, "development"), 100) + .await + .expect("queued run"); assert_eq!(runs.len(), test_case.expected_runs); let (stop, shutdown) = tokio::sync::watch::channel(false); let (_changed, changes) = tokio::sync::watch::channel(()); diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs index 34e9c4b198..1dd53ec524 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs @@ -1,24 +1,24 @@ //! Operator trigger facts remain consistent across publication and retry boundaries. -use sqlbuild_orchestration::credentials::models::AuditContext; -use sqlbuild_orchestration::declarations::main::change_trigger::change_trigger; -use sqlbuild_orchestration::declarations::main::list_trigger_events::list_trigger_events; -use sqlbuild_orchestration::declarations::main::list_triggers::list_triggers; -use sqlbuild_orchestration::declarations::models::{ - TriggerChange, TriggerChangeRequest, TriggerStatus, -}; -use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::scheduler::main::evaluate_trigger::evaluate_trigger; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use crate::credentials::models::AuditContext; +use crate::declarations::main::change_trigger::change_trigger; +use crate::declarations::main::list_trigger_events::list_trigger_events; +use crate::declarations::main::list_triggers::list_triggers; +use crate::declarations::models::{TriggerChange, TriggerChangeRequest, TriggerStatus}; +use crate::scheduler::main::_acquire_lease::acquire_lease; +use crate::scheduler::main::_evaluate_trigger::evaluate_trigger; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; use uuid::Uuid; -use crate::fault_helpers::{install_fault, remove_fault}; -use crate::location_helpers::create_location; -use crate::store_helpers::{close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::ticks::helpers::{snapshot, trigger_audits}; -use crate::ticks::test_types::{TransactionFault, TriggerControlsTestCase}; +use crate::credentials::tests::location_helpers::create_location; +use crate::scheduler::tests::contracts::ticks::helpers::{snapshot, trigger_audits}; +use crate::scheduler::tests::contracts::ticks::test_types::{ + TransactionFault, TriggerControlsTestCase, +}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; #[test] fn given_operator_changes_when_retried_and_republished_then_facts_and_effective_state_remain_consistent() diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs index 5f10deea0d..d523cedb8e 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs @@ -1,7 +1,7 @@ //! Database cases and expected scheduler tick outcomes. -pub(crate) use crate::fault_types::TransactionFault; -use crate::store_test_types::Backend; +pub(crate) use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::store_test_types::Backend; pub(crate) struct TickTransactionTestCase { pub(crate) description: &'static str, @@ -18,20 +18,20 @@ pub(crate) struct TriggerControlsTestCase { pub(crate) struct IdleLauncher; -impl sqlbuild_orchestration::scheduler::types::RunLauncher for IdleLauncher { +impl crate::scheduler::types::RunLauncher for IdleLauncher { async fn poll( &mut self, - _store: &sqlbuild_orchestration::store::models::ControlStore, - _lease: Option<&sqlbuild_orchestration::scheduler::models::SchedulerLease>, - ) -> Result<(), sqlbuild_orchestration::store::errors::ControlError> { + _store: &crate::store::models::ControlStore, + _lease: Option<&crate::scheduler::models::SchedulerLease>, + ) -> Result<(), crate::store::errors::ControlError> { Ok(()) } async fn shutdown( &mut self, - _store: &sqlbuild_orchestration::store::models::ControlStore, - _lease: Option<&sqlbuild_orchestration::scheduler::models::SchedulerLease>, - ) -> Result<(), sqlbuild_orchestration::store::errors::ControlError> { + _store: &crate::store::models::ControlStore, + _lease: Option<&crate::scheduler::models::SchedulerLease>, + ) -> Result<(), crate::store::errors::ControlError> { Ok(()) } } diff --git a/crates/sqlbuild-orchestration/tests/snapshots.rs b/crates/sqlbuild-orchestration/tests/snapshots.rs deleted file mode 100644 index 13be1a8abc..0000000000 --- a/crates/sqlbuild-orchestration/tests/snapshots.rs +++ /dev/null @@ -1,14 +0,0 @@ -#[path = "snapshots/test_contract.rs"] -mod contract; -#[path = "snapshots/helpers.rs"] -mod helpers; -#[path = "credentials/locations/helpers.rs"] -mod location_helpers; -#[path = "store/replicas/helpers.rs"] -mod replica_helpers; -#[path = "store/helpers.rs"] -mod store_helpers; -#[path = "store/test_types.rs"] -mod store_test_types; -#[path = "snapshots/test_types.rs"] -mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index e486bb4182..7c093b292f 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -2,21 +2,21 @@ use std::collections::BTreeMap; +use crate::snapshots::main::current_snapshot::current_snapshot; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::models::{SnapshotInput, SnapshotNode}; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::main::open_store::open_store; +use crate::store::models::ControlStore; use serde_json::json; -use sqlbuild_orchestration::snapshots::main::current_snapshot::current_snapshot; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; -use sqlbuild_orchestration::store::errors::ControlError; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; -use sqlbuild_orchestration::store::main::open_store::open_store; -use sqlbuild_orchestration::store::models::ControlStore; use uuid::Uuid; -use crate::location_helpers::create_location; -use crate::replica_helpers::second_store; -use crate::store_helpers::{close_store, close_test_store, open_test_store}; -use crate::store_test_types::Backend; -use crate::test_types::SchemaRejectionCase; +use crate::credentials::tests::location_helpers::create_location; +use crate::snapshots::tests::test_types::SchemaRejectionCase; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store}; +use crate::store::tests::store_test_types::Backend; fn snapshot(location_id: Uuid) -> SnapshotInput { SnapshotInput { diff --git a/crates/sqlbuild-orchestration/tests/snapshots/test_contract.rs b/crates/sqlbuild-orchestration/tests/snapshots/test_contract.rs index 8096938089..708a6aff32 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/test_contract.rs @@ -1,7 +1,7 @@ -use crate::helpers::{snapshot_contract, sqlite_failed_publish_and_schema_floor}; -use crate::store_helpers::run; -use crate::store_test_types::Backend; -use crate::test_types::{RecoveryTestCase, SnapshotTestCase}; +use crate::snapshots::tests::helpers::{snapshot_contract, sqlite_failed_publish_and_schema_floor}; +use crate::snapshots::tests::test_types::{RecoveryTestCase, SnapshotTestCase}; +use crate::store::tests::store_helpers::run; +use crate::store::tests::store_test_types::Backend; #[test] fn given_control_store_when_snapshots_publish_then_generations_sources_and_changes_are_consistent() diff --git a/crates/sqlbuild-orchestration/tests/snapshots/test_contracts.rs b/crates/sqlbuild-orchestration/tests/snapshots/test_contracts.rs new file mode 100644 index 0000000000..6963cfddf6 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/snapshots/test_contracts.rs @@ -0,0 +1,8 @@ +//! Snapshot contracts compiled within the snapshot owner. + +#[path = "test_contract.rs"] +mod contract; +#[path = "helpers.rs"] +mod helpers; +#[path = "test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/snapshots/test_types.rs b/crates/sqlbuild-orchestration/tests/snapshots/test_types.rs index cd4379f770..61a2c6832b 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/test_types.rs @@ -1,6 +1,6 @@ //! Cases that alter a disposable schema to verify startup rejection. -use crate::store_test_types::Backend; +use crate::store::tests::store_test_types::Backend; #[derive(Debug)] pub(super) struct SchemaRejectionCase { diff --git a/crates/sqlbuild-orchestration/tests/store.rs b/crates/sqlbuild-orchestration/tests/store.rs deleted file mode 100644 index 5a79851f21..0000000000 --- a/crates/sqlbuild-orchestration/tests/store.rs +++ /dev/null @@ -1,16 +0,0 @@ -//! Recall storage integration contracts for SQLite and PostgreSQL. - -#[path = "store/recall/helpers.rs"] -mod helpers; -#[path = "store/replicas/helpers.rs"] -mod replica_helpers; -#[path = "store/helpers.rs"] -mod store_helpers; -#[path = "store/test_types.rs"] -mod store_test_types; -#[path = "store/recall/test_contract.rs"] -mod test_contract; -#[path = "store/recall/test_types.rs"] -mod test_types; -#[path = "store/recall/test_views.rs"] -mod test_views; diff --git a/crates/sqlbuild-orchestration/tests/store/faults/helpers.rs b/crates/sqlbuild-orchestration/tests/store/faults/helpers.rs index c7d5c81dd5..8c221edcc0 100644 --- a/crates/sqlbuild-orchestration/tests/store/faults/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/faults/helpers.rs @@ -1,8 +1,8 @@ //! Database-trigger fault injection at durable publication boundaries. -use sqlbuild_orchestration::store::models::ControlStore; +use crate::store::models::ControlStore; -use crate::fault_types::TransactionFault; +use crate::store::tests::fault_types::TransactionFault; pub(crate) async fn install_fault(store: &ControlStore, point: &TransactionFault) { let ControlStore::Postgres(pool) = store else { diff --git a/crates/sqlbuild-orchestration/tests/store/helpers.rs b/crates/sqlbuild-orchestration/tests/store/helpers.rs index f93f44f0f5..db2525d17d 100644 --- a/crates/sqlbuild-orchestration/tests/store/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/helpers.rs @@ -1,10 +1,10 @@ //! Real-database setup shared by storage, scheduling, and queue contracts. -use sqlbuild_orchestration::store::main::open_store::open_store; -use sqlbuild_orchestration::store::models::ControlStore; +use crate::store::main::open_store::open_store; +use crate::store::models::ControlStore; use uuid::Uuid; -use crate::store_test_types::{Backend, TestStore}; +use crate::store::tests::store_test_types::{Backend, TestStore}; pub(crate) fn run(future: impl std::future::Future) -> T { tokio::runtime::Builder::new_current_thread() diff --git a/crates/sqlbuild-orchestration/tests/store/leases/helpers.rs b/crates/sqlbuild-orchestration/tests/store/leases/helpers.rs index b813d957d3..59f067dc23 100644 --- a/crates/sqlbuild-orchestration/tests/store/leases/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/leases/helpers.rs @@ -1,6 +1,6 @@ //! Expire the persisted lease without introducing a fake control-store clock. -use sqlbuild_orchestration::store::models::ControlStore; +use crate::store::models::ControlStore; pub(crate) async fn expire_lease(store: &ControlStore) { let ControlStore::Postgres(pool) = store else { diff --git a/crates/sqlbuild-orchestration/tests/store/recall/helpers.rs b/crates/sqlbuild-orchestration/tests/store/recall/helpers.rs index 2fb0605a1d..a530b60e5c 100644 --- a/crates/sqlbuild-orchestration/tests/store/recall/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/recall/helpers.rs @@ -1,16 +1,55 @@ //! Recall publication fixtures exercise the control store's durable append boundary. +use crate::store::errors::ControlError; +use crate::store::main::append_recall::append_recall; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::migrate_store::migrate_store; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, RecallAppendOutcome, TransactionMode}; +use crate::store::types::Row; use serde_json::json; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; -use sqlbuild_orchestration::store::main::publish_recall::publish_recall; -use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; use sqlbuild_recall::records::main::artifact_rows::artifact_rows; use sqlbuild_recall::records::main::event_object_row::event_object_row; use sqlbuild_recall::records::models::{ArtifactInput, RecallRow, RowEnvelope}; -use crate::replica_helpers::second_store; -use crate::store_helpers::{close_store, close_test_store, open_test_store}; -use crate::store_test_types::Backend; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store}; +use crate::store::tests::store_test_types::Backend; + +pub(crate) async fn publish_recall( + store: &ControlStore, + rows: &[RecallRow], +) -> Result, ControlError> { + let mut transaction = begin(store, TransactionMode::Write).await?; + let mut outcomes = Vec::with_capacity(rows.len()); + for row in rows { + let outcome; + (transaction, outcome) = append_recall(transaction, row).await?; + outcomes.push(outcome); + } + commit(transaction).await?; + Ok(outcomes) +} + +pub(crate) async fn read_recall_events( + store: &ControlStore, + after: i64, + limit: u32, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query( + transaction, + "SELECT * FROM recall_events WHERE storage_id > $1 ORDER BY storage_id LIMIT $2", + vec![ + Parameter::Integer(after), + Parameter::Integer(i64::from(limit.min(1000))), + ], + ) + .await?; + commit(transaction).await?; + Ok(rows) +} pub(crate) fn event(id: &str, occurred_at: &str, duration_ms: f64) -> RecallRow { event_object_row(&RowEnvelope { invocation_id: "orders-build".into(), diff --git a/crates/sqlbuild-orchestration/tests/store/recall/test_contract.rs b/crates/sqlbuild-orchestration/tests/store/recall/test_contract.rs index d0ccf95c46..988e9542be 100644 --- a/crates/sqlbuild-orchestration/tests/store/recall/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/store/recall/test_contract.rs @@ -1,9 +1,9 @@ -//! Idempotency and commit-order paging through the public store API. +//! Idempotency and commit-order paging through real store transactions. -use crate::helpers::recall_contract; -use crate::store_helpers::run; -use crate::store_test_types::Backend; -use crate::test_types::RecallTestCase; +use crate::store::tests::helpers::recall_contract; +use crate::store::tests::store_helpers::run; +use crate::store::tests::store_test_types::Backend; +use crate::store::tests::test_types::RecallTestCase; #[test] fn given_native_recall_facts_when_retried_and_paged_then_content_and_commit_order_are_preserved() { diff --git a/crates/sqlbuild-orchestration/tests/store/recall/test_types.rs b/crates/sqlbuild-orchestration/tests/store/recall/test_types.rs index 481cac0601..53cbf105a6 100644 --- a/crates/sqlbuild-orchestration/tests/store/recall/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/store/recall/test_types.rs @@ -1,6 +1,6 @@ //! Expected Recall storage outcomes for both supported databases. -use crate::store_test_types::Backend; +use crate::store::tests::store_test_types::Backend; pub(crate) struct RecallTestCase { pub(crate) description: &'static str, diff --git a/crates/sqlbuild-orchestration/tests/store/recall/test_views.rs b/crates/sqlbuild-orchestration/tests/store/recall/test_views.rs index 75a8b9971d..a157ecf787 100644 --- a/crates/sqlbuild-orchestration/tests/store/recall/test_views.rs +++ b/crates/sqlbuild-orchestration/tests/store/recall/test_views.rs @@ -1,9 +1,8 @@ //! Durable Recall views preserve the published execution records. +use crate::store::main::migrate_store::migrate_store; +use crate::store::main::read_recall_view::read_recall_view; use serde_json::json; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; -use sqlbuild_orchestration::store::main::publish_recall::publish_recall; -use sqlbuild_orchestration::store::main::read_recall_view::read_recall_view; use sqlbuild_recall::records::main::artifact_rows::artifact_rows; use sqlbuild_recall::records::main::delivery_row::delivery_row; use sqlbuild_recall::records::main::event_object_row::event_object_row; @@ -14,9 +13,10 @@ use sqlbuild_recall::records::models::{ ArtifactInput, DeliveryCounts, InvocationContext, OutputChunk, OutputStream, RowEnvelope, }; -use crate::store_helpers::{close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::test_types::RecallViewTestCase; +use crate::store::tests::helpers::publish_recall; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; +use crate::store::tests::test_types::RecallViewTestCase; #[test] fn given_native_context_output_and_audits_when_reading_views_then_both_databases_preserve_their_meaning() diff --git a/crates/sqlbuild-orchestration/tests/store/replicas/helpers.rs b/crates/sqlbuild-orchestration/tests/store/replicas/helpers.rs index 22fe1b374b..443b0e5e26 100644 --- a/crates/sqlbuild-orchestration/tests/store/replicas/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/replicas/helpers.rs @@ -1,7 +1,7 @@ //! Independent database handles for multi-replica integration contracts. -use sqlbuild_orchestration::store::main::open_store::open_store; -use sqlbuild_orchestration::store::models::ControlStore; +use crate::store::main::open_store::open_store; +use crate::store::models::ControlStore; pub(crate) async fn second_store(store: &ControlStore) -> ControlStore { let ControlStore::Postgres(pool) = store else { diff --git a/crates/sqlbuild-orchestration/tests/store/test_contracts.rs b/crates/sqlbuild-orchestration/tests/store/test_contracts.rs new file mode 100644 index 0000000000..a257c9aeb9 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/test_contracts.rs @@ -0,0 +1,22 @@ +//! Recall storage integration contracts for SQLite and PostgreSQL. + +#[path = "faults/helpers.rs"] +pub(crate) mod fault_helpers; +#[path = "faults/test_types.rs"] +pub(crate) mod fault_types; +#[path = "recall/helpers.rs"] +pub(crate) mod helpers; +#[path = "leases/helpers.rs"] +pub(crate) mod lease_helpers; +#[path = "replicas/helpers.rs"] +pub(crate) mod replica_helpers; +#[path = "helpers.rs"] +pub(crate) mod store_helpers; +#[path = "test_types.rs"] +pub(crate) mod store_test_types; +#[path = "recall/test_contract.rs"] +mod test_contract; +#[path = "recall/test_types.rs"] +mod test_types; +#[path = "recall/test_views.rs"] +mod test_views; diff --git a/crates/sqlbuild-orchestration/tests/store/test_types.rs b/crates/sqlbuild-orchestration/tests/store/test_types.rs index 55a1a65092..1ffddfd679 100644 --- a/crates/sqlbuild-orchestration/tests/store/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/store/test_types.rs @@ -1,6 +1,6 @@ //! Store fixtures retain their isolated SQLite directory or PostgreSQL schema until cleanup. -use sqlbuild_orchestration::store::models::ControlStore; +use crate::store::models::ControlStore; #[derive(Clone, Copy, Debug)] pub(crate) enum Backend { From 5a37fcf72015f3483cf8f90f03764296aff97877 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 17:46:35 +0000 Subject: [PATCH 41/59] feat: execute explicit backfill windows Freeze inclusive typed cursor windows with run requests, derive execution overrides without mutating pinned job settings, and preserve the window through group children and reruns. Verify real local and Docker warehouse writes and the SQLite/PostgreSQL publication contracts. Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../orchestration/backfill_integer.sql | 16 ++ .../orchestration/backfill_timestamp.sql | 17 ++ .../sqlbuild-cli/src/argv/command_table.json | 28 +++ .../src/native_commands/_helpers/decision.rs | 47 ++++- .../src/runs/_helpers/requests/job.rs | 7 +- .../src/runs/main/request_job_run.rs | 10 +- .../sqlbuild-orchestration/src/runs/models.rs | 18 ++ .../src/scheduler/_helpers/persistence.rs | 1 + .../tests/runs/helpers.rs | 1 + .../tests/runs/test_group_admission.rs | 5 + .../tests/runs/test_groups.rs | 2 + .../tests/runs/test_requests.rs | 29 +++- .../tests/runs/test_reruns.rs | 7 +- .../_helpers/location_command/worker.rs | 23 ++- .../publication/_test_types.py | 7 + .../publication/test_backfill.py | 164 ++++++++++++++++++ 16 files changed, 375 insertions(+), 7 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/backfill_integer.sql create mode 100644 benchmarks/duckshop/edge_cases/orchestration/backfill_timestamp.sql create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_backfill.py diff --git a/benchmarks/duckshop/edge_cases/orchestration/backfill_integer.sql b/benchmarks/duckshop/edge_cases/orchestration/backfill_integer.sql new file mode 100644 index 0000000000..2d882a92ae --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/backfill_integer.sql @@ -0,0 +1,16 @@ +MODEL ( + description 'Backfill orders over an explicit integer window.', + materialized incremental, + incremental_strategy delete_insert, + incremental_mode microbatch, + microbatch_strategy watermark, + cursor batch_id, + cursor_type integer, + cursor_start 0, + cursor_watermark_mode all, + cursor_inputs (raw_orders (column batch_id, roles [filter, watermark])), + batch_size '2', + batch_concurrency 1, +); +SELECT amount, batch_id FROM __source("raw_orders") +WHERE batch_id >= __cursor_start() AND batch_id < __cursor_end() diff --git a/benchmarks/duckshop/edge_cases/orchestration/backfill_timestamp.sql b/benchmarks/duckshop/edge_cases/orchestration/backfill_timestamp.sql new file mode 100644 index 0000000000..fa8b2bcd25 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/backfill_timestamp.sql @@ -0,0 +1,17 @@ +MODEL ( + description 'Backfill orders over an explicit timestamp window.', + materialized incremental, + incremental_strategy delete_insert, + incremental_mode microbatch, + microbatch_strategy watermark, + cursor ordered_at, + cursor_type timestamp, + cursor_grain day, + cursor_start '2026-07-01', + cursor_watermark_mode all, + cursor_inputs (raw_orders (column ordered_at, roles [filter, watermark])), + batch_size 1d, + batch_concurrency 1, +); +SELECT amount, ordered_at FROM __source("raw_orders") +WHERE ordered_at >= __cursor_start() AND ordered_at < __cursor_end() diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index fea155edee..038ceefb5b 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -583,6 +583,34 @@ "help": "run UUID; reuse it with identical arguments after an uncertain result", "group": 1 }, + { + "kind": "store", + "options": ["--start-cursor-ts"], + "dest": "admin_start_cursor_ts", + "help": "backfill timestamp start (RFC3339); requires --end-cursor-ts", + "group": 1 + }, + { + "kind": "store", + "options": ["--end-cursor-ts"], + "dest": "admin_end_cursor_ts", + "help": "inclusive backfill timestamp end (RFC3339); requires --start-cursor-ts", + "group": 1 + }, + { + "kind": "store", + "options": ["--start-cursor-int"], + "dest": "admin_start_cursor_int", + "help": "backfill integer start; requires --end-cursor-int", + "group": 1 + }, + { + "kind": "store", + "options": ["--end-cursor-int"], + "dest": "admin_end_cursor_int", + "help": "inclusive backfill integer end; requires --start-cursor-int", + "group": 1 + }, { "kind": "store", "options": [ diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index c0e69b8143..f78d6f1874 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -8,7 +8,7 @@ use serde_json::json; use sqlbuild_orchestration::credentials::models::{CredentialScope, PrincipalKind}; use sqlbuild_orchestration::declarations::models::TriggerChange; use sqlbuild_orchestration::platform::models::{LocationWork, LocationWorkRequest}; -use sqlbuild_orchestration::runs::models::{JobRunRequest, RerunRequest}; +use sqlbuild_orchestration::runs::models::{JobRunRequest, RerunRequest, RunWindow}; use sqlbuild_orchestration::store::errors::ControlError; use uuid::Uuid; @@ -125,6 +125,50 @@ pub(crate) fn control_admin(namespace: &Namespace) -> Result None, + (Some(start), Some(end), None, None) => Some(RunWindow::Timestamp { + start: DateTime::parse_from_rfc3339(&start) + .map_err(|_| { + ControlError::Invalid( + "backfill start must be an RFC3339 timestamp with timezone" + .into(), + ) + })? + .with_timezone(&Utc), + end: DateTime::parse_from_rfc3339(&end) + .map_err(|_| { + ControlError::Invalid( + "backfill end must be an RFC3339 timestamp with timezone" + .into(), + ) + })? + .with_timezone(&Utc), + }), + (None, None, Some(start), Some(end)) => Some(RunWindow::Integer { + start: start.parse().map_err(|_| { + ControlError::Invalid( + "backfill start must be a signed 64-bit integer".into(), + ) + })?, + end: end.parse().map_err(|_| { + ControlError::Invalid( + "backfill end must be a signed 64-bit integer".into(), + ) + })?, + }), + _ => { + return Err(ControlError::Invalid( + "backfill requires both start and end of exactly one cursor type" + .into(), + )); + } + }; return Ok(ControlAdmin::RequestJobRun(JobRunRequest { run_id: event_id, location_id: id, @@ -132,6 +176,7 @@ pub(crate) fn control_admin(namespace: &Namespace) -> Result start > end, + RunWindow::Integer { start, end } => start > end, + }) { + return Err(ControlError::Invalid( + "backfill window start must not follow end".into(), + )); + } let event_id = Uuid::new_v5(&request.run_id, b"manual-job-request"); let transaction = begin(store, TransactionMode::Write).await?; let (transaction, retry) = request_retry( diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index c049379a61..64ab1a370c 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -73,6 +73,20 @@ impl RunStatus { } } +/// Inclusive typed CLI cursor bounds; the native planner derives exclusive execution ends. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum RunWindow { + Timestamp { + start: DateTime, + end: DateTime, + }, + Integer { + start: i64, + end: i64, + }, +} + /// The authorized selection and pinned execution inputs are immutable once requested. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] pub struct RunRequest { @@ -94,6 +108,8 @@ pub struct RunRequest { pub selection: BTreeSet, pub write_set: BTreeSet, pub settings: BTreeMap, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub window: Option, pub concurrency_groups: BTreeSet, pub overlap: crate::scheduler::models::Overlap, } @@ -108,6 +124,8 @@ pub struct JobRunRequest { pub job_key: String, pub reason: String, pub priority: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub window: Option, } /// A retry derives its execution scope from a terminal run, never from caller selectors. diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs index 3a69b46ea5..0f659487c0 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs @@ -61,6 +61,7 @@ pub(crate) async fn persist_ticks( write_set.clone() }, settings: trigger.job_json.settings.clone(), + window: None, concurrency_groups: trigger.job_json.concurrency_groups.clone(), overlap: trigger.job_json.overlap, }; diff --git a/crates/sqlbuild-orchestration/tests/runs/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/helpers.rs index c86ac7fe47..12663d3921 100644 --- a/crates/sqlbuild-orchestration/tests/runs/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/runs/helpers.rs @@ -80,6 +80,7 @@ pub(crate) fn request( write_set: selection.clone(), selection, settings: BTreeMap::new(), + window: None, concurrency_groups: BTreeSet::new(), overlap: Overlap::Allow, } diff --git a/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs b/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs index a57ce0a896..015e0ca74c 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs @@ -90,6 +90,7 @@ fn given_mixed_physical_and_parent_waiters_when_admitting_then_priority_and_queu job_key: "daily".into(), reason: "Exercise mixed queue ordering".into(), priority: test_case.parent_priority, + window: None, }, ) .await @@ -184,6 +185,7 @@ fn given_an_unserved_queued_group_when_dispatching_an_eligible_scope_then_unknow job_key: "daily".into(), reason: "Exercise launcher eligibility".into(), priority: 1, + window: None, }, ) .await @@ -426,6 +428,7 @@ fn given_capacity_and_multistage_cycles_when_replicas_admit_and_resume_groups_th job_key: format!("orders_group_{index}"), reason: "Exercise capacity-aware admission".into(), priority: index as i64 + 1, + window: None, }, ) .await @@ -585,6 +588,7 @@ fn given_an_older_blocked_allow_group_when_an_independent_generation_is_queued_t job_key: "daily".into(), reason: "Older group".into(), priority: 1, + window: None, }; request_job_run(store, &actor, &older) .await @@ -758,6 +762,7 @@ fn given_a_large_healthy_project_with_blocked_groups_when_claiming_then_unrelate job_key: format!("group_{index}"), reason: "Exercise blocked group queue".into(), priority: 1, + window: None, }, ) .await diff --git a/crates/sqlbuild-orchestration/tests/runs/test_groups.rs b/crates/sqlbuild-orchestration/tests/runs/test_groups.rs index b2fe494aef..10f1b53bf8 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_groups.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_groups.rs @@ -78,6 +78,7 @@ fn given_a_pinned_group_when_redeployed_and_cancelled_then_children_keep_scope_a job_key: "daily_orders".into(), reason: "Build the group".into(), priority: 1, + window: None, }; assert!( request_job_run(store, &context, &request) @@ -351,6 +352,7 @@ fn given_opposing_parent_and_child_limits_when_advancing_groups_then_both_groups job_key: job_key.into(), reason: "Build independent groups".into(), priority: 1, + window: None, }, ) .await diff --git a/crates/sqlbuild-orchestration/tests/runs/test_requests.rs b/crates/sqlbuild-orchestration/tests/runs/test_requests.rs index 5ad011d62f..df6994d8cf 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_requests.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_requests.rs @@ -6,7 +6,7 @@ use crate::credentials::models::AuditContext; use crate::runs::main::claim_next_run::claim_next_run; use crate::runs::main::list_runs::list_runs; use crate::runs::main::request_job_run::request_job_run; -use crate::runs::models::{JobRunRequest, RunStatus}; +use crate::runs::models::{JobRunRequest, RunOrigin, RunStatus, RunWindow}; use crate::scheduler::tests::contracts::helpers::acquire_lease; use crate::snapshots::main::publish_snapshot::publish_snapshot; use crate::store::main::migrate_store::migrate_store; @@ -68,6 +68,10 @@ fn given_a_published_job_when_requested_and_retried_after_redeploy_then_audit_an job_key: "daily_orders".into(), reason: "Refresh orders".into(), priority: 7, + window: Some(RunWindow::Timestamp { + start: "2026-07-03T01:17:23+05:30".parse().expect("start"), + end: "2026-07-04T01:17:23+05:30".parse().expect("end"), + }), }; for invalid in [ JobRunRequest { @@ -86,6 +90,17 @@ fn given_a_published_job_when_requested_and_retried_after_redeploy_then_audit_an reason: " ".into(), ..request.clone() }, + JobRunRequest { + window: Some(RunWindow::Integer { start: 11, end: 3 }), + ..request.clone() + }, + JobRunRequest { + window: Some(RunWindow::Timestamp { + start: "2026-07-04T01:17:23Z".parse().expect("start"), + end: "2026-07-03T01:17:23Z".parse().expect("end"), + }), + ..request.clone() + }, ] { assert!(request_job_run(store, &context, &invalid).await.is_err()); } @@ -139,6 +154,9 @@ fn given_a_published_job_when_requested_and_retried_after_redeploy_then_audit_an let mut changed = request.clone(); changed.reason = "Different request".into(); assert!(request_job_run(store, &context, &changed).await.is_err()); + changed = request.clone(); + changed.window = Some(RunWindow::Integer { start: 3, end: 11 }); + assert!(request_job_run(store, &context, &changed).await.is_err()); let other_actor = AuditContext { actor: "other-operator".into(), ..context.clone() @@ -168,6 +186,14 @@ fn given_a_published_job_when_requested_and_retried_after_redeploy_then_audit_an .expect("queued request"); assert_eq!(claim.request.run_id, request.run_id); assert_eq!(claim.request.snapshot_id, published.snapshot_id); + assert_eq!(claim.request.origin, RunOrigin::Backfill); + assert_eq!( + claim.request.window, + Some(RunWindow::Timestamp { + start: "2026-07-02T19:47:23Z".parse().expect("normalized start"), + end: "2026-07-03T19:47:23Z".parse().expect("normalized end"), + }) + ); assert_eq!( claim.request.image_id.as_deref(), Some("sha256:synthetic-orders-image") @@ -189,6 +215,7 @@ fn given_a_published_job_when_requested_and_retried_after_redeploy_then_audit_an assert_eq!(claim.request.requested_by, "test-operator"); let fresh = JobRunRequest { run_id: Uuid::now_v7(), + window: Some(RunWindow::Integer { start: 7, end: 7 }), ..request.clone() }; assert!( diff --git a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs index 4c50700634..75339ea1fe 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs @@ -9,7 +9,7 @@ use crate::runs::main::{ }; use crate::runs::models::{ JobRunRequest, NodeOutcome, RerunRequest, RunAuthority, RunChange, RunChangeRequest, RunOrigin, - RunOutcome, RunStatus, + RunOutcome, RunStatus, RunWindow, }; use crate::scheduler::tests::contracts::helpers::acquire_lease; use crate::snapshots::main::publish_snapshot::publish_snapshot; @@ -80,6 +80,7 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ job_key: "daily_orders".into(), reason: "Build orders".into(), priority: 1, + window: Some(RunWindow::Integer { start: -3, end: 11 }), }, ) .await @@ -285,6 +286,10 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ assert_eq!(claim.request.origin, RunOrigin::Rerun); assert_eq!(claim.request.priority, 7); assert_eq!(claim.request.settings, settings[expected_job]); + assert_eq!( + claim.request.window, + Some(RunWindow::Integer { start: -3, end: 11 }) + ); selected.extend(claim.request.selection.iter().cloned()); succeed(store, &claim).await; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs index 52da11300d..b7876f07f4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -10,7 +10,7 @@ use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; use sqlbuild_cli::dispatch::main::command_dispatch::command_dispatch; use sqlbuild_cli::dispatch::models::Dispatch; use sqlbuild_host::entries::models::ExecutorHost; -use sqlbuild_orchestration::runs::models::RunOutcome; +use sqlbuild_orchestration::runs::models::{RunOutcome, RunWindow}; use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with, within}; use crate::bindings::_helpers::build_command::command::run_build; @@ -31,8 +31,27 @@ fn build_request<'py>( ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { let execution = worker.get().execution(py)?; let frozen = execution.claim.request; + let mut settings = frozen.settings; + if let Some(window) = frozen.window { + for key in [ + "start_cursor", + "end_cursor", + "start_cursor_ts", + "end_cursor_ts", + "start_cursor_int", + "end_cursor_int", + ] { + settings.remove(key); + } + let (kind, start, end) = match window { + RunWindow::Timestamp { start, end } => ("ts", start.to_rfc3339(), end.to_rfc3339()), + RunWindow::Integer { start, end } => ("int", start.to_string(), end.to_string()), + }; + settings.insert(format!("start_cursor_{kind}"), serde_json::json!(start)); + settings.insert(format!("end_cursor_{kind}"), serde_json::json!(end)); + } let mut argv: Vec> = std::iter::once("build".to_owned()) - .chain(job_settings_arguments(&frozen.settings).map_err(PyValueError::new_err)?) + .chain(job_settings_arguments(&settings).map_err(PyValueError::new_err)?) .map(|argument| PyString::new(py, &argument)) .collect(); argv.extend([ diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 135c64aaf6..e6b239fffe 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -120,3 +120,10 @@ class WorkerRestartTestCase(WorkerExecutionTestCase): description: str shutdown_signal: int = 15 expected_scheduler_exit: int = 0 + + +@dataclass(frozen=True) +class BackfillExecutionTestCase(WorkerExecutionTestCase): + description: str + window_arguments: tuple[str, ...] = () + expected_window: dict[str, Any] = field(default_factory=dict) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_backfill.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_backfill.py new file mode 100644 index 0000000000..9d54256fc6 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_backfill.py @@ -0,0 +1,164 @@ +"""Explicit backfills and their reruns execute the same pinned window through real workers.""" + +import json +import sqlite3 +import subprocess +import sys +from dataclasses import replace +from pathlib import Path +from typing import Any +from uuid import uuid4 + +import duckdb +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + BackfillExecutionTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + scheduled_worker, + wait_for_run, +) + + +@pytest.mark.parametrize( + "test_case", + [ + BackfillExecutionTestCase( + description="local integer window overrides authored bounds and survives group rerun", + model_sql="SELECT 1 AS amount", + warehouse_query="SELECT amount FROM orders ORDER BY amount", + expected_status="success", + expected_rows=[(23,), (37,)], + files={ + "models/orders.sql": Path( + "benchmarks/duckshop/edge_cases/orchestration/backfill_integer.sql" + ).read_text() + }, + window_arguments=("--start-cursor-int", "3", "--end-cursor-int", "6"), + expected_window={"kind": "integer", "start": 3, "end": 6}, + ), + BackfillExecutionTestCase( + description="Docker timestamp window normalizes offsets and survives group rerun", + model_sql="SELECT 1 AS amount", + warehouse_query="SELECT amount FROM orders ORDER BY amount", + expected_status="success", + expected_rows=[(23,), (37,)], + launcher="docker", + files={ + "models/orders.sql": Path( + "benchmarks/duckshop/edge_cases/orchestration/backfill_timestamp.sql" + ).read_text() + }, + window_arguments=( + "--start-cursor-ts", + "2026-07-03T05:30:00+05:30", + "--end-cursor-ts", + "2026-07-04T05:30:00+05:30", + ), + expected_window={ + "kind": "timestamp", + "start": "2026-07-03T00:00:00Z", + "end": "2026-07-04T00:00:00Z", + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_explicit_backfill_bounds_when_running_and_rerunning_a_group_then_only_its_frozen_window_is_written( + test_case: BackfillExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + test_case = replace( + test_case, + files=test_case.files + | { + "sources/raw_orders.yml": "sources:\n - name: raw_orders\n description: Synthetic order windows.\n schema: main\n table: raw_orders\n", + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, job_group\n" + "orders = job('orders', description='Windowed orders.', select='orders', " + "settings={'start_cursor': '0', 'end_cursor': '1'})\n" + "annual_orders = job_group('annual_orders', description='Windowed group.', jobs=[orders])\n" + ), + }, + ) + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + with duckdb.connect(str(worker.project / "warehouse.duckdb")) as warehouse: + warehouse.execute( + "CREATE TABLE raw_orders AS SELECT * FROM (VALUES (11, 2, TIMESTAMP '2026-07-02'), (23, 3, TIMESTAMP '2026-07-03'), (37, 6, TIMESTAMP '2026-07-04'), (41, 7, TIMESTAMP '2026-07-05')) t(amount, batch_id, ordered_at)" + ) + executable: str = str(Path(sys.executable).with_name("sqb")) + run_id: str = str(uuid4()) + arguments: list[str] = [ + executable, + "admin", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "request-run", + worker.api.location_id, + "annual_orders", + "--target", + "development", + "--run-id", + run_id, + "--reason", + "Backfill an explicit order window", + *test_case.window_arguments, + ] + rejected: subprocess.CompletedProcess[str] = subprocess.run( + arguments[:-2], capture_output=True, text=True, timeout=10, check=False + ) + assert rejected.returncode != 0 + assert "both start and end of exactly one cursor type" in rejected.stderr + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_runs").fetchone() == (0,) + for expected in [True, False]: + requested: subprocess.CompletedProcess[str] = subprocess.run( + arguments, capture_output=True, text=True, timeout=10, check=False + ) + assert requested.returncode == 0, requested.stderr + assert json.loads(requested.stdout) == {"run_id": run_id, "applied": expected} + completed: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, run_id=run_id + ) + assert completed["status"] == test_case.expected_status, worker.log.read_text() + with duckdb.connect(str(worker.project / "warehouse.duckdb")) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + warehouse.execute("UPDATE raw_orders SET amount=amount+100") + retry_id: str = str(uuid4()) + retried: subprocess.CompletedProcess[str] = subprocess.run( + [ + executable, + "admin", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "rerun-run", + run_id, + "--run-id", + retry_id, + "--reason", + "Repeat the same explicit order window", + ], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + assert retried.returncode == 0, retried.stderr + retry: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, run_id=retry_id + ) + assert retry["status"] == test_case.expected_status, worker.log.read_text() + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == [(123,), (137,)] + with sqlite3.connect(worker.control_store) as control: + requests: list[dict[str, Any]] = [ + json.loads(row[0]) + for row in control.execute("SELECT request_json FROM sqb_runs").fetchall() + ] + assert len(requests) == 4 + assert all(request["window"] == test_case.expected_window for request in requests) + assert {request["origin"] for request in requests} == {"backfill", "rerun"} + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + assert control.execute( + "SELECT count(*) FROM sqb_security_audit WHERE action='job.run_requested'" + ).fetchone() == (1,) From 004088c8f05030c8afdf09540eaca5fe576d0409 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 18:34:20 +0000 Subject: [PATCH 42/59] feat: publish and refresh authored calendars Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- Cargo.lock | 2 + .../orchestration/calendar_orders.py | 17 ++ .../src/native_commands/_helpers/execution.rs | 15 +- .../src/native_commands/main/_run_location.rs | 7 +- .../src/native_commands/models.rs | 10 +- .../declarations/main/publish_definitions.rs | 44 ++- .../src/declarations/models.rs | 1 + .../tests/scheduler/ticks/test_triggers.rs | 123 ++++++++ crates/sqlbuild-python/Cargo.toml | 2 + .../_helpers/compiled_project/snapshot.rs | 16 +- .../compiled_project/snapshot_declarations.rs | 104 ++----- .../_helpers/location_command/command.rs | 32 +- .../src/bindings/_helpers/native_host/mod.rs | 1 + .../native_host/orchestration_declarations.rs | 84 +++++- .../native_host/orchestration_triggers.rs | 243 +++++++++++++++ fensu.toml | 1 + src/sqlbuild/orchestration.py | 22 +- .../main/compile_snapshot.py | 4 +- .../orchestration_declarations/models.py | 32 +- .../publication/_test_types.py | 15 + .../publication/test_calendar.py | 285 ++++++++++++++++++ 21 files changed, 934 insertions(+), 126 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py diff --git a/Cargo.lock b/Cargo.lock index d3c2ae6c05..8a5982d252 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2631,6 +2631,8 @@ dependencies = [ name = "sqlbuild-python" version = "0.186.2" dependencies = [ + "chrono", + "chrono-tz", "fensu-policy", "mimalloc", "pyo3", diff --git a/benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py b/benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py new file mode 100644 index 0000000000..7beff48f02 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py @@ -0,0 +1,17 @@ +"""Provider-backed fulfillment times evaluated in the code location, never by the scheduler.""" + +from sqlbuild.orchestration import calendar_schedule, job + +orders = job("orders", description="Build order reporting.", select="orders") + + +@calendar_schedule( + job=orders, + description="Refresh orders at upcoming fulfillment times.", + horizon="8s", + timezone="America/New_York", + catch_up="all", + max_catch_up=10, +) +def fulfillment_calendar(window, fulfillment): + return fulfillment.upcoming(window) diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 7de7f07935..ee24726afd 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -6,11 +6,11 @@ use std::path::{Path, PathBuf}; use std::process::Stdio; use std::time::{Duration, SystemTime}; +use chrono::{DateTime, Utc}; use sqlbuild_orchestration::api::errors::ClientError; use sqlbuild_orchestration::platform::main::report_remote_location_health::report_remote_location_health; use sqlbuild_orchestration::platform::models::{LocationHealth, LocationHealthReport}; use sqlbuild_orchestration::snapshots::main::next_remote_generation::next_remote_generation; -use sqlbuild_orchestration::snapshots::models::SnapshotRecord; use sqlbuild_orchestration::store::errors::ControlError; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::open_store::open_store; @@ -45,8 +45,8 @@ use crate::native_commands::constants::{ use crate::native_commands::errors::PythonNeeded; use crate::native_commands::main::_run_cost::run_cost; use crate::native_commands::models::{ - CommandOutput, InstalledPackage, LocationCompileRequest, LocationCompiler, LocationServe, - LostStdout, NativeCommand, NativeInvocation, ProjectLocation, + CommandOutput, InstalledPackage, LocationCompileRequest, LocationCompiler, LocationPublication, + LocationServe, LostStdout, NativeCommand, NativeInvocation, ProjectLocation, }; use crate::workers::main::open_workers::open_workers; use crate::workers::models::Workers; @@ -57,7 +57,7 @@ pub(crate) async fn compile_generation( generation: i64, sources: &[sqlbuild_cache::project_snapshot::models::PathStamp], stopped: impl Future>, -) -> Result, ControlError> { +) -> Result>)>, ControlError> { let (reply, publication) = oneshot::channel(); requests .send(LocationCompileRequest { @@ -89,7 +89,8 @@ pub(crate) async fn compile_generation( } }; let published = record.is_some(); - if let Some(record) = record { + let refresh_at = record.as_ref().and_then(|(_, refresh_at)| *refresh_at); + if let Some((record, _)) = record { let output = serde_json::to_string(&record) .map_err(|error| ControlError::Invalid(error.to_string()))?; if write_output(&[(Stream::Stdout, &format!("{output}\n"))]).is_some() { @@ -99,14 +100,14 @@ pub(crate) async fn compile_generation( ))); } } - Ok(Some(published)) + Ok(Some((published, refresh_at))) } /// The child is retained by its owning task while this exchange can be cancelled. pub(crate) async fn exchange_location_generation( compiler: &mut LocationCompiler, generation: i64, -) -> Result, ControlError> { +) -> Result, ControlError> { compiler .stdin .write_all(format!("{generation}\n").as_bytes()) diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs index 7fc5eb449e..5a8082353a 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs @@ -4,6 +4,7 @@ use std::future::Future; use std::path::{Path, PathBuf}; use std::time::Duration; +use chrono::Utc; use sqlbuild_orchestration::platform::main::connect_location_work::connect_location_work; use sqlbuild_orchestration::platform::models::{ LocationHealth, LocationWorkClient, LocationWorkPoll, @@ -110,13 +111,15 @@ async fn watch_location( Stream::Stderr, &format!("Compiling location generation {generation} in an isolated interpreter...\n"), )])); - let Some(published) = + let Some((published, refresh_at)) = compile_generation(&requests, generation, &watch.observed.0, stopped.as_mut()).await? else { return Ok(()); }; if published { - watch.retry_at = None; + watch.retry_at = refresh_at.map(|deadline| { + Instant::now() + (deadline - Utc::now()).to_std().unwrap_or_default() + }); published_generation = Some(generation); let Some(files) = check_published_location( options, diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index c8e6917938..627cf9baa9 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -152,14 +152,18 @@ pub(crate) struct LocationWatch { pub(crate) retry_at: Option, } +/// Private interpreter response: publication metadata and its earliest calendar refresh deadline. +pub(crate) type LocationPublication = ( + sqlbuild_orchestration::snapshots::models::SnapshotRecord, + Option>, +); + /// A bounded supervisor request; dropping the reply cancels and reaps an active compiler. #[derive(Debug)] pub(crate) struct LocationCompileRequest { pub(crate) generation: i64, pub(crate) python_sources: Vec, - pub(crate) reply: tokio::sync::oneshot::Sender< - Result, String>, - >, + pub(crate) reply: tokio::sync::oneshot::Sender, String>>, } /// Only this task owns the isolated interpreter and its private stdio protocol. diff --git a/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs b/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs index 72e6e9b2db..b9754a724f 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs @@ -3,7 +3,8 @@ use uuid::Uuid; use crate::declarations::_helpers::trigger_state::trigger_timing; -use crate::declarations::models::{Definitions, PreviousTriggerState}; +use crate::declarations::models::{Definitions, PreviousTriggerState, Schedule}; +use crate::scheduler::constants::MAX_TICKS_PER_BATCH; use crate::snapshots::models::SnapshotRecord; use crate::store::errors::ControlError; use crate::store::main::decode_row::decode_row; @@ -52,12 +53,44 @@ pub(crate) async fn publish_definitions( (transaction, _) = query(transaction, "UPDATE sqb_triggers SET removed_in_snapshot_id=$1 WHERE location_id=$2 AND target=$3 AND removed_in_snapshot_id IS NULL", vec![Parameter::Id(snapshot.snapshot_id), Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone())]).await?; - for (name, trigger) in definitions.triggers { + for (name, mut trigger) in definitions.triggers { let id = Uuid::new_v5( &snapshot.location_id, &serde_json::to_vec(&(&snapshot.target, "schedule", &name)) .map_err(|error| ControlError::Invalid(error.to_string()))?, ); + let previous; + (transaction, previous) = query(transaction, + "SELECT t.definition_json, s.started_at, s.last_tick_for, s.status_override FROM sqb_trigger_state s JOIN sqb_triggers t ON t.trigger_id=s.trigger_id WHERE s.trigger_id=$1", + vec![Parameter::Id(id)]).await?; + let state = previous + .into_iter() + .next() + .map(decode_row::) + .transpose()?; + if let Some(state) = &state + && let Schedule::Calendar { + times: previous_times, + .. + } = &state.definition_json.schedule + && let Schedule::Calendar { times, .. } = &mut trigger.schedule + { + times.extend( + previous_times + .iter() + .filter(|time| { + **time >= state.started_at + && **time <= snapshot.published_at + && state.last_tick_for.is_none_or(|last| **time > last) + }) + .copied(), + ); + if times.len() > MAX_TICKS_PER_BATCH { + return Err(ControlError::Invalid(format!( + "calendar {name} pending and upcoming times exceed {MAX_TICKS_PER_BATCH}; let the scheduler drain pending ticks before refreshing" + ))); + } + } let definition = serde_json::to_value(&trigger) .map_err(|error| ControlError::Invalid(error.to_string()))?; (transaction, _) = query(transaction, @@ -65,13 +98,6 @@ pub(crate) async fn publish_definitions( ON CONFLICT (trigger_id) DO UPDATE SET job_key=excluded.job_key, definition_json=excluded.definition_json, snapshot_id=excluded.snapshot_id, removed_in_snapshot_id=NULL", vec![Parameter::Id(id), Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone()), Parameter::Text(name), Parameter::Text(trigger.job_key.clone()), Parameter::Json(definition), Parameter::Id(snapshot.snapshot_id)]).await?; - let previous; - (transaction, previous) = query(transaction, "SELECT started_at, last_tick_for, status_override FROM sqb_trigger_state WHERE trigger_id=$1", vec![Parameter::Id(id)]).await?; - let state = previous - .into_iter() - .next() - .map(decode_row::) - .transpose()?; let started_at = state.as_ref().map_or( trigger.starts_at.unwrap_or(snapshot.published_at), |state| state.started_at, diff --git a/crates/sqlbuild-orchestration/src/declarations/models.rs b/crates/sqlbuild-orchestration/src/declarations/models.rs index 623d581f43..98f516c0d3 100644 --- a/crates/sqlbuild-orchestration/src/declarations/models.rs +++ b/crates/sqlbuild-orchestration/src/declarations/models.rs @@ -142,6 +142,7 @@ pub(crate) struct EditableTriggerState { #[derive(Clone, Debug, Deserialize)] pub(crate) struct PreviousTriggerState { + pub(crate) definition_json: TriggerDefinition, pub(crate) started_at: DateTime, pub(crate) last_tick_for: Option>, pub(crate) status_override: Option, diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs index 1dd53ec524..2c05267eb7 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs @@ -247,3 +247,126 @@ fn given_operator_changes_when_retried_and_republished_then_facts_and_effective_ }); } } + +#[test] +fn given_a_calendar_with_unprocessed_due_times_when_refreshed_then_due_ticks_survive_and_only_future_times_are_replaced() + { + use crate::runs::main::list_runs::list_runs; + use crate::scheduler::main::list_ticks::list_ticks; + use crate::scheduler::tests::contracts::ticks::test_types::TickTransactionTestCase; + use serde_json::json; + + for test_case in [ + TickTransactionTestCase { + description: "SQLite retains due calendar ticks", + backend: Backend::Sqlite, + expected_ticks: 3, + expected_runs: 1, + }, + TickTransactionTestCase { + description: "PostgreSQL retains due calendar ticks", + backend: Backend::Postgres, + expected_ticks: 3, + expected_runs: 1, + }, + ] { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + publish_snapshot(store, &input) + .await + .expect("publish pending calendar"); + input.generation = 2; + let future: chrono::DateTime = + "2031-01-01T00:00:00Z".parse().expect("future"); + input.documents.get_mut("triggers").expect("triggers")["daily_orders"]["schedule"]["times"] = json!( + (0..10_000) + .map(|seconds| future + chrono::TimeDelta::seconds(seconds)) + .collect::>() + ); + assert!( + publish_snapshot(store, &input) + .await + .expect_err("pending ticks cannot be truncated to fit a refreshed horizon") + .to_string() + .contains("let the scheduler drain pending ticks") + ); + input.documents.get_mut("triggers").expect("triggers")["daily_orders"]["schedule"]["times"] = + json!(["2031-01-01T00:00:00Z"]); + publish_snapshot(store, &input) + .await + .expect("refresh while scheduler is offline"); + let trigger = list_triggers(store, location, "development") + .await + .expect("refreshed trigger") + .remove(0); + assert_eq!( + serde_json::to_value(&trigger.definition_json.schedule).expect("calendar")["times"], + json!([ + "2020-01-01T00:00:00Z", + "2020-01-02T00:00:00Z", + "2020-01-03T00:00:00Z", + "2031-01-01T00:00:00Z" + ]), + "{}", + test_case.description + ); + assert_eq!( + trigger.next_due_at, + Some("2020-01-01T00:00:00Z".parse().expect("due")) + ); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("election") + .expect("lease"); + let plan = evaluate_trigger(store, &lease, trigger.trigger_id) + .await + .expect("catch up") + .expect("trigger"); + assert_eq!(plan.actions.len(), test_case.expected_ticks); + assert_eq!( + list_ticks(store, trigger.trigger_id, 100) + .await + .expect("ticks") + .len(), + test_case.expected_ticks + ); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("coalesced run") + .len(), + test_case.expected_runs + ); + input.generation = 3; + input.documents.get_mut("triggers").expect("triggers")["daily_orders"]["schedule"]["times"] = + json!(["2032-01-01T00:00:00Z"]); + publish_snapshot(store, &input) + .await + .expect("refresh after catch-up"); + let trigger = list_triggers(store, location, "development") + .await + .expect("drained trigger") + .remove(0); + assert_eq!( + serde_json::to_value(&trigger.definition_json.schedule).expect("calendar")["times"], + json!(["2032-01-01T00:00:00Z"]) + ); + assert_eq!( + list_ticks(store, trigger.trigger_id, 100) + .await + .expect("immutable tick history") + .len(), + test_case.expected_ticks + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index 270745994d..621a30947c 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -16,6 +16,8 @@ default = [] extension-module = ["pyo3/extension-module"] [dependencies] +chrono.workspace = true +chrono-tz.workspace = true fensu-policy.workspace = true mimalloc.workspace = true pyo3.workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs index 0d95fcf560..bbb369fa17 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs @@ -15,7 +15,7 @@ use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_re use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode, SnapshotRecord}; use crate::bindings::_helpers::compiled_project::snapshot_declarations::{ - AuthoredDeclaration, compile_declarations, discover_declarations, + compile_declarations, discover_declarations, }; use crate::bindings::_helpers::compiled_project::snapshot_models::model_nodes; use crate::bindings::_helpers::compiled_project::snapshot_resources::resource_nodes; @@ -126,13 +126,15 @@ pub(crate) fn build_orchestration_snapshot( None => discover_declarations(py, &project_dir)?, }; if !declarations.is_empty() { - let declarations = collect_declarations(py, declarations)?; - let authored = declarations - .iter() - .map(|declaration| declaration.extract::()) - .collect::>>()?; + let authored = collect_declarations(py, declarations)?; let graph = project.lineage_graph(py)?; - input = compile_declarations(input, graph.bind(py).get().graph(), authored)?; + input = compile_declarations( + py, + input, + &project_dir, + graph.bind(py).get().graph(), + authored, + )?; } Ok(NativeOrchestrationSnapshot { input }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index d5922477fc..7bc75bb723 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -3,7 +3,6 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; -use pyo3::FromPyObject; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use serde_json::Value; @@ -17,13 +16,16 @@ use sqlbuild_discovery::tree::models::ProjectTree; use sqlbuild_orchestration::declarations::main::group_dependencies::group_dependencies; use sqlbuild_orchestration::declarations::main::snapshot_definitions::snapshot_definitions; use sqlbuild_orchestration::declarations::models::{ - GroupDefinition, JobDefinition, OnFailure, Schedule, TriggerDefinition, + Definitions, GroupDefinition, JobDefinition, OnFailure, TriggerDefinition, }; use sqlbuild_orchestration::scheduler::models::Overlap; use sqlbuild_orchestration::snapshots::models::SnapshotInput; use crate::bindings::_helpers::native_host::orchestration_declarations::{ - authored_setting, module_jobs, + AuthoredDeclaration, AuthoredGroup, AuthoredJob, module_jobs, +}; +use crate::bindings::_helpers::native_host::orchestration_triggers::{ + compile_calendars, compile_cron, }; /// Dedicated declaration modules are imported only in the code-location interpreter. @@ -55,82 +57,10 @@ pub(crate) fn discover_declarations<'py>( Ok(declarations) } -#[derive(FromPyObject)] -struct AuthoredSchedule { - cron: String, - timezone: String, - #[pyo3(from_py_with = authored_setting)] - starts_at: Value, - default_status: String, - catch_up: String, - #[pyo3(from_py_with = authored_setting)] - max_catch_up: Value, -} - -#[derive(FromPyObject)] -pub(crate) struct AuthoredJob { - name: String, - description: String, - select: String, - exclude: String, - overlap: String, - on_failure: String, - #[pyo3(from_py_with = authored_setting)] - settings: Value, - #[pyo3(from_py_with = authored_setting)] - concurrency: Value, - concurrency_groups: Vec, - schedule: Option, - targets: Vec, -} - -#[derive(FromPyObject, PartialEq, Eq, PartialOrd, Ord)] -struct AuthoredJobReference { - name: String, -} - -#[derive(FromPyObject)] -pub(crate) struct AuthoredGroup { - name: String, - description: String, - jobs: Vec, - after: BTreeMap>, - overlap: String, - concurrency_groups: Vec, - schedule: Option, - targets: Vec, -} - -#[derive(FromPyObject)] -pub(crate) enum AuthoredDeclaration { - Job(AuthoredJob), - Group(AuthoredGroup), -} - -impl AuthoredSchedule { - fn compile(self, name: &str, description: &str) -> PyResult { - let catch_up = if self.max_catch_up.is_null() { - Value::String(self.catch_up) - } else { - serde_json::json!({self.catch_up: {"max_runs": self.max_catch_up}}) - }; - Ok(TriggerDefinition { - description: description.into(), - job_key: name.into(), - schedule: Schedule::Cron { expression: self.cron, timezone: self.timezone }, - starts_at: serde_json::from_value(self.starts_at) - .map_err(|error| PyValueError::new_err(format!("starts_at: {error}")))?, - default_status: serde_json::from_value(Value::String(self.default_status)) - .map_err(|error| PyValueError::new_err(format!("default_status: {error}")))?, - catch_up: serde_json::from_value(catch_up).map_err(|error| PyValueError::new_err(format!( - "catch_up: {error}; use 'latest' without a bound or 'all' with a positive max_catch_up" - )))?, - }) - } -} - pub(crate) fn compile_declarations( + py: Python<'_>, mut input: SnapshotInput, + project_dir: &Path, graph: &ProjectGraph, declarations: Vec, ) -> PyResult { @@ -148,10 +78,12 @@ pub(crate) fn compile_declarations( let mut jobs: BTreeMap = BTreeMap::new(); let mut authored_groups: Vec = Vec::new(); let mut authored_jobs: Vec = Vec::new(); + let mut authored_calendars = Vec::new(); for declaration in declarations { match declaration { AuthoredDeclaration::Job(job) => authored_jobs.push(job), AuthoredDeclaration::Group(group) => authored_groups.push(group), + AuthoredDeclaration::Calendar(calendar) => authored_calendars.push(calendar), } } let mut triggers: BTreeMap = BTreeMap::new(); @@ -193,7 +125,7 @@ pub(crate) fn compile_declarations( if let Some(schedule) = job.schedule { triggers.insert( job.name.clone(), - schedule.compile(&job.name, &job.description)?, + compile_cron(schedule.bind(py), &job.name, &job.description)?, ); } jobs.insert( @@ -244,7 +176,7 @@ pub(crate) fn compile_declarations( if let Some(schedule) = group.schedule { triggers.insert( group.name.clone(), - schedule.compile(&group.name, &group.description)?, + compile_cron(schedule.bind(py), &group.name, &group.description)?, ); } groups.insert( @@ -263,6 +195,20 @@ pub(crate) fn compile_declarations( }, ); } + let Definitions { + jobs, + groups, + triggers, + } = compile_calendars( + py, + (project_dir, &input.target), + authored_calendars, + Definitions { + jobs, + groups, + triggers, + }, + )?; for (key, document) in [ ("jobs", serde_json::to_value(jobs)), ("groups", serde_json::to_value(groups)), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs index fdd7526b6b..6f9e92f4f8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs @@ -1,13 +1,16 @@ //! Compile in the location interpreter and publish data through its scoped HTTP credential. +use std::collections::BTreeMap; use std::path::Path; use std::process::{Command, Stdio}; +use chrono::{DateTime, Utc}; use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::PyDict; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::git_revisions::main::head_commit::head_commit; +use sqlbuild_orchestration::declarations::models::{Schedule, TriggerDefinition}; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotRecord}; use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; @@ -129,7 +132,12 @@ fn compile_location<'py>( view, &serde_json::to_string(input).map_err(|error| PyValueError::new_err(error.to_string()))?, project_dir.extract()?, - None, + // Workers verify nodes and environment, not dynamic calendar declarations. + arguments + .value("action")? + .extract::>()? + .is_some_and(|action| action == "execute") + .then(Vec::new), )?; status(py, "Location snapshot compiled.")?; Ok((snapshot, discovered)) @@ -159,7 +167,7 @@ pub(super) fn location_arguments<'py>( pub(super) fn publish_location( py: Python<'_>, arguments: &KeywordArguments<'_>, -) -> PyResult { +) -> PyResult<(SnapshotRecord, Option>)> { if arguments .value("action")? .extract::>()? @@ -182,6 +190,22 @@ pub(super) fn publish_location( .extract::()?; let input = metadata(py, arguments, &directory)?; let (snapshot, _) = compile_location(py, arguments, &input)?; + let triggers: BTreeMap = serde_json::from_value( + snapshot + .input + .documents + .get("triggers") + .cloned() + .unwrap_or_else(|| serde_json::json!({})), + ) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + let refresh_at = triggers + .values() + .filter_map(|trigger| match &trigger.schedule { + Schedule::Calendar { refresh_at, .. } => Some(*refresh_at), + Schedule::Cron { .. } => None, + }) + .min(); status(py, "Publishing location snapshot...")?; let record = snapshot.publication( py, @@ -196,7 +220,7 @@ pub(super) fn publish_location( PyValueError::new_err(message) })?; status(py, "Location snapshot published.")?; - Ok(record) + Ok((record, refresh_at)) } /// One-shot publication and the supervisor's private stdio runtime share the same compiler path. @@ -249,7 +273,7 @@ pub(crate) fn run_location<'py>( return run_runtime(py, options.unwrap_or_else(|| PyDict::new(py))); } let publication = (|| { - let record = publish_location(py, &arguments)?; + let (record, _) = publish_location(py, &arguments)?; let output = serde_json::to_string(&record) .map_err(|error| PyValueError::new_err(error.to_string()))?; write_console(py, "stdout", &format!("{output}\n")).map_err(|error| { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index 25adcc06b4..e604e71354 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -6,6 +6,7 @@ pub(crate) mod host; pub(crate) mod model_lookup; pub(crate) mod orchestration_declarations; pub(crate) mod orchestration_environment; +pub(crate) mod orchestration_triggers; pub(crate) mod registration; pub(crate) mod results; pub(crate) mod sqlite_connection; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs index d758b406a0..1b04b08ad8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs @@ -1,15 +1,57 @@ //! The public authoring records enter native compilation here, never scheduler execution. -use std::collections::HashSet; +use std::collections::{BTreeMap, HashSet}; use std::path::Path; +use pyo3::FromPyObject; use pyo3::exceptions::{PyTypeError, PyValueError}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; +use serde_json::Value; use sqlbuild_host::entries::models::UserCodeHost; use crate::bindings::_helpers::native_host::host::entry; +#[derive(FromPyObject)] +pub(crate) struct AuthoredJob { + pub(crate) name: String, + pub(crate) description: String, + pub(crate) select: String, + pub(crate) exclude: String, + pub(crate) overlap: String, + pub(crate) on_failure: String, + #[pyo3(from_py_with = authored_setting)] + pub(crate) settings: Value, + #[pyo3(from_py_with = authored_setting)] + pub(crate) concurrency: Value, + pub(crate) concurrency_groups: Vec, + pub(crate) schedule: Option>, + pub(crate) targets: Vec, +} + +#[derive(FromPyObject, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) struct AuthoredJobReference { + pub(crate) name: String, +} + +#[derive(FromPyObject)] +pub(crate) struct AuthoredGroup { + pub(crate) name: String, + pub(crate) description: String, + pub(crate) jobs: Vec, + pub(crate) after: BTreeMap>, + pub(crate) overlap: String, + pub(crate) concurrency_groups: Vec, + pub(crate) schedule: Option>, + pub(crate) targets: Vec, +} + +pub(crate) enum AuthoredDeclaration { + Job(AuthoredJob), + Group(AuthoredGroup), + Calendar(Py), +} + /// JSON authoring values cross once; the interpreter rejects circular user containers safely. pub(crate) fn authored_setting(value: &Bound<'_, PyAny>) -> PyResult { let text: String = value @@ -32,6 +74,9 @@ pub(crate) fn module_jobs<'py>( let module = entry(py, UserCodeHost::LoadOrchestrationModule)?.call((), Some(&options))?; let job_class = py.import("sqlbuild.orchestration")?.getattr("Job")?; let group_class = py.import("sqlbuild.orchestration")?.getattr("JobGroup")?; + let calendar_class = py + .import("sqlbuild.orchestration")? + .getattr("CalendarSchedule")?; let attributes = module.getattr("__dict__")?.cast_into::()?; let mut pending: Vec> = attributes.values().iter().collect(); let mut seen: HashSet = HashSet::new(); @@ -40,7 +85,10 @@ pub(crate) fn module_jobs<'py>( if !seen.insert(value.as_ptr() as usize) { continue; } - if value.is_instance(&job_class)? || value.is_instance(&group_class)? { + if value.is_instance(&job_class)? + || value.is_instance(&group_class)? + || value.is_instance(&calendar_class)? + { jobs.push(value); } else if value.is_instance_of::() || value.is_instance_of::() { pending.extend(value.try_iter()?.collect::>>()?); @@ -52,12 +100,18 @@ pub(crate) fn module_jobs<'py>( pub(crate) fn collect_declarations<'py>( py: Python<'py>, declarations: Vec>, -) -> PyResult>> { +) -> PyResult> { let job = py.import("sqlbuild.orchestration")?.getattr("Job")?; let group = py.import("sqlbuild.orchestration")?.getattr("JobGroup")?; + let calendar = py + .import("sqlbuild.orchestration")? + .getattr("CalendarSchedule")?; let mut roots: HashSet = HashSet::new(); for declaration in &declarations { if !roots.insert(declaration.as_ptr() as usize) { + if declaration.is_instance(&calendar)? { + return Err(PyValueError::new_err("duplicate calendar declaration")); + } return Err(PyValueError::new_err(format!( "duplicate job {}", declaration.getattr("name")?.extract::()? @@ -66,12 +120,21 @@ pub(crate) fn collect_declarations<'py>( } let mut pending = declarations; let mut seen: HashSet = HashSet::new(); - let mut collected: Vec> = Vec::new(); + let mut collected: Vec = Vec::new(); while let Some(declaration) = pending.pop() { if !seen.insert(declaration.as_ptr() as usize) { continue; } - if declaration.is_instance(&group)? { + let authored = if declaration.is_instance(&calendar)? { + let target = declaration.getattr("job")?; + if !target.is_instance(&job)? && !target.is_instance(&group)? { + return Err(PyTypeError::new_err( + "calendar job must be a Job or JobGroup object", + )); + } + pending.push(target); + AuthoredDeclaration::Calendar(declaration.unbind()) + } else if declaration.is_instance(&group)? { for member in declaration.getattr("jobs")?.try_iter()? { let member = member?; if !member.is_instance(&job)? { @@ -93,12 +156,15 @@ pub(crate) fn collect_declarations<'py>( } } } + AuthoredDeclaration::Group(declaration.extract()?) } else if !declaration.is_instance(&job)? { return Err(PyTypeError::new_err( - "snapshot declarations must be sqlbuild.orchestration.Job or JobGroup objects", + "snapshot declarations must be Job, JobGroup or CalendarSchedule objects", )); - } - collected.push(declaration); + } else { + AuthoredDeclaration::Job(declaration.extract()?) + }; + collected.push(authored); } Ok(collected) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs new file mode 100644 index 0000000000..198541d498 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -0,0 +1,243 @@ +//! Authored trigger controls and bounded calendars evaluated only in the location interpreter. + +use std::collections::BTreeSet; +use std::path::Path; + +use chrono::{DateTime, TimeDelta, Timelike, Utc}; +use chrono_tz::Tz; +use pyo3::FromPyObject; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods}; +use serde_json::Value; +use sqlbuild_cursor::algebra::main::parse_duration::parse_duration; +use sqlbuild_host::entries::models::UserCodeHost; +use sqlbuild_orchestration::declarations::models::{Definitions, Schedule, TriggerDefinition}; +use sqlbuild_orchestration::scheduler::constants::MAX_TICKS_PER_BATCH; + +use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, within}; +use crate::bindings::_helpers::discovery::imported_modules::{provider_by_name, provider_usages}; +use crate::bindings::_helpers::discovery::imported_providers::discover_provider_classes; +use crate::bindings::_helpers::native_host::host::entry; +use crate::bindings::_helpers::native_host::orchestration_declarations::authored_setting; + +#[derive(FromPyObject)] +struct AuthoredControls { + #[pyo3(from_py_with = authored_setting)] + starts_at: Value, + default_status: String, + catch_up: String, + #[pyo3(from_py_with = authored_setting)] + max_catch_up: Value, +} + +fn compile_trigger( + authored: &Bound<'_, PyAny>, + job_key: &str, + description: &str, + schedule: Schedule, +) -> PyResult { + let controls = authored.extract::()?; + let catch_up = if controls.max_catch_up.is_null() { + Value::String(controls.catch_up) + } else { + serde_json::json!({controls.catch_up: {"max_runs": controls.max_catch_up}}) + }; + Ok(TriggerDefinition { + description: description.into(), + job_key: job_key.into(), + schedule, + starts_at: serde_json::from_value(controls.starts_at) + .map_err(|error| PyValueError::new_err(format!("starts_at: {error}")))?, + default_status: serde_json::from_value(Value::String(controls.default_status)) + .map_err(|error| PyValueError::new_err(format!("default_status: {error}")))?, + catch_up: serde_json::from_value(catch_up).map_err(|error| PyValueError::new_err(format!( + "catch_up: {error}; use 'latest' without a bound or 'all' with a positive max_catch_up" + )))?, + }) +} + +pub(crate) fn compile_cron( + authored: &Bound<'_, PyAny>, + name: &str, + description: &str, +) -> PyResult { + compile_trigger( + authored, + name, + description, + Schedule::Cron { + expression: authored.getattr("cron")?.extract()?, + timezone: authored.getattr("timezone")?.extract()?, + }, + ) +} + +fn horizon(value: &str) -> PyResult { + let invalid = || { + PyValueError::new_err( + "calendar horizon must be a positive fixed duration using d, h, m or s, within datetime range", + ) + }; + let digits = parse_duration(value).ok_or_else(invalid)?; + if [&digits.years, &digits.months] + .iter() + .any(|digits| !digits.trim_start_matches('0').is_empty()) + { + return Err(invalid()); + } + let seconds = [ + (digits.days, 86_400_i64), + (digits.hours, 3_600), + (digits.minutes, 60), + (digits.seconds, 1), + ] + .into_iter() + .try_fold(0_i64, |total, (digits, factor)| { + total.checked_add(digits.parse::().ok()?.checked_mul(factor)?) + }) + .ok_or_else(invalid)?; + TimeDelta::try_seconds(seconds).ok_or_else(invalid) +} + +fn calendar_times( + authored: &Bound<'_, PyAny>, + session: &Bound<'_, PyAny>, + providers: &Bound<'_, PyDict>, + start: DateTime, +) -> PyResult { + let py = authored.py(); + let timezone: String = authored.getattr("timezone")?.extract()?; + timezone + .parse::() + .map_err(|_| PyValueError::new_err("calendar timezone must be an IANA timezone"))?; + let duration = horizon(&authored.getattr("horizon")?.extract::()?)?; + let end = start + .checked_add_signed(duration) + .ok_or_else(|| PyValueError::new_err("calendar horizon is out of datetime range"))?; + let datetime = imported(py, "datetime", "datetime")?; + let zone = imported(py, "zoneinfo", "ZoneInfo")?.call1((timezone,))?; + let window = PyDict::new(py); + for (name, instant) in [("start", start), ("end", end)] { + window.set_item( + name, + datetime + .call_method1("fromisoformat", (instant.to_rfc3339(),))? + .call_method1("astimezone", (&zone,))?, + )?; + } + let window = py + .import("sqlbuild.orchestration")? + .getattr("CalendarWindow")? + .call((), Some(&window))?; + let function = authored.getattr("function")?; + let arguments = PyDict::new(py); + arguments.set_item("window", window)?; + for usage in provider_usages(&function, providers)?.try_iter()? { + let usage = usage?; + if usage.getattr("parameter_name")?.extract::()? == "window" { + continue; + } + arguments.set_item( + usage.getattr("parameter_name")?, + session.call_method1("get", (usage.getattr("provider_name")?,))?, + )?; + } + let mut times = BTreeSet::new(); + for (index, time) in function + .call((), Some(&arguments))? + .try_iter()? + .take(MAX_TICKS_PER_BATCH + 1) + .enumerate() + { + if index == MAX_TICKS_PER_BATCH { + return Err(PyValueError::new_err(format!( + "calendar exceeds {MAX_TICKS_PER_BATCH} upcoming times" + ))); + } + let time = time?; + if !time.is_instance(&datetime)? { + return Err(PyValueError::new_err( + "calendar times must be timezone-aware datetimes", + )); + } + let time = + DateTime::parse_from_rfc3339(&time.call_method0("isoformat")?.extract::()?) + .map_err(|_| { + PyValueError::new_err("calendar times must be timezone-aware datetimes") + })? + .with_timezone(&Utc); + if time < start || time >= end { + return Err(PyValueError::new_err( + "calendar time lies outside its half-open evaluation window", + )); + } + times.insert(time); + } + Ok(Schedule::Calendar { + times, + refresh_at: start + duration / 2, + }) +} + +pub(crate) fn compile_calendars( + py: Python<'_>, + (project_dir, target): (&Path, &str), + authored: Vec>, + mut definitions: Definitions, +) -> PyResult { + let mut calendars = Vec::new(); + for calendar in authored { + let targets = calendar + .bind(py) + .getattr("targets")? + .extract::>()?; + if targets.is_empty() || targets.iter().any(|value| value == target) { + calendars.push(calendar); + } + } + if calendars.is_empty() { + return Ok(definitions); + } + let providers = + discover_provider_classes(imported(py, "pathlib", "Path")?.call1((project_dir,))?)?; + let by_name = provider_by_name(&providers)?; + let session = call_with( + &entry(py, UserCodeHost::BuildProviderSession)?, + &[ + ("discovered_providers", providers), + ("allow_shared", PyBool::new(py, false).to_owned().into_any()), + ], + )?; + let now = Utc::now(); + let start = now + .with_nanosecond(now.timestamp_subsec_micros() * 1_000) + .ok_or_else(|| PyValueError::new_err("calendar clock is out of range"))?; + within(&session, |_| { + for calendar in calendars { + let calendar = calendar.bind(py); + let name: String = calendar + .getattr("function")? + .getattr("__name__")? + .extract()?; + let job_key: String = calendar.getattr("job")?.getattr("name")?.extract()?; + let description: String = calendar.getattr("description")?.extract()?; + if description.trim().is_empty() + || (!definitions.jobs.contains_key(&job_key) + && !definitions.groups.contains_key(&job_key)) + { + return Err(PyValueError::new_err(format!( + "calendar {name} needs a description and a published job" + ))); + } + if definitions.triggers.contains_key(&name) { + return Err(PyValueError::new_err(format!("duplicate trigger {name}"))); + } + let schedule = calendar_times(calendar, &session, &by_name, start)?; + let mut trigger = compile_trigger(calendar, &job_key, &description, schedule)?; + trigger.starts_at.get_or_insert(start); + definitions.triggers.insert(name, trigger); + } + Ok(definitions) + }) +} diff --git a/fensu.toml b/fensu.toml index 104f5bec85..67c1d437d9 100644 --- a/fensu.toml +++ b/fensu.toml @@ -2661,6 +2661,7 @@ module_layers = [ "model_config=front", "native_host=infra", "native_host/model_lookup=project", + "native_host/orchestration_triggers=project", "native_threads=infra", "output_capture=output", "plan_command=commands", diff --git a/src/sqlbuild/orchestration.py b/src/sqlbuild/orchestration.py index d305deef89..997e6d0181 100644 --- a/src/sqlbuild/orchestration.py +++ b/src/sqlbuild/orchestration.py @@ -1,9 +1,27 @@ """Public authoring and snapshot compilation API for orchestration declarations.""" from sqlbuild.orchestration_declarations.main.compile_snapshot import compile_snapshot -from sqlbuild.orchestration_declarations.models import Job, JobGroup, Schedule +from sqlbuild.orchestration_declarations.models import ( + CalendarSchedule, + CalendarWindow, + Job, + JobGroup, + Schedule, +) +from sqlbuild.orchestration_declarations.models import CalendarSchedule as calendar_schedule from sqlbuild.orchestration_declarations.models import Job as job from sqlbuild.orchestration_declarations.models import JobGroup as job_group from sqlbuild.orchestration_declarations.models import Schedule as schedule -__all__ = ("Job", "JobGroup", "Schedule", "compile_snapshot", "job", "job_group", "schedule") +__all__ = ( + "CalendarSchedule", + "CalendarWindow", + "Job", + "JobGroup", + "Schedule", + "calendar_schedule", + "compile_snapshot", + "job", + "job_group", + "schedule", +) diff --git a/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py b/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py index cc10a5eb4a..b9e09facb3 100644 --- a/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py +++ b/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py @@ -7,7 +7,7 @@ from sqlbuild._native import NativeOrchestrationSnapshot, build_orchestration_snapshot from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.orchestration_declarations.models import Job, JobGroup +from sqlbuild.orchestration_declarations.models import CalendarSchedule, Job, JobGroup def compile_snapshot( @@ -15,7 +15,7 @@ def compile_snapshot( project: CompiledProjectView, metadata_json: str, project_dir: Path, - declarations: Sequence[Job | JobGroup] | None = None, + declarations: Sequence[Job | JobGroup | CalendarSchedule] | None = None, ) -> NativeOrchestrationSnapshot: """Compile authoring records and retained project facts into a data-only snapshot.""" return build_orchestration_snapshot( diff --git a/src/sqlbuild/orchestration_declarations/models.py b/src/sqlbuild/orchestration_declarations/models.py index e113ce4aa4..b9617e4ffe 100644 --- a/src/sqlbuild/orchestration_declarations/models.py +++ b/src/sqlbuild/orchestration_declarations/models.py @@ -2,8 +2,9 @@ from __future__ import annotations -from collections.abc import Sequence -from dataclasses import dataclass, field +from collections.abc import Callable, Iterable, Sequence +from dataclasses import dataclass, field, replace +from datetime import datetime from typing import Literal @@ -50,3 +51,30 @@ class JobGroup: concurrency_groups: tuple[str, ...] = field(default=(), kw_only=True) schedule: Schedule | None = field(default=None, kw_only=True) targets: tuple[str, ...] = field(default=(), kw_only=True) + + +@dataclass(frozen=True, kw_only=True) +class CalendarWindow: + """Timezone-aware, half-open interval for an upcoming calendar evaluation.""" + + start: datetime + end: datetime + + +@dataclass(frozen=True, kw_only=True, eq=False) +class CalendarSchedule: + """Decorate a location callback returning upcoming timezone-aware datetimes.""" + + job: Job | JobGroup + description: str + horizon: str + timezone: str + starts_at: str | None = None + default_status: Literal["running", "paused"] = "running" + catch_up: Literal["latest", "all"] = "latest" + max_catch_up: int | None = None + targets: tuple[str, ...] = () + function: Callable[..., Iterable[datetime]] | None = field(default=None, repr=False) + + def __call__(self, function: Callable[..., Iterable[datetime]]) -> CalendarSchedule: + return replace(self, function=function) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index e6b239fffe..67d5f94e18 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -127,3 +127,18 @@ class BackfillExecutionTestCase(WorkerExecutionTestCase): description: str window_arguments: tuple[str, ...] = () expected_window: dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class CalendarRejectionTestCase: + description: str + expression: str + expected_message: str + horizon: str = "8s" + timezone: str = "UTC" + + +@dataclass(frozen=True) +class CalendarRefreshTestCase: + description: str + expected_timezone: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py new file mode 100644 index 0000000000..f15bea15b5 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py @@ -0,0 +1,285 @@ +"""Calendar callbacks remain in locations; native publication and refresh preserve good state.""" + +import json +import sqlite3 +import subprocess +import sys +from dataclasses import replace +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any +from urllib.error import HTTPError + +import duckdb +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import TYPE_PROOF_FILES +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + CalendarRefreshTestCase, + CalendarRejectionTestCase, + WorkerExecutionTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + current_snapshot, + location_process, + scheduled_worker, + snapshot_api, + wait_for_generation, + wait_for_location_health, + wait_for_run, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + + +@pytest.mark.parametrize( + "test_case", + [ + CalendarRejectionTestCase( + "naive time", "[datetime(2026, 1, 1)]", "timezone-aware datetimes" + ), + CalendarRejectionTestCase("exclusive end", "[window.end]", "half-open evaluation window"), + CalendarRejectionTestCase( + "before start", + "[window.start - timedelta(microseconds=1)]", + "half-open evaluation window", + ), + CalendarRejectionTestCase( + "non-datetime", "['2026-01-01T00:00:00Z']", "timezone-aware datetimes" + ), + CalendarRejectionTestCase( + "unbounded generator", "repeat(window.start)", "exceeds 10000 upcoming times" + ), + CalendarRejectionTestCase("zero horizon", "[]", "positive fixed duration", horizon="0s"), + CalendarRejectionTestCase( + "overflowing horizon", + "[]", + "positive fixed duration", + horizon="999999999999999999999999999d", + ), + CalendarRejectionTestCase("unknown zone", "[]", "IANA timezone", timezone="Unknown/Orders"), + ], + ids=lambda case: case.description, +) +def test_given_invalid_calendar_results_when_publishing_through_the_cli_then_validation_is_bounded_and_no_snapshot_is_committed( + test_case: CalendarRejectionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project: Path = tmp_path / "project" + write_files( + project_dir=project, + files=TYPE_PROOF_FILES + | { + "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + + '\n[targets.development]\nschema = "main"\n', + "orchestration/orders.py": ( + "from datetime import datetime, timedelta\nfrom itertools import repeat\n" + "from sqlbuild.orchestration import calendar_schedule, job\n" + "orders = job('orders', description='Order reporting.', select='orders')\n" + f"@calendar_schedule(job=orders, description='Delivery times.', horizon={test_case.horizon!r}, timezone={test_case.timezone!r})\n" + f"def deliveries(window):\n return {test_case.expression}\n" + ), + }, + ) + with snapshot_api(tmp_path / "control") as api: + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", api.bearer) + result: subprocess.CompletedProcess[str] = subprocess.run( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "publish", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + "--generation", + "1", + ], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + assert result.returncode != 0 + assert test_case.expected_message in result.stderr + with pytest.raises(HTTPError) as failure: + current_snapshot(api, "development") + assert failure.value.code == 404 + + +@pytest.mark.parametrize( + "test_case", + [ + CalendarRefreshTestCase("offset-aware calendar refresh", "America/New_York"), + CalendarRefreshTestCase("UTC calendar refresh", "UTC"), + ], + ids=lambda case: case.description, +) +def test_given_a_served_calendar_when_time_passes_and_a_provider_fails_then_refresh_reuses_the_interpreter_and_keeps_last_good( + test_case: CalendarRefreshTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project: Path = tmp_path / "project" + data: Path = tmp_path / "fulfillment.json" + evaluations: Path = tmp_path / "evaluations.jsonl" + data.write_text(json.dumps({"offsets": [3, 0, 3]})) + monkeypatch.setenv("SQLBUILD_FULFILLMENT_DATA", str(data)) + monkeypatch.setenv("SQLBUILD_CALENDAR_EVALUATIONS", str(evaluations)) + write_files( + project_dir=project, + files=TYPE_PROOF_FILES + | { + "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + + '\n[targets.development]\nschema = "main"\n', + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py" + ) + .read_text() + .replace("America/New_York", test_case.expected_timezone) + + ( + "\n@calendar_schedule(job=orders, description='Other target.', horizon='8s', timezone='UTC', targets=('production',))\n" + "def excluded_calendar(window):\n raise ValueError('excluded calendar must not evaluate')\n" + ), + "providers/fulfillment.py": ( + "import json, os\nfrom pathlib import Path\nfrom datetime import timedelta\n" + "from sqlbuild.provider.classes.provider import Provider\n" + "class Fulfillment(Provider):\n" + ' """Upcoming synthetic fulfillment times."""\n' + " def setup(self, ctx):\n self.record({'phase': 'setup'})\n" + " def teardown(self):\n self.record({'phase': 'teardown'})\n" + " def record(self, value):\n" + " with Path(os.environ['SQLBUILD_CALENDAR_EVALUATIONS']).open('a') as out:\n" + " out.write(json.dumps(value | {'pid': os.getpid()}) + '\\n')\n" + " def upcoming(self, window):\n" + " self.record({'phase': 'evaluate', 'start': window.start.isoformat(), 'end': window.end.isoformat(), 'zone': window.start.tzinfo.key})\n" + " data = json.loads(Path(os.environ['SQLBUILD_FULFILLMENT_DATA']).read_text())\n" + " if data.get('fail'):\n raise ValueError('synthetic calendar unavailable')\n" + " return [window.start + timedelta(seconds=offset) for offset in data['offsets']]\n" + ), + }, + ) + with snapshot_api(tmp_path / "control") as api: + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", api.bearer) + monkeypatch.setenv("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://must-not-open") + log: Path = tmp_path / "location.log" + publications: Path = tmp_path / "publications.jsonl" + arguments: list[str] = [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + ] + with location_process(arguments, log, publications) as process: + first: dict[str, Any] = wait_for_generation(api, "development", 1, process, log) + second: dict[str, Any] = wait_for_generation(api, "development", 2, process, log) + assert first["nodes"] == second["nodes"] + assert second["changes"] == [] + assert first["documents"]["triggers"].keys() == {"fulfillment_calendar"} + with sqlite3.connect(tmp_path / "control/.sqlbuild/control.sqlite3") as control: + stored, next_due = control.execute( + "SELECT definition_json, next_due_at FROM sqb_triggers JOIN sqb_trigger_state USING (trigger_id)" + ).fetchone() + original_times: list[str] = first["documents"]["triggers"]["fulfillment_calendar"][ + "schedule" + ]["times"] + assert json.loads(stored)["schedule"]["times"] == sorted( + original_times + + second["documents"]["triggers"]["fulfillment_calendar"]["schedule"]["times"] + ) + assert datetime.fromisoformat(next_due) == datetime.fromisoformat(original_times[0]) + observations: list[dict[str, Any]] = [ + json.loads(line) for line in evaluations.read_text().splitlines() + ] + assert [row["phase"] for row in observations] == ["setup", "evaluate", "teardown"] * 2 + successful: list[dict[str, Any]] = observations[1::3] + assert len(successful) == 2 + assert len({row["pid"] for row in observations}) == 1 + assert all(row["zone"] == test_case.expected_timezone for row in successful) + for snapshot, evaluation in zip([first, second], successful, strict=True): + start: datetime = datetime.fromisoformat(evaluation["start"]) + end: datetime = datetime.fromisoformat(evaluation["end"]) + assert end - start == timedelta(seconds=8) + schedule: dict[str, Any] = snapshot["documents"]["triggers"][ + "fulfillment_calendar" + ]["schedule"] + assert schedule["kind"] == "calendar" + assert [datetime.fromisoformat(value) for value in schedule["times"]] == [ + start + timedelta(seconds=offset) for offset in [0, 3] + ] + assert datetime.fromisoformat(schedule["refresh_at"]) == start + timedelta( + seconds=4 + ) + data.write_text(json.dumps({"fail": True})) + _ = wait_for_location_health( + api, "development", {"kind": "compile_failed", "generation": 3}, process, log + ) + assert current_snapshot(api, "development")["record"] == second["record"] + observations = [json.loads(line) for line in evaluations.read_text().splitlines()] + assert [row["phase"] for row in observations] == ["setup", "evaluate", "teardown"] * 3 + data.write_text(json.dumps({"offsets": [1]})) + (project / "models/stg_orders.sql").write_text( + TYPE_PROOF_FILES["models/stg_orders.sql"].replace( + "CAST(5 AS INTEGER)", "CAST(19 AS INTEGER)" + ) + ) + third: dict[str, Any] = wait_for_generation(api, "development", 3, process, log) + assert "CAST(19 AS INTEGER)" in third["nodes"]["model:stg_orders"]["raw_sql"] + assert ( + len(third["documents"]["triggers"]["fulfillment_calendar"]["schedule"]["times"]) + == 1 + ) + assert [ + json.loads(line)["generation"] for line in publications.read_text().splitlines() + ] == [1, 2, 3] + assert api.bearer not in log.read_text() + publications.read_text() + assert not (project / ".sqlbuild/control.sqlite3").exists() + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + description="calendar evaluation belongs only to the location", + model_sql="SELECT 29 AS amount", + warehouse_query="SELECT amount FROM orders", + expected_status="success", + expected_rows=[(29,)], + ), + ], + ids=lambda case: case.description, +) +def test_given_a_published_calendar_when_a_real_worker_compiles_then_it_does_not_evaluate_the_calendar_again( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + evaluated: Path = tmp_path / "evaluated" + test_case = replace( + test_case, + files=test_case.files + | { + "orchestration/orders.py": ( + "from pathlib import Path\nfrom sqlbuild.orchestration import calendar_schedule, job, schedule\n" + "orders = job('orders', description='Order reporting.', select='orders', schedule=schedule(cron='0 0 1 1 *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" + "@calendar_schedule(job=orders, description='Upcoming deliveries.', horizon='1d', timezone='UTC')\n" + "def deliveries(window):\n" + f" marker = Path({str(evaluated)!r})\n" + " if marker.exists():\n raise ValueError('calendar evaluated inside a run worker')\n" + " marker.write_text('evaluated')\n return []\n" + ) + }, + ) + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + assert evaluated.read_text() == "evaluated" + completed: dict[str, Any] = wait_for_run(worker.api, worker.process, worker.log) + assert completed["status"] == test_case.expected_status, worker.log.read_text() + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows From b01f456b51d3128bb9683124a1609dbf764c0981 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 18:34:28 +0000 Subject: [PATCH 43/59] test: avoid writable native executable copies --- crates/sqlbuild-cli/tests/native_commands/helpers.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/sqlbuild-cli/tests/native_commands/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/helpers.rs index 606246903b..ec2421c381 100644 --- a/crates/sqlbuild-cli/tests/native_commands/helpers.rs +++ b/crates/sqlbuild-cli/tests/native_commands/helpers.rs @@ -6,8 +6,10 @@ use crate::test_types::{ServiceTestCase, TestControl}; pub(crate) fn isolated_binary(directory: &Path) -> std::path::PathBuf { let binary = directory.join(format!("sqb{}", std::env::consts::EXE_SUFFIX)); - std::fs::copy(env!("CARGO_BIN_EXE_sqb"), &binary) - .expect("copy standalone binary without Python CLI"); + // Do not open the executable for writing: concurrent process launches can inherit + // the copy's writable descriptor until exec, making this launch fail with ETXTBSY. + std::fs::hard_link(env!("CARGO_BIN_EXE_sqb"), &binary) + .expect("link standalone binary without Python CLI"); binary } From 063f2e84461a59768da8165a56a4ddc0331fb0cd Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 02:00:28 +0000 Subject: [PATCH 44/59] fix: distinguish PyO3 metadata from Python crossings Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- fensu.toml | 204 +++++++++--------- pyproject.toml | 2 +- .../fensu_policy/_helpers/native_crossings.py | 8 +- .../scripts/fensu_policy/__init__.py | 1 + .../scripts/fensu_policy/_test_types.py | 10 + .../fensu_policy/test_native_boundary.py | 78 +++++++ .../rules/test_native_boundary.py | 31 +++ uv.lock | 30 +-- 8 files changed, 239 insertions(+), 125 deletions(-) create mode 100644 tests/integration/scripts/fensu_policy/__init__.py create mode 100644 tests/integration/scripts/fensu_policy/_test_types.py create mode 100644 tests/integration/scripts/fensu_policy/test_native_boundary.py diff --git a/fensu.toml b/fensu.toml index 67c1d437d9..aefc72b446 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1570,18 +1570,6 @@ path = "crates/sqlbuild-orchestration/src/credentials/models.rs" symbols = ["IssuedToken"] reason = "IssuedToken implements Debug manually to redact its bearer credential; deriving Debug would expose the secret." -[[targets.rust.rule_exceptions]] -rule = "XSB072" -path = "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs" -symbols = ["NativeOrchestrationSnapshot"] -reason = "The pyclass module attribute names sqlbuild._native but executes no Python call. The struct retains only SnapshotInput; binding functions remain checked." - -[[targets.rust.rule_exceptions]] -rule = "XSB072" -path = "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_worker.rs" -symbols = ["NativeOrchestrationWorker"] -reason = "The pyclass module attribute names sqlbuild._native but executes no Python call. The struct retains only native worker ownership; binding functions remain checked." - [[targets.rust.rule_exceptions]] rule = "FPRSS201" path = "crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs" @@ -1687,17 +1675,17 @@ edge_paths = [ "crates/sqlbuild-python/src/bindings/_helpers/records/registry.rs", ] pending_budgets = [ - "crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_test_comparisons.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_inferences.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_inferences.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/nullability_rules.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/attachments/reference_markers.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs=20", + "crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs=19", "crates/sqlbuild-python/src/bindings/_helpers/auditing/identity.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/auditing/measurement.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/auditing/schema_audits.rs=1", @@ -1712,43 +1700,43 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs=22", + "crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs=21", "crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/build_command/policies.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/build_command/support.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/audit_progress.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs=14", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/nested_progress.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/build_command/support.rs=7", + "crates/sqlbuild-python/src/bindings/_helpers/build_progress/audit_progress.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs=13", + "crates/sqlbuild-python/src/bindings/_helpers/build_progress/nested_progress.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/build_progress/plan_facts.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress.rs=12", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_reporters.rs=11", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_text.rs=6", + "crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress.rs=11", + "crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_reporters.rs=8", + "crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_text.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/build_progress/result_text.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/build_progress/terminal.rs=23", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_lines.rs=10", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs=9", + "crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_lines.rs=9", + "crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/cache/native_store.rs=13", - "crates/sqlbuild-python/src/bindings/_helpers/cache/sql_test_scan_store.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs=21", + "crates/sqlbuild-python/src/bindings/_helpers/cache/sql_test_scan_store.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs=20", "crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/cli/argv.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs=27", "crates/sqlbuild-python/src/bindings/_helpers/cli/launch_state.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/cli/namespace.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs=21", + "crates/sqlbuild-python/src/bindings/_helpers/cli/namespace.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs=19", "crates/sqlbuild-python/src/bindings/_helpers/cli/surrogates.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/cli/termination.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_progress.rs=7", + "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_progress.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/adapters.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/runtime_errors.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs=8", + "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs=9", @@ -1757,19 +1745,19 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/storage_policies.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/compile_command/models.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/compile_command/prepared_artifacts.rs=42", + "crates/sqlbuild-python/src/bindings/_helpers/compile_command/models.rs=0", + "crates/sqlbuild-python/src/bindings/_helpers/compile_command/prepared_artifacts.rs=41", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs=30", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs=45", - "crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs=40", + "crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs=38", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/runtime_identity.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/startup_inputs.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/surrogates.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs=23", + "crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs=22", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/test_artifact_cache.rs=39", "crates/sqlbuild-python/src/bindings/_helpers/compile_context/compile_timings.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/compile_context/diagnostic_collector.rs=10", - "crates/sqlbuild-python/src/bindings/_helpers/compile_context/input_reads.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/compile_context/diagnostic_collector.rs=9", + "crates/sqlbuild-python/src/bindings/_helpers/compile_context/input_reads.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs=20", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs=3", @@ -1797,22 +1785,22 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/artifacts.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs=10", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_views.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_views.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_analysis.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/inputs_view.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/inputs_view.rs=0", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs=7", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/records.rs=14", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/source_views.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/source_views.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/compute_logs/conversion.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs=8", @@ -1829,8 +1817,8 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/store.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/allocation.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs=7", - "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs=23", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs=9", @@ -1852,11 +1840,11 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init_project.rs=27", "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/support.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/dbt_events/stream.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/dbt_events/stream.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/event_stream.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/execution.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/manifest.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/orchestration.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs=19", @@ -1868,8 +1856,8 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_coercion.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_connection.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_load.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs=11", - "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runner.rs=7", + "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs=10", + "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runner.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_config.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_execution.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_results.rs=8", @@ -1878,7 +1866,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/console_records.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/log_records.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/redaction.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs=15", + "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/facts.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/output.rs=4", @@ -1894,7 +1882,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/discovery/locations.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/discovery/project_config.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs=6", @@ -1909,16 +1897,16 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs=29", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/models.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs=28", + "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/models.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/registration.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs=17", + "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/values.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs=12", + "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/conversion.rs=19", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/functions.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/execution_history/postgres.rs=28", - "crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs=28", + "crates/sqlbuild-python/src/bindings/_helpers/execution_history/postgres.rs=27", + "crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs=27", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/statements.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/exported_constants/registration.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs=16", @@ -1943,7 +1931,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/graph/python_lifecycle.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs=5", @@ -1954,7 +1942,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/loop_classes.rs=11", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs=7", + "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_calls.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_contracts.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_external.rs=4", @@ -1982,7 +1970,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/envelopes.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/payloads.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/validation.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs=9", + "crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_cache.rs=28", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_facts.rs=3", @@ -1992,11 +1980,11 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/load/ingestr_command.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs=17", - "crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs=19", + "crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs=17", "crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/module_digests.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/store_keys.rs=27", - "crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs=9", + "crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/manifest/rows.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/microbatches/conversion.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/microbatches/projection.rs=1", @@ -2007,16 +1995,16 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/errors.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/model_config/errors.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/model_config/header_columns.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs=0", + "crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/typed_config.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/unicode_environment.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/output_capture/dispatcher.rs=33", + "crates/sqlbuild-python/src/bindings/_helpers/model_config/typed_config.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/model_config/unicode_environment.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/output_capture/dispatcher.rs=31", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/values.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs=12", @@ -2036,10 +2024,10 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/hashing.rs=1", @@ -2059,7 +2047,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/known_input_columns.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/known_input_columns.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs=4", @@ -2073,9 +2061,9 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_execution.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/projections.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs=2", @@ -2108,11 +2096,11 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_sql.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs=14", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs=15", + "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs=1", @@ -2120,12 +2108,12 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/document.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/functions.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/style.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/document.rs=0", + "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/functions.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/style.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/terminal.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/models.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/models.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs=1", @@ -2137,8 +2125,8 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/recall/console.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/recall/replacements.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/recall/writer.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/recall/writer.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/auditing.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/cursors.rs=1", @@ -2149,8 +2137,8 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/migrations.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/planner.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/rule_shapes.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/records/error_classes.rs=11", - "crates/sqlbuild-python/src/bindings/_helpers/records/models.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/records/error_classes.rs=9", + "crates/sqlbuild-python/src/bindings/_helpers/records/models.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs=26", "crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_edits.rs=1", @@ -2165,24 +2153,24 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs=24", "crates/sqlbuild-python/src/bindings/_helpers/render/declaration_indexes.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs=13", + "crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_arguments.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_classes.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_evaluation.rs=14", - "crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs=17", + "crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_generated_references.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/result_output/aggregation.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/result_output/checks.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs=13", - "crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs=20", - "crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_envelope.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs=19", + "crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_envelope.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_records.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_values.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/result_output/resource_ids.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs=14", + "crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/review_command/command.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/review_command/comparison.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/review_command/execution.rs=1", @@ -2190,11 +2178,11 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_observation.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_observation.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_hosts.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs=21", - "crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/rules/fact_encoder.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/rules/findings.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/rules/fix_facts.rs=8", @@ -2209,7 +2197,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/rules/native_header_parser.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/rules/project_tree_queries.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs=6", + "crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs=1", @@ -2221,26 +2209,26 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/rules/sql_test_objects.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/capture_command.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/conversions.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/snapshot_sync.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_contexts.rs=6", + "crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_contexts.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_resolution.rs=15", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_cache.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_errors.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs=33", + "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs=32", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs=9", + "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs=23", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_tolerant.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs=16", + "crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs=6", @@ -2248,7 +2236,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs=13", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs=101", + "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs=100", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/authored_ctes.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs=4", @@ -2261,8 +2249,8 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_values.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/sqltext/code_scanner.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/sqltext/code_scanner.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/state_rows/events.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs=4", @@ -2275,7 +2263,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/type_system/equality.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/type_system/normalization.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs=8", - "crates/sqlbuild-python/src/bindings/models.rs=7", + "crates/sqlbuild-python/src/bindings/models.rs=0", ] [targets.rust.rule_options.XSB073] diff --git a/pyproject.toml b/pyproject.toml index f66e346fcf..c29137b916 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,7 @@ dev = [ "pytest-cov>=7.0.0", "pytest-xdist>=3.8.0", "ruff>=0.12.0", - "fensu>=0.28.0", + "fensu>=0.28.1", "testcontainers>=4.8.1", "ty>=0.0.1a19", ] diff --git a/scripts/fensu_policy/_helpers/native_crossings.py b/scripts/fensu_policy/_helpers/native_crossings.py index 430cc3fb78..27df706697 100644 --- a/scripts/fensu_policy/_helpers/native_crossings.py +++ b/scripts/fensu_policy/_helpers/native_crossings.py @@ -27,6 +27,11 @@ _PYTHON_FIELD: re.Pattern[str] = re.compile(NATIVE_PYTHON_FIELD_PATTERN) _STRUCT_OPEN: re.Pattern[str] = re.compile(r"^\s*(?:pub(?:\([^)]*\))?\s+)?struct\s+\w+[^;]*\{\s*$") _FIELD: re.Pattern[str] = re.compile(r"^\s*(?:pub(?:\([^)]*\))?\s+)?\w+\s*:") +_PYCLASS_MODULE: re.Pattern[str] = re.compile( + r"(^[ \t]*#\[\s*(?:pyo3::)?pyclass\s*\([^]]*?\bmodule\s*=\s*(?:r#*)?)" + r'"(?:\\.|[^"\\\n])*"', + re.MULTILINE, +) def is_checked_binding(*, path: str) -> bool: @@ -72,8 +77,9 @@ def _code_lines(*, source: str) -> list[tuple[int, str]]: def crossing_lines(*, source: str) -> list[int]: - """Line numbers of Rust-to-Python crossings, one entry per crossing.""" + """Line numbers of Rust-to-Python crossings, excluding inert PyO3 module metadata.""" + source = _PYCLASS_MODULE.sub(r'\1""', source) found: list[int] = [] number: int line: str diff --git a/tests/integration/scripts/fensu_policy/__init__.py b/tests/integration/scripts/fensu_policy/__init__.py new file mode 100644 index 0000000000..6aab21b701 --- /dev/null +++ b/tests/integration/scripts/fensu_policy/__init__.py @@ -0,0 +1 @@ +"""Integration checks for repository-authored Fensu policies.""" diff --git a/tests/integration/scripts/fensu_policy/_test_types.py b/tests/integration/scripts/fensu_policy/_test_types.py new file mode 100644 index 0000000000..ef287d1b65 --- /dev/null +++ b/tests/integration/scripts/fensu_policy/_test_types.py @@ -0,0 +1,10 @@ +"""Native boundary metadata cases for the real Fensu CLI.""" + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class NativeBoundaryTestCase: + description: str + metadata: str + expected_fault_count: int diff --git a/tests/integration/scripts/fensu_policy/test_native_boundary.py b/tests/integration/scripts/fensu_policy/test_native_boundary.py new file mode 100644 index 0000000000..1b996fae04 --- /dev/null +++ b/tests/integration/scripts/fensu_policy/test_native_boundary.py @@ -0,0 +1,78 @@ +"""Exercise native boundary policy through Fensu's real Rust analyzer and rule host.""" + +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +from tests.integration.scripts.fensu_policy._test_types import NativeBoundaryTestCase + + +@pytest.mark.parametrize( + "test_case", + [ + NativeBoundaryTestCase( + description="pyclass module metadata", + metadata='#[pyclass(frozen, module = "sqlbuild._native")]\npub struct Snapshot;\n', + expected_fault_count=2, + ), + NativeBoundaryTestCase( + description="namespaced pyclass raw module metadata", + metadata='#[pyo3::pyclass(frozen, module = r#"sqlbuild._native"#)]\npub struct Snapshot;\n', + expected_fault_count=2, + ), + ], + ids=lambda case: case.description, +) +def test_given_pyclass_metadata_when_checking_rust_then_only_real_python_crossings_fault( + tmp_path: Path, + test_case: NativeBoundaryTestCase, +) -> None: + repository: Path = Path(__file__).resolve().parents[4] + shutil.copytree( + repository / "scripts/fensu_policy", + tmp_path / "scripts/fensu_policy", + ignore=shutil.ignore_patterns("__pycache__"), + ) + (tmp_path / "scripts/__init__.py").write_text("") + (tmp_path / "Cargo.toml").write_text( + '[workspace]\nmembers = ["crates/sqlbuild-python"]\nresolver = "2"\n' + ) + crate: Path = tmp_path / "crates/sqlbuild-python" + (crate / "src").mkdir(parents=True) + (crate / "Cargo.toml").write_text( + '[package]\nname = "sqlbuild-python"\nversion = "0.1.0"\nedition = "2021"\n' + ) + (tmp_path / "fensu.toml").write_text( + '[targets.rust]\nanalyzer = "rust"\nroots = ["crates"]\n' + 'rule_modules = ["scripts.fensu_policy.rules.native_boundary"]\n' + 'select = ["XSB072"]\n' + ) + source: Path = crate / "src/lib.rs" + source.write_text(test_case.metadata) + command: list[str] = [ + str(Path(sys.executable).with_name("fensu")), + "check", + "--target", + "rust", + "--no-cache", + "--no-color", + ] + clean: subprocess.CompletedProcess[str] = subprocess.run( + command, cwd=tmp_path, capture_output=True, text=True, timeout=15, check=False + ) + assert clean.returncode == 0, clean.stdout + clean.stderr + assert "Found 0 faults" in clean.stdout + source.write_text( + test_case.metadata + 'fn read(py: Python<\'_>) { py.import("sqlbuild.compiler"); }\n' + ) + crossing: subprocess.CompletedProcess[str] = subprocess.run( + command, cwd=tmp_path, capture_output=True, text=True, timeout=15, check=False + ) + assert crossing.returncode == 1, crossing.stdout + crossing.stderr + assert crossing.stdout.count("XSB072") == test_case.expected_fault_count, crossing.stdout + assert crossing.stdout.count(" --> crates/sqlbuild-python/src/lib.rs:3:0") == 2 + assert " --> crates/sqlbuild-python/src/lib.rs:1:0" not in crossing.stdout + assert source.read_text().startswith(test_case.metadata) diff --git a/tests/unit/scripts/fensu_policy/rules/test_native_boundary.py b/tests/unit/scripts/fensu_policy/rules/test_native_boundary.py index dd3e4ba20f..9422bcd69a 100644 --- a/tests/unit/scripts/fensu_policy/rules/test_native_boundary.py +++ b/tests/unit/scripts/fensu_policy/rules/test_native_boundary.py @@ -135,6 +135,37 @@ def test_given_test_module_when_checking_native_patches_then_matches_contract( ), expected_lines=(3, 4, 5, 5, 6, 6), ), + NativeLinesTestCase( + description="pyclass module metadata is not a Python crossing", + source='#[pyclass(frozen, module = "sqlbuild._native")]\nstruct Snapshot;\n', + expected_lines=(), + ), + NativeLinesTestCase( + description="multiline namespaced pyclass metadata preserves callback line numbers", + source=dedent( + """ + #[pyo3::pyclass( + frozen, + module = "sqlbuild._native", + )] + struct Snapshot; + fn read(py: Python<'_>) { + py.import("sqlbuild.compiler"); + } + """ + ), + expected_lines=(8, 8), + ), + NativeLinesTestCase( + description="raw pyclass module metadata does not hide an adjacent real crossing", + source='#[pyclass(module = r#"sqlbuild._native"#)] struct Snapshot; fn read() { imported(py, "sqlbuild.compiler", "compile"); }\n', + expected_lines=(1, 1), + ), + NativeLinesTestCase( + description="module assignments outside pyclass remain crossings", + source='fn read() { let module = "sqlbuild.compiler"; load(module); }\n', + expected_lines=(1,), + ), NativeLinesTestCase( description="comments, imports and test modules are skipped", source=dedent( diff --git a/uv.lock b/uv.lock index 7f698d1833..17225a2936 100644 --- a/uv.lock +++ b/uv.lock @@ -1343,31 +1343,31 @@ wheels = [ [[package]] name = "fensu" -version = "0.28.0" +version = "0.28.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "fensu-cli" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/17/69/a132ffc407be97a0fdcfa052cb8b49408091db748e6afe0f7a6c77e6fa7d/fensu-0.28.0.tar.gz", hash = "sha256:11bcea3207f7fd92babc200f40d074e3da789ee11e7fc7024575a415de7c1176", size = 538990, upload-time = "2026-10-27T00:39:16.19Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ab/c7/baf72530c3a67a6d26957658475d32e11adbeb2dbb452425fd5d4af1f575/fensu-0.28.1.tar.gz", hash = "sha256:b81119f0301580367799b77312d5fd82f3ce5a14613b5b709537bca4c070b912", size = 539035, upload-time = "2026-10-31T19:33:41.927Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/59/e9/7434bd0565d54ff58796105728b3ccc8b838c7200997184978e6ebfdec4d/fensu-0.28.0-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:29df31aff900d20b0c845940af0cd91eead55e158a2511e682e3e887a22dada8", size = 4154207, upload-time = "2026-10-27T00:38:52.311Z" }, - { url = "https://files.pythonhosted.org/packages/7f/d8/e0875b269515d9a75643d5488bdc6f70a4b6c346995f12ce07304a3610f7/fensu-0.28.0-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:33a6d8b7d2f734c48f48087462c0e5a1ff94535c0e27e6a4820bc10303ff3653", size = 4004243, upload-time = "2026-10-27T00:38:54.994Z" }, - { url = "https://files.pythonhosted.org/packages/e8/05/b87c0ca5cf3f724517d6958f1ebd68948f9399306bd6d6814783635bad9a/fensu-0.28.0-cp312-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:de07b3fee5d908d2a766ca136753c8040d6fe25a1346d9b3b5a98483fc23d874", size = 4230192, upload-time = "2026-10-27T00:38:57.321Z" }, - { url = "https://files.pythonhosted.org/packages/cb/d2/fdfc7fd322c3497c53c22754a33433e33678a6de84832fd41add09481a21/fensu-0.28.0-cp312-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f44293a2da6bf4558d00eb4f859b28f3fb69d396dad1c25ac466a92a7d1dd734", size = 4307267, upload-time = "2026-10-27T00:38:59.679Z" }, - { url = "https://files.pythonhosted.org/packages/56/c0/2ab2c2c1af4ddd1b45c750679bd9f0b777d7eca3c0e00a233c991aaa13e7/fensu-0.28.0-cp312-abi3-win_amd64.whl", hash = "sha256:ffa88f1d259a1830d513ae930ff856d5b5e4941ba433889dea04d3db5753f281", size = 3977828, upload-time = "2026-10-27T00:39:01.761Z" }, + { url = "https://files.pythonhosted.org/packages/e8/02/8288dc1ecb853941750d25c2b1effc6fe68d9eeeb188f3d289823731cf69/fensu-0.28.1-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:bf1d49d2c800e7c8acdba031c12e6ea1a38544f45c0b75fbe8c35369b63ee98b", size = 4154268, upload-time = "2026-10-31T19:33:24.239Z" }, + { url = "https://files.pythonhosted.org/packages/4c/70/f21686ef9b89f70c84ea3a94dcdbda47839263496c36aa962c15c2eb093e/fensu-0.28.1-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:7e7564b2c4ffb9a031659b3bd82368be70deb97c3f4f620ad05ddfb5f8086e9c", size = 4004303, upload-time = "2026-10-31T19:33:25.807Z" }, + { url = "https://files.pythonhosted.org/packages/3c/50/558ac8fadf943aad5a8a5ba500a3f63e0f5245fd1f66480b6cd8590ce599/fensu-0.28.1-cp312-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:70da92dad5e5dfbe927bfe870893fe8dc612f28e72780de32014ca42703ba7db", size = 4230255, upload-time = "2026-10-31T19:33:27.649Z" }, + { url = "https://files.pythonhosted.org/packages/5b/5f/b7659a2d6489293a6c16cafc5f8c500b5d5de22536767f468147bed813e3/fensu-0.28.1-cp312-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:207a911faee0e14f40005060281ea624b6360f7f35cf6440a02dd0870c4f0bfc", size = 4307330, upload-time = "2026-10-31T19:33:29.361Z" }, + { url = "https://files.pythonhosted.org/packages/69/a6/aa98ad24bb27e3c829c9675c2b753f5d793fb5f546e8b266ade494d6e044/fensu-0.28.1-cp312-abi3-win_amd64.whl", hash = "sha256:88089637c5a52a2e074dd7b29fb76d4e35d1d9d28e8ef4398be30c3bed7d8071", size = 3977891, upload-time = "2026-10-31T19:33:30.84Z" }, ] [[package]] name = "fensu-cli" -version = "0.28.0" +version = "0.28.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/79/d9/f347f514fb224f6a1f4ca6e616ac49b573f9dceba4ff8d9efcd334f48bea/fensu_cli-0.28.0.tar.gz", hash = "sha256:1feb379ed92e193cbd4d28ab3d7729798b88120aaee2f59fd7f341519b6ee945", size = 765638, upload-time = "2026-10-27T00:39:17.752Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6d/da/a4ef081ab0ec23cb719dfbce2747c11ee3832a2dda2aaef1f7722548d1e6/fensu_cli-0.28.1.tar.gz", hash = "sha256:4857b2ef850d35bae19462cd6e26fdc0b33c00d97f13f20bb5b00b5c879ef2e9", size = 765839, upload-time = "2026-10-31T19:33:43.051Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3b/b3/57f1de89e1cb7534b0ef6415822c8982a247b9d2efe8d2fddd85e35389c8/fensu_cli-0.28.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:13ecc7a224a4b32eef19456f21659e682e636ef53f305919aa57cae27c8e3fec", size = 7978247, upload-time = "2026-10-27T00:39:04.644Z" }, - { url = "https://files.pythonhosted.org/packages/ff/ff/2e887d727f82c61ea7cb6773429b61d4711b93f7f94814eb9a7e891384dc/fensu_cli-0.28.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:149e5e22c8895c0a3ed3956b30e693e78df7dfb67ce8e0c5682b84c77e811e7e", size = 7490546, upload-time = "2026-10-27T00:39:07.384Z" }, - { url = "https://files.pythonhosted.org/packages/76/4e/daa4c2d70e5b0398382f84c846aacc4a937c4d00b3938d00a67351b0f212/fensu_cli-0.28.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:99932d17956bb10da1ac677867567b504a961d0565fcda730c1e682434244113", size = 7570220, upload-time = "2026-10-27T00:39:09.852Z" }, - { url = "https://files.pythonhosted.org/packages/f4/f9/c9e1095c507da3a53b3e7bf3f7c95f75824429ea45a72cb2c7af67a605f7/fensu_cli-0.28.0-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ee51c5d6d0d5fa0b36c0f70d3d6fde429793b643f0075cdfdf3fc8b8d73e3ccb", size = 8086364, upload-time = "2026-10-27T00:39:12.052Z" }, - { url = "https://files.pythonhosted.org/packages/9c/89/ece32e452d2b4ade243dd66a2612619be7400f6acf59f854ce674bb78e30/fensu_cli-0.28.0-py3-none-win_amd64.whl", hash = "sha256:acca6ac39cbe0614d3a8e4ce7c2c12cc5fd3c800fc4c66b5484c54dc19ef4a66", size = 7700477, upload-time = "2026-10-27T00:39:14.15Z" }, + { url = "https://files.pythonhosted.org/packages/33/03/1c5a56b3d233360d1fb699ff7a92cee005faf8f8fda8c92a7688cc0be167/fensu_cli-0.28.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:67b0bc97cdaeb08e633e68336b1d82b3e598d615b54b9b6f840772929cf6929e", size = 7975688, upload-time = "2026-10-31T19:33:32.569Z" }, + { url = "https://files.pythonhosted.org/packages/f2/a7/d90a388b17a89d9e34238e16e03ce306331c212d215f02ff4c43d6d93189/fensu_cli-0.28.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e6a9d97525aea16e079b773b7141a1167bd799bc07ff8bfc720b8f311c91d20b", size = 7492140, upload-time = "2026-10-31T19:33:34.634Z" }, + { url = "https://files.pythonhosted.org/packages/62/f3/4a5247e312d4725ead4ab37d0076371b731a8858c5819cfcceb1d6ce0b69/fensu_cli-0.28.1-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b8fd61d84ffd2be3c023335e374cabc72eac74dfe5cd90103bde52eba6c40da5", size = 7860157, upload-time = "2026-10-31T19:33:36.407Z" }, + { url = "https://files.pythonhosted.org/packages/d4/71/a954282a9f3f372fe77c942ff6acc8cc2f797b7da82b3b40a6c69715b668/fensu_cli-0.28.1-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1115a28579b917d6d8c0010e99488318e4cd264236642b5081bb79e8378bdc15", size = 8323299, upload-time = "2026-10-31T19:33:38.357Z" }, + { url = "https://files.pythonhosted.org/packages/35/cb/aa860d0afcb17a7931ab14dcab061d234143caef12e2791d17f27d0816c3/fensu_cli-0.28.1-py3-none-win_amd64.whl", hash = "sha256:4294965dcf4c317b66e4e489c9b9b38e8f8d82000c59bf9f3275fe887a6b45d8", size = 7705331, upload-time = "2026-10-31T19:33:40.314Z" }, ] [[package]] @@ -4346,7 +4346,7 @@ dev = [ { name = "dbt-snowflake", specifier = "==1.12.0b1" }, { name = "dbt-sqlserver", specifier = ">=1.9.0,<1.11" }, { name = "dlt", extras = ["duckdb", "filesystem", "sql-database"], specifier = ">=1.28.0,<2" }, - { name = "fensu", specifier = ">=0.28.0" }, + { name = "fensu", specifier = ">=0.28.1" }, { name = "ingestr", specifier = ">=1.1.73" }, { name = "jsonschema", specifier = ">=4.23.0" }, { name = "maturin", specifier = ">=1.9,<2" }, From 180c4af5899c6ebfe4a971b014e5e0f873630000 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 02:00:34 +0000 Subject: [PATCH 45/59] refactor: align calendar backend ownership boundaries Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../src/native_commands/_helpers/execution.rs | 5 +- .../sqlbuild-cli/src/native_commands/mod.rs | 1 + .../src/native_commands/models.rs | 7 +- .../sqlbuild-cli/src/native_commands/types.rs | 7 ++ .../declarations/_helpers/trigger_state.rs | 39 ++++++- .../declarations/main/publish_definitions.rs | 106 ++++++++---------- .../tests/scheduler/ticks/test_triggers.rs | 8 +- .../_helpers/compiled_project/snapshot.rs | 3 +- .../compiled_project/snapshot_declarations.rs | 7 +- .../_helpers/location_command/command.rs | 7 +- .../_helpers/location_command/constants.rs | 3 + .../_helpers/native_host/constants.rs | 4 + .../src/bindings/_helpers/native_host/mod.rs | 1 + .../native_host/orchestration_triggers.rs | 16 +-- 14 files changed, 124 insertions(+), 90 deletions(-) create mode 100644 crates/sqlbuild-cli/src/native_commands/types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index ee24726afd..dac117b46c 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -45,9 +45,10 @@ use crate::native_commands::constants::{ use crate::native_commands::errors::PythonNeeded; use crate::native_commands::main::_run_cost::run_cost; use crate::native_commands::models::{ - CommandOutput, InstalledPackage, LocationCompileRequest, LocationCompiler, LocationPublication, - LocationServe, LostStdout, NativeCommand, NativeInvocation, ProjectLocation, + CommandOutput, InstalledPackage, LocationCompileRequest, LocationCompiler, LocationServe, + LostStdout, NativeCommand, NativeInvocation, ProjectLocation, }; +use crate::native_commands::types::LocationPublication; use crate::workers::main::open_workers::open_workers; use crate::workers::models::Workers; diff --git a/crates/sqlbuild-cli/src/native_commands/mod.rs b/crates/sqlbuild-cli/src/native_commands/mod.rs index d99ca200f2..5ac20489d5 100644 --- a/crates/sqlbuild-cli/src/native_commands/mod.rs +++ b/crates/sqlbuild-cli/src/native_commands/mod.rs @@ -5,6 +5,7 @@ pub(crate) mod constants; pub mod errors; pub mod main; pub(crate) mod models; +pub(crate) mod types; #[cfg(test)] mod tests; diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 627cf9baa9..b16504cefa 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -11,6 +11,7 @@ use sqlbuild_orchestration::runs::models::{JobRunRequest, RerunRequest}; use uuid::Uuid; use crate::argv::models::Namespace; +use crate::native_commands::types::LocationPublication; /// Privileged local operations require direct ownership of the control database. pub(crate) enum ControlAdmin { @@ -152,12 +153,6 @@ pub(crate) struct LocationWatch { pub(crate) retry_at: Option, } -/// Private interpreter response: publication metadata and its earliest calendar refresh deadline. -pub(crate) type LocationPublication = ( - sqlbuild_orchestration::snapshots::models::SnapshotRecord, - Option>, -); - /// A bounded supervisor request; dropping the reply cancels and reaps an active compiler. #[derive(Debug)] pub(crate) struct LocationCompileRequest { diff --git a/crates/sqlbuild-cli/src/native_commands/types.rs b/crates/sqlbuild-cli/src/native_commands/types.rs new file mode 100644 index 0000000000..438f742268 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/types.rs @@ -0,0 +1,7 @@ +//! Private interpreter protocol shapes owned by the location supervisor. + +use chrono::{DateTime, Utc}; +use sqlbuild_orchestration::snapshots::models::SnapshotRecord; + +/// Publication metadata and the earliest calendar refresh deadline. +pub(crate) type LocationPublication = (SnapshotRecord, Option>); diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_state.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_state.rs index fccc6753f0..b258882455 100644 --- a/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_state.rs +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_state.rs @@ -2,10 +2,47 @@ use chrono::{DateTime, TimeDelta, Utc}; -use crate::declarations::models::{TriggerDefinition, TriggerStatus}; +use crate::declarations::models::{ + PreviousTriggerState, Schedule, TriggerDefinition, TriggerStatus, +}; +use crate::scheduler::constants::MAX_TICKS_PER_BATCH; use crate::scheduler::main::next_schedule_time::next_schedule_time; +use crate::snapshots::models::SnapshotRecord; use crate::store::errors::ControlError; +/// Replacing a future horizon must retain due ticks not yet consumed by the scheduler. +pub(crate) fn retain_calendar_ticks( + name: &str, + trigger: &mut TriggerDefinition, + state: Option<&PreviousTriggerState>, + snapshot: &SnapshotRecord, +) -> Result<(), ControlError> { + if let Some(state) = state + && let Schedule::Calendar { + times: previous_times, + .. + } = &state.definition_json.schedule + && let Schedule::Calendar { times, .. } = &mut trigger.schedule + { + times.extend( + previous_times + .iter() + .filter(|time| { + **time >= state.started_at + && **time <= snapshot.published_at + && state.last_tick_for.is_none_or(|last| **time > last) + }) + .copied(), + ); + if times.len() > MAX_TICKS_PER_BATCH { + return Err(ControlError::Invalid(format!( + "calendar {name} pending and upcoming times exceed {MAX_TICKS_PER_BATCH}; let the scheduler drain pending ticks before refreshing" + ))); + } + } + Ok(()) +} + pub(crate) fn trigger_timing( definition: &TriggerDefinition, started_at: DateTime, diff --git a/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs b/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs index b9754a724f..9dd685ac6c 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs @@ -2,9 +2,8 @@ use uuid::Uuid; -use crate::declarations::_helpers::trigger_state::trigger_timing; -use crate::declarations::models::{Definitions, PreviousTriggerState, Schedule}; -use crate::scheduler::constants::MAX_TICKS_PER_BATCH; +use crate::declarations::_helpers::trigger_state::{retain_calendar_ticks, trigger_timing}; +use crate::declarations::models::{Definitions, PreviousTriggerState}; use crate::snapshots::models::SnapshotRecord; use crate::store::errors::ControlError; use crate::store::main::decode_row::decode_row; @@ -16,40 +15,7 @@ pub(crate) async fn publish_definitions( snapshot: &SnapshotRecord, definitions: Definitions, ) -> Result { - for table in ["sqb_job_nodes", "sqb_jobs"] { - (transaction, _) = query( - transaction, - &format!("DELETE FROM {table} WHERE location_id=$1 AND target=$2"), - vec![ - Parameter::Id(snapshot.location_id), - Parameter::Text(snapshot.target.clone()), - ], - ) - .await?; - } - let jobs = definitions - .jobs - .into_iter() - .map(|(name, job)| (name, "job", job)) - .chain( - definitions - .groups - .into_iter() - .map(|(name, group)| (name, "group", group.job)), - ); - for (name, kind, job) in jobs { - let definition = - serde_json::to_value(&job).map_err(|error| ControlError::Invalid(error.to_string()))?; - (transaction, _) = query(transaction, - "INSERT INTO sqb_jobs (location_id, target, job_key, snapshot_id, kind, definition_json, overlap_policy) VALUES ($1,$2,$3,$4,$7,$5,$6)", - vec![Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone()), Parameter::Text(name.clone()), - Parameter::Id(snapshot.snapshot_id), Parameter::Json(definition), Parameter::Text(job.overlap.as_str().into()), Parameter::Text(kind.into())]).await?; - for node in job.selection { - (transaction, _) = query(transaction, - "INSERT INTO sqb_job_nodes (location_id, target, job_key, node_key) VALUES ($1,$2,$3,$4)", - vec![Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone()), Parameter::Text(name.clone()), Parameter::Text(node)]).await?; - } - } + transaction = publish_jobs(transaction, snapshot, &definitions).await?; (transaction, _) = query(transaction, "UPDATE sqb_triggers SET removed_in_snapshot_id=$1 WHERE location_id=$2 AND target=$3 AND removed_in_snapshot_id IS NULL", vec![Parameter::Id(snapshot.snapshot_id), Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone())]).await?; @@ -68,29 +34,7 @@ pub(crate) async fn publish_definitions( .next() .map(decode_row::) .transpose()?; - if let Some(state) = &state - && let Schedule::Calendar { - times: previous_times, - .. - } = &state.definition_json.schedule - && let Schedule::Calendar { times, .. } = &mut trigger.schedule - { - times.extend( - previous_times - .iter() - .filter(|time| { - **time >= state.started_at - && **time <= snapshot.published_at - && state.last_tick_for.is_none_or(|last| **time > last) - }) - .copied(), - ); - if times.len() > MAX_TICKS_PER_BATCH { - return Err(ControlError::Invalid(format!( - "calendar {name} pending and upcoming times exceed {MAX_TICKS_PER_BATCH}; let the scheduler drain pending ticks before refreshing" - ))); - } - } + retain_calendar_ticks(&name, &mut trigger, state.as_ref(), snapshot)?; let definition = serde_json::to_value(&trigger) .map_err(|error| ControlError::Invalid(error.to_string()))?; (transaction, _) = query(transaction, @@ -115,3 +59,45 @@ pub(crate) async fn publish_definitions( } Ok(transaction) } + +async fn publish_jobs( + mut transaction: ControlTransaction, + snapshot: &SnapshotRecord, + definitions: &Definitions, +) -> Result { + for table in ["sqb_job_nodes", "sqb_jobs"] { + (transaction, _) = query( + transaction, + &format!("DELETE FROM {table} WHERE location_id=$1 AND target=$2"), + vec![ + Parameter::Id(snapshot.location_id), + Parameter::Text(snapshot.target.clone()), + ], + ) + .await?; + } + let jobs = definitions + .jobs + .iter() + .map(|(name, job)| (name, "job", job)) + .chain( + definitions + .groups + .iter() + .map(|(name, group)| (name, "group", &group.job)), + ); + for (name, kind, job) in jobs { + let definition = + serde_json::to_value(job).map_err(|error| ControlError::Invalid(error.to_string()))?; + (transaction, _) = query(transaction, + "INSERT INTO sqb_jobs (location_id, target, job_key, snapshot_id, kind, definition_json, overlap_policy) VALUES ($1,$2,$3,$4,$7,$5,$6)", + vec![Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone()), Parameter::Text(name.clone()), + Parameter::Id(snapshot.snapshot_id), Parameter::Json(definition), Parameter::Text(job.overlap.as_str().into()), Parameter::Text(kind.into())]).await?; + for node in &job.selection { + (transaction, _) = query(transaction, + "INSERT INTO sqb_job_nodes (location_id, target, job_key, node_key) VALUES ($1,$2,$3,$4)", + vec![Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone()), Parameter::Text(name.clone()), Parameter::Text(node.clone())]).await?; + } + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs index 2c05267eb7..4c388072ab 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs @@ -14,7 +14,7 @@ use uuid::Uuid; use crate::credentials::tests::location_helpers::create_location; use crate::scheduler::tests::contracts::ticks::helpers::{snapshot, trigger_audits}; use crate::scheduler::tests::contracts::ticks::test_types::{ - TransactionFault, TriggerControlsTestCase, + TickTransactionTestCase, TransactionFault, TriggerControlsTestCase, }; use crate::store::tests::fault_helpers::{install_fault, remove_fault}; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; @@ -253,10 +253,9 @@ fn given_a_calendar_with_unprocessed_due_times_when_refreshed_then_due_ticks_sur { use crate::runs::main::list_runs::list_runs; use crate::scheduler::main::list_ticks::list_ticks; - use crate::scheduler::tests::contracts::ticks::test_types::TickTransactionTestCase; use serde_json::json; - for test_case in [ + let test_cases = [ TickTransactionTestCase { description: "SQLite retains due calendar ticks", backend: Backend::Sqlite, @@ -269,7 +268,8 @@ fn given_a_calendar_with_unprocessed_due_times_when_refreshed_then_due_ticks_sur expected_ticks: 3, expected_runs: 1, }, - ] { + ]; + for test_case in test_cases { run(async { let Some(fixture) = open_test_store(test_case.backend).await else { return; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs index bbb369fa17..8edfb3a580 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs @@ -131,8 +131,7 @@ pub(crate) fn build_orchestration_snapshot( input = compile_declarations( py, input, - &project_dir, - graph.bind(py).get().graph(), + (&project_dir, graph.bind(py).get().graph()), authored, )?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index 7bc75bb723..9706fcc399 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -4,7 +4,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; use serde_json::Value; use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; use sqlbuild_analysis::graph::models::ProjectGraph; @@ -60,8 +60,7 @@ pub(crate) fn discover_declarations<'py>( pub(crate) fn compile_declarations( py: Python<'_>, mut input: SnapshotInput, - project_dir: &Path, - graph: &ProjectGraph, + (project_dir, graph): (&Path, &ProjectGraph), declarations: Vec, ) -> PyResult { if declarations.is_empty() { @@ -78,7 +77,7 @@ pub(crate) fn compile_declarations( let mut jobs: BTreeMap = BTreeMap::new(); let mut authored_groups: Vec = Vec::new(); let mut authored_jobs: Vec = Vec::new(); - let mut authored_calendars = Vec::new(); + let mut authored_calendars: Vec> = Vec::new(); for declaration in declarations { match declaration { AuthoredDeclaration::Job(job) => authored_jobs.push(job), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs index 6f9e92f4f8..d2efd5db0b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs @@ -19,7 +19,7 @@ use crate::bindings::_helpers::compiled_project::snapshot::{ NativeOrchestrationSnapshot, build_orchestration_snapshot, }; use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; -use crate::bindings::_helpers::location_command::constants::PUBLISH_ACTION; +use crate::bindings::_helpers::location_command::constants::{EXECUTE_ACTION, PUBLISH_ACTION}; use crate::bindings::_helpers::location_command::runtime::run_runtime; use crate::bindings::_helpers::location_command::worker::execute_worker; use crate::bindings::_helpers::native_host::host::write_console; @@ -50,7 +50,7 @@ fn metadata( let generation = arguments .value("generation")? .extract::>()? - .or_else(|| (action.as_deref() == Some("execute")).then_some(1)) + .or_else(|| (action.as_deref() == Some(EXECUTE_ACTION)).then_some(1)) .ok_or_else(|| PyValueError::new_err("publish requires --generation"))?; if target.trim().is_empty() || generation < 1 { return Err(PyValueError::new_err( @@ -132,11 +132,10 @@ fn compile_location<'py>( view, &serde_json::to_string(input).map_err(|error| PyValueError::new_err(error.to_string()))?, project_dir.extract()?, - // Workers verify nodes and environment, not dynamic calendar declarations. arguments .value("action")? .extract::>()? - .is_some_and(|action| action == "execute") + .is_some_and(|action| action == EXECUTE_ACTION) .then(Vec::new), )?; status(py, "Location snapshot compiled.")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs index dc0a1d6e2d..ffee78c649 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs @@ -1,2 +1,5 @@ /// Location actions accepted by the interpreter-side one-shot publisher. pub(crate) const PUBLISH_ACTION: &str = "publish"; + +/// Workers verify the pinned nodes and environment without reevaluating declarations. +pub(crate) const EXECUTE_ACTION: &str = "execute"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs new file mode 100644 index 0000000000..5c0e4cbe43 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs @@ -0,0 +1,4 @@ +//! Authoring parameters reserved by the code-location host. + +/// The calendar's evaluation window is supplied by the host, not a provider. +pub(crate) const CALENDAR_WINDOW_PARAMETER: &str = "window"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index e604e71354..ad5619410e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -1,6 +1,7 @@ //! Permanent Python host calls and compatibility views of native-owned execution data. pub(crate) mod connections; +pub(crate) mod constants; pub(crate) mod header_projection; pub(crate) mod host; pub(crate) mod model_lookup; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs index 198541d498..c5366827ed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -18,6 +18,7 @@ use sqlbuild_orchestration::scheduler::constants::MAX_TICKS_PER_BATCH; use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, within}; use crate::bindings::_helpers::discovery::imported_modules::{provider_by_name, provider_usages}; use crate::bindings::_helpers::discovery::imported_providers::discover_provider_classes; +use crate::bindings::_helpers::native_host::constants::CALENDAR_WINDOW_PARAMETER; use crate::bindings::_helpers::native_host::host::entry; use crate::bindings::_helpers::native_host::orchestration_declarations::authored_setting; @@ -94,9 +95,10 @@ fn horizon(value: &str) -> PyResult { ] .into_iter() .try_fold(0_i64, |total, (digits, factor)| { - total.checked_add(digits.parse::().ok()?.checked_mul(factor)?) - }) - .ok_or_else(invalid)?; + let digits = digits.parse::().map_err(|_| invalid())?; + let seconds = digits.checked_mul(factor).ok_or_else(invalid)?; + total.checked_add(seconds).ok_or_else(invalid) + })?; TimeDelta::try_seconds(seconds).ok_or_else(invalid) } @@ -132,10 +134,10 @@ fn calendar_times( .call((), Some(&window))?; let function = authored.getattr("function")?; let arguments = PyDict::new(py); - arguments.set_item("window", window)?; + arguments.set_item(CALENDAR_WINDOW_PARAMETER, window)?; for usage in provider_usages(&function, providers)?.try_iter()? { let usage = usage?; - if usage.getattr("parameter_name")?.extract::()? == "window" { + if usage.getattr("parameter_name")?.extract::()? == CALENDAR_WINDOW_PARAMETER { continue; } arguments.set_item( @@ -143,7 +145,7 @@ fn calendar_times( session.call_method1("get", (usage.getattr("provider_name")?,))?, )?; } - let mut times = BTreeSet::new(); + let mut times: BTreeSet> = BTreeSet::new(); for (index, time) in function .call((), Some(&arguments))? .try_iter()? @@ -186,7 +188,7 @@ pub(crate) fn compile_calendars( authored: Vec>, mut definitions: Definitions, ) -> PyResult { - let mut calendars = Vec::new(); + let mut calendars: Vec> = Vec::new(); for calendar in authored { let targets = calendar .bind(py) From c87bdda2a1771f29a1a3ab29e6cc827738e88e5b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 02:23:59 +0000 Subject: [PATCH 46/59] feat: deliver platform health to server webhook Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../src/native_commands/_helpers/execution.rs | 47 +++++ .../src/native_commands/main/_run_control.rs | 17 +- .../src/workers/_helpers/invocation.rs | 2 + .../src/platform/_helpers/mod.rs | 1 + .../src/platform/_helpers/webhook_delivery.rs | 61 ++++++ .../src/platform/constants.rs | 8 + .../platform/main/connect_platform_webhook.rs | 31 +++ .../src/platform/main/mod.rs | 2 + .../src/platform/main/run_platform_sink.rs | 48 +++++ .../src/platform/mod.rs | 1 + .../src/platform/models.rs | 30 +++ .../src/store/_helpers/migration.rs | 1 + .../src/store/main/migrate_store.rs | 3 +- .../src/store/platform_sink.sql | 7 + .../tests/api/test_location_work.rs | 4 +- .../tests/platform/helpers.rs | 104 +++++++++- .../tests/platform/test_contracts.rs | 2 + .../tests/platform/test_types.rs | 23 ++- .../tests/platform/test_webhook.rs | 193 ++++++++++++++++++ .../tests/snapshots/helpers.rs | 3 +- fensu.toml | 6 + .../publication/docker/helpers.py | 2 + .../compiled_project/publication/helpers.py | 9 +- .../publication/test_docker_worker.py | 2 +- 24 files changed, 583 insertions(+), 24 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/platform/_helpers/webhook_delivery.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/constants.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/connect_platform_webhook.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/run_platform_sink.rs create mode 100644 crates/sqlbuild-orchestration/src/store/platform_sink.sql create mode 100644 crates/sqlbuild-orchestration/tests/platform/test_webhook.rs diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index dac117b46c..27c20ee723 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -8,7 +8,11 @@ use std::time::{Duration, SystemTime}; use chrono::{DateTime, Utc}; use sqlbuild_orchestration::api::errors::ClientError; +use sqlbuild_orchestration::platform::constants::PLATFORM_WEBHOOK_VARIABLE; +use sqlbuild_orchestration::platform::main::connect_platform_webhook::connect_platform_webhook; use sqlbuild_orchestration::platform::main::report_remote_location_health::report_remote_location_health; +use sqlbuild_orchestration::platform::main::run_platform_sink::run_platform_sink; +use sqlbuild_orchestration::platform::main::run_watchdog::run_watchdog; use sqlbuild_orchestration::platform::models::{LocationHealth, LocationHealthReport}; use sqlbuild_orchestration::snapshots::main::next_remote_generation::next_remote_generation; use sqlbuild_orchestration::store::errors::ControlError; @@ -300,6 +304,49 @@ fn location_child(options: &LocationServe, script: &Path) -> Command { command } +/// Health production and delivery remain native and independent of scheduler leadership. +pub(crate) fn start_platform_roles( + tasks: &mut JoinSet>, + (store, shutdown): (&ControlStore, &watch::Receiver), +) -> Result<(), ControlError> { + let sink = match std::env::var(PLATFORM_WEBHOOK_VARIABLE) { + Ok(endpoint) => Some(connect_platform_webhook(&endpoint)?), + Err(std::env::VarError::NotPresent) => None, + Err(_) => { + return Err(ControlError::Invalid( + "platform webhook configuration must be UTF-8".into(), + )); + } + }; + drop(write_output(&[( + Stream::Stderr, + "Starting platform watchdog...\n", + )])); + let watchdog_store = store.clone(); + let watchdog_shutdown = shutdown.clone(); + tasks.spawn( + async move { run_watchdog(&watchdog_store, Uuid::now_v7(), watchdog_shutdown).await }, + ); + drop(write_output(&[( + Stream::Stderr, + "Platform watchdog started\n", + )])); + if let Some(sink) = sink { + drop(write_output(&[( + Stream::Stderr, + "Starting platform webhook...\n", + )])); + let sink_store = store.clone(); + let sink_shutdown = shutdown.clone(); + tasks.spawn(async move { run_platform_sink(&sink_store, sink, sink_shutdown).await }); + drop(write_output(&[( + Stream::Stderr, + "Platform webhook started\n", + )])); + } + Ok(()) +} + /// Open and migrate the privileged store, reporting progress without contaminating JSON stdout. pub(crate) async fn open_control_store( namespace: &Namespace, diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs index 1ca4f4f3b1..02e6928bd2 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs @@ -1,7 +1,6 @@ //! Supervise native API/scheduler roles together; a failed role stops its siblings. use sqlbuild_orchestration::api::main::serve_api::serve_api; -use sqlbuild_orchestration::platform::main::run_watchdog::run_watchdog; use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler; use sqlbuild_orchestration::scheduler::models::SchedulerState; use sqlbuild_orchestration::store::errors::ControlError; @@ -11,7 +10,7 @@ use uuid::Uuid; use crate::argv::models::{Namespace, Stream}; use crate::native_commands::_helpers::decision::{control_address, flag}; use crate::native_commands::_helpers::execution::{ - open_control_store, open_control_workers, supervise_control_roles, + open_control_store, open_control_workers, start_platform_roles, supervise_control_roles, }; use crate::native_commands::_helpers::terminal::write_output; @@ -48,19 +47,7 @@ pub(crate) async fn run_control( listener, stopped(shutdown.clone()), )); - drop(write_output(&[( - Stream::Stderr, - "Starting platform watchdog...\n", - )])); - let watchdog_store = store.clone(); - let watchdog_shutdown = shutdown.clone(); - tasks.spawn(async move { - run_watchdog(&watchdog_store, Uuid::now_v7(), watchdog_shutdown).await - }); - drop(write_output(&[( - Stream::Stderr, - "Platform watchdog started\n", - )])); + start_platform_roles(&mut tasks, (&store, &shutdown))?; } if standalone || !flag(namespace, "control_no_scheduler") { drop(write_output(&[(Stream::Stderr, "Starting scheduler...\n")])); diff --git a/crates/sqlbuild-cli/src/workers/_helpers/invocation.rs b/crates/sqlbuild-cli/src/workers/_helpers/invocation.rs index 67f3054326..a951cc74d6 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/invocation.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/invocation.rs @@ -1,6 +1,7 @@ //! Both launchers hand off the same scoped invocation and exclude privileged credentials. use sqlbuild_core::command_output::constants::INVOCATION_CONTEXT_ENV; +use sqlbuild_orchestration::platform::constants::PLATFORM_WEBHOOK_VARIABLE; use sqlbuild_orchestration::runs::models::RunClaim; use crate::workers::models::LauncherScope; @@ -26,6 +27,7 @@ pub(in crate::workers) fn arguments(scope: &LauncherScope<'_>, claim: &RunClaim) pub(in crate::workers) fn excluded(name: &str, control_variable: &str) -> bool { name == control_variable + || name == PLATFORM_WEBHOOK_VARIABLE || matches!( name, "SQLBUILD_CONTROL_DATABASE_URL" | "SQLBUILD_LOCATION_TOKEN" | "SQLBUILD_RUN_TOKEN" diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs index 8f5f6285fa..06cdb8c059 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs @@ -2,4 +2,5 @@ pub(crate) mod location_heartbeat; pub(crate) mod location_work; pub(crate) mod observations; pub(crate) mod watchdog; +pub(crate) mod webhook_delivery; pub(crate) mod work_client; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/webhook_delivery.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/webhook_delivery.rs new file mode 100644 index 0000000000..73879b7c63 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/webhook_delivery.rs @@ -0,0 +1,61 @@ +//! Ordered, at-least-once delivery; a response is acknowledged only after HTTP success. + +use crate::platform::constants::SINK_PAGE_SIZE; +use crate::platform::models::{PlatformEventEnvelope, PlatformWebhook}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// A lost acknowledgement may resend the same event_id; receivers must deduplicate by that ID. +pub(crate) async fn deliver_platform_events( + store: &ControlStore, + sink: &PlatformWebhook, +) -> Result { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query( + transaction, + "SELECT storage_id, event_id, occurred_at, payload FROM recall_events \ + WHERE producer='platform' AND invocation_id LIKE 'platform/%' AND storage_id > \ + (SELECT COALESCE(MAX(storage_id), 0) FROM sqb_platform_sink_deliveries WHERE sink_id=$1) \ + ORDER BY storage_id LIMIT $2", + vec![ + Parameter::Text(sink.sink_id.clone()), + Parameter::Integer(SINK_PAGE_SIZE), + ], + ) + .await?; + let events: Vec = + rows.into_iter().map(decode_row).collect::>()?; + commit(transaction).await?; + let count = events.len(); + for event in events { + let response = sink + .client + .post(sink.url.clone()) + .header("Idempotency-Key", event.event_id.to_string()) + .json(&event) + .send() + .await + .map_err(|_| { + ControlError::Invalid("platform webhook delivery failed; event retained".into()) + })?; + if !response.status().is_success() { + return Err(ControlError::Invalid(format!( + "platform webhook returned HTTP {}; event retained", + response.status().as_u16() + ))); + } + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_platform_sink_deliveries (sink_id, storage_id, event_id, delivered_at) \ + VALUES ($1,$2,$3,$4) ON CONFLICT (sink_id, storage_id) DO NOTHING", + vec![Parameter::Text(sink.sink_id.clone()), Parameter::Integer(event.storage_id), + Parameter::Text(event.event_id.to_string()), Parameter::Time(now)]).await?; + commit(transaction).await?; + } + Ok(count) +} diff --git a/crates/sqlbuild-orchestration/src/platform/constants.rs b/crates/sqlbuild-orchestration/src/platform/constants.rs new file mode 100644 index 0000000000..f405071bd8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/constants.rs @@ -0,0 +1,8 @@ +//! Bounds shared by the platform event reader and webhook dispatcher. + +/// Operator-owned delivery endpoint; never inherited by project workers or printed in logs. +pub const PLATFORM_WEBHOOK_VARIABLE: &str = "SQLBUILD_PLATFORM_WEBHOOK_URL"; + +pub(crate) const SINK_PAGE_SIZE: i64 = 100; +pub(crate) const SINK_REQUEST_TIMEOUT_SECONDS: u64 = 10; +pub(crate) const SINK_POLL_SECONDS: u64 = 5; diff --git a/crates/sqlbuild-orchestration/src/platform/main/connect_platform_webhook.rs b/crates/sqlbuild-orchestration/src/platform/main/connect_platform_webhook.rs new file mode 100644 index 0000000000..4a715b9ec8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/connect_platform_webhook.rs @@ -0,0 +1,31 @@ +//! Validate operator-owned delivery configuration without exposing credentials in errors. + +use std::time::Duration; + +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; + +use crate::platform::constants::SINK_REQUEST_TIMEOUT_SECONDS; +use crate::platform::models::PlatformWebhook; +use crate::store::errors::ControlError; + +/// Changing the endpoint creates a new consumer, starting from the retained event history. +pub fn connect_platform_webhook(endpoint: &str) -> Result { + let url = reqwest::Url::parse(endpoint) + .map_err(|_| ControlError::Invalid("platform webhook needs an HTTP(S) URL".into()))?; + if !matches!(url.scheme(), "http" | "https") || url.host_str().is_none() { + return Err(ControlError::Invalid( + "platform webhook needs an HTTP(S) URL".into(), + )); + } + let sink_id = sha256_hex(url.as_str().as_bytes()); + let client = reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(SINK_REQUEST_TIMEOUT_SECONDS)) + .build() + .map_err(|_| ControlError::Invalid("platform webhook client could not start".into()))?; + Ok(PlatformWebhook { + client, + url, + sink_id, + }) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/mod.rs b/crates/sqlbuild-orchestration/src/platform/main/mod.rs index 095c42262e..e9b14619b3 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/mod.rs @@ -1,9 +1,11 @@ pub mod claim_location_work; pub mod complete_location_work; pub mod connect_location_work; +pub mod connect_platform_webhook; pub mod location_health; pub mod location_work; pub mod report_location_health; pub mod report_remote_location_health; pub mod request_location_work; +pub mod run_platform_sink; pub mod run_watchdog; diff --git a/crates/sqlbuild-orchestration/src/platform/main/run_platform_sink.rs b/crates/sqlbuild-orchestration/src/platform/main/run_platform_sink.rs new file mode 100644 index 0000000000..02757c1627 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/run_platform_sink.rs @@ -0,0 +1,48 @@ +//! Deliver platform health independently of user interpreters and scheduler leadership. + +use std::time::Duration; + +use tokio::sync::watch; + +use crate::platform::_helpers::webhook_delivery::deliver_platform_events; +use crate::platform::constants::SINK_POLL_SECONDS; +use crate::platform::models::PlatformWebhook; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; + +pub async fn run_platform_sink( + store: &ControlStore, + sink: PlatformWebhook, + mut shutdown: watch::Receiver, +) -> Result<(), ControlError> { + let mut failed = false; + while !*shutdown.borrow() { + tokio::select! { + result = deliver_platform_events(store, &sink) => { + match result { + Ok(count) => { + if failed { + eprintln!("Platform webhook delivery recovered"); + } + failed = false; + if count > 0 { continue; } + }, + Err(_) => { + if !failed { + eprintln!("Platform webhook delivery failed; retaining events and retrying"); + } + failed = true; + }, + } + }, + result = shutdown.changed() => { + if result.is_err() || *shutdown.borrow() { break; } + }, + } + tokio::select! { + _ = tokio::time::sleep(Duration::from_secs(SINK_POLL_SECONDS)) => {}, + result = shutdown.changed() => { if result.is_err() { break; } }, + } + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/platform/mod.rs b/crates/sqlbuild-orchestration/src/platform/mod.rs index 32d92d3cc2..d58521e0f6 100644 --- a/crates/sqlbuild-orchestration/src/platform/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod _helpers; +pub mod constants; pub mod main; pub mod models; diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index 29f902dff0..72340bfd81 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -152,3 +152,33 @@ pub(crate) struct WatchdogTrigger { pub(crate) definition_json: TriggerDefinition, pub(crate) next_due_at: Option>, } + +/// The endpoint may contain credentials; neither errors nor Debug reveal it. +#[derive(Clone)] +pub struct PlatformWebhook { + pub(crate) client: reqwest::Client, + pub(crate) url: reqwest::Url, + pub(crate) sink_id: String, +} + +impl std::fmt::Debug for PlatformWebhook { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("PlatformWebhook") + .finish_non_exhaustive() + } +} + +/// Stable event identity permits receiver-side deduplication after uncertain acknowledgements. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct PlatformEventEnvelope { + pub storage_id: i64, + pub event_id: Uuid, + pub occurred_at: DateTime, + pub payload: PlatformEventPayload, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct PlatformEventPayload { + pub platform: PlatformEvent, +} diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index cd44c4e59a..fd922e7d2f 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -9,6 +9,7 @@ use crate::store::types::Row; pub(crate) const SCHEMA: &str = include_str!("../schema.sql"); pub(crate) const LOCATION_INSTANCES: &str = include_str!("../location_instances.sql"); pub(crate) const LOCATION_WORK: &str = include_str!("../location_work.sql"); +pub(crate) const PLATFORM_SINK: &str = include_str!("../platform_sink.sql"); pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index 31fefb1c4e..56f2194ffb 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -4,7 +4,7 @@ use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; use crate::store::_helpers::migration::{ - LOCATION_INSTANCES, LOCATION_WORK, SCHEMA, render, verify_migrations, + LOCATION_INSTANCES, LOCATION_WORK, PLATFORM_SINK, SCHEMA, render, verify_migrations, }; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; @@ -30,6 +30,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { let additions = [ ("location_instances", LOCATION_INSTANCES), ("location_work", LOCATION_WORK), + ("platform_sink", PLATFORM_SINK), ]; let mut digests = vec![digest.clone()]; digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); diff --git a/crates/sqlbuild-orchestration/src/store/platform_sink.sql b/crates/sqlbuild-orchestration/src/store/platform_sink.sql new file mode 100644 index 0000000000..ece8bb3b6f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/platform_sink.sql @@ -0,0 +1,7 @@ +CREATE TABLE sqb_platform_sink_deliveries ( + sink_id TEXT NOT NULL, + storage_id BIGINT NOT NULL, + event_id TEXT NOT NULL, + delivered_at {time} NOT NULL, + PRIMARY KEY (sink_id, storage_id) +); diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs index 03d21e27b5..7d77cdf9fa 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs @@ -46,9 +46,9 @@ fn given_outbound_work_when_claimed_retried_or_replaced_then_scope_and_immutable let store = &fixture.store; migrate_store(store).await.expect("fresh migration"); // Exercise both deployed predecessors, including checksums and repeated startup. - execute(store, "DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DELETE FROM sqb_schema_migrations WHERE version=3").await; + execute(store, "DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DELETE FROM sqb_schema_migrations WHERE version>=3").await; migrate_store(store).await.expect("schema 2 upgrade"); - execute(store, "DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2").await; + execute(store, "DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2").await; migrate_store(store).await.expect("schema 1 upgrade"); migrate_store(store).await.expect("repeat migration"); let location_id = create_location(store, "orders", "test-admin") diff --git a/crates/sqlbuild-orchestration/tests/platform/helpers.rs b/crates/sqlbuild-orchestration/tests/platform/helpers.rs index a00d9adf24..e2da7ca896 100644 --- a/crates/sqlbuild-orchestration/tests/platform/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/platform/helpers.rs @@ -1,17 +1,29 @@ use std::collections::{BTreeMap, BTreeSet}; use std::time::Duration; -use crate::platform::models::PlatformEvent; +use axum::Json; +use axum::extract::State; +use axum::http::{HeaderMap, StatusCode}; +use axum::response::IntoResponse; + +use crate::platform::models::{PlatformEvent, PlatformEventEnvelope}; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::publish_run_recall::publish_run_recall; +use crate::runs::models::{RecallBatch, RecallRecord}; use crate::snapshots::models::{SnapshotInput, SnapshotNode}; use crate::store::errors::ControlError; use crate::store::models::ControlStore; use crate::store::tests::helpers::read_recall_events; use chrono::{TimeDelta, Utc}; use serde_json::json; +use sqlbuild_recall::records::models::RecallValue; +use sqlbuild_recall::schema::models::Table; use tokio::task::JoinHandle; use uuid::Uuid; -use crate::platform::tests::test_types::Observation; +use crate::platform::tests::test_types::{Observation, WebhookReceiver}; +use crate::runs::tests::helpers::{queue, request}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; pub(crate) fn snapshot(location_id: Uuid) -> SnapshotInput { let future = Utc::now() + TimeDelta::days(10); @@ -109,3 +121,91 @@ pub(crate) async fn wait_for_scans( .await .expect("both replicas finish a paged scan without a scheduler") } + +pub(crate) async fn receive_webhook( + State(receiver): State, + headers: HeaderMap, + Json(event): Json, +) -> impl IntoResponse { + assert_eq!( + headers.get("Idempotency-Key").expect("deduplication key"), + event.event_id.to_string().as_str() + ); + let mut state = receiver.state.lock().expect("receiver state"); + state.requests.push(event); + receiver.delivered.send_replace(state.requests.len()); + ( + StatusCode::from_u16(state.status).expect("configured response status"), + [(axum::http::header::LOCATION, "/events")], + ) +} + +pub(crate) async fn wait_for_webhook( + receiver: &WebhookReceiver, + dispatcher: &JoinHandle>, + count: usize, +) { + let mut delivered = receiver.delivered.subscribe(); + tokio::time::timeout( + Duration::from_secs(10), + delivered.wait_for(|received| *received >= count), + ) + .await + .expect("dispatcher drains more than one bounded page") + .expect("webhook receiver remains available"); + assert!( + !dispatcher.is_finished(), + "delivery loop stopped before draining" + ); +} + +pub(crate) async fn publish_platform_impostor(store: &ControlStore, identity: (Uuid, Uuid)) { + let mut requested = request(identity, 42, 0, &["orders"]); + requested.image_id = None; + queue(store, &requested).await; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let claim = claim_next_run(store, &lease, &BTreeMap::new(), None) + .await + .expect("claim") + .expect("work"); + let batch = RecallBatch { + invocation_id: "platform/impostor".into(), + records: vec![RecallRecord { + table: Table::Events, + values: BTreeMap::from([ + ( + "event_id".into(), + RecallValue::Text(Uuid::now_v7().to_string()), + ), + ( + "event_type".into(), + RecallValue::Text("platform_heartbeat".into()), + ), + ("event_schema_version".into(), RecallValue::Integer(1)), + ("event_kind".into(), RecallValue::Text("runtime".into())), + ("severity".into(), RecallValue::Text("info".into())), + ( + "occurred_at".into(), + RecallValue::Text("2026-01-01T00:00:00Z".into()), + ), + ("producer".into(), RecallValue::Text("platform".into())), + ("invocation_sequence".into(), RecallValue::Integer(0)), + ( + "payload".into(), + RecallValue::Text( + json!({"platform": {"kind": "heartbeat", "replica_id": Uuid::now_v7()}}) + .to_string(), + ), + ), + ]), + }], + }; + let result = publish_run_recall(store, (claim.request.run_id, claim.claim_id), &batch) + .await + .expect("publish untrusted producer"); + assert_eq!(result.delivered, [(Table::Events, 1)]); + assert!(result.conflicts.is_empty()); +} diff --git a/crates/sqlbuild-orchestration/tests/platform/test_contracts.rs b/crates/sqlbuild-orchestration/tests/platform/test_contracts.rs index 1a231c8e58..0f5c53a679 100644 --- a/crates/sqlbuild-orchestration/tests/platform/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/platform/test_contracts.rs @@ -6,3 +6,5 @@ mod helpers; mod test_types; #[path = "test_watchdog.rs"] mod test_watchdog; +#[path = "test_webhook.rs"] +mod test_webhook; diff --git a/crates/sqlbuild-orchestration/tests/platform/test_types.rs b/crates/sqlbuild-orchestration/tests/platform/test_types.rs index 01cf50f419..970ec7a8c0 100644 --- a/crates/sqlbuild-orchestration/tests/platform/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/platform/test_types.rs @@ -1,4 +1,6 @@ -use crate::platform::models::PlatformEvent; +use std::sync::{Arc, Mutex}; + +use crate::platform::models::{PlatformEvent, PlatformEventEnvelope}; use chrono::{DateTime, Utc}; use serde::Deserialize; @@ -24,3 +26,22 @@ pub(crate) struct Observation { pub(crate) struct PlatformPayload { pub(crate) platform: PlatformEvent, } + +pub(crate) struct PlatformSinkTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) initial_status: u16, + pub(crate) expected_events: usize, +} + +#[derive(Clone, Debug)] +pub(crate) struct WebhookReceiver { + pub(crate) state: Arc>, + pub(crate) delivered: tokio::sync::watch::Sender, +} + +#[derive(Debug)] +pub(crate) struct WebhookState { + pub(crate) status: u16, + pub(crate) requests: Vec, +} diff --git a/crates/sqlbuild-orchestration/tests/platform/test_webhook.rs b/crates/sqlbuild-orchestration/tests/platform/test_webhook.rs new file mode 100644 index 0000000000..d2aeeea1c8 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/platform/test_webhook.rs @@ -0,0 +1,193 @@ +use std::collections::BTreeSet; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use axum::Router; +use axum::routing::post; +use uuid::Uuid; + +use crate::platform::_helpers::webhook_delivery::deliver_platform_events; +use crate::platform::main::connect_platform_webhook::connect_platform_webhook; +use crate::platform::main::run_platform_sink::run_platform_sink; +use crate::platform::main::run_watchdog::run_watchdog; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; + +use crate::credentials::tests::location_helpers::create_location; +use crate::platform::tests::helpers::{ + publish_platform_impostor, receive_webhook, snapshot, wait_for_scans, wait_for_webhook, +}; +use crate::platform::tests::test_types::{PlatformSinkTestCase, WebhookReceiver, WebhookState}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_location_outage_when_webhook_delivery_retries_then_all_health_facts_survive_restart_and_lost_acknowledgements() + { + let test_cases = [ + PlatformSinkTestCase { + description: "SQLite durable platform delivery", + backend: Backend::Sqlite, + initial_status: 503, + expected_events: 105, + }, + PlatformSinkTestCase { + description: "PostgreSQL durable platform delivery", + backend: Backend::Postgres, + initial_status: 307, + expected_events: 105, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate delivery store"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("publish schedules"); + // No location process and no scheduler are running. + let replica_id = Uuid::now_v7(); + let (stop, shutdown) = tokio::sync::watch::channel(false); + let watchdog_store = store.clone(); + let tasks = [tokio::spawn(async move { + run_watchdog(&watchdog_store, replica_id, shutdown).await + })]; + let observations = wait_for_scans(store, &BTreeSet::from([replica_id]), &tasks).await; + stop.send(true).expect("stop watchdog"); + for task in tasks { + task.await + .expect("watchdog task") + .expect("watchdog stopped"); + } + assert_eq!( + observations.len(), + test_case.expected_events, + "{}", + test_case.description + ); + publish_platform_impostor(store, (location, published.snapshot_id)).await; + + let state = Arc::new(Mutex::new(WebhookState { + status: test_case.initial_status, + requests: Vec::new(), + })); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("webhook listener"); + let endpoint = format!( + "http://delivery:sample-password@{}/events?token=sample-token", + listener.local_addr().expect("address") + ); + let receiver = WebhookReceiver { + state: state.clone(), + delivered: tokio::sync::watch::channel(0).0, + }; + let app = Router::new() + .route("/events", post(receive_webhook)) + .with_state(receiver.clone()); + let (http_stop, http_shutdown) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + axum::serve(listener, app) + .with_graceful_shutdown(async { + drop(http_shutdown.await); + }) + .await + }); + let sink = connect_platform_webhook(&endpoint).expect("configure webhook"); + let diagnostic = format!("{sink:?}"); + assert!(!diagnostic.contains("sample-password")); + assert!(!diagnostic.contains("sample-token")); + assert!(deliver_platform_events(store, &sink).await.is_err()); + assert_eq!( + state.lock().expect("receiver").requests.len(), + 1, + "failure stops the page" + ); + state.lock().expect("receiver").status = 204; + let fault = TransactionFault { + description: "lose a successful webhook acknowledgement", + table: "sqb_platform_sink_deliveries", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'test receipt failure')", + postgres_body: "RAISE EXCEPTION 'test receipt failure'", + }; + install_fault(store, &fault).await; + assert!(deliver_platform_events(store, &sink).await.is_err()); + remove_fault(store, &fault).await; + let second = second_store(store).await; + let restarted = connect_platform_webhook(&endpoint).expect("restart webhook client"); + let (sink_stop, sink_shutdown) = tokio::sync::watch::channel(false); + let sink_store = second.clone(); + let dispatcher = tokio::spawn(async move { + run_platform_sink(&sink_store, restarted, sink_shutdown).await + }); + wait_for_webhook(&receiver, &dispatcher, test_case.expected_events + 2).await; + sink_stop.send(true).expect("stop dispatcher"); + tokio::time::timeout(Duration::from_secs(2), dispatcher) + .await + .expect("bounded dispatcher shutdown") + .expect("dispatcher task") + .expect("dispatcher stopped"); + // Shutdown may cancel the last receipt after the remote endpoint accepted it. + assert!( + deliver_platform_events(store, &sink) + .await + .expect("reconcile in-flight delivery") + <= 1 + ); + assert_eq!( + deliver_platform_events(store, &sink) + .await + .expect("durable cursor"), + 0 + ); + { + let receiver = state.lock().expect("receiver"); + let requests = &receiver.requests; + assert!( + (test_case.expected_events + 2..=test_case.expected_events + 3) + .contains(&requests.len()) + ); + assert_eq!(requests[0], requests[1], "HTTP retry retains full identity"); + assert_eq!( + requests[1], requests[2], + "lost receipt retries the same fact" + ); + assert_eq!( + requests[2..] + .iter() + .map(|event| event.event_id) + .collect::>() + .len(), + test_case.expected_events + ); + let mut delivered = requests[2..].to_vec(); + delivered.dedup(); + assert_eq!(delivered.len(), observations.len()); + for (event, observation) in delivered.iter().zip(&observations) { + assert_eq!(event.storage_id, observation.storage_id); + assert_eq!(event.occurred_at, observation.occurred_at); + assert_eq!(event.payload.platform, observation.payload.platform); + } + } + http_stop.send(()).expect("stop receiver"); + tokio::time::timeout(Duration::from_secs(2), server) + .await + .expect("bounded shutdown") + .expect("server task") + .expect("server stopped"); + close_store(second).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index 7c093b292f..7b2283fbd1 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -243,7 +243,8 @@ pub(super) async fn sqlite_failed_publish_and_schema_floor() -> i64 { let connection = rusqlite::Connection::open(&path).expect("open fault injection connection"); connection .execute_batch( - "DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; \ + "DROP TABLE sqb_platform_sink_deliveries; \ + DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; \ DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2", ) .expect("reconstruct the previous schema without changing existing facts"); diff --git a/fensu.toml b/fensu.toml index aefc72b446..8c5126e9c4 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1570,6 +1570,12 @@ path = "crates/sqlbuild-orchestration/src/credentials/models.rs" symbols = ["IssuedToken"] reason = "IssuedToken implements Debug manually to redact its bearer credential; deriving Debug would expose the secret." +[[targets.rust.rule_exceptions]] +rule = "FPRSS201" +path = "crates/sqlbuild-orchestration/src/platform/models.rs" +symbols = ["PlatformWebhook"] +reason = "PlatformWebhook implements Debug manually to redact its endpoint, which may contain delivery credentials; other models derive Debug." + [[targets.rust.rule_exceptions]] rule = "FPRSS201" path = "crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py index 58135a9dec..d1ac635fdd 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py @@ -89,6 +89,8 @@ def docker_deployment(directory: Path, executable: str) -> Iterator[WorkerDeploy "--env", "SQLBUILD_CONTROL_DATABASE_URL=synthetic-control-store", "--env", + "SQLBUILD_PLATFORM_WEBHOOK_URL=http://127.0.0.1:1/events?token=synthetic-delivery-credential", + "--env", "SQLBUILD_SCHEDULER_TEST_STORE=synthetic-configured-store", *mounts, image_id, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index f0717f1000..bcc8855eca 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -169,6 +169,7 @@ def snapshot_api(directory: Path) -> Iterator[SnapshotApi]: log: Path = directory / "server.log" environment: dict[str, str] = os.environ.copy() environment.pop("SQLBUILD_CONTROL_DATABASE_URL", None) + environment.pop("SQLBUILD_PLATFORM_WEBHOOK_URL", None) with log.open("wb") as stderr: process: subprocess.Popen[bytes] = subprocess.Popen( [str(Path(sys.executable).with_name("sqb")), "server", "--port", "0", "--no-scheduler"], @@ -271,7 +272,9 @@ def scheduled_worker( "MODEL (description 'Orders.', materialized table);\n" + test_case.model_sql + "\n" ), "orchestration/orders.py": ( - "from sqlbuild.orchestration import job, schedule\n" + "import os\nfrom sqlbuild.orchestration import job, schedule\n" + "if 'SQLBUILD_RUN_TOKEN' in os.environ:\n" + " assert 'SQLBUILD_PLATFORM_WEBHOOK_URL' not in os.environ\n" 'annual_orders = job("annual_orders", select="orders", ' 'description="Build orders.", ' 'schedule=schedule(cron="0 0 1 1 *", timezone="UTC", ' @@ -334,6 +337,10 @@ def scheduled_worker( with sqlite3.connect(control_store) as control: _ = control.executescript(test_case.store_setup) monkeypatch.setenv("SQLBUILD_SCHEDULER_TEST_STORE", f"sqlite://{control_store}") + monkeypatch.setenv( + "SQLBUILD_PLATFORM_WEBHOOK_URL", + "http://127.0.0.1:1/events?token=synthetic-delivery-credential", + ) log: Path = directory / "scheduler.log" with location_process( [ diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py index a09544ebf1..8d54cf22ee 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py @@ -47,7 +47,7 @@ "if 'SQLBUILD_RUN_TOKEN' in os.environ:\n" " assert os.getenv('SQLBUILD_WORKER_TEST_VALUE') == 'orders-context'\n" " for name in ('SQLBUILD_LOCATION_TOKEN', 'SQLBUILD_CONTROL_DATABASE_URL', " - "'SQLBUILD_SCHEDULER_TEST_STORE'):\n" + "'SQLBUILD_SCHEDULER_TEST_STORE', 'SQLBUILD_PLATFORM_WEBHOOK_URL'):\n" " assert name not in os.environ, name\n" 'annual_orders = job("annual_orders", select="orders", ' 'description="Build orders.", ' From 45b44e3128089692d6837d0916db7bdb137c1e28 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 04:31:17 +0000 Subject: [PATCH 47/59] test: share disposable PostgreSQL harness across native gates Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- CONTRIBUTING.md | 13 ++++++++ Makefile | 8 ++++- .../tests/store/helpers.rs | 11 ++----- scripts/native_tests/__init__.py | 1 + scripts/native_tests/constants.py | 6 ++++ scripts/native_tests/main/__init__.py | 1 + scripts/native_tests/main/run.py | 33 +++++++++++++++++++ scripts/run_native_tests.py | 12 +++++++ 8 files changed, 76 insertions(+), 9 deletions(-) create mode 100644 scripts/native_tests/__init__.py create mode 100644 scripts/native_tests/constants.py create mode 100644 scripts/native_tests/main/__init__.py create mode 100644 scripts/native_tests/main/run.py create mode 100644 scripts/run_native_tests.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f0ab39173e..ba45c64ca6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -42,6 +42,19 @@ For a change confined to one Rust crate, run its focused tests before rebuilding cargo test -p sqlbuild-sqltext given_model_files_when_matching_headers_then_offsets_follow_the_python_pattern ``` +Control-plane integration tests require Docker and run both SQLite and PostgreSQL: + +```bash +make test-native NATIVE_TEST_ARGS='-p sqlbuild-orchestration --lib' +``` + +The same Testcontainers harness runs under `make rust-check` in CI. Each invocation starts one +digest-pinned PostgreSQL container, waits for database readiness, shares it across the suite with +an isolated schema per fixture, and removes it after success or failure. No host PostgreSQL setup +is needed. Missing Docker, startup failures or missing database configuration fail the gate; +PostgreSQL cases never silently skip. `NATIVE_TEST_ARGS` accepts Cargo test arguments; its default +runs the workspace with all features. + Keep the toolchain, `CARGO_TARGET_DIR`, profile and `RUSTFLAGS` consistent between iterations. Changing them can invalidate compiled artifacts. Coordinate builds that share a target directory; do not delete another task's artifacts or change global compiler-cache settings. The first build diff --git a/Makefile b/Makefile index 1753e39f05..911e79832f 100644 --- a/Makefile +++ b/Makefile @@ -39,7 +39,13 @@ fast-native: rust-check: rust-lint uv run fensu check - cargo test --workspace --all-features + $(MAKE) test-native + +NATIVE_TEST_ARGS ?= --workspace --all-features + +.PHONY: test-native +test-native: + uv run --no-sync python -m scripts.run_native_tests $(NATIVE_TEST_ARGS) .PHONY: rust-lint rust-lint: diff --git a/crates/sqlbuild-orchestration/tests/store/helpers.rs b/crates/sqlbuild-orchestration/tests/store/helpers.rs index db2525d17d..ba5fd692cc 100644 --- a/crates/sqlbuild-orchestration/tests/store/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/helpers.rs @@ -32,14 +32,9 @@ pub(crate) async fn open_test_store(backend: Backend) -> Option { postgres: None, }); }; - let Ok(url) = std::env::var("SQLBUILD_TEST_CONTROL_POSTGRES_URL") else { - assert!( - std::env::var_os("SQLBUILD_REQUIRE_POSTGRES_TESTS").is_none(), - "PostgreSQL test URL is required" - ); - eprintln!("PostgreSQL integration requires SQLBUILD_TEST_CONTROL_POSTGRES_URL"); - return None; - }; + let url = std::env::var("SQLBUILD_TEST_CONTROL_POSTGRES_URL").expect( + "PostgreSQL integration requires the disposable database harness; run make test-native", + ); let admin = sqlx_postgres::PgPool::connect(&url) .await .expect("connect to test PostgreSQL"); diff --git a/scripts/native_tests/__init__.py b/scripts/native_tests/__init__.py new file mode 100644 index 0000000000..4175a3327b --- /dev/null +++ b/scripts/native_tests/__init__.py @@ -0,0 +1 @@ +"""Shared native integration-test infrastructure.""" diff --git a/scripts/native_tests/constants.py b/scripts/native_tests/constants.py new file mode 100644 index 0000000000..3473a3cec7 --- /dev/null +++ b/scripts/native_tests/constants.py @@ -0,0 +1,6 @@ +"""Reproducible native integration database image.""" + +POSTGRES_IMAGE: str = ( + "postgres:17.6-alpine3.22@" + "sha256:ef257d85f76e48da1c64832459b59fcaba1a4dac97bf5d7450c77753542eee94" +) diff --git a/scripts/native_tests/main/__init__.py b/scripts/native_tests/main/__init__.py new file mode 100644 index 0000000000..22055e8817 --- /dev/null +++ b/scripts/native_tests/main/__init__.py @@ -0,0 +1 @@ +"""Native test command entrypoints.""" diff --git a/scripts/native_tests/main/run.py b/scripts/native_tests/main/run.py new file mode 100644 index 0000000000..fa3a52fc5e --- /dev/null +++ b/scripts/native_tests/main/run.py @@ -0,0 +1,33 @@ +"""One disposable database per native suite; Rust fixtures isolate their own schemas.""" + +from __future__ import annotations + +import os +import subprocess +import sys + +from testcontainers.postgres import PostgresContainer + +from scripts.native_tests.constants import POSTGRES_IMAGE + + +def run_native_tests(argv: list[str] | None = None) -> int: + """Fail closed on Docker/startup errors and propagate Cargo's result after cleanup.""" + arguments: list[str] = sys.argv[1:] if argv is None else argv + print("Starting disposable PostgreSQL for native integration tests...", flush=True) + try: + with PostgresContainer(POSTGRES_IMAGE, driver=None) as postgres: + print("PostgreSQL ready. Running native tests with isolated schemas...", flush=True) + result: subprocess.CompletedProcess[bytes] = subprocess.run( + ["cargo", "test", *arguments], + env=os.environ + | { + "SQLBUILD_TEST_CONTROL_POSTGRES_URL": postgres.get_connection_url(), + }, + check=False, + ) + print(f"PostgreSQL removed. Native tests exited {result.returncode}.", flush=True) + return result.returncode + except Exception as error: + print(f"Native integration tests failed: {error}", file=sys.stderr, flush=True) + return 1 diff --git a/scripts/run_native_tests.py b/scripts/run_native_tests.py new file mode 100644 index 0000000000..2f673d1fc7 --- /dev/null +++ b/scripts/run_native_tests.py @@ -0,0 +1,12 @@ +"""Direct wrapper for native tests with disposable PostgreSQL.""" + +from scripts.native_tests.main.run import run_native_tests + + +def main(argv: list[str] | None = None) -> int: + """Run Cargo tests with a shared Testcontainers PostgreSQL instance.""" + return run_native_tests(argv) + + +if __name__ == "__main__": + raise SystemExit(main()) From ac9bc0acf15a7be67cf4f81541e86a42f29671b8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 04:53:53 +0000 Subject: [PATCH 48/59] feat: schedule durable built-in platform sensors Queue sensor runs, commit cursors and publish outcomes atomically under scheduler fencing. Preserve deduplication, recovery and operator overrides across deployments. Place the public PlatformEventKind enum in types.py with an enum-only rule-3 facade entry, retaining remaining_lines=0 and native implementation ownership. Cover mixed schedule/sensor publication and real worker execution without relaxing existing regression assertions. Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../orchestration/platform_orders.py | 30 +++ .../orchestration/edge_cases/definitions.py | 18 +- .../src/declarations/_helpers/documents.rs | 22 +- .../src/declarations/_helpers/mod.rs | 1 + .../declarations/_helpers/operator_state.rs | 81 +++++++ .../src/declarations/main/change_trigger.rs | 44 +--- .../declarations/main/publish_definitions.rs | 2 + .../declarations/main/snapshot_definitions.rs | 2 + .../src/declarations/models.rs | 14 +- crates/sqlbuild-orchestration/src/lib.rs | 1 + .../src/platform/_helpers/watchdog.rs | 2 +- .../src/platform/_helpers/webhook_delivery.rs | 20 +- .../src/platform/constants.rs | 2 +- .../src/platform/main/mod.rs | 1 + .../src/platform/main/read_platform_events.rs | 22 ++ .../src/platform/models.rs | 21 ++ .../src/runs/_helpers/requests/job.rs | 68 +----- .../src/runs/main/frozen_job_request.rs | 70 ++++++ .../src/runs/main/mod.rs | 1 + .../src/runs/main/request_job_run.rs | 2 +- .../scheduler/_helpers/drain_due_triggers.rs | 5 +- .../src/scheduler/_helpers/drain_sensors.rs | 36 +++ .../src/scheduler/_helpers/mod.rs | 1 + .../src/sensors/_helpers/attempts.rs | 65 ++++++ .../src/sensors/_helpers/facts.rs | 65 ++++++ .../src/sensors/_helpers/mod.rs | 4 + .../src/sensors/_helpers/platform.rs | 34 +++ .../src/sensors/_helpers/requests.rs | 62 +++++ .../sensors/main/begin_sensor_evaluation.rs | 80 +++++++ .../sensors/main/complete_builtin_sensor.rs | 73 ++++++ .../src/sensors/main/mod.rs | 6 + .../src/sensors/main/publish_sensors.rs | 37 +++ .../src/sensors/main/read_sensor_cursor.rs | 24 ++ .../src/sensors/main/reset_sensor_cursor.rs | 41 ++++ .../src/sensors/main/validate_sensors.rs | 22 ++ .../sqlbuild-orchestration/src/sensors/mod.rs | 9 + .../src/sensors/models.rs | 114 +++++++++ .../src/store/_helpers/migration.rs | 1 + .../src/store/main/migrate_store.rs | 3 +- .../src/store/sensors.sql | 25 ++ .../tests/api/test_location_work.rs | 4 +- .../tests/scheduler/ticks/test_triggers.rs | 4 +- .../tests/sensors/helpers.rs | 114 +++++++++ .../tests/sensors/test_contracts.rs | 10 + .../tests/sensors/test_delivery.rs | 183 +++++++++++++++ .../tests/sensors/test_recovery.rs | 218 ++++++++++++++++++ .../tests/sensors/test_types.rs | 7 + .../tests/snapshots/helpers.rs | 3 +- .../compiled_project/snapshot_declarations.rs | 41 +++- .../_helpers/location_command/command.rs | 15 +- .../native_host/orchestration_declarations.rs | 35 ++- .../native_host/orchestration_triggers.rs | 36 ++- python-boundary.toml | 5 + src/sqlbuild/_native.pyi | 2 +- src/sqlbuild/orchestration.py | 10 + .../main/compile_snapshot.py | 4 +- .../orchestration_declarations/models.py | 23 ++ .../orchestration_declarations/types.py | 12 + .../compiler/compiled_project/_test_types.py | 2 + .../publication/_test_types.py | 1 + .../compiled_project/publication/helpers.py | 3 + .../publication/test_calendar.py | 10 +- .../publication/test_publication.py | 19 +- .../publication/test_worker.py | 41 ++++ .../test_orchestration_snapshot.py | 82 +++++++ 65 files changed, 1854 insertions(+), 161 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/platform_orders.py create mode 100644 crates/sqlbuild-orchestration/src/declarations/_helpers/operator_state.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/read_platform_events.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_sensors.rs create mode 100644 crates/sqlbuild-orchestration/src/sensors/_helpers/attempts.rs create mode 100644 crates/sqlbuild-orchestration/src/sensors/_helpers/facts.rs create mode 100644 crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs create mode 100644 crates/sqlbuild-orchestration/src/sensors/_helpers/requests.rs create mode 100644 crates/sqlbuild-orchestration/src/sensors/main/begin_sensor_evaluation.rs create mode 100644 crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs create mode 100644 crates/sqlbuild-orchestration/src/sensors/main/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/sensors/main/publish_sensors.rs create mode 100644 crates/sqlbuild-orchestration/src/sensors/main/read_sensor_cursor.rs create mode 100644 crates/sqlbuild-orchestration/src/sensors/main/reset_sensor_cursor.rs create mode 100644 crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs create mode 100644 crates/sqlbuild-orchestration/src/sensors/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/sensors/models.rs create mode 100644 crates/sqlbuild-orchestration/src/store/sensors.sql create mode 100644 crates/sqlbuild-orchestration/tests/sensors/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs create mode 100644 crates/sqlbuild-orchestration/tests/sensors/test_delivery.rs create mode 100644 crates/sqlbuild-orchestration/tests/sensors/test_recovery.rs create mode 100644 crates/sqlbuild-orchestration/tests/sensors/test_types.rs create mode 100644 src/sqlbuild/orchestration_declarations/types.py diff --git a/benchmarks/duckshop/edge_cases/orchestration/platform_orders.py b/benchmarks/duckshop/edge_cases/orchestration/platform_orders.py new file mode 100644 index 0000000000..4814f4ae1e --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/platform_orders.py @@ -0,0 +1,30 @@ +"""Sensor-only roots retain referenced jobs while target filters and event enums stay native.""" + +from sqlbuild.orchestration import PlatformEventKind, job, platform_event, sensor + + +def definitions(): + refresh = job("health_orders", description="Refresh orders.", select="orders") + return [ + sensor( + "overdue_orders", + description="Catch up overdue order reporting.", + when=platform_event(kinds=[PlatformEventKind.TICK_OVERDUE]), + run=refresh, + minimum_interval_seconds=7, + default_status="stopped", + targets=("development",), + ), + sensor( + "other_target_orders", + description="Refresh another target.", + when=platform_event(kinds=[PlatformEventKind.HEARTBEAT]), + run=refresh, + targets=("production",), + ), + ] + + +triggers = definitions() +alias = triggers[0] +triggers.append(triggers) diff --git a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py index af566e9dff..ef8caed097 100644 --- a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py +++ b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py @@ -1,6 +1,13 @@ """Exercise alias deduplication and cyclic factory containers during native discovery.""" -from sqlbuild.orchestration import job, job_group, schedule +from sqlbuild.orchestration import ( + PlatformEventKind, + job, + job_group, + platform_event, + schedule, + sensor, +) edge_orders = job( "edge_snapshot_orders", @@ -51,3 +58,12 @@ jobs=[edge_products], concurrency_groups=("edge_products_limit",), ) +edge_health = sensor( + "edge_platform_orders", + description="Catch up retained platform observations with a stopped group sensor.", + when=platform_event(kinds=[PlatformEventKind.TICK_OVERDUE]), + run=edge_group, + minimum_interval_seconds=7, + default_status="stopped", +) +edge_factory_results.append(edge_health) diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/documents.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/documents.rs index cf9e048bb0..112ceab0af 100644 --- a/crates/sqlbuild-orchestration/src/declarations/_helpers/documents.rs +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/documents.rs @@ -24,9 +24,25 @@ fn document( pub(crate) fn decode_definitions( documents: &BTreeMap, ) -> Result { - Ok(Definitions { + let mut definitions = Definitions { jobs: document(documents, "jobs")?, groups: document(documents, "groups")?, - triggers: document(documents, "triggers")?, - }) + ..Definitions::default() + }; + for (name, value) in document::(documents, "triggers")? { + if value.get("when").is_some() { + definitions.sensors.insert( + name, + serde_json::from_value(value) + .map_err(|error| ControlError::Invalid(format!("triggers: {error}")))?, + ); + } else { + definitions.triggers.insert( + name, + serde_json::from_value(value) + .map_err(|error| ControlError::Invalid(format!("triggers: {error}")))?, + ); + } + } + Ok(definitions) } diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs index fda8521da8..75c383f6bc 100644 --- a/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs @@ -1,2 +1,3 @@ pub(crate) mod documents; +pub(crate) mod operator_state; pub(crate) mod trigger_state; diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/operator_state.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/operator_state.rs new file mode 100644 index 0000000000..b07ce5e73b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/operator_state.rs @@ -0,0 +1,81 @@ +//! Apply the operator's projection and return the before/after audit images in the same transaction. + +use serde_json::{Value, json}; + +use crate::declarations::_helpers::trigger_state::trigger_timing; +use crate::declarations::models::{ + EditableTriggerState, TriggerChange, TriggerChangeRequest, TriggerDeclaration, TriggerStatus, +}; +use crate::sensors::main::read_sensor_cursor::read_sensor_cursor; +use crate::sensors::main::reset_sensor_cursor::reset_sensor_cursor; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn apply_operator_change( + transaction: ControlTransaction, + mut state: EditableTriggerState, + request: &TriggerChangeRequest, +) -> Result<(ControlTransaction, (Value, Value)), ControlError> { + let (mut transaction, cursor) = match &state.definition_json { + TriggerDeclaration::Sensor(_) => { + if matches!(request.change, TriggerChange::SetCursor { .. }) { + return Err(ControlError::Invalid( + "sensor cursors use storage order; use reset-cursor to replay retained events" + .into(), + )); + } + let (transaction, cursor) = read_sensor_cursor(transaction, request.trigger_id).await?; + (transaction, Some(cursor)) + } + TriggerDeclaration::Schedule(_) => (transaction, None), + }; + let mut before = + json!({"status_override": state.status_override, "last_tick_for": state.last_tick_for}); + if let Some(cursor) = &cursor { + before["cursor"] = json!(cursor); + } + match request.change { + TriggerChange::Enable => state.status_override = Some(TriggerStatus::Running), + TriggerChange::Pause => state.status_override = Some(TriggerStatus::Paused), + TriggerChange::ResetDefault => state.status_override = None, + TriggerChange::SetCursor { time } => state.last_tick_for = Some(time), + TriggerChange::ResetCursor => state.last_tick_for = None, + } + let (status, next) = match &state.definition_json { + TriggerDeclaration::Schedule(definition) => trigger_timing( + definition, + state.started_at, + state.last_tick_for, + state.status_override, + )?, + TriggerDeclaration::Sensor(definition) => { + if matches!(request.change, TriggerChange::ResetCursor) { + transaction = + reset_sensor_cursor(transaction, (request.trigger_id, request.event_id)) + .await?; + } + let status = state + .status_override + .unwrap_or(definition.default_status.projection()); + ( + status, + (status == TriggerStatus::Running).then_some(transaction.now), + ) + } + }; + let mut after = json!({"status_override": state.status_override, "last_tick_for": state.last_tick_for, "reason": request.reason}); + if let Some(cursor) = cursor { + after["cursor"] = json!(if matches!(request.change, TriggerChange::ResetCursor) { + Default::default() + } else { + cursor + }); + } + let now = transaction.now; + let (transaction, _) = query(transaction, + "UPDATE sqb_trigger_state SET status=$1, status_override=$2, last_tick_for=$3, next_due_at=$4, updated_at=$5 WHERE trigger_id=$6", + vec![Parameter::Text(status.as_str().into()), Parameter::OptionalText(state.status_override.map(|value| value.as_str().into())), Parameter::OptionalTime(state.last_tick_for), + Parameter::OptionalTime(next), Parameter::Time(now), Parameter::Id(request.trigger_id)]).await?; + Ok((transaction, (before, after))) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs b/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs index 0977da705b..095353123a 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs @@ -4,10 +4,8 @@ use serde_json::json; use crate::credentials::main::append_audit::append_audit; use crate::credentials::models::AuditContext; -use crate::declarations::_helpers::trigger_state::trigger_timing; -use crate::declarations::models::{ - EditableTriggerState, TriggerChange, TriggerChangeRequest, TriggerStatus, -}; +use crate::declarations::_helpers::operator_state::apply_operator_change; +use crate::declarations::models::{EditableTriggerState, TriggerChangeRequest}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; @@ -52,48 +50,24 @@ pub async fn change_trigger( vec![Parameter::Id(request.trigger_id)], ) .await?; - let mut state: EditableTriggerState = - decode_row(rows.into_iter().next().ok_or_else(|| { - ControlError::Invalid("trigger does not exist in the current snapshot".into()) - })?)?; - let before = json!({ - "status_override": state.status_override, - "last_tick_for": state.last_tick_for, - }); - match request.change { - TriggerChange::Enable => state.status_override = Some(TriggerStatus::Running), - TriggerChange::Pause => state.status_override = Some(TriggerStatus::Paused), - TriggerChange::ResetDefault => state.status_override = None, - TriggerChange::SetCursor { time } => state.last_tick_for = Some(time), - TriggerChange::ResetCursor => state.last_tick_for = None, - } - let (status, next) = trigger_timing( - &state.definition_json, - state.started_at, - state.last_tick_for, - state.status_override, - )?; + let state: EditableTriggerState = decode_row(rows.into_iter().next().ok_or_else(|| { + ControlError::Invalid("trigger does not exist in the current snapshot".into()) + })?)?; + let snapshot_id = state.snapshot_id; let now = transaction.now; let (transaction, _) = query(transaction, "INSERT INTO sqb_trigger_state_events (event_id, trigger_id, occurred_at, snapshot_id, request_json) VALUES ($1,$2,$3,$4,$5)", vec![Parameter::Id(request.event_id), Parameter::Id(request.trigger_id), Parameter::Time(now), Parameter::Id(state.snapshot_id), Parameter::Json(body)]).await?; - let (transaction, _) = query(transaction, - "UPDATE sqb_trigger_state SET status=$1, status_override=$2, last_tick_for=$3, next_due_at=$4, updated_at=$5 WHERE trigger_id=$6", - vec![Parameter::Text(status.as_str().into()), Parameter::OptionalText(state.status_override.map(|value| value.as_str().into())), Parameter::OptionalTime(state.last_tick_for), - Parameter::OptionalTime(next), Parameter::Time(now), Parameter::Id(request.trigger_id)]).await?; + let (transaction, (before, after)) = apply_operator_change(transaction, state, request).await?; commit( append_audit( transaction, context, (request.event_id, "trigger.changed"), ( - json!({"trigger_id": request.trigger_id, "snapshot_id": state.snapshot_id}), + json!({"trigger_id": request.trigger_id, "snapshot_id": snapshot_id}), before, - json!({ - "status_override": state.status_override, - "last_tick_for": state.last_tick_for, - "reason": request.reason, - }), + after, ), ) .await?, diff --git a/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs b/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs index 9dd685ac6c..230f3c116d 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs @@ -4,6 +4,7 @@ use uuid::Uuid; use crate::declarations::_helpers::trigger_state::{retain_calendar_ticks, trigger_timing}; use crate::declarations::models::{Definitions, PreviousTriggerState}; +use crate::sensors::main::publish_sensors::publish_sensors; use crate::snapshots::models::SnapshotRecord; use crate::store::errors::ControlError; use crate::store::main::decode_row::decode_row; @@ -19,6 +20,7 @@ pub(crate) async fn publish_definitions( (transaction, _) = query(transaction, "UPDATE sqb_triggers SET removed_in_snapshot_id=$1 WHERE location_id=$2 AND target=$3 AND removed_in_snapshot_id IS NULL", vec![Parameter::Id(snapshot.snapshot_id), Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone())]).await?; + transaction = publish_sensors(transaction, snapshot, &definitions).await?; for (name, mut trigger) in definitions.triggers { let id = Uuid::new_v5( &snapshot.location_id, diff --git a/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs b/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs index 5966cd18b3..cbe7c7a152 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs @@ -8,6 +8,7 @@ use crate::declarations::models::{Definitions, Schedule}; use crate::scheduler::constants::MAX_TICKS_PER_BATCH; use crate::scheduler::main::parse_cron::parse_cron; use crate::scheduler::models::CatchUp; +use crate::sensors::main::validate_sensors::validate_sensors; use crate::snapshots::models::SnapshotInput; use crate::store::errors::ControlError; @@ -80,5 +81,6 @@ pub fn snapshot_definitions(input: &SnapshotInput) -> Result {} } } + validate_sensors(&definitions)?; Ok(definitions) } diff --git a/crates/sqlbuild-orchestration/src/declarations/models.rs b/crates/sqlbuild-orchestration/src/declarations/models.rs index 98f516c0d3..5231762051 100644 --- a/crates/sqlbuild-orchestration/src/declarations/models.rs +++ b/crates/sqlbuild-orchestration/src/declarations/models.rs @@ -6,6 +6,7 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use crate::scheduler::models::{CatchUp, Overlap}; +use crate::sensors::models::SensorDefinition; #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -90,6 +91,15 @@ pub struct Definitions { pub jobs: BTreeMap, pub groups: BTreeMap, pub triggers: BTreeMap, + pub sensors: BTreeMap, +} + +/// Legacy schedule documents retain their stored shape; sensors add a distinct condition shape. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(untagged)] +pub enum TriggerDeclaration { + Schedule(TriggerDefinition), + Sensor(SensorDefinition), } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] @@ -97,7 +107,7 @@ pub struct TriggerRecord { pub trigger_id: uuid::Uuid, pub name: String, pub job_key: String, - pub definition_json: TriggerDefinition, + pub definition_json: TriggerDeclaration, pub status: TriggerStatus, pub last_tick_for: Option>, pub next_due_at: Option>, @@ -133,7 +143,7 @@ pub struct TriggerChangeRecord { #[derive(Clone, Debug, Deserialize)] pub(crate) struct EditableTriggerState { - pub(crate) definition_json: TriggerDefinition, + pub(crate) definition_json: TriggerDeclaration, pub(crate) snapshot_id: uuid::Uuid, pub(crate) started_at: DateTime, pub(crate) last_tick_for: Option>, diff --git a/crates/sqlbuild-orchestration/src/lib.rs b/crates/sqlbuild-orchestration/src/lib.rs index 2546686e46..cc435f1701 100644 --- a/crates/sqlbuild-orchestration/src/lib.rs +++ b/crates/sqlbuild-orchestration/src/lib.rs @@ -8,6 +8,7 @@ pub mod declarations; pub mod platform; pub mod runs; pub mod scheduler; +pub mod sensors; pub mod snapshots; pub mod store; pub mod worker; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs index 1757b07c66..16ded206d3 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs @@ -78,7 +78,7 @@ async fn scan_trigger_page( "SELECT t.trigger_id, t.definition_json, s.next_due_at \ FROM sqb_triggers t JOIN sqb_trigger_state s ON s.trigger_id=t.trigger_id \ JOIN sqb_locations l ON l.location_id=t.location_id \ - WHERE s.status='running' AND t.removed_in_snapshot_id IS NULL AND l.archived_at IS NULL \ + WHERE t.kind='schedule' AND s.status='running' AND t.removed_in_snapshot_id IS NULL AND l.archived_at IS NULL \ {cursor} ORDER BY t.trigger_id LIMIT 100" ), after.into_iter().map(Parameter::Id).collect(), diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/webhook_delivery.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/webhook_delivery.rs index 73879b7c63..eb5547281b 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/webhook_delivery.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/webhook_delivery.rs @@ -1,11 +1,10 @@ //! Ordered, at-least-once delivery; a response is acknowledged only after HTTP success. -use crate::platform::constants::SINK_PAGE_SIZE; -use crate::platform::models::{PlatformEventEnvelope, PlatformWebhook}; +use crate::platform::main::read_platform_events::read_platform_events; +use crate::platform::models::PlatformWebhook; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; -use crate::store::main::decode_row::decode_row; use crate::store::main::query::query; use crate::store::models::{ControlStore, Parameter, TransactionMode}; @@ -17,18 +16,13 @@ pub(crate) async fn deliver_platform_events( let transaction = begin(store, TransactionMode::Read).await?; let (transaction, rows) = query( transaction, - "SELECT storage_id, event_id, occurred_at, payload FROM recall_events \ - WHERE producer='platform' AND invocation_id LIKE 'platform/%' AND storage_id > \ - (SELECT COALESCE(MAX(storage_id), 0) FROM sqb_platform_sink_deliveries WHERE sink_id=$1) \ - ORDER BY storage_id LIMIT $2", - vec![ - Parameter::Text(sink.sink_id.clone()), - Parameter::Integer(SINK_PAGE_SIZE), - ], + "SELECT COALESCE(MAX(storage_id), 0) AS cursor FROM sqb_platform_sink_deliveries WHERE sink_id=$1", + vec![Parameter::Text(sink.sink_id.clone())], ) .await?; - let events: Vec = - rows.into_iter().map(decode_row).collect::>()?; + let cursor = serde_json::from_value(rows[0]["cursor"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let (transaction, events) = read_platform_events(transaction, cursor).await?; commit(transaction).await?; let count = events.len(); for event in events { diff --git a/crates/sqlbuild-orchestration/src/platform/constants.rs b/crates/sqlbuild-orchestration/src/platform/constants.rs index f405071bd8..75d8a07df7 100644 --- a/crates/sqlbuild-orchestration/src/platform/constants.rs +++ b/crates/sqlbuild-orchestration/src/platform/constants.rs @@ -3,6 +3,6 @@ /// Operator-owned delivery endpoint; never inherited by project workers or printed in logs. pub const PLATFORM_WEBHOOK_VARIABLE: &str = "SQLBUILD_PLATFORM_WEBHOOK_URL"; -pub(crate) const SINK_PAGE_SIZE: i64 = 100; +pub(crate) const PLATFORM_EVENT_PAGE_SIZE: i64 = 100; pub(crate) const SINK_REQUEST_TIMEOUT_SECONDS: u64 = 10; pub(crate) const SINK_POLL_SECONDS: u64 = 5; diff --git a/crates/sqlbuild-orchestration/src/platform/main/mod.rs b/crates/sqlbuild-orchestration/src/platform/main/mod.rs index e9b14619b3..850f9e7c04 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/mod.rs @@ -4,6 +4,7 @@ pub mod connect_location_work; pub mod connect_platform_webhook; pub mod location_health; pub mod location_work; +pub(crate) mod read_platform_events; pub mod report_location_health; pub mod report_remote_location_health; pub mod request_location_work; diff --git a/crates/sqlbuild-orchestration/src/platform/main/read_platform_events.rs b/crates/sqlbuild-orchestration/src/platform/main/read_platform_events.rs new file mode 100644 index 0000000000..48a38850e6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/read_platform_events.rs @@ -0,0 +1,22 @@ +//! One authoritative namespace filter protects every platform stream consumer from worker input. + +use crate::platform::constants::PLATFORM_EVENT_PAGE_SIZE; +use crate::platform::models::PlatformEventEnvelope; +use crate::store::errors::ControlError; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn read_platform_events( + transaction: ControlTransaction, + after: i64, +) -> Result<(ControlTransaction, Vec), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT storage_id,event_id,occurred_at,payload FROM recall_events WHERE producer='platform' \ + AND invocation_id LIKE 'platform/%' AND storage_id>$1 ORDER BY storage_id LIMIT $2", + vec![Parameter::Integer(after), Parameter::Integer(PLATFORM_EVENT_PAGE_SIZE)]).await?; + Ok(( + transaction, + rows.into_iter().map(decode_row).collect::>()?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index 72340bfd81..09ded290f5 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -110,6 +110,17 @@ pub(crate) struct StoredLocationWork { pub(crate) claimed_until: Option>, } +#[derive( + Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, utoipa::ToSchema, +)] +#[serde(rename_all = "snake_case")] +pub enum PlatformEventKind { + Heartbeat, + SchedulerNoLeader, + TickOverdue, + CalendarHorizonLow, +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "kind", rename_all = "snake_case")] pub enum PlatformEvent { @@ -129,6 +140,16 @@ pub enum PlatformEvent { } impl PlatformEvent { + #[must_use] + pub const fn kind(&self) -> PlatformEventKind { + match self { + Self::Heartbeat { .. } => PlatformEventKind::Heartbeat, + Self::SchedulerNoLeader => PlatformEventKind::SchedulerNoLeader, + Self::TickOverdue { .. } => PlatformEventKind::TickOverdue, + Self::CalendarHorizonLow { .. } => PlatformEventKind::CalendarHorizonLow, + } + } + #[must_use] pub const fn event_type(&self) -> &'static str { match self { diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs index 4d4785b696..a2c223fbb9 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs @@ -4,75 +4,11 @@ use std::collections::BTreeSet; use uuid::Uuid; -use crate::credentials::models::AuditContext; use crate::declarations::main::frozen_definitions::frozen_definitions; -use crate::declarations::models::JobDefinition; use crate::runs::main::job_write_set::job_write_set; -use crate::runs::models::{JobRunRequest, RunKind, RunOrigin, RunRequest}; +use crate::runs::models::{RunKind, RunOrigin, RunRequest}; use crate::store::errors::ControlError; -use crate::store::main::query::query; -use crate::store::models::{ControlTransaction, Parameter}; - -pub(crate) async fn frozen_job_request( - transaction: ControlTransaction, - context: &AuditContext, - request: &JobRunRequest, -) -> Result<(ControlTransaction, RunRequest), ControlError> { - let (transaction, jobs) = query( - transaction, - "SELECT j.snapshot_id, j.definition_json, j.kind, s.image_id FROM sqb_jobs j \ - JOIN sqb_current_snapshots c ON c.location_id=j.location_id AND c.target=j.target AND c.snapshot_id=j.snapshot_id \ - JOIN sqb_snapshots s ON s.snapshot_id=j.snapshot_id WHERE j.location_id=$1 AND j.target=$2 AND j.job_key=$3", - vec![Parameter::Id(request.location_id), Parameter::Text(request.target.clone()), Parameter::Text(request.job_key.clone())], - ).await?; - let row = jobs.first().ok_or_else(|| { - ControlError::Invalid("published job not found in this location and target".into()) - })?; - let snapshot_id = serde_json::from_value(row["snapshot_id"].clone()) - .map_err(|error| ControlError::Invalid(error.to_string()))?; - let definition: JobDefinition = serde_json::from_value(row["definition_json"].clone()) - .map_err(|error| ControlError::Invalid(error.to_string()))?; - let image_id = serde_json::from_value(row["image_id"].clone()) - .map_err(|error| ControlError::Invalid(error.to_string()))?; - let kind: RunKind = serde_json::from_value(row["kind"].clone()) - .map_err(|error| ControlError::Invalid(error.to_string()))?; - let (transaction, write_set) = - job_write_set(transaction, snapshot_id, &definition.selection).await?; - Ok(( - transaction, - RunRequest { - run_id: request.run_id, - location_id: request.location_id, - target: request.target.clone(), - snapshot_id, - image_id, - kind, - origin: if request.window.is_some() { - RunOrigin::Backfill - } else { - RunOrigin::Manual - }, - job_key: Some(request.job_key.clone()), - parent_run_id: None, - root_run_id: request.run_id, - trigger_id: None, - tick_id: None, - rerun_of_run_id: None, - requested_by: context.actor.clone(), - priority: request.priority, - selection: definition.selection, - write_set: if kind == RunKind::Group { - Default::default() - } else { - write_set - }, - settings: definition.settings, - window: request.window.clone(), - concurrency_groups: definition.concurrency_groups, - overlap: definition.overlap, - }, - )) -} +use crate::store::models::ControlTransaction; pub(crate) async fn frozen_group_children( transaction: ControlTransaction, diff --git a/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs b/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs new file mode 100644 index 0000000000..e91697d05d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs @@ -0,0 +1,70 @@ +//! Freeze published job inputs for manual requests and sensor evaluations alike. + +use crate::credentials::models::AuditContext; +use crate::declarations::models::JobDefinition; +use crate::runs::main::job_write_set::job_write_set; +use crate::runs::models::{JobRunRequest, RunKind, RunOrigin, RunRequest}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn frozen_job_request( + transaction: ControlTransaction, + context: &AuditContext, + request: &JobRunRequest, +) -> Result<(ControlTransaction, RunRequest), ControlError> { + let (transaction, jobs) = query( + transaction, + "SELECT j.snapshot_id, j.definition_json, j.kind, s.image_id FROM sqb_jobs j \ + JOIN sqb_current_snapshots c ON c.location_id=j.location_id AND c.target=j.target AND c.snapshot_id=j.snapshot_id \ + JOIN sqb_snapshots s ON s.snapshot_id=j.snapshot_id WHERE j.location_id=$1 AND j.target=$2 AND j.job_key=$3", + vec![Parameter::Id(request.location_id), Parameter::Text(request.target.clone()), Parameter::Text(request.job_key.clone())], + ).await?; + let row = jobs.first().ok_or_else(|| { + ControlError::Invalid("published job not found in this location and target".into()) + })?; + let snapshot_id = serde_json::from_value(row["snapshot_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let definition: JobDefinition = serde_json::from_value(row["definition_json"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let image_id = serde_json::from_value(row["image_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let kind: RunKind = serde_json::from_value(row["kind"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let (transaction, write_set) = + job_write_set(transaction, snapshot_id, &definition.selection).await?; + Ok(( + transaction, + RunRequest { + run_id: request.run_id, + location_id: request.location_id, + target: request.target.clone(), + snapshot_id, + image_id, + kind, + origin: if request.window.is_some() { + RunOrigin::Backfill + } else { + RunOrigin::Manual + }, + job_key: Some(request.job_key.clone()), + parent_run_id: None, + root_run_id: request.run_id, + trigger_id: None, + tick_id: None, + rerun_of_run_id: None, + requested_by: context.actor.clone(), + priority: request.priority, + selection: definition.selection, + write_set: if kind == RunKind::Group { + Default::default() + } else { + write_set + }, + settings: definition.settings, + window: request.window.clone(), + concurrency_groups: definition.concurrency_groups, + overlap: definition.overlap, + }, + )) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/mod.rs b/crates/sqlbuild-orchestration/src/runs/main/mod.rs index 6359a19779..6027f5ea35 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/mod.rs @@ -1,6 +1,7 @@ pub mod advance_groups; pub mod change_run; pub mod claim_next_run; +pub(crate) mod frozen_job_request; pub mod heartbeat_run; pub(crate) mod job_write_set; pub mod list_run_claims; diff --git a/crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs b/crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs index 5df18cde15..8593e77c41 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs @@ -5,8 +5,8 @@ use uuid::Uuid; use crate::credentials::main::append_audit::append_audit; use crate::credentials::models::AuditContext; -use crate::runs::_helpers::requests::job::frozen_job_request; use crate::runs::_helpers::requests::publication::{publish_request, request_retry}; +use crate::runs::main::frozen_job_request::frozen_job_request; use crate::runs::models::{JobRunRequest, RunWindow}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs index f2edb08a87..ba0d626c09 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs @@ -2,6 +2,7 @@ use uuid::Uuid; +use crate::scheduler::_helpers::drain_sensors::drain_sensors; use crate::scheduler::main::_evaluate_trigger::evaluate_trigger; use crate::scheduler::main::renew_lease::renew_lease; use crate::scheduler::models::{SchedulerCycle, SchedulerLease}; @@ -21,7 +22,7 @@ pub(crate) async fn drain_due_triggers( let (transaction, rows) = query(transaction, "SELECT s.trigger_id FROM sqb_trigger_state s JOIN sqb_triggers t ON t.trigger_id=s.trigger_id \ JOIN sqb_locations l ON l.location_id=t.location_id \ - WHERE s.status='running' AND s.next_due_at<=$1 AND t.removed_in_snapshot_id IS NULL AND l.archived_at IS NULL \ + WHERE t.kind='schedule' AND s.status='running' AND s.next_due_at<=$1 AND t.removed_in_snapshot_id IS NULL AND l.archived_at IS NULL \ ORDER BY s.next_due_at, s.trigger_id LIMIT 100", vec![Parameter::Time(now)]).await?; commit(transaction).await?; for row in rows { @@ -33,5 +34,7 @@ pub(crate) async fn drain_due_triggers( } lease = renew_lease(store, &lease).await?; } + let (lease, sensor_evaluations) = drain_sensors(store, lease).await?; + cycle.evaluated += sensor_evaluations; Ok((lease, cycle)) } diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_sensors.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_sensors.rs new file mode 100644 index 0000000000..2456383857 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_sensors.rs @@ -0,0 +1,36 @@ +//! The scheduler drains built-in sensors without importing project code. + +use crate::scheduler::main::renew_lease::renew_lease; +use crate::scheduler::models::SchedulerLease; +use crate::sensors::main::begin_sensor_evaluation::begin_sensor_evaluation; +use crate::sensors::main::complete_builtin_sensor::complete_builtin_sensor; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub(crate) async fn drain_sensors( + store: &ControlStore, + mut lease: SchedulerLease, +) -> Result<(SchedulerLease, usize), ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "SELECT t.trigger_id FROM sqb_triggers t JOIN sqb_trigger_state s ON s.trigger_id=t.trigger_id \ + JOIN sqb_locations l ON l.location_id=t.location_id WHERE t.kind='sensor' AND s.status='running' \ + AND s.next_due_at<=$1 AND t.removed_in_snapshot_id IS NULL AND l.archived_at IS NULL \ + ORDER BY s.next_due_at,t.trigger_id LIMIT 100", vec![Parameter::Time(now)]).await?; + commit(transaction).await?; + let mut evaluated = 0; + for row in rows { + let trigger_id = serde_json::from_value(row["trigger_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if let Some(request) = begin_sensor_evaluation(store, &lease, trigger_id).await? { + complete_builtin_sensor(store, &lease, &request).await?; + evaluated += 1; + } + lease = renew_lease(store, &lease).await?; + } + Ok((lease, evaluated)) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/mod.rs index 5102baaac9..c007a30d78 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod calendar; pub(crate) mod drain_due_triggers; +pub(crate) mod drain_sensors; pub(crate) mod expansion; pub(crate) mod persistence; pub(crate) mod ticks; diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/attempts.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/attempts.rs new file mode 100644 index 0000000000..310fde7a9b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/attempts.rs @@ -0,0 +1,65 @@ +//! Indexed reads recover attempts and schedule projections without mutable phase transitions. + +use chrono::{DateTime, Utc}; +use uuid::Uuid; + +use crate::sensors::models::{DueSensor, SensorEvaluationFact}; +use crate::store::errors::ControlError; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn read_sensor( + transaction: ControlTransaction, + trigger_id: Uuid, +) -> Result<(ControlTransaction, Option), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT t.trigger_id,t.location_id,t.target,t.snapshot_id,t.definition_json,s.status \ + FROM sqb_triggers t JOIN sqb_trigger_state s ON s.trigger_id=t.trigger_id \ + JOIN sqb_locations l ON l.location_id=t.location_id WHERE t.trigger_id=$1 AND t.kind='sensor' \ + AND t.removed_in_snapshot_id IS NULL AND l.archived_at IS NULL", + vec![Parameter::Id(trigger_id)]).await?; + Ok(( + transaction, + rows.into_iter().next().map(decode_row).transpose()?, + )) +} + +pub(crate) async fn last_attempt( + transaction: ControlTransaction, + trigger_id: Uuid, +) -> Result<(ControlTransaction, Option<(SensorEvaluationFact, bool)>), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT e.fact_json, CASE WHEN EXISTS (SELECT 1 FROM sqb_sensor_evaluation_events terminal \ + WHERE terminal.evaluation_id=e.evaluation_id AND terminal.phase<>'requested') THEN 1 ELSE 0 END AS terminal \ + FROM sqb_sensor_evaluation_events e WHERE e.trigger_id=$1 AND e.phase='requested' ORDER BY e.storage_id DESC LIMIT 1", + vec![Parameter::Id(trigger_id)]).await?; + let last = rows + .first() + .map(|row| { + let fact = serde_json::from_value(row["fact_json"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + Ok::<_, ControlError>((fact, row["terminal"].as_i64() == Some(1))) + }) + .transpose()?; + Ok((transaction, last)) +} + +pub(crate) async fn defer_sensor( + transaction: ControlTransaction, + trigger_id: Uuid, + next: DateTime, +) -> Result { + let now = transaction.now; + let (transaction, _) = query( + transaction, + "UPDATE sqb_trigger_state SET next_due_at=$1,updated_at=$2 WHERE trigger_id=$3", + vec![ + Parameter::Time(next), + Parameter::Time(now), + Parameter::Id(trigger_id), + ], + ) + .await?; + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/facts.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/facts.rs new file mode 100644 index 0000000000..c94e5577c5 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/facts.rs @@ -0,0 +1,65 @@ +//! Attempts and cursor commits use Recall's serialized commit-order sequence. + +use serde_json::json; +use uuid::Uuid; + +use crate::sensors::models::{ + SensorCursor, SensorEvaluationFact, SensorEvaluationOutcome, SensorEvaluationRequest, +}; +use crate::store::errors::ControlError; +use crate::store::main::append_control_event::append_control_event; +use crate::store::main::query::query; +use crate::store::models::{ControlEvent, ControlTransaction, Parameter}; + +pub(crate) async fn append_evaluation( + transaction: ControlTransaction, + request: &SensorEvaluationRequest, + outcome: SensorEvaluationOutcome, +) -> Result<(ControlTransaction, (Uuid, i64)), ControlError> { + let phase = outcome.phase(); + let event_id = Uuid::new_v5(&request.evaluation_id, phase.as_bytes()); + let occurred_at = transaction.now; + let fact = SensorEvaluationFact { + request: request.clone(), + outcome, + }; + let payload = json!(fact); + let (transaction, appended) = append_control_event( + transaction, + ControlEvent { + event_id, + invocation_id: format!("sensor/{}", request.trigger_id), + event_type: "sensor_evaluation", + producer: "scheduler", + occurred_at, + run_id: None, + payload: payload.clone(), + }, + ) + .await?; + if appended.conflict.is_some() { + return Err(ControlError::Invalid( + "sensor evaluation conflicts with a durable fact".into(), + )); + } + let storage_id = appended + .storage_id + .ok_or_else(|| ControlError::Invalid("sensor evaluation has no storage cursor".into()))?; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_sensor_evaluation_events (event_id,storage_id,evaluation_id,trigger_id,phase,occurred_at,fact_json) \ + VALUES ($1,$2,$3,$4,$5,$6,$7)", + vec![Parameter::Id(event_id), Parameter::Integer(storage_id), Parameter::Id(request.evaluation_id), + Parameter::Id(request.trigger_id), Parameter::Text(phase.into()), Parameter::Time(occurred_at), Parameter::Json(payload)]).await?; + Ok((transaction, (event_id, storage_id))) +} + +pub(crate) async fn commit_cursor( + transaction: ControlTransaction, + (trigger_id, event_id, storage_id): (Uuid, Uuid, i64), + cursor: &SensorCursor, +) -> Result { + let (transaction, _) = query(transaction, + "INSERT INTO sqb_sensor_cursor_commits (event_id,storage_id,trigger_id,cursor_json) VALUES ($1,$2,$3,$4)", + vec![Parameter::Id(event_id), Parameter::Integer(storage_id), Parameter::Id(trigger_id), Parameter::Json(json!(cursor))]).await?; + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs new file mode 100644 index 0000000000..498b74207a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod attempts; +pub(crate) mod facts; +pub(crate) mod platform; +pub(crate) mod requests; diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs new file mode 100644 index 0000000000..8215ce4379 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs @@ -0,0 +1,34 @@ +//! One bounded platform page yields matched run requests and a storage-order cursor. + +use uuid::Uuid; + +use crate::platform::main::read_platform_events::read_platform_events; +use crate::sensors::_helpers::requests::queue_sensor_run; +use crate::sensors::models::{SensorCondition, SensorCursor, SensorEvaluationRequest}; +use crate::store::errors::ControlError; +use crate::store::models::ControlTransaction; + +pub(crate) async fn consume_platform_events( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, +) -> Result<(ControlTransaction, (SensorCursor, Vec)), ControlError> { + let (mut transaction, events) = + read_platform_events(transaction, evaluation.cursor.event_storage_id).await?; + let cursor = SensorCursor { + event_storage_id: events + .last() + .map_or(evaluation.cursor.event_storage_id, |event| event.storage_id), + }; + let SensorCondition::PlatformEvent { kinds } = &evaluation.definition.when; + let mut run_ids: Vec = Vec::new(); + for event in events + .iter() + .filter(|event| kinds.contains(&event.payload.platform.kind())) + { + let run_id; + (transaction, run_id) = + queue_sensor_run(transaction, evaluation, &event.event_id.to_string()).await?; + run_ids.push(run_id); + } + Ok((transaction, (cursor, run_ids))) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/requests.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/requests.rs new file mode 100644 index 0000000000..237959031b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/requests.rs @@ -0,0 +1,62 @@ +//! A sensor run key gates queue publication within the cursor/outcome transaction. + +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::runs::main::frozen_job_request::frozen_job_request; +use crate::runs::main::queue_run::queue_run; +use crate::runs::models::{JobRunRequest, RunOrigin}; +use crate::sensors::models::SensorEvaluationRequest; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn queue_sensor_run( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, + run_key: &str, +) -> Result<(ControlTransaction, Uuid), ControlError> { + let (transaction, rows) = query( + transaction, + "SELECT run_id FROM sqb_run_requests WHERE trigger_id=$1 AND run_key=$2", + vec![ + Parameter::Id(evaluation.trigger_id), + Parameter::Text(run_key.into()), + ], + ) + .await?; + if let Some(row) = rows.into_iter().next() { + return Ok(( + transaction, + serde_json::from_value(row["run_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + )); + } + let run_id = Uuid::new_v5(&evaluation.trigger_id, run_key.as_bytes()); + let context = AuditContext { + actor: "system:sensor".into(), + granting_role: "system:scheduler".into(), + request_id: evaluation.evaluation_id, + ip_address: None, + }; + let authored = JobRunRequest { + run_id, + location_id: evaluation.location_id, + target: evaluation.target.clone(), + job_key: evaluation.definition.job_key.clone(), + reason: "Sensor condition matched".into(), + priority: 0, + window: None, + }; + let (transaction, mut frozen) = frozen_job_request(transaction, &context, &authored).await?; + if frozen.snapshot_id != evaluation.snapshot_id { + return Err(ControlError::Fenced); + } + frozen.origin = RunOrigin::Sensor; + frozen.trigger_id = Some(evaluation.trigger_id); + let transaction = queue_run(transaction, &frozen).await?; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_run_requests (trigger_id,run_key,evaluation_id,run_id) VALUES ($1,$2,$3,$4)", + vec![Parameter::Id(evaluation.trigger_id), Parameter::Text(run_key.into()), Parameter::Id(evaluation.evaluation_id), Parameter::Id(run_id)]).await?; + Ok((transaction, run_id)) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/main/begin_sensor_evaluation.rs b/crates/sqlbuild-orchestration/src/sensors/main/begin_sensor_evaluation.rs new file mode 100644 index 0000000000..45b82e4cbb --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/main/begin_sensor_evaluation.rs @@ -0,0 +1,80 @@ +//! Commit the requested phase before evaluation, so crashes count as attempts. + +use chrono::TimeDelta; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use uuid::Uuid; + +use crate::declarations::models::TriggerStatus; +use crate::scheduler::main::check_lease::check_lease; +use crate::scheduler::models::SchedulerLease; +use crate::sensors::_helpers::attempts::{defer_sensor, last_attempt, read_sensor}; +use crate::sensors::_helpers::facts::append_evaluation; +use crate::sensors::main::read_sensor_cursor::read_sensor_cursor; +use crate::sensors::models::{SensorEvaluationOutcome, SensorEvaluationRequest}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn begin_sensor_evaluation( + store: &ControlStore, + lease: &SchedulerLease, + trigger_id: Uuid, +) -> Result, ControlError> { + let transaction = check_lease(begin(store, TransactionMode::Write).await?, lease).await?; + let (transaction, sensor) = read_sensor(transaction, trigger_id).await?; + let Some(sensor) = sensor else { + return Ok(None); + }; + if sensor.status != TriggerStatus::Running { + return Ok(None); + } + let now = transaction.now; + let (transaction, cursor) = read_sensor_cursor(transaction, trigger_id).await?; + let (mut transaction, previous) = last_attempt(transaction, trigger_id).await?; + if let Some((previous, terminal)) = previous { + if !terminal { + if previous.request.epoch == lease.epoch + && previous.request.snapshot_id == sensor.snapshot_id + && previous.request.cursor == cursor + { + commit(transaction).await?; + return Ok(Some(previous.request)); + } + (transaction, _) = append_evaluation( + transaction, + &previous.request, + SensorEvaluationOutcome::Abandoned, + ) + .await?; + } + let next = previous.request.requested_at + + TimeDelta::seconds(i64::from( + sensor.definition_json.minimum_interval_seconds.get(), + )); + if next > now { + commit(defer_sensor(transaction, trigger_id, next).await?).await?; + return Ok(None); + } + } + let definition_fingerprint = sha256_hex( + &serde_json::to_vec(&sensor.definition_json) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + ); + let request = SensorEvaluationRequest { + evaluation_id: Uuid::now_v7(), + trigger_id: sensor.trigger_id, + location_id: sensor.location_id, + target: sensor.target, + snapshot_id: sensor.snapshot_id, + definition: sensor.definition_json, + definition_fingerprint, + cursor, + epoch: lease.epoch, + requested_at: now, + }; + let (transaction, _) = + append_evaluation(transaction, &request, SensorEvaluationOutcome::Requested).await?; + commit(check_lease(transaction, lease).await?).await?; + Ok(Some(request)) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs b/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs new file mode 100644 index 0000000000..5fcf9286c3 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs @@ -0,0 +1,73 @@ +//! Cursor, outcome, deduplicated requests and queued runs commit together. + +use chrono::TimeDelta; + +use crate::declarations::models::TriggerStatus; +use crate::scheduler::main::check_lease::check_lease; +use crate::scheduler::models::SchedulerLease; +use crate::sensors::_helpers::attempts::{defer_sensor, last_attempt, read_sensor}; +use crate::sensors::_helpers::facts::{append_evaluation, commit_cursor}; +use crate::sensors::_helpers::platform::consume_platform_events; +use crate::sensors::main::read_sensor_cursor::read_sensor_cursor; +use crate::sensors::models::{SensorEvaluationOutcome, SensorEvaluationRequest}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn complete_builtin_sensor( + store: &ControlStore, + lease: &SchedulerLease, + evaluation: &SensorEvaluationRequest, +) -> Result { + let transaction = check_lease(begin(store, TransactionMode::Write).await?, lease).await?; + if evaluation.epoch != lease.epoch { + return Err(ControlError::Fenced); + } + let (transaction, previous) = last_attempt(transaction, evaluation.trigger_id).await?; + let Some((previous, terminal)) = previous else { + return Err(ControlError::Fenced); + }; + if previous.request != *evaluation { + return Err(ControlError::Fenced); + } + if terminal { + commit(transaction).await?; + return Ok(0); + } + let (transaction, sensor) = read_sensor(transaction, evaluation.trigger_id).await?; + let (transaction, cursor) = read_sensor_cursor(transaction, evaluation.trigger_id).await?; + if sensor.is_none_or(|sensor| { + sensor.status != TriggerStatus::Running || sensor.snapshot_id != evaluation.snapshot_id + }) || cursor != evaluation.cursor + { + let (transaction, _) = + append_evaluation(transaction, evaluation, SensorEvaluationOutcome::Abandoned).await?; + commit(transaction).await?; + return Ok(0); + } + let (transaction, (cursor, run_ids)) = consume_platform_events(transaction, evaluation).await?; + let count = run_ids.len(); + let (transaction, (event_id, storage_id)) = append_evaluation( + transaction, + evaluation, + SensorEvaluationOutcome::Completed { + cursor: cursor.clone(), + run_ids, + }, + ) + .await?; + let transaction = commit_cursor( + transaction, + (evaluation.trigger_id, event_id, storage_id), + &cursor, + ) + .await?; + let next = evaluation.requested_at + + TimeDelta::seconds(i64::from( + evaluation.definition.minimum_interval_seconds.get(), + )); + let transaction = defer_sensor(transaction, evaluation.trigger_id, next).await?; + commit(check_lease(transaction, lease).await?).await?; + Ok(count) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/main/mod.rs b/crates/sqlbuild-orchestration/src/sensors/main/mod.rs new file mode 100644 index 0000000000..a57c8e3058 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/main/mod.rs @@ -0,0 +1,6 @@ +pub mod begin_sensor_evaluation; +pub mod complete_builtin_sensor; +pub(crate) mod publish_sensors; +pub(crate) mod read_sensor_cursor; +pub(crate) mod reset_sensor_cursor; +pub(crate) mod validate_sensors; diff --git a/crates/sqlbuild-orchestration/src/sensors/main/publish_sensors.rs b/crates/sqlbuild-orchestration/src/sensors/main/publish_sensors.rs new file mode 100644 index 0000000000..3f2029c481 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/main/publish_sensors.rs @@ -0,0 +1,37 @@ +//! Sensor publication keeps operator overrides and immutable cursor history across deployments. + +use serde_json::json; +use uuid::Uuid; + +use crate::declarations::models::Definitions; +use crate::snapshots::models::SnapshotRecord; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn publish_sensors( + mut transaction: ControlTransaction, + snapshot: &SnapshotRecord, + definitions: &Definitions, +) -> Result { + for (name, sensor) in &definitions.sensors { + let trigger_id = Uuid::new_v5( + &snapshot.location_id, + &serde_json::to_vec(&(&snapshot.target, "sensor", name)) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + ); + (transaction, _) = query(transaction, + "INSERT INTO sqb_triggers (trigger_id,location_id,target,kind,name,job_key,definition_json,snapshot_id) \ + VALUES ($1,$2,$3,'sensor',$4,$5,$6,$7) ON CONFLICT (trigger_id) DO UPDATE \ + SET job_key=excluded.job_key,definition_json=excluded.definition_json,snapshot_id=excluded.snapshot_id,removed_in_snapshot_id=NULL", + vec![Parameter::Id(trigger_id), Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone()), + Parameter::Text(name.clone()), Parameter::Text(sensor.job_key.clone()), Parameter::Json(json!(sensor)), Parameter::Id(snapshot.snapshot_id)]).await?; + let status = sensor.default_status.projection(); + (transaction, _) = query(transaction, + "INSERT INTO sqb_trigger_state (trigger_id,status,started_at,next_due_at,updated_at) VALUES ($1,$2,$3,$3,$3) \ + ON CONFLICT (trigger_id) DO UPDATE SET status=COALESCE(sqb_trigger_state.status_override,excluded.status), \ + next_due_at=excluded.next_due_at,updated_at=excluded.updated_at", + vec![Parameter::Id(trigger_id), Parameter::Text(status.as_str().into()), Parameter::Time(snapshot.published_at)]).await?; + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/main/read_sensor_cursor.rs b/crates/sqlbuild-orchestration/src/sensors/main/read_sensor_cursor.rs new file mode 100644 index 0000000000..ab2c060dd5 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/main/read_sensor_cursor.rs @@ -0,0 +1,24 @@ +//! Operator reads and evaluations share the latest immutable cursor commit. + +use uuid::Uuid; + +use crate::sensors::models::SensorCursor; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn read_sensor_cursor( + transaction: ControlTransaction, + trigger_id: Uuid, +) -> Result<(ControlTransaction, SensorCursor), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT cursor_json FROM sqb_sensor_cursor_commits WHERE trigger_id=$1 ORDER BY storage_id DESC LIMIT 1", + vec![Parameter::Id(trigger_id)]).await?; + let cursor = rows + .first() + .map(|row| serde_json::from_value(row["cursor_json"].clone())) + .transpose() + .map_err(|error| ControlError::Invalid(error.to_string()))? + .unwrap_or_default(); + Ok((transaction, cursor)) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/main/reset_sensor_cursor.rs b/crates/sqlbuild-orchestration/src/sensors/main/reset_sensor_cursor.rs new file mode 100644 index 0000000000..e447028c30 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/main/reset_sensor_cursor.rs @@ -0,0 +1,41 @@ +//! Operator reset appends a cursor fact without mutating previous commits or run keys. + +use serde_json::json; +use uuid::Uuid; + +use crate::sensors::_helpers::facts::commit_cursor; +use crate::sensors::models::SensorCursor; +use crate::store::errors::ControlError; +use crate::store::main::append_control_event::append_control_event; +use crate::store::models::{ControlEvent, ControlTransaction}; + +pub(crate) async fn reset_sensor_cursor( + transaction: ControlTransaction, + (trigger_id, request_id): (Uuid, Uuid), +) -> Result { + let cursor = SensorCursor::default(); + let event_id = Uuid::new_v5(&request_id, b"sensor-cursor-reset"); + let occurred_at = transaction.now; + let (transaction, appended) = append_control_event( + transaction, + ControlEvent { + event_id, + invocation_id: format!("sensor/{trigger_id}"), + event_type: "sensor_cursor_reset", + producer: "scheduler", + occurred_at, + run_id: None, + payload: json!({"trigger_id": trigger_id, "cursor": cursor}), + }, + ) + .await?; + if appended.conflict.is_some() { + return Err(ControlError::Invalid( + "sensor cursor reset conflicts with a durable fact".into(), + )); + } + let storage_id = appended + .storage_id + .ok_or_else(|| ControlError::Invalid("sensor cursor reset has no storage cursor".into()))?; + commit_cursor(transaction, (trigger_id, event_id, storage_id), &cursor).await +} diff --git a/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs b/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs new file mode 100644 index 0000000000..06cd779823 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs @@ -0,0 +1,22 @@ +//! Native publication validates every sensor before any snapshot pointer moves. + +use crate::declarations::models::Definitions; +use crate::sensors::models::SensorCondition; +use crate::store::errors::ControlError; + +pub(crate) fn validate_sensors(definitions: &Definitions) -> Result<(), ControlError> { + for (name, sensor) in &definitions.sensors { + let SensorCondition::PlatformEvent { kinds } = &sensor.when; + if name.trim().is_empty() + || sensor.description.trim().is_empty() + || kinds.is_empty() + || (!definitions.jobs.contains_key(&sensor.job_key) + && !definitions.groups.contains_key(&sensor.job_key)) + { + return Err(ControlError::Invalid(format!( + "sensor {name} needs a description, a known job and nonempty event kinds" + ))); + } + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/mod.rs b/crates/sqlbuild-orchestration/src/sensors/mod.rs new file mode 100644 index 0000000000..ca1039eddc --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/mod.rs @@ -0,0 +1,9 @@ +//! Sensor attempts and cursor commits are immutable facts, not mutable lifecycle rows. + +pub(crate) mod _helpers; +pub mod main; +pub mod models; + +#[cfg(test)] +#[path = "../../tests/sensors/test_contracts.rs"] +mod tests; diff --git a/crates/sqlbuild-orchestration/src/sensors/models.rs b/crates/sqlbuild-orchestration/src/sensors/models.rs new file mode 100644 index 0000000000..f39cf43d2f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/models.rs @@ -0,0 +1,114 @@ +//! Published conditions and immutable attempts share one sensor abstraction. + +use std::collections::BTreeSet; +use std::num::NonZeroU32; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::declarations::models::TriggerStatus; +use crate::platform::models::PlatformEventKind; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum SensorStatus { + #[default] + Running, + Stopped, +} + +impl SensorStatus { + #[must_use] + pub const fn projection(self) -> TriggerStatus { + match self { + Self::Running => TriggerStatus::Running, + Self::Stopped => TriggerStatus::Paused, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum SensorCondition { + PlatformEvent { kinds: BTreeSet }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct SensorDefinition { + pub description: String, + pub job_key: String, + pub when: SensorCondition, + #[schema(value_type = u32, minimum = 1)] + pub minimum_interval_seconds: NonZeroU32, + #[serde(default)] + pub default_status: SensorStatus, +} + +/// Storage order is authoritative even when a fact's occurrence time predates earlier pages. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SensorCursor { + pub event_storage_id: i64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SensorEvaluationRequest { + pub evaluation_id: Uuid, + pub trigger_id: Uuid, + pub location_id: Uuid, + pub target: String, + pub snapshot_id: Uuid, + pub definition: SensorDefinition, + pub definition_fingerprint: String, + pub cursor: SensorCursor, + pub epoch: i64, + pub requested_at: DateTime, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "phase", rename_all = "snake_case", deny_unknown_fields)] +pub enum SensorEvaluationOutcome { + Requested, + Completed { + cursor: SensorCursor, + run_ids: Vec, + }, + Failed { + code: String, + }, + TimedOut, + Abandoned, +} + +impl SensorEvaluationOutcome { + #[must_use] + pub const fn phase(&self) -> &'static str { + match self { + Self::Requested => "requested", + Self::Completed { .. } => "completed", + Self::Failed { .. } => "failed", + Self::TimedOut => "timed_out", + Self::Abandoned => "abandoned", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SensorEvaluationFact { + pub request: SensorEvaluationRequest, + pub outcome: SensorEvaluationOutcome, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct DueSensor { + pub(crate) trigger_id: Uuid, + pub(crate) location_id: Uuid, + pub(crate) target: String, + pub(crate) snapshot_id: Uuid, + pub(crate) definition_json: SensorDefinition, + pub(crate) status: TriggerStatus, +} diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index fd922e7d2f..5ae6a26780 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -10,6 +10,7 @@ pub(crate) const SCHEMA: &str = include_str!("../schema.sql"); pub(crate) const LOCATION_INSTANCES: &str = include_str!("../location_instances.sql"); pub(crate) const LOCATION_WORK: &str = include_str!("../location_work.sql"); pub(crate) const PLATFORM_SINK: &str = include_str!("../platform_sink.sql"); +pub(crate) const SENSORS: &str = include_str!("../sensors.sql"); pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index 56f2194ffb..dcc49f4793 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -4,7 +4,7 @@ use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; use crate::store::_helpers::migration::{ - LOCATION_INSTANCES, LOCATION_WORK, PLATFORM_SINK, SCHEMA, render, verify_migrations, + LOCATION_INSTANCES, LOCATION_WORK, PLATFORM_SINK, SCHEMA, SENSORS, render, verify_migrations, }; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; @@ -31,6 +31,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { ("location_instances", LOCATION_INSTANCES), ("location_work", LOCATION_WORK), ("platform_sink", PLATFORM_SINK), + ("sensors", SENSORS), ]; let mut digests = vec![digest.clone()]; digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); diff --git a/crates/sqlbuild-orchestration/src/store/sensors.sql b/crates/sqlbuild-orchestration/src/store/sensors.sql new file mode 100644 index 0000000000..9e1ab95213 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/sensors.sql @@ -0,0 +1,25 @@ +CREATE TABLE sqb_sensor_evaluation_events ( + event_id {uuid} PRIMARY KEY, + storage_id BIGINT NOT NULL UNIQUE, + evaluation_id {uuid} NOT NULL, + trigger_id {uuid} NOT NULL REFERENCES sqb_triggers(trigger_id), + phase TEXT NOT NULL CHECK (phase IN ('requested', 'completed', 'failed', 'timed_out', 'abandoned')), + occurred_at {time} NOT NULL, + fact_json {json} NOT NULL, + UNIQUE (evaluation_id, phase) +); +CREATE INDEX sqb_sensor_evaluation_history ON sqb_sensor_evaluation_events(trigger_id, storage_id); +CREATE TABLE sqb_sensor_cursor_commits ( + event_id {uuid} PRIMARY KEY, + storage_id BIGINT NOT NULL UNIQUE, + trigger_id {uuid} NOT NULL REFERENCES sqb_triggers(trigger_id), + cursor_json {json} NOT NULL +); +CREATE INDEX sqb_sensor_cursor_history ON sqb_sensor_cursor_commits(trigger_id, storage_id); +CREATE TABLE sqb_run_requests ( + trigger_id {uuid} NOT NULL REFERENCES sqb_triggers(trigger_id), + run_key TEXT NOT NULL, + evaluation_id {uuid} NOT NULL, + run_id {uuid} NOT NULL REFERENCES sqb_runs(run_id), + PRIMARY KEY (trigger_id, run_key) +); diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs index 7d77cdf9fa..4af6d90bef 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs @@ -46,9 +46,9 @@ fn given_outbound_work_when_claimed_retried_or_replaced_then_scope_and_immutable let store = &fixture.store; migrate_store(store).await.expect("fresh migration"); // Exercise both deployed predecessors, including checksums and repeated startup. - execute(store, "DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DELETE FROM sqb_schema_migrations WHERE version>=3").await; + execute(store, "DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DELETE FROM sqb_schema_migrations WHERE version>=3").await; migrate_store(store).await.expect("schema 2 upgrade"); - execute(store, "DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2").await; + execute(store, "DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2").await; migrate_store(store).await.expect("schema 1 upgrade"); migrate_store(store).await.expect("repeat migration"); let location_id = create_location(store, "orders", "test-admin") diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs index 4c388072ab..41610c718b 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs @@ -308,7 +308,7 @@ fn given_a_calendar_with_unprocessed_due_times_when_refreshed_then_due_ticks_sur .expect("refreshed trigger") .remove(0); assert_eq!( - serde_json::to_value(&trigger.definition_json.schedule).expect("calendar")["times"], + serde_json::to_value(&trigger.definition_json).expect("calendar")["schedule"]["times"], json!([ "2020-01-01T00:00:00Z", "2020-01-02T00:00:00Z", @@ -356,7 +356,7 @@ fn given_a_calendar_with_unprocessed_due_times_when_refreshed_then_due_ticks_sur .expect("drained trigger") .remove(0); assert_eq!( - serde_json::to_value(&trigger.definition_json.schedule).expect("calendar")["times"], + serde_json::to_value(&trigger.definition_json).expect("calendar")["schedule"]["times"], json!(["2032-01-01T00:00:00Z"]) ); assert_eq!( diff --git a/crates/sqlbuild-orchestration/tests/sensors/helpers.rs b/crates/sqlbuild-orchestration/tests/sensors/helpers.rs new file mode 100644 index 0000000000..7aea4d6287 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/helpers.rs @@ -0,0 +1,114 @@ +use chrono::{DateTime, Utc}; +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::declarations::main::change_trigger::change_trigger; +use crate::declarations::models::{TriggerChange, TriggerChangeRequest}; +use crate::platform::models::PlatformEvent; +use crate::sensors::models::SensorEvaluationFact; +use crate::snapshots::models::SnapshotInput; +use crate::store::main::append_control_event::append_control_event; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{ControlEvent, ControlStore, TransactionMode}; + +use crate::runs::tests::helpers::snapshot as base_snapshot; +use crate::store::tests::helpers::read_recall_events; + +pub(crate) fn snapshot(location_id: Uuid) -> SnapshotInput { + let mut input = base_snapshot(location_id); + input.documents.insert( + "jobs".into(), + json!({ + "orders": {"description": "Refresh orders", "selection": ["orders"], "overlap": "allow"} + }), + ); + input.documents.insert( + "triggers".into(), + json!({ + "health_orders": { + "description": "Refresh reporting on overdue ticks", "job_key": "orders", + "when": {"kind": "platform_event", "kinds": ["tick_overdue"]}, + "minimum_interval_seconds": 1 + } + }), + ); + input +} + +pub(crate) async fn append_events( + store: &ControlStore, + indices: std::ops::Range, + occurred_at: DateTime, +) { + let mut transaction = begin(store, TransactionMode::Write) + .await + .expect("event transaction"); + let payloads = [ + PlatformEvent::TickOverdue { + trigger_id: Uuid::from_u128(7), + scheduled_for: occurred_at, + }, + PlatformEvent::Heartbeat { + replica_id: Uuid::from_u128(8), + }, + ]; + for index in indices { + let platform = &payloads[(index % 2) as usize]; + let (next, appended) = append_control_event( + transaction, + ControlEvent { + event_id: Uuid::from_u128(1000 + index), + invocation_id: "platform/test-health".into(), + event_type: platform.event_type(), + producer: "platform", + occurred_at, + run_id: None, + payload: json!({"platform": platform}), + }, + ) + .await + .expect("append platform observation"); + assert!(appended.inserted); + transaction = next; + } + commit(transaction).await.expect("commit observations"); +} + +pub(crate) async fn evaluations(store: &ControlStore) -> Vec { + read_recall_events(store, 0, 1000) + .await + .expect("read durable history") + .into_iter() + .filter(|row| row["event_type"] == "sensor_evaluation") + .map(|row| serde_json::from_value(row["payload"].clone()).expect("typed attempt fact")) + .collect() +} + +pub(crate) async fn change(store: &ControlStore, trigger_id: Uuid, change: TriggerChange) { + let event_id = Uuid::now_v7(); + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: event_id, + ip_address: None, + }; + let request = TriggerChangeRequest { + event_id, + trigger_id, + actor: context.actor.clone(), + reason: "Test operator change".into(), + change, + }; + assert!( + change_trigger(store, &context, &request) + .await + .expect("apply operator fact") + ); + assert!( + !change_trigger(store, &context, &request) + .await + .expect("retry operator fact") + ); +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs b/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs new file mode 100644 index 0000000000..04f4485f1e --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs @@ -0,0 +1,10 @@ +//! Real stores prove atomic sensor consumption and recovery across scheduler ownership changes. + +#[path = "helpers.rs"] +mod helpers; +#[path = "test_delivery.rs"] +mod test_delivery; +#[path = "test_recovery.rs"] +mod test_recovery; +#[path = "test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_delivery.rs b/crates/sqlbuild-orchestration/tests/sensors/test_delivery.rs new file mode 100644 index 0000000000..de5581844b --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_delivery.rs @@ -0,0 +1,183 @@ +use std::collections::BTreeSet; +use std::time::Duration; + +use chrono::{TimeDelta, Utc}; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::declarations::models::TriggerChange; +use crate::runs::main::list_runs::list_runs; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::sensors::main::begin_sensor_evaluation::begin_sensor_evaluation; +use crate::sensors::main::complete_builtin_sensor::complete_builtin_sensor; +use crate::sensors::models::SensorEvaluationOutcome; +use crate::sensors::tests::helpers::{append_events, change, evaluations, snapshot}; +use crate::sensors::tests::test_types::SensorTestCase; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_paged_platform_events_when_sensor_publication_fails_then_retry_is_atomic_and_reset_deduplicates() + { + let test_cases = [ + SensorTestCase { + description: "SQLite sensor delivery", + backend: Backend::Sqlite, + expected_runs: 54, + }, + SensorTestCase { + description: "PostgreSQL sensor delivery", + backend: Backend::Postgres, + expected_runs: 54, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + publish_snapshot(store, &snapshot(location)) + .await + .expect("publish sensor"); + let trigger = list_triggers(store, location, "development") + .await + .expect("triggers") + .remove(0); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let now = Utc::now(); + append_events(store, 0..105, now).await; + let requested = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("request") + .expect("due"); + let replica = second_store(store).await; + assert_eq!( + begin_sensor_evaluation(&replica, &lease, trigger.trigger_id) + .await + .expect("recover pending"), + Some(requested.clone()) + ); + for table in [ + "sqb_runs", + "sqb_run_requests", + "sqb_sensor_evaluation_events", + "sqb_sensor_cursor_commits", + ] { + let point = TransactionFault { + description: "fail sensor transaction", + table, + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'injected sensor failure')", + postgres_body: "RAISE EXCEPTION 'injected sensor failure'", + }; + install_fault(store, &point).await; + assert!( + complete_builtin_sensor(store, &lease, &requested) + .await + .is_err(), + "{table}" + ); + remove_fault(store, &point).await; + assert!( + list_runs(store, (location, "development"), 1000) + .await + .expect("rolled back runs") + .is_empty(), + "{table}" + ); + assert_eq!( + evaluations(store).await.len(), + 1, + "only requested survives {table}" + ); + } + assert_eq!( + complete_builtin_sensor(&replica, &lease, &requested) + .await + .expect("retry first page"), + 50 + ); + assert_eq!( + complete_builtin_sensor(store, &lease, &requested) + .await + .expect("retry completion"), + 0 + ); + tokio::time::sleep(Duration::from_secs(1)).await; + let second = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("next page") + .expect("due"); + assert!(second.cursor.event_storage_id > 0); + assert_eq!( + complete_builtin_sensor(store, &lease, &second) + .await + .expect("remaining page"), + 3 + ); + // Committed later, but its occurrence precedes the entire first page. + append_events(store, 106..107, now - TimeDelta::days(1)).await; + tokio::time::sleep(Duration::from_secs(1)).await; + let late = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("late request") + .expect("due"); + assert!(late.cursor.event_storage_id > second.cursor.event_storage_id); + assert_eq!( + complete_builtin_sensor(store, &lease, &late) + .await + .expect("late arrival"), + 1 + ); + change(store, trigger.trigger_id, TriggerChange::ResetCursor).await; + tokio::time::sleep(Duration::from_secs(1)).await; + let replay = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("replay") + .expect("due"); + assert_eq!(replay.cursor.event_storage_id, 0); + assert_eq!( + complete_builtin_sensor(store, &lease, &replay) + .await + .expect("replay retained first page"), + 50 + ); + let runs = list_runs(store, (location, "development"), 1000) + .await + .expect("queued runs"); + assert_eq!( + runs.len(), + test_case.expected_runs, + "{}: reset never requeues an existing run key", + test_case.description + ); + assert!( + runs.iter() + .all(|run| run.selection_json == BTreeSet::from(["orders".into()])) + ); + let facts = evaluations(store).await; + assert_eq!(facts.len(), 8); + assert!(facts.iter().all(|fact| matches!( + fact.outcome, + SensorEvaluationOutcome::Requested | SensorEvaluationOutcome::Completed { .. } + ))); + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_recovery.rs b/crates/sqlbuild-orchestration/tests/sensors/test_recovery.rs new file mode 100644 index 0000000000..426fc861a4 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_recovery.rs @@ -0,0 +1,218 @@ +use std::time::Duration; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::declarations::models::{TriggerChange, TriggerStatus}; +use crate::runs::main::list_runs::list_runs; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::sensors::main::begin_sensor_evaluation::begin_sensor_evaluation; +use crate::sensors::main::complete_builtin_sensor::complete_builtin_sensor; +use crate::sensors::models::SensorEvaluationOutcome; +use crate::sensors::tests::helpers::{append_events, change, evaluations, snapshot}; +use crate::sensors::tests::test_types::SensorTestCase; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::lease_helpers::expire_lease; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_crashed_sensor_when_leadership_or_snapshot_changes_then_stale_results_cannot_queue_runs() +{ + let test_cases = [ + SensorTestCase { + description: "SQLite sensor recovery", + backend: Backend::Sqlite, + expected_runs: 1, + }, + SensorTestCase { + description: "PostgreSQL sensor recovery", + backend: Backend::Postgres, + expected_runs: 1, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + publish_snapshot(store, &input) + .await + .expect("publish sensor"); + let trigger = list_triggers(store, location, "development") + .await + .expect("triggers") + .remove(0); + append_events(store, 0..1, chrono::Utc::now()).await; + let first = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let pending = begin_sensor_evaluation(store, &first, trigger.trigger_id) + .await + .expect("request") + .expect("due"); + expire_lease(store).await; + let replica = second_store(store).await; + let successor = acquire_lease(&replica, Uuid::now_v7()) + .await + .expect("takeover") + .expect("new leader"); + assert!(matches!( + complete_builtin_sensor(store, &first, &pending).await, + Err(ControlError::Fenced) + )); + assert!( + begin_sensor_evaluation(&replica, &successor, trigger.trigger_id) + .await + .expect("recover abandoned attempt") + .is_none(), + "a crash still counts toward the minimum interval" + ); + assert!(matches!( + evaluations(store) + .await + .last() + .expect("abandoned fact") + .outcome, + SensorEvaluationOutcome::Abandoned + )); + tokio::time::sleep(Duration::from_secs(1)).await; + let requested = begin_sensor_evaluation(store, &successor, trigger.trigger_id) + .await + .expect("new request") + .expect("due"); + assert_ne!(pending.evaluation_id, requested.evaluation_id); + let mut forged = requested.clone(); + forged.cursor.event_storage_id = 999; + assert!(matches!( + complete_builtin_sensor(store, &successor, &forged).await, + Err(ControlError::Fenced) + )); + input.generation += 1; + publish_snapshot(store, &input).await.expect("redeploy"); + assert_eq!( + complete_builtin_sensor(store, &successor, &requested) + .await + .expect("abandon stale snapshot"), + 0 + ); + assert!( + list_runs(store, (location, "development"), 100) + .await + .expect("no stale runs") + .is_empty() + ); + change(store, trigger.trigger_id, TriggerChange::Pause).await; + input.generation += 1; + publish_snapshot(store, &input) + .await + .expect("redeploy paused sensor"); + assert_eq!( + list_triggers(store, location, "development") + .await + .expect("preserved override")[0] + .status, + TriggerStatus::Paused + ); + assert!( + begin_sensor_evaluation(store, &successor, trigger.trigger_id) + .await + .expect("paused") + .is_none() + ); + change(store, trigger.trigger_id, TriggerChange::ResetDefault).await; + tokio::time::sleep(Duration::from_secs(1)).await; + let current = begin_sensor_evaluation(store, &successor, trigger.trigger_id) + .await + .expect("code default") + .expect("running"); + assert_eq!( + complete_builtin_sensor(store, &successor, ¤t) + .await + .expect("catch up retained health"), + test_case.expected_runs, + "{}", + test_case.description + ); + tokio::time::sleep(Duration::from_secs(1)).await; + let before_reset = begin_sensor_evaluation(store, &successor, trigger.trigger_id) + .await + .expect("request before reset") + .expect("due"); + change(store, trigger.trigger_id, TriggerChange::ResetCursor).await; + assert_eq!( + complete_builtin_sensor(store, &successor, &before_reset) + .await + .expect("reset fences old cursor"), + 0 + ); + assert!(matches!( + evaluations(store) + .await + .last() + .expect("reset abandonment") + .outcome, + SensorEvaluationOutcome::Abandoned + )); + tokio::time::sleep(Duration::from_secs(1)).await; + let before_pause = begin_sensor_evaluation(store, &successor, trigger.trigger_id) + .await + .expect("request before pause") + .expect("due"); + change(store, trigger.trigger_id, TriggerChange::Pause).await; + assert_eq!( + complete_builtin_sensor(store, &successor, &before_pause) + .await + .expect("pause fences pending result"), + 0 + ); + assert!(matches!( + evaluations(store) + .await + .last() + .expect("pause abandonment") + .outcome, + SensorEvaluationOutcome::Abandoned + )); + change(store, trigger.trigger_id, TriggerChange::ResetDefault).await; + tokio::time::sleep(Duration::from_secs(1)).await; + let delayed = begin_sensor_evaluation(store, &successor, trigger.trigger_id) + .await + .expect("request delayed evaluation") + .expect("due"); + tokio::time::sleep(Duration::from_secs(1)).await; + complete_builtin_sensor(store, &successor, &delayed) + .await + .expect("complete delayed evaluation"); + let immediate = begin_sensor_evaluation(store, &successor, trigger.trigger_id) + .await + .expect("interval measured from start, not completion") + .expect("already due"); + assert_ne!(immediate.evaluation_id, delayed.evaluation_id); + complete_builtin_sensor(store, &successor, &immediate) + .await + .expect("complete next attempt"); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("deduplicated runs") + .len(), + test_case.expected_runs, + "{}", + test_case.description + ); + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_types.rs b/crates/sqlbuild-orchestration/tests/sensors/test_types.rs new file mode 100644 index 0000000000..b762f2f4c5 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_types.rs @@ -0,0 +1,7 @@ +use crate::store::tests::store_test_types::Backend; + +pub(crate) struct SensorTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_runs: usize, +} diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index 7b2283fbd1..5507104d12 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -243,7 +243,8 @@ pub(super) async fn sqlite_failed_publish_and_schema_floor() -> i64 { let connection = rusqlite::Connection::open(&path).expect("open fault injection connection"); connection .execute_batch( - "DROP TABLE sqb_platform_sink_deliveries; \ + "DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; \ + DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; \ DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; \ DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2", ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index 9706fcc399..be7fa9ac3b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -16,16 +16,16 @@ use sqlbuild_discovery::tree::models::ProjectTree; use sqlbuild_orchestration::declarations::main::group_dependencies::group_dependencies; use sqlbuild_orchestration::declarations::main::snapshot_definitions::snapshot_definitions; use sqlbuild_orchestration::declarations::models::{ - Definitions, GroupDefinition, JobDefinition, OnFailure, TriggerDefinition, + Definitions, GroupDefinition, JobDefinition, OnFailure, TriggerDeclaration, TriggerDefinition, }; use sqlbuild_orchestration::scheduler::models::Overlap; use sqlbuild_orchestration::snapshots::models::SnapshotInput; use crate::bindings::_helpers::native_host::orchestration_declarations::{ - AuthoredDeclaration, AuthoredGroup, AuthoredJob, module_jobs, + AuthoredDeclaration, AuthoredGroup, AuthoredJob, AuthoredSensor, module_jobs, }; use crate::bindings::_helpers::native_host::orchestration_triggers::{ - compile_calendars, compile_cron, + compile_calendars, compile_cron, compile_sensors, }; /// Dedicated declaration modules are imported only in the code-location interpreter. @@ -78,11 +78,13 @@ pub(crate) fn compile_declarations( let mut authored_groups: Vec = Vec::new(); let mut authored_jobs: Vec = Vec::new(); let mut authored_calendars: Vec> = Vec::new(); + let mut authored_sensors: Vec = Vec::new(); for declaration in declarations { match declaration { AuthoredDeclaration::Job(job) => authored_jobs.push(job), AuthoredDeclaration::Group(group) => authored_groups.push(group), AuthoredDeclaration::Calendar(calendar) => authored_calendars.push(calendar), + AuthoredDeclaration::Sensor(sensor) => authored_sensors.push(sensor), } } let mut triggers: BTreeMap = BTreeMap::new(); @@ -198,16 +200,31 @@ pub(crate) fn compile_declarations( jobs, groups, triggers, - } = compile_calendars( - py, - (project_dir, &input.target), - authored_calendars, - Definitions { - jobs, - groups, - triggers, - }, + sensors, + } = compile_sensors( + &input.target, + authored_sensors, + compile_calendars( + py, + (project_dir, &input.target), + authored_calendars, + Definitions { + jobs, + groups, + triggers, + ..Definitions::default() + }, + )?, )?; + let triggers: BTreeMap<_, _> = triggers + .into_iter() + .map(|(name, trigger)| (name, TriggerDeclaration::Schedule(trigger))) + .chain( + sensors + .into_iter() + .map(|(name, sensor)| (name, TriggerDeclaration::Sensor(sensor))), + ) + .collect(); for (key, document) in [ ("jobs", serde_json::to_value(jobs)), ("groups", serde_json::to_value(groups)), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs index d2efd5db0b..84031a99e7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs @@ -10,7 +10,9 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::PyDict; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::git_revisions::main::head_commit::head_commit; -use sqlbuild_orchestration::declarations::models::{Schedule, TriggerDefinition}; +use sqlbuild_orchestration::declarations::models::{ + Schedule, TriggerDeclaration, TriggerDefinition, +}; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotRecord}; use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; @@ -189,7 +191,7 @@ pub(super) fn publish_location( .extract::()?; let input = metadata(py, arguments, &directory)?; let (snapshot, _) = compile_location(py, arguments, &input)?; - let triggers: BTreeMap = serde_json::from_value( + let triggers: BTreeMap = serde_json::from_value( snapshot .input .documents @@ -200,9 +202,12 @@ pub(super) fn publish_location( .map_err(|error| PyValueError::new_err(error.to_string()))?; let refresh_at = triggers .values() - .filter_map(|trigger| match &trigger.schedule { - Schedule::Calendar { refresh_at, .. } => Some(*refresh_at), - Schedule::Cron { .. } => None, + .filter_map(|trigger| match trigger { + TriggerDeclaration::Schedule(TriggerDefinition { + schedule: Schedule::Calendar { refresh_at, .. }, + .. + }) => Some(*refresh_at), + TriggerDeclaration::Schedule(_) | TriggerDeclaration::Sensor(_) => None, }) .min(); status(py, "Publishing location snapshot...")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs index 1b04b08ad8..24a1f9f20e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs @@ -46,10 +46,29 @@ pub(crate) struct AuthoredGroup { pub(crate) targets: Vec, } +#[derive(FromPyObject)] +pub(crate) struct AuthoredPlatformCondition { + pub(crate) kind: String, + pub(crate) kinds: Vec, +} + +#[derive(FromPyObject)] +pub(crate) struct AuthoredSensor { + pub(crate) name: String, + pub(crate) description: String, + pub(crate) when: AuthoredPlatformCondition, + pub(crate) run: AuthoredJobReference, + #[pyo3(from_py_with = authored_setting)] + pub(crate) minimum_interval_seconds: Value, + pub(crate) default_status: String, + pub(crate) targets: Vec, +} + pub(crate) enum AuthoredDeclaration { Job(AuthoredJob), Group(AuthoredGroup), Calendar(Py), + Sensor(AuthoredSensor), } /// JSON authoring values cross once; the interpreter rejects circular user containers safely. @@ -77,6 +96,7 @@ pub(crate) fn module_jobs<'py>( let calendar_class = py .import("sqlbuild.orchestration")? .getattr("CalendarSchedule")?; + let sensor_class = py.import("sqlbuild.orchestration")?.getattr("Sensor")?; let attributes = module.getattr("__dict__")?.cast_into::()?; let mut pending: Vec> = attributes.values().iter().collect(); let mut seen: HashSet = HashSet::new(); @@ -88,6 +108,7 @@ pub(crate) fn module_jobs<'py>( if value.is_instance(&job_class)? || value.is_instance(&group_class)? || value.is_instance(&calendar_class)? + || value.is_instance(&sensor_class)? { jobs.push(value); } else if value.is_instance_of::() || value.is_instance_of::() { @@ -106,6 +127,7 @@ pub(crate) fn collect_declarations<'py>( let calendar = py .import("sqlbuild.orchestration")? .getattr("CalendarSchedule")?; + let sensor = py.import("sqlbuild.orchestration")?.getattr("Sensor")?; let mut roots: HashSet = HashSet::new(); for declaration in &declarations { if !roots.insert(declaration.as_ptr() as usize) { @@ -125,7 +147,16 @@ pub(crate) fn collect_declarations<'py>( if !seen.insert(declaration.as_ptr() as usize) { continue; } - let authored = if declaration.is_instance(&calendar)? { + let authored = if declaration.is_instance(&sensor)? { + let target = declaration.getattr("run")?; + if !target.is_instance(&job)? && !target.is_instance(&group)? { + return Err(PyTypeError::new_err( + "sensor run must be a Job or JobGroup object", + )); + } + pending.push(target); + AuthoredDeclaration::Sensor(declaration.extract()?) + } else if declaration.is_instance(&calendar)? { let target = declaration.getattr("job")?; if !target.is_instance(&job)? && !target.is_instance(&group)? { return Err(PyTypeError::new_err( @@ -159,7 +190,7 @@ pub(crate) fn collect_declarations<'py>( AuthoredDeclaration::Group(declaration.extract()?) } else if !declaration.is_instance(&job)? { return Err(PyTypeError::new_err( - "snapshot declarations must be Job, JobGroup or CalendarSchedule objects", + "snapshot declarations must be Job, JobGroup, CalendarSchedule or Sensor objects", )); } else { AuthoredDeclaration::Job(declaration.extract()?) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs index c5366827ed..b787901fc2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -9,18 +9,21 @@ use pyo3::FromPyObject; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods}; -use serde_json::Value; +use serde_json::{Value, json}; use sqlbuild_cursor::algebra::main::parse_duration::parse_duration; use sqlbuild_host::entries::models::UserCodeHost; use sqlbuild_orchestration::declarations::models::{Definitions, Schedule, TriggerDefinition}; use sqlbuild_orchestration::scheduler::constants::MAX_TICKS_PER_BATCH; +use sqlbuild_orchestration::sensors::models::SensorDefinition; use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, within}; use crate::bindings::_helpers::discovery::imported_modules::{provider_by_name, provider_usages}; use crate::bindings::_helpers::discovery::imported_providers::discover_provider_classes; use crate::bindings::_helpers::native_host::constants::CALENDAR_WINDOW_PARAMETER; use crate::bindings::_helpers::native_host::host::entry; -use crate::bindings::_helpers::native_host::orchestration_declarations::authored_setting; +use crate::bindings::_helpers::native_host::orchestration_declarations::{ + AuthoredSensor, authored_setting, +}; #[derive(FromPyObject)] struct AuthoredControls { @@ -243,3 +246,32 @@ pub(crate) fn compile_calendars( Ok(definitions) }) } + +pub(crate) fn compile_sensors( + target: &str, + authored: Vec, + mut definitions: Definitions, +) -> PyResult { + let mut names: BTreeSet = BTreeSet::new(); + for sensor in authored { + if !names.insert(sensor.name.clone()) || definitions.triggers.contains_key(&sensor.name) { + return Err(PyValueError::new_err(format!( + "duplicate trigger {}", + sensor.name + ))); + } + if !sensor.targets.is_empty() && !sensor.targets.iter().any(|value| value == target) { + continue; + } + let definition: SensorDefinition = serde_json::from_value(json!({ + "description": sensor.description, + "job_key": sensor.run.name, + "when": {"kind": sensor.when.kind, "kinds": sensor.when.kinds}, + "minimum_interval_seconds": sensor.minimum_interval_seconds, + "default_status": sensor.default_status, + })) + .map_err(|error| PyValueError::new_err(format!("sensor {}: {error}", sensor.name)))?; + definitions.sensors.insert(sensor.name, definition); + } + Ok(definitions) +} diff --git a/python-boundary.toml b/python-boundary.toml index b02d91a88d..c5d7d33fc7 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -22,6 +22,11 @@ path = "orchestration_declarations/models.py" rule = 3 note = "Public job and schedule authoring records only; selection resolution, validation and scheduling stay native." +[[stays]] +path = "orchestration_declarations/types.py" +rule = 3 +note = "Public PlatformEventKind enum only, placed in types.py to satisfy enum ownership; no Python event processing or implementation allowance." + [[stays]] path = "orchestration_declarations/main/compile_snapshot.py" rule = 3 diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index b235862344..84509d6ae7 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1739,7 +1739,7 @@ class NativeCompiledAudit: def meta(self) -> dict[str, object] | None: ... class NativeOrchestrationSnapshot: - """A compiled data-only publication retained natively until transport.""" + """Compiled jobs, schedules and sensors retained as data until native publication.""" def to_json(self) -> str: ... def publish(self, server_url: str, bearer: str) -> str: ... diff --git a/src/sqlbuild/orchestration.py b/src/sqlbuild/orchestration.py index 997e6d0181..0920f49d17 100644 --- a/src/sqlbuild/orchestration.py +++ b/src/sqlbuild/orchestration.py @@ -6,22 +6,32 @@ CalendarWindow, Job, JobGroup, + PlatformEventCondition, Schedule, + Sensor, ) from sqlbuild.orchestration_declarations.models import CalendarSchedule as calendar_schedule from sqlbuild.orchestration_declarations.models import Job as job from sqlbuild.orchestration_declarations.models import JobGroup as job_group +from sqlbuild.orchestration_declarations.models import PlatformEventCondition as platform_event from sqlbuild.orchestration_declarations.models import Schedule as schedule +from sqlbuild.orchestration_declarations.models import Sensor as sensor +from sqlbuild.orchestration_declarations.types import PlatformEventKind __all__ = ( "CalendarSchedule", "CalendarWindow", "Job", "JobGroup", + "PlatformEventCondition", + "PlatformEventKind", "Schedule", + "Sensor", "calendar_schedule", "compile_snapshot", "job", "job_group", + "platform_event", "schedule", + "sensor", ) diff --git a/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py b/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py index b9e09facb3..cc7f6c83f2 100644 --- a/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py +++ b/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py @@ -7,7 +7,7 @@ from sqlbuild._native import NativeOrchestrationSnapshot, build_orchestration_snapshot from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.orchestration_declarations.models import CalendarSchedule, Job, JobGroup +from sqlbuild.orchestration_declarations.models import CalendarSchedule, Job, JobGroup, Sensor def compile_snapshot( @@ -15,7 +15,7 @@ def compile_snapshot( project: CompiledProjectView, metadata_json: str, project_dir: Path, - declarations: Sequence[Job | JobGroup | CalendarSchedule] | None = None, + declarations: Sequence[Job | JobGroup | CalendarSchedule | Sensor] | None = None, ) -> NativeOrchestrationSnapshot: """Compile authoring records and retained project facts into a data-only snapshot.""" return build_orchestration_snapshot( diff --git a/src/sqlbuild/orchestration_declarations/models.py b/src/sqlbuild/orchestration_declarations/models.py index b9617e4ffe..8086e34bf6 100644 --- a/src/sqlbuild/orchestration_declarations/models.py +++ b/src/sqlbuild/orchestration_declarations/models.py @@ -7,6 +7,8 @@ from datetime import datetime from typing import Literal +from sqlbuild.orchestration_declarations.types import PlatformEventKind + @dataclass(frozen=True, kw_only=True) class Schedule: @@ -78,3 +80,24 @@ class CalendarSchedule: def __call__(self, function: Callable[..., Iterable[datetime]]) -> CalendarSchedule: return replace(self, function=function) + + +@dataclass(frozen=True, kw_only=True) +class PlatformEventCondition: + """Consume matching server observations in durable storage order.""" + + kinds: Sequence[PlatformEventKind] + kind: Literal["platform_event"] = field(default="platform_event", init=False) + + +@dataclass(frozen=True, eq=False) +class Sensor: + """Start a job for each matching event; event identity deduplicates replayed requests.""" + + name: str + description: str = field(kw_only=True) + when: PlatformEventCondition = field(kw_only=True) + run: Job | JobGroup = field(kw_only=True) + minimum_interval_seconds: int = field(default=30, kw_only=True) + default_status: Literal["running", "stopped"] = field(default="running", kw_only=True) + targets: tuple[str, ...] = field(default=(), kw_only=True) diff --git a/src/sqlbuild/orchestration_declarations/types.py b/src/sqlbuild/orchestration_declarations/types.py new file mode 100644 index 0000000000..c41199fb86 --- /dev/null +++ b/src/sqlbuild/orchestration_declarations/types.py @@ -0,0 +1,12 @@ +"""Public enum vocabulary only; event processing belongs to Rust.""" + +from enum import StrEnum + + +class PlatformEventKind(StrEnum): + """Closed vocabulary of durable platform observations.""" + + HEARTBEAT = "heartbeat" + SCHEDULER_NO_LEADER = "scheduler_no_leader" + TICK_OVERDUE = "tick_overdue" + CALENDAR_HORIZON_LOW = "calendar_horizon_low" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index 293ae24452..87a99a43a4 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -253,6 +253,8 @@ class SnapshotDiscoveryTestCase: expected_jobs: tuple[str, ...] declarations: tuple[()] | None = None expected_groups: dict[str, dict[str, list[str]]] = field(default_factory=dict) + expected_sensors: dict[str, dict[str, object]] = field(default_factory=dict) + expected_schedules: tuple[str, ...] = () @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 67d5f94e18..ee942f38ab 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -99,6 +99,7 @@ class WorkerExecutionTestCase: expected_status: str expected_rows: list[tuple[int]] expected_node_count: int = 1 + expected_origin: Literal["schedule", "sensor"] = "schedule" files: dict[str, str] = field(default_factory=dict) files_after_publication: dict[str, str] = field(default_factory=dict) store_setup: str = "" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index bcc8855eca..ae3edf17d5 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -375,6 +375,9 @@ def worker_result(worker: ScheduledWorker, test_case: WorkerExecutionTestCase) - assert nodes[0]["started_at"] is not None assert nodes[0]["ended_at"] is not None with sqlite3.connect(control_store) as control: + assert control.execute( + "SELECT origin FROM sqb_runs WHERE run_id=?", (run["run_id"],) + ).fetchone() == (test_case.expected_origin,) assert control.execute( "SELECT count(*) FROM recall_events " "WHERE invocation_id=? AND event_type='run_launched'", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py index f15bea15b5..7eb241a5fe 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py @@ -143,6 +143,9 @@ def test_given_a_served_calendar_when_time_passes_and_a_provider_fails_then_refr "\n@calendar_schedule(job=orders, description='Other target.', horizon='8s', timezone='UTC', targets=('production',))\n" "def excluded_calendar(window):\n raise ValueError('excluded calendar must not evaluate')\n" ), + "orchestration/platform_orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" + ).read_text(), "providers/fulfillment.py": ( "import json, os\nfrom pathlib import Path\nfrom datetime import timedelta\n" "from sqlbuild.provider.classes.provider import Provider\n" @@ -184,10 +187,13 @@ def test_given_a_served_calendar_when_time_passes_and_a_provider_fails_then_refr second: dict[str, Any] = wait_for_generation(api, "development", 2, process, log) assert first["nodes"] == second["nodes"] assert second["changes"] == [] - assert first["documents"]["triggers"].keys() == {"fulfillment_calendar"} + assert first["documents"]["triggers"].keys() == { + "fulfillment_calendar", + "overdue_orders", + } with sqlite3.connect(tmp_path / "control/.sqlbuild/control.sqlite3") as control: stored, next_due = control.execute( - "SELECT definition_json, next_due_at FROM sqb_triggers JOIN sqb_trigger_state USING (trigger_id)" + "SELECT definition_json, next_due_at FROM sqb_triggers JOIN sqb_trigger_state USING (trigger_id) WHERE kind = 'schedule'" ).fetchone() original_times: list[str] = first["documents"]["triggers"]["fulfillment_calendar"][ "schedule" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py index 31caf9be2d..0c49401511 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py @@ -121,11 +121,20 @@ def test_given_retained_compiler_data_when_publishing_then_http_updates_atomical ) with urlopen(request, timeout=5) as response: triggers: list[dict[str, Any]] = json.load(response) - assert len(triggers) == 1 - assert triggers[0]["status"] == "paused" - assert triggers[0]["definition_json"]["starts_at"] == "2026-07-02T19:47:23.456789Z" - assert triggers[0]["definition_json"]["default_status"] == "paused" - assert triggers[0]["definition_json"]["catch_up"] == {"all": {"max_runs": 3}} + by_name: dict[str, dict[str, Any]] = {trigger["name"]: trigger for trigger in triggers} + assert len(triggers) == 2 + assert set(by_name) == {"edge_snapshot_orders", "edge_platform_orders"} + scheduled: dict[str, Any] = by_name["edge_snapshot_orders"] + assert scheduled["status"] == "paused" + assert scheduled["definition_json"]["starts_at"] == "2026-07-02T19:47:23.456789Z" + assert scheduled["definition_json"]["default_status"] == "paused" + assert scheduled["definition_json"]["catch_up"] == {"all": {"max_runs": 3}} + assert by_name["edge_platform_orders"]["status"] == "paused" + assert by_name["edge_platform_orders"]["definition_json"]["default_status"] == "stopped" + assert by_name["edge_platform_orders"]["definition_json"]["when"] == { + "kind": "platform_event", + "kinds": ["tick_overdue"], + } @pytest.mark.parametrize( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index a95849fa86..edae5b6127 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -64,6 +64,27 @@ expected_child_statuses={"staging": ("success",), "marts": ("success",)}, expected_totals=("success", None), ), + GroupExecutionTestCase( + "a platform sensor starts group parents and children", + "SELECT 17 AS amount", + "SELECT amount FROM totals", + "success", + [(17,)], + expected_origin="sensor", + expected_child_statuses={"staging": ("success",), "marts": ("success",)}, + expected_totals=("success", None), + files={ + "orchestration/orders.py": ( + "from sqlbuild.orchestration import PlatformEventKind, job, job_group, platform_event, sensor\n" + "staging = job('staging', description='Stage orders.', select='stg_orders broken_orders')\n" + "marts = job('marts', description='Build independent marts.', select='orders totals')\n" + "annual_orders = job_group('annual_orders', description='Stage then build marts.', jobs=[marts, staging])\n" + "health_orders = sensor('health_orders', description='Refresh on platform heartbeat.', " + "when=platform_event(kinds=[PlatformEventKind.HEARTBEAT]), run=annual_orders, " + "minimum_interval_seconds=60)\n" + ) + }, + ), GroupExecutionTestCase( "whole downstream job is blocked without building an empty selection", "SELECT CAST(value AS INTEGER) AS amount FROM (VALUES ('invalid')) AS source(value)", @@ -161,6 +182,9 @@ def test_given_a_scheduled_group_when_workers_execute_then_dependencies_and_fail runs["marts"]["started_at"] or runs["staging"]["ended_at"] ) with sqlite3.connect(worker.control_store) as control: + assert control.execute( + "SELECT origin FROM sqb_runs WHERE run_id=?", (parent["run_id"],) + ).fetchone() == (test_case.expected_origin,) assert ( control.execute( "SELECT parent_run_id, root_run_id FROM sqb_runs WHERE kind='job' ORDER BY job_key" @@ -379,6 +403,23 @@ def test_given_a_retained_baseline_when_starting_a_worker_then_verification_and_ "success", [(23,)], ), + WorkerExecutionTestCase( + "a platform sensor catches up retained events through a real worker", + "SELECT 47 AS amount", + "SELECT amount FROM orders", + "success", + [(47,)], + expected_origin="sensor", + files={ + "orchestration/orders.py": ( + "from sqlbuild.orchestration import PlatformEventKind, job, platform_event, sensor\n" + "annual_orders = job('annual_orders', select='orders', description='Build orders.')\n" + "health_orders = sensor('health_orders', description='Refresh on platform heartbeat.', " + "when=platform_event(kinds=[PlatformEventKind.HEARTBEAT]), run=annual_orders, " + "minimum_interval_seconds=60)\n" + ) + }, + ), WorkerExecutionTestCase( "authored variables and reload use ordinary build argument conversion", "SELECT @@batch AS amount", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index f274c11f7e..68b97e7b34 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -603,6 +603,34 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject "edge_inventory_group": {"edge_snapshot_inventory": []}, "edge_products_group": {"edge_snapshot_products": []}, }, + expected_sensors={ + "edge_platform_orders": { + "description": "Catch up retained platform observations with a stopped group sensor.", + "job_key": "edge_snapshot_group", + "when": {"kind": "platform_event", "kinds": ["tick_overdue"]}, + "minimum_interval_seconds": 7, + "default_status": "stopped", + } + }, + expected_schedules=("edge_snapshot_orders",), + ), + SnapshotDiscoveryTestCase( + description="sensor-only roots preserve referenced jobs and filter targets", + files={ + "orchestration/health.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" + ).read_text() + }, + expected_jobs=("health_orders",), + expected_sensors={ + "overdue_orders": { + "description": "Catch up overdue order reporting.", + "job_key": "health_orders", + "when": {"kind": "platform_event", "kinds": ["tick_overdue"]}, + "minimum_interval_seconds": 7, + "default_status": "stopped", + } + }, ), SnapshotDiscoveryTestCase( description="a project without declarations has no job document", @@ -634,6 +662,12 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects name: definition["jobs"] for name, definition in publication["documents"].get("groups", {}).items() } == test_case.expected_groups + assert { + name: publication["documents"]["triggers"][name] for name in test_case.expected_sensors + } == test_case.expected_sensors + assert tuple(publication["documents"].get("triggers", {})) == tuple( + sorted((*test_case.expected_schedules, *test_case.expected_sensors)) + ) for definition in publication["documents"].get("jobs", {}).values(): assert set(definition["selection"]) <= publication["nodes"].keys() @@ -720,6 +754,54 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects expected_error=ValueError, expected_message="mutually exclusive", ), + SnapshotDiscoveryRejectionTestCase( + description="a zero sensor interval cannot publish", + files={ + "orchestration/health.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" + ) + .read_text() + .replace("minimum_interval_seconds=7", "minimum_interval_seconds=0") + }, + expected_error=ValueError, + expected_message="nonzero u32", + ), + SnapshotDiscoveryRejectionTestCase( + description="empty platform conditions cannot publish", + files={ + "orchestration/health.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" + ) + .read_text() + .replace("kinds=[PlatformEventKind.TICK_OVERDUE]", "kinds=[]") + }, + expected_error=ValueError, + expected_message="nonempty event kinds", + ), + SnapshotDiscoveryRejectionTestCase( + description="platform kinds are a closed vocabulary", + files={ + "orchestration/health.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" + ) + .read_text() + .replace("kinds=[PlatformEventKind.TICK_OVERDUE]", "kinds=['unknown_event']") + }, + expected_error=ValueError, + expected_message="unknown variant", + ), + SnapshotDiscoveryRejectionTestCase( + description="sensor code defaults use running or stopped", + files={ + "orchestration/health.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" + ) + .read_text() + .replace('default_status="stopped"', 'default_status="paused"') + }, + expected_error=ValueError, + expected_message="unknown variant", + ), ], ids=lambda case: case.description, ) From 0621fe02016c51a83d951c4005eac91dafc8ddea Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 05:11:11 +0000 Subject: [PATCH 49/59] feat: observe expired code location heartbeats Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../orchestration/edge_cases/definitions.py | 2 +- .../src/platform/_helpers/liveness.rs | 59 ++++++ .../src/platform/_helpers/mod.rs | 1 + .../src/platform/_helpers/watchdog.rs | 11 +- .../src/platform/models.rs | 16 ++ .../tests/platform/test_types.rs | 6 + .../tests/platform/test_watchdog.rs | 176 +++++++++++++++++- .../tests/sensors/helpers.rs | 12 +- .../orchestration_declarations/types.py | 1 + .../publication/test_publication.py | 2 +- .../test_orchestration_snapshot.py | 5 +- 11 files changed, 283 insertions(+), 8 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/platform/_helpers/liveness.rs diff --git a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py index ef8caed097..f88b56bb70 100644 --- a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py +++ b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py @@ -61,7 +61,7 @@ edge_health = sensor( "edge_platform_orders", description="Catch up retained platform observations with a stopped group sensor.", - when=platform_event(kinds=[PlatformEventKind.TICK_OVERDUE]), + when=platform_event(kinds=[PlatformEventKind.TICK_OVERDUE, PlatformEventKind.LOCATION_BROKEN]), run=edge_group, minimum_interval_seconds=7, default_status="stopped", diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/liveness.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/liveness.rs new file mode 100644 index 0000000000..7239f2d718 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/liveness.rs @@ -0,0 +1,59 @@ +//! Liveness is derived across all sessions; an old instance cannot mask a healthy successor. + +use chrono::TimeDelta; +use uuid::Uuid; + +use crate::platform::_helpers::observations::observe; +use crate::platform::models::{PlatformEvent, WatchdogLocation}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Locations report every 30 seconds; tolerate two missed reports. +pub(crate) async fn scan_location_page( + store: &ControlStore, + installation_id: Uuid, + after: Option<(Uuid, String)>, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Write).await?; + let mut parameters = vec![Parameter::Time(transaction.now - TimeDelta::seconds(90))]; + let cursor = if let Some((location_id, target)) = after { + parameters.extend([Parameter::Id(location_id), Parameter::Text(target)]); + "AND (i.location_id>$2 OR (i.location_id=$2 AND i.target>$3))" + } else { + "" + }; + let (mut transaction, rows) = query( + transaction, + &format!( + "SELECT i.location_id, i.target, MAX(i.last_seen_at) AS last_seen_at \ + FROM sqb_location_instances i JOIN sqb_locations l ON l.location_id=i.location_id \ + WHERE l.archived_at IS NULL {cursor} GROUP BY i.location_id,i.target \ + HAVING MAX(i.last_seen_at)<$1 ORDER BY i.location_id,i.target LIMIT 100" + ), + parameters, + ) + .await?; + let locations: Vec = + rows.into_iter().map(decode_row).collect::>()?; + let next = locations + .last() + .map(|row| (row.location_id, row.target.clone())); + for location in locations { + transaction = observe( + transaction, + installation_id, + PlatformEvent::LocationBroken { + location_id: location.location_id, + target: location.target, + last_seen_at: location.last_seen_at, + }, + ) + .await?; + } + commit(transaction).await?; + Ok(next) +} diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs index 06cdb8c059..574aaa4680 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod liveness; pub(crate) mod location_heartbeat; pub(crate) mod location_work; pub(crate) mod observations; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs index 16ded206d3..89ca089b53 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs @@ -1,10 +1,11 @@ -//! Health scanning releases the writer lock between bounded trigger pages. +//! Health scanning releases the writer lock between location and trigger pages. use chrono::TimeDelta; use tokio::sync::watch; use uuid::Uuid; use crate::declarations::models::Schedule; +use crate::platform::_helpers::liveness::scan_location_page; use crate::platform::_helpers::observations::observe; use crate::platform::models::{InstallationHealth, PlatformEvent, WatchdogTrigger}; use crate::store::errors::ControlError; @@ -40,6 +41,14 @@ pub(crate) async fn scan_health( .await?; } commit(transaction).await?; + let mut location_after = None; + while !*shutdown.borrow() { + location_after = scan_location_page(store, health.installation_id, location_after).await?; + if location_after.is_none() { + break; + } + tokio::task::yield_now().await; + } let mut after = None; while !*shutdown.borrow() { after = scan_trigger_page(store, health.installation_id, after).await?; diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index 09ded290f5..39f6851841 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -119,6 +119,7 @@ pub enum PlatformEventKind { SchedulerNoLeader, TickOverdue, CalendarHorizonLow, + LocationBroken, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -137,6 +138,12 @@ pub enum PlatformEvent { refresh_at: DateTime, horizon_end: Option>, }, + /// All observed instances of this location/target have missed three heartbeats. + LocationBroken { + location_id: Uuid, + target: String, + last_seen_at: DateTime, + }, } impl PlatformEvent { @@ -147,6 +154,7 @@ impl PlatformEvent { Self::SchedulerNoLeader => PlatformEventKind::SchedulerNoLeader, Self::TickOverdue { .. } => PlatformEventKind::TickOverdue, Self::CalendarHorizonLow { .. } => PlatformEventKind::CalendarHorizonLow, + Self::LocationBroken { .. } => PlatformEventKind::LocationBroken, } } @@ -157,6 +165,7 @@ impl PlatformEvent { Self::SchedulerNoLeader => "platform_scheduler_no_leader", Self::TickOverdue { .. } => "platform_tick_overdue", Self::CalendarHorizonLow { .. } => "platform_calendar_horizon_low", + Self::LocationBroken { .. } => "platform_location_broken", } } } @@ -174,6 +183,13 @@ pub(crate) struct WatchdogTrigger { pub(crate) next_due_at: Option>, } +#[derive(Debug, Deserialize)] +pub(crate) struct WatchdogLocation { + pub(crate) location_id: Uuid, + pub(crate) target: String, + pub(crate) last_seen_at: DateTime, +} + /// The endpoint may contain credentials; neither errors nor Debug reveal it. #[derive(Clone)] pub struct PlatformWebhook { diff --git a/crates/sqlbuild-orchestration/tests/platform/test_types.rs b/crates/sqlbuild-orchestration/tests/platform/test_types.rs index 970ec7a8c0..52ecb92e0c 100644 --- a/crates/sqlbuild-orchestration/tests/platform/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/platform/test_types.rs @@ -15,6 +15,12 @@ pub(crate) struct WatchdogTestCase { pub(crate) expected_low_horizons: usize, } +pub(crate) struct LocationWatchdogTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_broken: usize, +} + #[derive(Debug, Deserialize)] pub(crate) struct Observation { pub(crate) storage_id: i64, diff --git a/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs b/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs index 0d03b0e495..fa3574df89 100644 --- a/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs +++ b/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs @@ -1,16 +1,22 @@ use std::collections::BTreeSet; use std::time::Duration; +use crate::platform::main::report_location_health::report_location_health; use crate::platform::main::run_watchdog::run_watchdog; -use crate::platform::models::PlatformEvent; +use crate::platform::models::{LocationHealth, LocationHealthReport, PlatformEvent}; use crate::scheduler::tests::contracts::helpers::acquire_lease; use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; use crate::store::main::migrate_store::migrate_store; +use crate::store::main::query::query; +use crate::store::models::{Parameter, TransactionMode}; +use chrono::TimeDelta; use uuid::Uuid; use crate::credentials::tests::location_helpers::create_location; use crate::platform::tests::helpers::{snapshot, wait_for_scans}; -use crate::platform::tests::test_types::WatchdogTestCase; +use crate::platform::tests::test_types::{LocationWatchdogTestCase, WatchdogTestCase}; use crate::store::tests::replica_helpers::second_store; use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; use crate::store::tests::store_test_types::Backend; @@ -169,3 +175,169 @@ fn given_two_server_replicas_when_scheduling_is_stalled_then_health_facts_are_du }); } } + +#[test] +fn given_stale_location_sessions_when_watchdogs_scan_then_latest_target_liveness_controls_paged_outage_events() + { + let test_cases = [ + LocationWatchdogTestCase { + description: "SQLite latest target receipts", + backend: Backend::Sqlite, + expected_broken: 104, + }, + LocationWatchdogTestCase { + description: "PostgreSQL latest target receipts", + backend: Backend::Postgres, + expected_broken: 104, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required backend"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate store"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let archived = create_location(store, "inventory", "test-admin") + .await + .expect("archived location"); + create_location(store, "products", "test-admin") + .await + .expect("never observed location"); + for (location_id, count) in [(location, 105), (archived, 1)] { + for index in 0..count { + for _ in 0..2 { + report_location_health( + store, + location_id, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id: Uuid::now_v7(), + target: format!("orders_{index:03}"), + health: LocationHealth::Heartbeat {}, + }, + ) + .await + .expect("register old session"); + } + } + } + let transaction = begin(store, TransactionMode::Write) + .await + .expect("age sessions"); + let stale_at = transaction.now - TimeDelta::seconds(105); + let (transaction, _) = query( + transaction, + "UPDATE sqb_location_instances SET last_seen_at=$1", + vec![Parameter::Time(stale_at)], + ) + .await + .expect("age all old sessions"); + let now = transaction.now; + let (transaction, _) = query( + transaction, + "UPDATE sqb_locations SET archived_at=$1 WHERE location_id=$2", + vec![Parameter::Time(now), Parameter::Id(archived)], + ) + .await + .expect("archive location"); + commit(transaction).await.expect("commit fixture"); + // A fresh successor protects only its target, not another target of the same location. + let healthy_session = Uuid::now_v7(); + report_location_health( + store, + location, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id: healthy_session, + target: "orders_104".into(), + health: LocationHealth::Heartbeat {}, + }, + ) + .await + .expect("healthy successor"); + let transaction = begin(store, TransactionMode::Write) + .await + .expect("age successor"); + let recent_at = transaction.now - TimeDelta::seconds(75); + let (transaction, _) = query( + transaction, + "UPDATE sqb_location_instances SET last_seen_at=$1 WHERE instance_id=$2", + vec![Parameter::Time(recent_at), Parameter::Id(healthy_session)], + ) + .await + .expect("two missed heartbeats"); + commit(transaction).await.expect("commit successor time"); + let second = second_store(store).await; + let replicas = BTreeSet::from([Uuid::now_v7(), Uuid::now_v7()]); + let (stop, shutdown) = tokio::sync::watch::channel(false); + let tasks: Vec<_> = [store.clone(), second.clone()] + .into_iter() + .zip(replicas.iter()) + .map(|(store, id)| { + let id = *id; + let shutdown = shutdown.clone(); + tokio::spawn(async move { run_watchdog(&store, id, shutdown).await }) + }) + .collect(); + let observations = wait_for_scans(store, &replicas, &tasks).await; + let broken: BTreeSet<_> = observations + .iter() + .filter_map(|row| { + let PlatformEvent::LocationBroken { + location_id, + ref target, + last_seen_at, + } = row.payload.platform + else { + return None; + }; + assert_eq!( + location_id, location, + "archived and never-observed locations are excluded" + ); + assert_eq!( + last_seen_at, stale_at, + "payload retains the latest server receipt time" + ); + Some(target.clone()) + }) + .collect(); + assert_eq!( + broken, + (0..test_case.expected_broken) + .map(|index| format!("orders_{index:03}")) + .collect(), + "{}: both pages are read, old replicas deduplicate, and the healthy target is excluded", + test_case.description + ); + let keys: BTreeSet<_> = observations + .iter() + .map(|row| { + ( + row.occurred_at, + serde_json::to_string(&row.payload.platform).expect("identity"), + ) + }) + .collect(); + assert_eq!( + keys.len(), + observations.len(), + "replicas share observation keys" + ); + stop.send(true).expect("stop replicas"); + for task in tasks { + tokio::time::timeout(Duration::from_secs(2), task) + .await + .expect("prompt shutdown") + .expect("join") + .expect("watchdog stopped"); + } + close_store(second).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/helpers.rs b/crates/sqlbuild-orchestration/tests/sensors/helpers.rs index 7aea4d6287..5152689454 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/helpers.rs @@ -29,7 +29,7 @@ pub(crate) fn snapshot(location_id: Uuid) -> SnapshotInput { json!({ "health_orders": { "description": "Refresh reporting on overdue ticks", "job_key": "orders", - "when": {"kind": "platform_event", "kinds": ["tick_overdue"]}, + "when": {"kind": "platform_event", "kinds": ["tick_overdue", "location_broken"]}, "minimum_interval_seconds": 1 } }), @@ -53,9 +53,17 @@ pub(crate) async fn append_events( PlatformEvent::Heartbeat { replica_id: Uuid::from_u128(8), }, + PlatformEvent::LocationBroken { + location_id: Uuid::from_u128(9), + target: "development".into(), + last_seen_at: occurred_at, + }, + PlatformEvent::Heartbeat { + replica_id: Uuid::from_u128(10), + }, ]; for index in indices { - let platform = &payloads[(index % 2) as usize]; + let platform = &payloads[(index % 4) as usize]; let (next, appended) = append_control_event( transaction, ControlEvent { diff --git a/src/sqlbuild/orchestration_declarations/types.py b/src/sqlbuild/orchestration_declarations/types.py index c41199fb86..edba630aa1 100644 --- a/src/sqlbuild/orchestration_declarations/types.py +++ b/src/sqlbuild/orchestration_declarations/types.py @@ -10,3 +10,4 @@ class PlatformEventKind(StrEnum): SCHEDULER_NO_LEADER = "scheduler_no_leader" TICK_OVERDUE = "tick_overdue" CALENDAR_HORIZON_LOW = "calendar_horizon_low" + LOCATION_BROKEN = "location_broken" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py index 0c49401511..1f33e90df2 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py @@ -133,7 +133,7 @@ def test_given_retained_compiler_data_when_publishing_then_http_updates_atomical assert by_name["edge_platform_orders"]["definition_json"]["default_status"] == "stopped" assert by_name["edge_platform_orders"]["definition_json"]["when"] == { "kind": "platform_event", - "kinds": ["tick_overdue"], + "kinds": ["tick_overdue", "location_broken"], } diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 68b97e7b34..72f6ded1d2 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -607,7 +607,10 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject "edge_platform_orders": { "description": "Catch up retained platform observations with a stopped group sensor.", "job_key": "edge_snapshot_group", - "when": {"kind": "platform_event", "kinds": ["tick_overdue"]}, + "when": { + "kind": "platform_event", + "kinds": ["tick_overdue", "location_broken"], + }, "minimum_interval_seconds": 7, "default_status": "stopped", } From 6f8f8cb3728e5a2af390e4f7460270520407a447 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 05:41:09 +0000 Subject: [PATCH 50/59] feat: retain typed launch failure observations Publish scheduler launch observations atomically with their source facts and retain pending observations across publication failures. Allow late observations only with immutable proof of the exact recorded claim and current scheduler fencing. Terminal projections and physical worker ownership remain unchanged. Cover local launch failures, real Docker response-loss and worker-completion ordering, transactional retry faults, and SQLite/PostgreSQL authority boundaries. Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../orchestration/edge_cases/definitions.py | 8 +- .../src/workers/_helpers/recovery.rs | 8 + .../src/workers/_helpers/shutdown.rs | 18 ++ .../src/workers/_helpers/supervisor.rs | 50 ++++- crates/sqlbuild-cli/src/workers/models.rs | 1 + .../native_commands/worker/test_recovery.rs | 41 +++- .../native_commands/worker/test_types.rs | 2 + .../src/platform/models.rs | 9 + .../src/runs/_helpers/lifecycle/changes.rs | 34 +++ .../src/runs/_helpers/lifecycle/projection.rs | 1 + .../runs/_helpers/lifecycle/publish_change.rs | 37 ++- .../src/runs/main/change_run.rs | 10 +- .../sqlbuild-orchestration/src/runs/models.rs | 5 + .../tests/runs/test_lifecycle.rs | 210 ++++++++++++++++++ .../tests/sensors/helpers.rs | 12 +- .../orchestration_declarations/types.py | 3 +- .../publication/docker/helpers.py | 66 ++++++ .../publication/test_docker_worker.py | 81 +++++++ .../publication/test_publication.py | 2 +- .../test_orchestration_snapshot.py | 2 +- 20 files changed, 580 insertions(+), 20 deletions(-) diff --git a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py index f88b56bb70..6762839dd4 100644 --- a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py +++ b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py @@ -61,7 +61,13 @@ edge_health = sensor( "edge_platform_orders", description="Catch up retained platform observations with a stopped group sensor.", - when=platform_event(kinds=[PlatformEventKind.TICK_OVERDUE, PlatformEventKind.LOCATION_BROKEN]), + when=platform_event( + kinds=[ + PlatformEventKind.TICK_OVERDUE, + PlatformEventKind.LOCATION_BROKEN, + PlatformEventKind.LAUNCH_FAILED, + ] + ), run=edge_group, minimum_interval_seconds=7, default_status="stopped", diff --git a/crates/sqlbuild-cli/src/workers/_helpers/recovery.rs b/crates/sqlbuild-cli/src/workers/_helpers/recovery.rs index c54d87b613..f439a58dd6 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/recovery.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/recovery.rs @@ -63,12 +63,20 @@ impl Workers { "Worker supervision recovered for run {}", claim.request.run_id )); + let rejected = child.is_none(); + let index = self.active.len(); self.active.push(ClaimedWorker { claim, child, stopping: None, exit: None, + launch_failure: rejected.then_some("docker_configuration_rejected"), }); + if rejected { + self.active[index] + .publish_launch_failure(store, lease) + .await?; + } } } } diff --git a/crates/sqlbuild-cli/src/workers/_helpers/shutdown.rs b/crates/sqlbuild-cli/src/workers/_helpers/shutdown.rs index 9613633420..7eafd6f876 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/shutdown.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/shutdown.rs @@ -114,6 +114,24 @@ impl Workers { } renewed = Instant::now(); } + if self.active[index].launch_failure.is_some() { + let reported = match lease { + Some(lease) => { + self.active[index] + .publish_launch_failure(store, lease) + .await + } + None => Err(ControlError::Fenced), + }; + if let Err(error) = reported { + if matches!(error, ControlError::Fenced) { + failure = Some(error); + } else { + failure.get_or_insert(error); + } + continue; + } + } if let Some(WorkerChild::Docker(child)) = &self.active[index].child { match child.try_wait().await { Ok(None) => { diff --git a/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs b/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs index b9aa97e091..ab053c0d10 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs @@ -177,10 +177,14 @@ impl Workers { child: None, stopping: None, exit: None, + launch_failure: None, }); let worker = &mut self.active[index]; let claim = &worker.claim; if matches!(location, Launcher::Local(_)) && claim.request.image_id.is_some() { + worker.launch_failure = Some("local_image_rejected"); + worker.publish_launch_failure(store, lease).await?; + let claim = &worker.claim; publish(store, lease, claim, ("local-image-rejected", RunChange::Finished { outcome: RunOutcome::Failed, reason: "image-pinned runs require a Docker launcher, not a local interpreter".into(), exit_code: None })).await?; self.active.swap_remove(index); continue; @@ -193,6 +197,9 @@ impl Workers { Some(name) } Err(_) => { + worker.launch_failure = Some("docker_configuration_rejected"); + worker.publish_launch_failure(store, lease).await?; + let claim = &worker.claim; publish(store, lease, claim, ("docker-image-rejected", RunChange::Finished { outcome: RunOutcome::Failed, reason: "Docker worker requires a pinned image and supported launcher configuration".into(), exit_code: None, })).await?; @@ -220,12 +227,19 @@ impl Workers { ) .await?; let local_command = match location { - Launcher::Local(location) => Some(prepare_spawn( + Launcher::Local(location) => match prepare_spawn( location, claim, &credential.bearer, &self.control_variable, - )?), + ) { + Ok(command) => Some(command), + Err(error) => { + worker.launch_failure = Some("local_preparation_failed"); + worker.publish_launch_failure(store, lease).await?; + return Err(error); + } + }, Launcher::Docker(_) => None, }; publish( @@ -245,10 +259,13 @@ impl Workers { None => { if let (Launcher::Docker(location), Some(WorkerChild::Docker(child))) = (location, &worker.child) - { - child + && let Err(error) = child .launch(location, claim, &credential.bearer, &self.control_variable) - .await?; + .await + { + worker.launch_failure = Some("docker_launch_failed"); + worker.publish_launch_failure(store, lease).await?; + return Err(error); } Ok(()) } @@ -262,6 +279,9 @@ impl Workers { )); } Err(_) => { + worker.launch_failure = Some("local_spawn_failed"); + worker.publish_launch_failure(store, lease).await?; + let claim = &worker.claim; publish( store, lease, @@ -311,6 +331,26 @@ fn prepare_spawn( Ok(command) } +impl ClaimedWorker { + pub(in crate::workers) async fn publish_launch_failure( + &mut self, + store: &ControlStore, + lease: &SchedulerLease, + ) -> Result<(), ControlError> { + if let Some(code) = self.launch_failure { + publish( + store, + lease, + &self.claim, + (code, RunChange::LaunchFailed { code: code.into() }), + ) + .await?; + self.launch_failure = None; + } + Ok(()) + } +} + pub(in crate::workers) async fn publish( store: &ControlStore, lease: &SchedulerLease, diff --git a/crates/sqlbuild-cli/src/workers/models.rs b/crates/sqlbuild-cli/src/workers/models.rs index 415247a7f7..35639fe18e 100644 --- a/crates/sqlbuild-cli/src/workers/models.rs +++ b/crates/sqlbuild-cli/src/workers/models.rs @@ -98,6 +98,7 @@ pub(super) struct ClaimedWorker { pub(super) child: Option, pub(super) stopping: Option, pub(super) exit: Option, + pub(super) launch_failure: Option<&'static str>, } /// Physical ownership survives leadership loss; only the current fence changes durable lifecycle. diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/test_recovery.rs b/crates/sqlbuild-cli/tests/native_commands/worker/test_recovery.rs index 0cdf1611ca..0a723a1db4 100644 --- a/crates/sqlbuild-cli/tests/native_commands/worker/test_recovery.rs +++ b/crates/sqlbuild-cli/tests/native_commands/worker/test_recovery.rs @@ -14,11 +14,29 @@ fn given_an_unlaunchable_committed_claim_when_recovering_then_rejection_retries_ WorkerRecoveryTestCase { description: "snapshot without an image pin", image_id: None, + failure_event: "run_terminated", + expected_pending_observations: 1, expected_status: "failed", }, WorkerRecoveryTestCase { description: "snapshot with a malformed image pin", image_id: Some("sha256:invalid"), + failure_event: "run_terminated", + expected_pending_observations: 1, + expected_status: "failed", + }, + WorkerRecoveryTestCase { + description: "missing image pin retains a rolled-back launch observation", + image_id: None, + failure_event: "platform_launch_failed", + expected_pending_observations: 0, + expected_status: "failed", + }, + WorkerRecoveryTestCase { + description: "invalid image pin retains a rolled-back launch observation", + image_id: Some("sha256:invalid"), + failure_event: "platform_launch_failed", + expected_pending_observations: 0, expected_status: "failed", }, ]; @@ -37,11 +55,11 @@ fn given_an_unlaunchable_committed_claim_when_recovering_then_rejection_retries_ connection .busy_timeout(Duration::from_secs(5)) .expect("database contention timeout"); - connection.execute_batch( + connection.execute_batch(&format!( "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00Z'; CREATE TRIGGER reject_recovery_publication BEFORE INSERT ON recall_events - WHEN NEW.event_type='run_terminated' BEGIN SELECT RAISE(ABORT, 'synthetic unavailable publication'); END;", - ).expect("expire the departed leader and interrupt terminal publication"); + WHEN NEW.event_type='{}' BEGIN SELECT RAISE(ABORT, 'synthetic unavailable publication'); END;", test_case.failure_event + )).expect("expire the departed leader and interrupt durable publication"); let mut replacement = start_recovery_scheduler(&control, &url, baseline.location_id); wait_for_worker_log(&mut replacement, "Retrying worker shutdown publication").await; let held: (i64, i64) = connection.query_row( @@ -49,6 +67,10 @@ fn given_an_unlaunchable_committed_claim_when_recovering_then_rejection_retries_ [], |row| Ok((row.get(0)?, row.get(1)?)), ).expect("retained claim while publication is unavailable"); assert_eq!(held, (1, 1), "{}", test_case.description); + for event_type in ["run_launch_failed", "platform_launch_failed"] { + let pending: i64 = connection.query_row("SELECT COUNT(*) FROM recall_events WHERE event_type=?1", [event_type], |row| row.get(0)).expect("pending source/platform pair"); + assert_eq!(pending, test_case.expected_pending_observations, "{}: {event_type}", test_case.description); + } connection .execute_batch("DROP TRIGGER reject_recovery_publication") .expect("restore terminal publication"); @@ -69,6 +91,19 @@ fn given_an_unlaunchable_committed_claim_when_recovering_then_rejection_retries_ "{}", test_case.description ); + let (producer, payload, count): (String, String, i64) = connection.query_row( + "SELECT producer, payload, COUNT(*) OVER () FROM recall_events WHERE event_type='platform_launch_failed'", + [], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ).expect("durable launch observation from the real replacement scheduler"); + assert_eq!(producer, "platform"); + assert_eq!(count, 1, "{}", test_case.description); + let source_count: i64 = connection.query_row("SELECT COUNT(*) FROM recall_events WHERE event_type='run_launch_failed'", [], |row| row.get(0)).expect("retried source fact"); + assert_eq!(source_count, 1, "{}", test_case.description); + let event: serde_json::Value = serde_json::from_str(&payload).expect("typed event payload"); + assert_eq!(event["platform"]["kind"], "launch_failed"); + assert_eq!(event["platform"]["run_id"], worker.identity.0.to_string()); + assert_eq!(event["platform"]["claim_id"], worker.identity.1.to_string()); + assert_eq!(event["platform"]["code"], "docker_configuration_rejected"); assert!( std::fs::read_to_string(&replacement.stderr) .expect("shutdown recovery log") diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs b/crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs index 81ecbc260f..0a6741cff7 100644 --- a/crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs +++ b/crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs @@ -16,6 +16,8 @@ pub(crate) struct WorkerHeartbeatTestCase { pub(crate) struct WorkerRecoveryTestCase { pub(crate) description: &'static str, pub(crate) image_id: Option<&'static str>, + pub(crate) failure_event: &'static str, + pub(crate) expected_pending_observations: i64, pub(crate) expected_status: &'static str, } diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index 39f6851841..7ab2762ac9 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -120,6 +120,7 @@ pub enum PlatformEventKind { TickOverdue, CalendarHorizonLow, LocationBroken, + LaunchFailed, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -144,6 +145,12 @@ pub enum PlatformEvent { target: String, last_seen_at: DateTime, }, + /// A launch attempt or its response failed; the worker may still start or finish. + LaunchFailed { + run_id: Uuid, + claim_id: Uuid, + code: String, + }, } impl PlatformEvent { @@ -155,6 +162,7 @@ impl PlatformEvent { Self::TickOverdue { .. } => PlatformEventKind::TickOverdue, Self::CalendarHorizonLow { .. } => PlatformEventKind::CalendarHorizonLow, Self::LocationBroken { .. } => PlatformEventKind::LocationBroken, + Self::LaunchFailed { .. } => PlatformEventKind::LaunchFailed, } } @@ -166,6 +174,7 @@ impl PlatformEvent { Self::TickOverdue { .. } => "platform_tick_overdue", Self::CalendarHorizonLow { .. } => "platform_calendar_horizon_low", Self::LocationBroken { .. } => "platform_location_broken", + Self::LaunchFailed { .. } => "platform_launch_failed", } } } diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/changes.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/changes.rs index 8310532774..f6a099cf26 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/changes.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/changes.rs @@ -4,8 +4,41 @@ use crate::runs::models::{ ChangingRun, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, }; use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; use crate::store::types::Row; +/// A launch-response observation is append-only even after completion, but requires recorded ownership. +pub(crate) async fn check_launch_claim( + transaction: ControlTransaction, + request: &RunChangeRequest, +) -> Result { + let claim_id = request.claim_id.ok_or_else(|| { + ControlError::Invalid("launch observation requires a recorded claim".into()) + })?; + let (transaction, rows) = query( + transaction, + "SELECT payload FROM recall_events WHERE event_id=$1 AND run_id=$2 \ + AND event_type='run_claimed' AND producer='scheduler' AND invocation_id=$3", + vec![ + Parameter::Text(claim_id.to_string()), + Parameter::Text(request.run_id.to_string()), + Parameter::Text(format!("scheduler/{}", request.run_id)), + ], + ) + .await?; + let recorded = rows.first().map(|row| &row["payload"]["claim"]); + if recorded.is_none_or(|claim| { + claim["claim_id"] != claim_id.to_string() + || claim["request"]["run_id"] != request.run_id.to_string() + }) { + return Err(ControlError::Invalid( + "launch observation does not belong to a recorded claim".into(), + )); + } + Ok(transaction) +} + pub(crate) fn validate_retry( previous: &Row, request: &RunChangeRequest, @@ -85,6 +118,7 @@ pub(crate) fn validate_change( )); } let valid = match request.change { + RunChange::LaunchFailed { .. } => false, RunChange::Launched { .. } => state.worker_instance.is_some(), RunChange::Started => { state.worker_instance.is_some() && matches!(state.status, Queued | Launched) diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/projection.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/projection.rs index 7f6ffb2de0..b430f35721 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/projection.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/projection.rs @@ -17,6 +17,7 @@ pub(crate) async fn project_change( ) -> Result { let now = transaction.now; match &request.change { + RunChange::LaunchFailed { .. } => {} RunChange::NodeStarted { .. } | RunChange::NodeFinished { .. } => { return project_node(transaction, request, &state.node_counts_json).await; } diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publish_change.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publish_change.rs index 68cc3edd32..c5445432e7 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publish_change.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publish_change.rs @@ -1,26 +1,57 @@ -//! Publish a lifecycle fact, its projection and terminal cleanup in the caller's transaction. +//! Publish facts atomically; launch observations never project state or release physical ownership. use serde_json::json; +use crate::platform::models::PlatformEvent; use crate::runs::_helpers::lifecycle::events::append_run_event; use crate::runs::_helpers::lifecycle::projection::project_change; use crate::runs::models::{ChangingRun, RunChange, RunChangeRequest, RunOutcome}; use crate::store::errors::ControlError; +use crate::store::main::append_control_event::append_control_event; use crate::store::main::query::query; -use crate::store::models::{ControlTransaction, Parameter}; +use crate::store::models::{ControlEvent, ControlTransaction, Parameter}; pub(crate) async fn publish_change( transaction: ControlTransaction, request: &RunChangeRequest, state: &ChangingRun, ) -> Result { - let (transaction, mut cursor) = append_run_event( + let (mut transaction, mut cursor) = append_run_event( transaction, (request.run_id, request.event_id), request.change.event_type(), json!({"change": request}), ) .await?; + if let (RunChange::LaunchFailed { code }, Some(claim_id)) = (&request.change, request.claim_id) + { + let platform = PlatformEvent::LaunchFailed { + run_id: request.run_id, + claim_id, + code: code.clone(), + }; + let occurred_at = transaction.now; + let outcome; + (transaction, outcome) = append_control_event( + transaction, + ControlEvent { + event_id: uuid::Uuid::new_v5(&request.event_id, b"platform-launch-failed"), + invocation_id: format!("platform/{}", request.run_id), + event_type: platform.event_type(), + producer: "platform", + occurred_at, + run_id: Some(request.run_id), + payload: json!({"platform": platform}), + }, + ) + .await?; + if outcome.conflict.is_some() { + return Err(ControlError::Invalid( + "launch failure conflicts with a durable fact".into(), + )); + } + return Ok(transaction); + } let mut transaction = project_change(transaction, request, state).await?; let mut cancel_unstarted = matches!(request.change, RunChange::CancelRequested { .. }) && state.worker_instance.is_none(); diff --git a/crates/sqlbuild-orchestration/src/runs/main/change_run.rs b/crates/sqlbuild-orchestration/src/runs/main/change_run.rs index c43086efe7..59cb4f2a40 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/change_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/change_run.rs @@ -4,7 +4,7 @@ use serde_json::json; use crate::credentials::main::append_audit::append_audit; use crate::runs::_helpers::lifecycle::changes::{ - validate_authority, validate_change, validate_retry, + check_launch_claim, validate_authority, validate_change, validate_retry, }; use crate::runs::_helpers::lifecycle::publish_change::publish_change; use crate::runs::models::{ChangingRun, RunAuthority, RunChange, RunChangeRequest}; @@ -39,7 +39,7 @@ pub async fn change_run( commit(transaction).await?; return Ok(false); } - let (transaction, rows) = query( + let (mut transaction, rows) = query( transaction, "SELECT r.status, r.cancel_requested_at, r.node_counts_json, l.worker_instance FROM sqb_runs r \ LEFT JOIN sqb_run_liveness l ON l.run_id=r.run_id WHERE r.run_id=$1", @@ -51,7 +51,11 @@ pub async fn change_run( .next() .ok_or_else(|| ControlError::Invalid("run does not exist".into()))?, )?; - validate_change(&state, request)?; + if matches!(request.change, RunChange::LaunchFailed { .. }) { + transaction = check_launch_claim(transaction, request).await?; + } else { + validate_change(&state, request)?; + } let mut transaction = publish_change(transaction, request, &state).await?; if let (RunAuthority::Operator(context), RunChange::CancelRequested { reason, .. }) = (authority, &request.change) diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index 64ab1a370c..16625f5cc1 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -250,6 +250,10 @@ pub enum RunChange { Launched { container_id: Option, }, + /// A failed launch response is not physical evidence that the worker has stopped. + LaunchFailed { + code: String, + }, Started, NodeStarted { node_key: String, @@ -279,6 +283,7 @@ impl RunChange { pub(crate) const fn event_type(&self) -> &'static str { match self { Self::Launched { .. } => "run_launched", + Self::LaunchFailed { .. } => "run_launch_failed", Self::Started => "run_started", Self::NodeStarted { .. } => "run_node_started", Self::NodeFinished { .. } => "run_node_finished", diff --git a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs index 2f98d185eb..21ec54e845 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs @@ -1,6 +1,7 @@ use std::collections::BTreeMap; use crate::credentials::models::AuditContext; +use crate::platform::models::PlatformEvent; use crate::runs::main::change_run::change_run; use crate::runs::main::claim_next_run::claim_next_run; use crate::runs::main::heartbeat_run::heartbeat_run; @@ -11,6 +12,7 @@ use crate::runs::models::{ }; use crate::scheduler::tests::contracts::helpers::acquire_lease; use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; use crate::store::main::migrate_store::migrate_store; use crate::store::tests::helpers::read_recall_events; use uuid::Uuid; @@ -19,6 +21,7 @@ use crate::credentials::tests::location_helpers::create_location; use crate::runs::tests::helpers::{queue, request, snapshot}; use crate::runs::tests::test_types::{RunLifecycleTestCase, TransactionFault}; use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::lease_helpers::expire_lease; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; use crate::store::tests::store_test_types::Backend; @@ -118,6 +121,96 @@ fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_ .status, RunStatus::Running ); + let launch_failed = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::LaunchFailed { + code: "docker_launch_failed".into(), + }, + ..started.clone() + }; + assert!(change_run(store, worker, &launch_failed).await.is_err()); + assert!( + change_run(store, RunAuthority::Operator(&operator), &launch_failed) + .await + .is_err() + ); + let wrong_claim = RunChangeRequest { + claim_id: Some(Uuid::now_v7()), + ..launch_failed.clone() + }; + assert!( + change_run(store, RunAuthority::Scheduler(&lease), &wrong_claim) + .await + .is_err() + ); + let before_launch_failure = read_recall_events(store, 0, 100) + .await + .expect("before launch failure"); + let fault = TransactionFault { + description: "platform launch observation must commit with its run fact", + table: "recall_events", + operation: "INSERT", + condition: "WHEN NEW.event_type='platform_launch_failed'", + sqlite_body: "SELECT RAISE(ABORT,'injected launch observation failure')", + postgres_body: "RAISE EXCEPTION 'injected launch observation failure'", + }; + install_fault(store, &fault).await; + assert!( + change_run(store, RunAuthority::Scheduler(&lease), &launch_failed) + .await + .is_err() + ); + remove_fault(store, &fault).await; + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("rolled back launch fact"), + before_launch_failure + ); + assert!( + change_run(store, RunAuthority::Scheduler(&lease), &launch_failed) + .await + .expect("launch observation") + ); + assert!( + !change_run(store, RunAuthority::Scheduler(&lease), &launch_failed) + .await + .expect("idempotent retry") + ); + let observations: Vec<_> = read_recall_events(store, 0, 100) + .await + .expect("health facts") + .into_iter() + .filter(|row| row["event_type"] == "platform_launch_failed") + .collect(); + assert_eq!(observations.len(), 1); + let platform: PlatformEvent = + serde_json::from_value(observations[0]["payload"]["platform"].clone()) + .expect("typed launch failure"); + assert_eq!( + platform, + PlatformEvent::LaunchFailed { + run_id: started.run_id, + claim_id: claim.claim_id, + code: "docker_launch_failed".into() + } + ); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("live run") + .into_iter() + .find(|run| run.run_id == started.run_id) + .expect("original run") + .status, + RunStatus::Running + ); + assert!( + claim_next_run(store, &lease, &limits, None) + .await + .expect("physical ownership retained") + .is_none() + ); let success = RunChangeRequest { event_id: Uuid::now_v7(), change: RunChange::Finished { @@ -345,6 +438,123 @@ fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_ assert_eq!(nodes[0].status, "success"); assert_eq!(nodes[0].rows_affected, Some(17)); assert_eq!(nodes[1].error_code.as_deref(), Some("E100")); + let late_failure = RunChangeRequest { + event_id: Uuid::now_v7(), + ..launch_failed.clone() + }; + let before_late = read_recall_events(store, 0, 100) + .await + .expect("terminal facts"); + let expiry = TransactionFault { + description: "late launch observation loses leadership before commit", + sqlite_body: "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + postgres_body: "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + ..fault + }; + for (point, fenced) in [(&fault, false), (&expiry, true)] { + install_fault(store, point).await; + let failed = + change_run(store, RunAuthority::Scheduler(&lease), &late_failure).await; + let error = failed.expect_err(point.description); + assert_eq!( + matches!(error, ControlError::Fenced), + fenced, + "{}", + point.description + ); + remove_fault(store, point).await; + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("terminal rollback"), + before_late + ); + } + assert!( + change_run(store, RunAuthority::Scheduler(&lease), &late_failure) + .await + .expect("late observation") + ); + assert!( + !change_run(store, RunAuthority::Scheduler(&lease), &late_failure) + .await + .expect("late retry") + ); + assert!(change_run(store, worker, &late_failure).await.is_err()); + assert!( + change_run(store, RunAuthority::Operator(&operator), &late_failure) + .await + .is_err() + ); + for invalid in [ + RunChangeRequest { + event_id: Uuid::now_v7(), + claim_id: Some(Uuid::now_v7()), + ..late_failure.clone() + }, + RunChangeRequest { + event_id: Uuid::now_v7(), + claim_id: None, + ..late_failure.clone() + }, + RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: Uuid::from_u128(2), + ..late_failure.clone() + }, + RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: Uuid::from_u128(2), + claim_id: Some(Uuid::new_v5(&Uuid::from_u128(2), b"run_claimed")), + ..late_failure.clone() + }, + RunChangeRequest { + event_id: Uuid::now_v7(), + ..started.clone() + }, + RunChangeRequest { + event_id: Uuid::now_v7(), + ..launched.clone() + }, + ] { + assert!( + change_run(store, RunAuthority::Scheduler(&lease), &invalid) + .await + .is_err(), + "terminal authority remains narrow: {invalid:?}" + ); + } + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("unchanged terminal projection") + .into_iter() + .find(|run| run.run_id == started.run_id) + .expect("original run"), + projected + ); + let recorded = read_recall_events(store, 0, 100) + .await + .expect("late fact pair"); + assert_eq!( + recorded + .iter() + .filter(|row| row["event_type"] == "platform_launch_failed") + .count(), + 2 + ); + expire_lease(store).await; + assert!( + matches!( + change_run(store, RunAuthority::Scheduler(&lease), &late_failure).await, + Err(ControlError::Fenced) + ), + "even duplicate observations require a live leader" + ); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect successor") + .expect("successor"); let following = claim_next_run(store, &lease, &limits, None) .await .expect("claim following run") diff --git a/crates/sqlbuild-orchestration/tests/sensors/helpers.rs b/crates/sqlbuild-orchestration/tests/sensors/helpers.rs index 5152689454..3f304e66c7 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/helpers.rs @@ -29,7 +29,7 @@ pub(crate) fn snapshot(location_id: Uuid) -> SnapshotInput { json!({ "health_orders": { "description": "Refresh reporting on overdue ticks", "job_key": "orders", - "when": {"kind": "platform_event", "kinds": ["tick_overdue", "location_broken"]}, + "when": {"kind": "platform_event", "kinds": ["tick_overdue", "location_broken", "launch_failed"]}, "minimum_interval_seconds": 1 } }), @@ -61,9 +61,17 @@ pub(crate) async fn append_events( PlatformEvent::Heartbeat { replica_id: Uuid::from_u128(10), }, + PlatformEvent::LaunchFailed { + run_id: Uuid::from_u128(11), + claim_id: Uuid::from_u128(12), + code: "local_spawn_failed".into(), + }, + PlatformEvent::Heartbeat { + replica_id: Uuid::from_u128(13), + }, ]; for index in indices { - let platform = &payloads[(index % 4) as usize]; + let platform = &payloads[(index % 6) as usize]; let (next, appended) = append_control_event( transaction, ControlEvent { diff --git a/src/sqlbuild/orchestration_declarations/types.py b/src/sqlbuild/orchestration_declarations/types.py index edba630aa1..913d56da06 100644 --- a/src/sqlbuild/orchestration_declarations/types.py +++ b/src/sqlbuild/orchestration_declarations/types.py @@ -4,10 +4,11 @@ class PlatformEventKind(StrEnum): - """Closed vocabulary of durable platform observations.""" + """Launch failures include response errors and do not prove worker absence.""" HEARTBEAT = "heartbeat" SCHEDULER_NO_LEADER = "scheduler_no_leader" TICK_OVERDUE = "tick_overdue" CALENDAR_HORIZON_LOW = "calendar_horizon_low" LOCATION_BROKEN = "location_broken" + LAUNCH_FAILED = "launch_failed" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py index d1ac635fdd..36d2ea7541 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py @@ -3,11 +3,16 @@ import json import os import shutil +import socket import subprocess import sys from collections.abc import Iterator from contextlib import contextmanager +from http.client import HTTPConnection, HTTPResponse +from http.server import BaseHTTPRequestHandler from pathlib import Path +from socketserver import UnixStreamServer +from threading import Event, Thread from uuid import uuid4 import pytest @@ -126,3 +131,64 @@ def worker_container(location: DockerLocation, location_id: str, run_id: str) -> == location.image_id ) return result + + +@contextmanager +def lost_start_response(directory: Path, upstream: Path, inspection: Event) -> Iterator[Path]: + """Fail the real start result and hold physical inspection until explicitly restored.""" + proxy: Path = directory / "docker-proxy.sock" + + class Handler(BaseHTTPRequestHandler): + def forward(self) -> None: + connection: HTTPConnection = HTTPConnection("localhost", timeout=5) + connection.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + connection.sock.settimeout(5) + connection.sock.connect(str(upstream)) + try: + body: bytes = self.rfile.read(int(self.headers.get("Content-Length", "0"))) + connection.request( + self.command, self.path, body, {"Content-Type": "application/json"} + ) + response: HTTPResponse = connection.getresponse() + payload: bytes = response.read() + status: int = response.status + started: bool = self.command == "POST" and self.path.endswith("/start") + unavailable: bool = ( + self.command == "GET" + and self.path.startswith("/containers/sqlbuild-") + and not inspection.is_set() + ) + assert not started or status == 204, ( + "The actual Docker start must succeed before faulting" + ) + status, payload = { + (False, False): (status, payload), + (True, False): (503, b'{"message":"synthetic lost start response"}'), + (False, True): (503, b'{"message":"synthetic inspection outage"}'), + }[(started, unavailable)] + self.send_response(status) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + finally: + connection.close() + + def do_GET(self) -> None: + self.forward() + + def do_POST(self) -> None: + self.forward() + + def log_message(self, format: str, *args: object) -> None: + """Never copy credential-bearing Docker requests to test output.""" + + with UnixStreamServer(str(proxy), Handler) as server: + thread: Thread = Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield proxy + finally: + server.shutdown() + thread.join(timeout=6) + assert not thread.is_alive(), "Docker fault proxy did not stop" + proxy.unlink(missing_ok=True) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py index 8d54cf22ee..05a6f35e07 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py @@ -4,19 +4,26 @@ import sqlite3 import sys import time +from collections.abc import Iterator +from contextlib import contextmanager from pathlib import Path +from threading import Event from typing import Any import pytest +from tests.integration.src.sqlbuild.compiler.compiled_project.publication import helpers from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( ScheduledWorker, + WorkerDeployment, WorkerExecutionTestCase, WorkerRestartTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.docker.helpers import ( REQUIRES_DOCKER, docker_command, + docker_deployment, + lost_start_response, worker_container, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( @@ -250,3 +257,77 @@ def test_given_an_executing_container_when_scheduler_restarts_then_the_same_work assert control.execute( "SELECT count(*) FROM recall_events WHERE event_type='run_claimed'" ).fetchone() == (1,) + + +@REQUIRES_DOCKER +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "worker completes before the lost launch response can be durably observed", + "SELECT 37 AS amount", + "SELECT amount FROM orders", + "success", + [(37,)], + launcher="docker", + store_setup=( + "CREATE TRIGGER reject_launch_observation BEFORE INSERT ON recall_events " + "WHEN NEW.event_type='platform_launch_failed' " + "BEGIN SELECT RAISE(ABORT,'synthetic observation outage'); END;" + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_a_lost_docker_start_response_when_the_worker_finishes_then_late_observation_retries_do_not_revive_it( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + inspection: Event = Event() + + @contextmanager + def deployment(directory: Path, executable: str) -> Iterator[WorkerDeployment]: + with docker_deployment(directory, executable) as actual: + assert actual.docker is not None + with lost_start_response(directory, actual.docker.socket, inspection) as proxy: + yield actual._replace(configuration=actual.configuration | {"socket": str(proxy)}) + + monkeypatch.setattr(helpers, "docker_deployment", deployment) + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + wait_for_log(worker.process, worker.log, "Retrying worker shutdown publication") + run: dict[str, Any] = worker_result(worker, test_case) + assert run["status"] == test_case.expected_status, worker.log.read_text() + with sqlite3.connect(worker.control_store) as control: + before: list[tuple[Any, ...]] = control.execute("SELECT * FROM sqb_runs").fetchall() + assert control.execute( + "SELECT COUNT(*) FROM recall_events WHERE event_type IN ('run_launch_failed','platform_launch_failed')" + ).fetchone() == (0,) + _ = control.execute("DROP TRIGGER reject_launch_observation") + deadline: float = time.monotonic() + 10 + while control.execute( + "SELECT COUNT(*) FROM recall_events WHERE event_type='platform_launch_failed'" + ).fetchone() != (1,): + assert worker.process.poll() is None, worker.log.read_text() + assert time.monotonic() < deadline, worker.log.read_text() + time.sleep(0.02) + # Terminal state alone cannot bypass the still-unavailable physical observation. + time.sleep(1.2) + assert worker.process.poll() is None, worker.log.read_text() + assert control.execute("SELECT * FROM sqb_runs").fetchall() == before + assert control.execute( + "SELECT event_type FROM recall_events WHERE event_type IN " + "('run_terminated','run_launch_failed','platform_launch_failed') ORDER BY storage_id" + ).fetchall() == [ + ("run_terminated",), + ("run_launch_failed",), + ("platform_launch_failed",), + ] + assert control.execute("SELECT COUNT(*) FROM sqb_write_claims").fetchone() == (0,) + assert control.execute("SELECT COUNT(*) FROM sqb_run_liveness").fetchone() == (0,) + inspection.set() + assert worker.process.wait(timeout=5) != 0, "Original response failure remains visible" + assert worker.docker is not None + state: dict[str, object] = worker_container( + worker.docker, worker.api.location_id, run["run_id"] + ) + assert state["Running"] is False + assert state["ExitCode"] == 0 diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py index 1f33e90df2..0812a7a2f5 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py @@ -133,7 +133,7 @@ def test_given_retained_compiler_data_when_publishing_then_http_updates_atomical assert by_name["edge_platform_orders"]["definition_json"]["default_status"] == "stopped" assert by_name["edge_platform_orders"]["definition_json"]["when"] == { "kind": "platform_event", - "kinds": ["tick_overdue", "location_broken"], + "kinds": ["tick_overdue", "location_broken", "launch_failed"], } diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 72f6ded1d2..48ab859069 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -609,7 +609,7 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject "job_key": "edge_snapshot_group", "when": { "kind": "platform_event", - "kinds": ["tick_overdue", "location_broken"], + "kinds": ["tick_overdue", "location_broken", "launch_failed"], }, "minimum_interval_seconds": 7, "default_status": "stopped", From c28d40603c68e7eab0ba2d8e0e9310abd03ad37f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 06:09:36 +0000 Subject: [PATCH 51/59] feat: add scoped job and group outcome sensors Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../orchestration/edge_cases/definitions.py | 18 + .../src/sensors/_helpers/mod.rs | 1 + .../src/sensors/_helpers/platform.rs | 5 +- .../src/sensors/_helpers/runs.rs | 49 +++ .../sensors/main/complete_builtin_sensor.rs | 12 +- .../src/sensors/main/validate_sensors.rs | 11 +- .../src/sensors/models.rs | 46 ++- .../src/store/_helpers/migration.rs | 1 + .../src/store/main/migrate_store.rs | 4 +- .../src/store/run_sensors.sql | 2 + .../tests/api/test_location_work.rs | 4 +- .../tests/sensors/helpers.rs | 109 ++++++ .../tests/sensors/test_contracts.rs | 2 + .../tests/sensors/test_runs.rs | 324 ++++++++++++++++++ .../tests/snapshots/helpers.rs | 3 +- .../_helpers/native_host/constants.rs | 2 + .../native_host/orchestration_declarations.rs | 17 +- .../native_host/orchestration_triggers.rs | 6 +- python-boundary.toml | 2 +- src/sqlbuild/orchestration.py | 7 +- .../orchestration_declarations/models.py | 13 +- .../orchestration_declarations/types.py | 12 + .../compiler/compiled_project/_test_types.py | 4 +- .../compiled_project/publication/helpers.py | 16 +- .../publication/test_publication.py | 21 +- .../publication/test_worker.py | 54 +++ .../test_orchestration_snapshot.py | 69 +++- 27 files changed, 785 insertions(+), 29 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/sensors/_helpers/runs.rs create mode 100644 crates/sqlbuild-orchestration/src/store/run_sensors.sql create mode 100644 crates/sqlbuild-orchestration/tests/sensors/test_runs.rs diff --git a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py index 6762839dd4..8895ffef72 100644 --- a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py +++ b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py @@ -2,9 +2,11 @@ from sqlbuild.orchestration import ( PlatformEventKind, + RunEventKind, job, job_group, platform_event, + run_event, schedule, sensor, ) @@ -73,3 +75,19 @@ default_status="stopped", ) edge_factory_results.append(edge_health) + +edge_job_outcomes = sensor( + "edge_job_outcomes", + description="Keep warning and partial outcomes distinct from success.", + when=run_event(job=edge_orders, kinds=[RunEventKind.WARN, RunEventKind.PARTIAL]), + run=edge_inventory, + default_status="stopped", +) +edge_group_outcomes = sensor( + "edge_group_outcomes", + description="Observe group completion including cancellation and presumed failure.", + when=run_event(job=edge_group, kinds=[RunEventKind.COMPLETED]), + run=edge_inventory, + default_status="stopped", +) +edge_factory_results.extend([edge_job_outcomes, edge_group_outcomes]) diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs index 498b74207a..bd9320484f 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs @@ -2,3 +2,4 @@ pub(crate) mod attempts; pub(crate) mod facts; pub(crate) mod platform; pub(crate) mod requests; +pub(crate) mod runs; diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs index 8215ce4379..208470a995 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs @@ -3,14 +3,16 @@ use uuid::Uuid; use crate::platform::main::read_platform_events::read_platform_events; +use crate::platform::models::PlatformEventKind; use crate::sensors::_helpers::requests::queue_sensor_run; -use crate::sensors::models::{SensorCondition, SensorCursor, SensorEvaluationRequest}; +use crate::sensors::models::{SensorCursor, SensorEvaluationRequest}; use crate::store::errors::ControlError; use crate::store::models::ControlTransaction; pub(crate) async fn consume_platform_events( transaction: ControlTransaction, evaluation: &SensorEvaluationRequest, + kinds: &std::collections::BTreeSet, ) -> Result<(ControlTransaction, (SensorCursor, Vec)), ControlError> { let (mut transaction, events) = read_platform_events(transaction, evaluation.cursor.event_storage_id).await?; @@ -19,7 +21,6 @@ pub(crate) async fn consume_platform_events( .last() .map_or(evaluation.cursor.event_storage_id, |event| event.storage_id), }; - let SensorCondition::PlatformEvent { kinds } = &evaluation.definition.when; let mut run_ids: Vec = Vec::new(); for event in events .iter() diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/runs.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/runs.rs new file mode 100644 index 0000000000..1ac8ccaa38 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/runs.rs @@ -0,0 +1,49 @@ +//! Terminal projections carry the durable finish cursor; late observations cannot move it. + +use std::collections::BTreeSet; + +use uuid::Uuid; + +use crate::sensors::_helpers::requests::queue_sensor_run; +use crate::sensors::models::{ + RunEventKind, SensorCursor, SensorEvaluationRequest, TerminalSensorRun, +}; +use crate::store::errors::ControlError; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn consume_run_events( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, + job_key: &str, + kinds: &BTreeSet, +) -> Result<(ControlTransaction, (SensorCursor, Vec)), ControlError> { + let (mut transaction, rows) = query( + transaction, + "SELECT run_id,status,last_event_storage_id FROM sqb_runs \ + WHERE location_id=$1 AND target=$2 AND job_key=$3 AND last_event_storage_id>$4 \ + AND status IN ('success','warn','partial','failed','cancelled','presumed_failed') \ + ORDER BY last_event_storage_id LIMIT 100", + vec![ + Parameter::Id(evaluation.location_id), + Parameter::Text(evaluation.target.clone()), + Parameter::Text(job_key.into()), + Parameter::Integer(evaluation.cursor.event_storage_id), + ], + ) + .await?; + let mut cursor = evaluation.cursor.clone(); + let mut run_ids: Vec = Vec::new(); + for row in rows { + let run: TerminalSensorRun = decode_row(row)?; + cursor.event_storage_id = run.last_event_storage_id; + if kinds.iter().any(|kind| kind.matches(run.status)) { + let run_id; + (transaction, run_id) = + queue_sensor_run(transaction, evaluation, &run.run_id.to_string()).await?; + run_ids.push(run_id); + } + } + Ok((transaction, (cursor, run_ids))) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs b/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs index 5fcf9286c3..fc35a312c2 100644 --- a/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs +++ b/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs @@ -8,8 +8,9 @@ use crate::scheduler::models::SchedulerLease; use crate::sensors::_helpers::attempts::{defer_sensor, last_attempt, read_sensor}; use crate::sensors::_helpers::facts::{append_evaluation, commit_cursor}; use crate::sensors::_helpers::platform::consume_platform_events; +use crate::sensors::_helpers::runs::consume_run_events; use crate::sensors::main::read_sensor_cursor::read_sensor_cursor; -use crate::sensors::models::{SensorEvaluationOutcome, SensorEvaluationRequest}; +use crate::sensors::models::{SensorCondition, SensorEvaluationOutcome, SensorEvaluationRequest}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; @@ -46,7 +47,14 @@ pub async fn complete_builtin_sensor( commit(transaction).await?; return Ok(0); } - let (transaction, (cursor, run_ids)) = consume_platform_events(transaction, evaluation).await?; + let (transaction, (cursor, run_ids)) = match &evaluation.definition.when { + SensorCondition::PlatformEvent { kinds } => { + consume_platform_events(transaction, evaluation, kinds).await? + } + SensorCondition::RunEvent { job_key, kinds } => { + consume_run_events(transaction, evaluation, job_key, kinds).await? + } + }; let count = run_ids.len(); let (transaction, (event_id, storage_id)) = append_evaluation( transaction, diff --git a/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs b/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs index 06cd779823..60aee8f0ac 100644 --- a/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs +++ b/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs @@ -6,10 +6,17 @@ use crate::store::errors::ControlError; pub(crate) fn validate_sensors(definitions: &Definitions) -> Result<(), ControlError> { for (name, sensor) in &definitions.sensors { - let SensorCondition::PlatformEvent { kinds } = &sensor.when; + let valid_condition = match &sensor.when { + SensorCondition::PlatformEvent { kinds } => !kinds.is_empty(), + SensorCondition::RunEvent { job_key, kinds } => { + !kinds.is_empty() + && (definitions.jobs.contains_key(job_key) + || definitions.groups.contains_key(job_key)) + } + }; if name.trim().is_empty() || sensor.description.trim().is_empty() - || kinds.is_empty() + || !valid_condition || (!definitions.jobs.contains_key(&sensor.job_key) && !definitions.groups.contains_key(&sensor.job_key)) { diff --git a/crates/sqlbuild-orchestration/src/sensors/models.rs b/crates/sqlbuild-orchestration/src/sensors/models.rs index f39cf43d2f..9dff71a198 100644 --- a/crates/sqlbuild-orchestration/src/sensors/models.rs +++ b/crates/sqlbuild-orchestration/src/sensors/models.rs @@ -9,6 +9,7 @@ use uuid::Uuid; use crate::declarations::models::TriggerStatus; use crate::platform::models::PlatformEventKind; +use crate::runs::models::RunOutcome; #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(rename_all = "snake_case")] @@ -31,7 +32,50 @@ impl SensorStatus { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum SensorCondition { - PlatformEvent { kinds: BTreeSet }, + PlatformEvent { + kinds: BTreeSet, + }, + RunEvent { + job_key: String, + kinds: BTreeSet, + }, +} + +#[derive( + Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, utoipa::ToSchema, +)] +#[serde(rename_all = "snake_case")] +pub enum RunEventKind { + Completed, + Success, + Warn, + Partial, + Failed, + Cancelled, + PresumedFailed, +} + +impl RunEventKind { + #[must_use] + pub const fn matches(self, outcome: RunOutcome) -> bool { + matches!( + (self, outcome), + (Self::Completed, _) + | (Self::Success, RunOutcome::Success) + | (Self::Warn, RunOutcome::Warn) + | (Self::Partial, RunOutcome::Partial) + | (Self::Failed, RunOutcome::Failed) + | (Self::Cancelled, RunOutcome::Cancelled) + | (Self::PresumedFailed, RunOutcome::PresumedFailed) + ) + } +} + +#[derive(Debug, Deserialize)] +pub(crate) struct TerminalSensorRun { + pub(crate) run_id: Uuid, + pub(crate) status: RunOutcome, + pub(crate) last_event_storage_id: i64, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index 5ae6a26780..560317b6ac 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -11,6 +11,7 @@ pub(crate) const LOCATION_INSTANCES: &str = include_str!("../location_instances. pub(crate) const LOCATION_WORK: &str = include_str!("../location_work.sql"); pub(crate) const PLATFORM_SINK: &str = include_str!("../platform_sink.sql"); pub(crate) const SENSORS: &str = include_str!("../sensors.sql"); +pub(crate) const RUN_SENSORS: &str = include_str!("../run_sensors.sql"); pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index dcc49f4793..59337ca99f 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -4,7 +4,8 @@ use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; use crate::store::_helpers::migration::{ - LOCATION_INSTANCES, LOCATION_WORK, PLATFORM_SINK, SCHEMA, SENSORS, render, verify_migrations, + LOCATION_INSTANCES, LOCATION_WORK, PLATFORM_SINK, RUN_SENSORS, SCHEMA, SENSORS, render, + verify_migrations, }; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; @@ -32,6 +33,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { ("location_work", LOCATION_WORK), ("platform_sink", PLATFORM_SINK), ("sensors", SENSORS), + ("run_sensors", RUN_SENSORS), ]; let mut digests = vec![digest.clone()]; digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); diff --git a/crates/sqlbuild-orchestration/src/store/run_sensors.sql b/crates/sqlbuild-orchestration/src/store/run_sensors.sql new file mode 100644 index 0000000000..de46a16b7b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/run_sensors.sql @@ -0,0 +1,2 @@ +CREATE INDEX sqb_runs_terminal_cursor ON sqb_runs(location_id, target, job_key, last_event_storage_id) +WHERE status IN ('success', 'warn', 'partial', 'failed', 'cancelled', 'presumed_failed'); diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs index 4af6d90bef..19aa6d5e83 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs @@ -46,9 +46,9 @@ fn given_outbound_work_when_claimed_retried_or_replaced_then_scope_and_immutable let store = &fixture.store; migrate_store(store).await.expect("fresh migration"); // Exercise both deployed predecessors, including checksums and repeated startup. - execute(store, "DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DELETE FROM sqb_schema_migrations WHERE version>=3").await; + execute(store, "DROP INDEX sqb_runs_terminal_cursor; DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DELETE FROM sqb_schema_migrations WHERE version>=3").await; migrate_store(store).await.expect("schema 2 upgrade"); - execute(store, "DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2").await; + execute(store, "DROP INDEX sqb_runs_terminal_cursor; DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2").await; migrate_store(store).await.expect("schema 1 upgrade"); migrate_store(store).await.expect("repeat migration"); let location_id = create_location(store, "orders", "test-admin") diff --git a/crates/sqlbuild-orchestration/tests/sensors/helpers.rs b/crates/sqlbuild-orchestration/tests/sensors/helpers.rs index 3f304e66c7..703b73d8e9 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/helpers.rs @@ -6,6 +6,14 @@ use crate::credentials::models::AuditContext; use crate::declarations::main::change_trigger::change_trigger; use crate::declarations::models::{TriggerChange, TriggerChangeRequest}; use crate::platform::models::PlatformEvent; +use crate::runs::main::{ + advance_groups::advance_groups, change_run::change_run, claim_next_run::claim_next_run, + request_job_run::request_job_run, +}; +use crate::runs::models::{ + JobRunRequest, NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, +}; +use crate::scheduler::models::SchedulerLease; use crate::sensors::models::SensorEvaluationFact; use crate::snapshots::models::SnapshotInput; use crate::store::main::append_control_event::append_control_event; @@ -128,3 +136,104 @@ pub(crate) async fn change(store: &ControlStore, trigger_id: Uuid, change: Trigg .expect("retry operator fact") ); } + +/// Real request, claim, node facts and finish, including the virtual group parent. +pub(crate) async fn finish_run( + store: &ControlStore, + lease: &SchedulerLease, + scope: (Uuid, &str), + job_key: &str, + outcome: RunOutcome, + nodes: [NodeOutcome; 2], +) -> Uuid { + let run_id = Uuid::now_v7(); + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + request_job_run( + store, + &context, + &JobRunRequest { + run_id, + location_id: scope.0, + target: scope.1.into(), + job_key: job_key.into(), + reason: "Exercise terminal sensor scope".into(), + priority: 10, + window: None, + }, + ) + .await + .expect("request source run"); + let limits = std::collections::BTreeMap::new(); + advance_groups(store, lease, (&limits, scope)) + .await + .expect("advance parent"); + let claim = claim_next_run(store, lease, &limits, Some(scope)) + .await + .expect("claim source") + .expect("source ready"); + assert!(claim.request.run_id == run_id || claim.request.parent_run_id == Some(run_id)); + let worker = RunAuthority::Worker { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }; + let changes = [ + RunChange::Started, + RunChange::NodeFinished { + node_key: "customers".into(), + outcome: nodes[0], + rows_affected: None, + error_code: None, + skipped_because_of: None, + }, + RunChange::NodeFinished { + node_key: "orders".into(), + outcome: nodes[1], + rows_affected: None, + error_code: None, + skipped_because_of: None, + }, + ]; + for change in changes { + change_run( + store, + worker, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change, + }, + ) + .await + .expect("publish worker evidence"); + } + let terminal = RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change: RunChange::Finished { + outcome, + reason: "Terminal outcome fixture".into(), + exit_code: None, + }, + }; + assert!( + change_run(store, RunAuthority::Scheduler(lease), &terminal) + .await + .expect("finish") + ); + assert!( + !change_run(store, RunAuthority::Scheduler(lease), &terminal) + .await + .expect("retry finish") + ); + advance_groups(store, lease, (&limits, scope)) + .await + .expect("complete parent"); + run_id +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs b/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs index 04f4485f1e..b2e64838ac 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs @@ -6,5 +6,7 @@ mod helpers; mod test_delivery; #[path = "test_recovery.rs"] mod test_recovery; +#[path = "test_runs.rs"] +mod test_runs; #[path = "test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs new file mode 100644 index 0000000000..00cdef84b8 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs @@ -0,0 +1,324 @@ +//! Actual lifecycle facts feed scoped outcome selectors and atomic recovery on both stores. + +use std::collections::{BTreeMap, BTreeSet}; +use std::time::Duration; + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::declarations::models::TriggerChange; +use crate::runs::main::list_runs::list_runs; +use crate::runs::models::{NodeOutcome, RunOutcome}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::sensors::main::{ + begin_sensor_evaluation::begin_sensor_evaluation, + complete_builtin_sensor::complete_builtin_sensor, +}; +use crate::sensors::tests::helpers::{change, finish_run, snapshot}; +use crate::sensors::tests::test_types::SensorTestCase; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::{Parameter, TransactionMode}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_terminal_jobs_and_groups_when_selectors_retry_then_outcomes_scope_and_run_keys_are_preserved() + { + let test_cases = [ + SensorTestCase { + description: "SQLite terminal outcome sensor recovery", + backend: Backend::Sqlite, + expected_runs: 16, + }, + SensorTestCase { + description: "PostgreSQL terminal outcome sensor recovery", + backend: Backend::Postgres, + expected_runs: 16, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required backend"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + migrate_store(store) + .await + .expect("warm migration preserves index"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "products", "test-admin") + .await + .expect("other location"); + let mut input = snapshot(location); + input.documents.insert("jobs".into(), json!({ + "orders": {"description": "Source orders", "selection": ["orders", "customers"], "overlap": "allow"}, + "inventory": {"description": "Sensor destination", "selection": ["orders", "customers"], "overlap": "allow"} + })); + input.documents.insert("groups".into(), json!({ + "daily_orders": {"description": "Source group", "selection": ["orders", "customers"], "jobs": {"orders": []}, "overlap": "allow"} + })); + let selectors = [ + "completed", + "success", + "warn", + "partial", + "failed", + "cancelled", + "presumed_failed", + ]; + let mut sensors = serde_json::Map::new(); + for job_key in ["orders", "daily_orders"] { + for selector in selectors { + sensors.insert(format!("{job_key}_{selector}"), json!({ + "description": "Consume exact terminal outcomes", "job_key": "inventory", + "when": {"kind": "run_event", "job_key": job_key, "kinds": [selector]}, + "minimum_interval_seconds": 1 + })); + } + } + input.documents.insert("triggers".into(), json!(sensors)); + publish_snapshot(store, &input) + .await + .expect("publish sensors"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let mut expected: BTreeMap> = BTreeMap::new(); + let outcomes = [ + ( + "success", + RunOutcome::Success, + [NodeOutcome::Success, NodeOutcome::Success], + ), + ( + "warn", + RunOutcome::Warn, + [NodeOutcome::Warn, NodeOutcome::Success], + ), + ( + "partial", + RunOutcome::Partial, + [NodeOutcome::Success, NodeOutcome::Failed], + ), + ( + "failed", + RunOutcome::Failed, + [NodeOutcome::Failed, NodeOutcome::Failed], + ), + ( + "cancelled", + RunOutcome::Cancelled, + [NodeOutcome::Cancelled, NodeOutcome::Cancelled], + ), + ( + "presumed_failed", + RunOutcome::PresumedFailed, + [NodeOutcome::Failed, NodeOutcome::Failed], + ), + ]; + for (selector, outcome, nodes) in outcomes { + let id = finish_run( + store, + &lease, + (location, "development"), + "orders", + outcome, + nodes, + ) + .await + .to_string(); + expected + .entry(format!("orders_{selector}")) + .or_default() + .insert(id.clone()); + expected + .entry("orders_completed".into()) + .or_default() + .insert(id); + } + let group = finish_run( + store, + &lease, + (location, "development"), + "daily_orders", + RunOutcome::Success, + [NodeOutcome::Success; 2], + ) + .await; + let sources = list_runs(store, (location, "development"), 100) + .await + .expect("sources"); + let child = sources + .iter() + .find(|record| { + record.run_id != group + && record.job_key.as_deref() == Some("orders") + && !expected["orders_completed"].contains(&record.run_id.to_string()) + }) + .expect("group child"); + for selector in ["completed", "success"] { + expected + .entry(format!("daily_orders_{selector}")) + .or_default() + .insert(group.to_string()); + expected + .entry(format!("orders_{selector}")) + .or_default() + .insert(child.run_id.to_string()); + } + // Same key in another target/location and a different job in this scope must not match. + let mut other_input = input.clone(); + other_input.location_id = other; + publish_snapshot(store, &other_input) + .await + .expect("other publication"); + other_input.location_id = location; + other_input.target = "preview".into(); + publish_snapshot(store, &other_input) + .await + .expect("other target"); + for scope in [(other, "development"), (location, "preview")] { + finish_run( + store, + &lease, + scope, + "orders", + RunOutcome::Success, + [NodeOutcome::Success; 2], + ) + .await; + } + finish_run( + store, + &lease, + (location, "development"), + "inventory", + RunOutcome::Success, + [NodeOutcome::Success; 2], + ) + .await; + // Recreate the exact pre-change index/migration state with populated run history. + let transaction = begin(store, TransactionMode::Write) + .await + .expect("upgrade fixture"); + let (transaction, _) = + query(transaction, "DROP INDEX sqb_runs_terminal_cursor", vec![]) + .await + .expect("remove new index"); + let (transaction, _) = query( + transaction, + "DELETE FROM sqb_schema_migrations WHERE name='run_sensors'", + vec![], + ) + .await + .expect("remove new migration record"); + commit(transaction).await.expect("pre-change schema ready"); + migrate_store(store) + .await + .expect("upgrade populated prior schema"); + let replica = second_store(store).await; + let fault = TransactionFault { + description: "fail atomic sensor cursor", + table: "sqb_sensor_cursor_commits", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'injected sensor failure')", + postgres_body: "RAISE EXCEPTION 'injected sensor failure'", + }; + let triggers = list_triggers(store, location, "development") + .await + .expect("triggers"); + for trigger in &triggers { + let requested = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("request") + .expect("due"); + assert_eq!( + begin_sensor_evaluation(&replica, &lease, trigger.trigger_id) + .await + .expect("recover"), + Some(requested.clone()) + ); + install_fault(store, &fault).await; + assert!( + complete_builtin_sensor(store, &lease, &requested) + .await + .is_err() + ); + remove_fault(store, &fault).await; + let wanted = expected.get(&trigger.name).cloned().unwrap_or_default(); + assert_eq!( + complete_builtin_sensor(&replica, &lease, &requested) + .await + .expect("retry"), + wanted.len(), + "{}", + trigger.name + ); + assert_eq!( + complete_builtin_sensor(store, &lease, &requested) + .await + .expect("duplicate completion"), + 0 + ); + let transaction = begin(store, TransactionMode::Read) + .await + .expect("read requests"); + let (transaction, rows) = query( + transaction, + "SELECT run_key FROM sqb_run_requests WHERE trigger_id=$1", + vec![Parameter::Id(trigger.trigger_id)], + ) + .await + .expect("keys"); + commit(transaction).await.expect("read commit"); + let actual: BTreeSet = rows + .iter() + .map(|row| row["run_key"].as_str().expect("run key").into()) + .collect(); + assert_eq!(actual, wanted, "{}", trigger.name); + change(store, trigger.trigger_id, TriggerChange::ResetCursor).await; + } + tokio::time::sleep(Duration::from_secs(1)).await; + for trigger in &triggers { + let replay = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("reset request") + .expect("due"); + complete_builtin_sensor(store, &lease, &replay) + .await + .expect("replay deduplicates"); + } + let after = list_runs(store, (location, "development"), 100) + .await + .expect("queued sensors"); + assert_eq!( + after.len(), + sources.len() + 1 + test_case.expected_runs, + "{}: reset adds no runs", + test_case.description + ); + for source in sources { + assert_eq!( + after.iter().find(|record| record.run_id == source.run_id), + Some(&source), + "sensor never changes source state" + ); + } + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index 5507104d12..2996933768 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -243,7 +243,8 @@ pub(super) async fn sqlite_failed_publish_and_schema_floor() -> i64 { let connection = rusqlite::Connection::open(&path).expect("open fault injection connection"); connection .execute_batch( - "DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; \ + "DROP INDEX sqb_runs_terminal_cursor; \ + DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; \ DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; \ DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; \ DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs index 5c0e4cbe43..b7e704b8c0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs @@ -2,3 +2,5 @@ /// The calendar's evaluation window is supplied by the host, not a provider. pub(crate) const CALENDAR_WINDOW_PARAMETER: &str = "window"; + +pub(crate) const RUN_EVENT_CONDITION: &str = "run_event"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs index 24a1f9f20e..0d9a84cfd8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs @@ -10,6 +10,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; use serde_json::Value; use sqlbuild_host::entries::models::UserCodeHost; +use crate::bindings::_helpers::native_host::constants::RUN_EVENT_CONDITION; use crate::bindings::_helpers::native_host::host::entry; #[derive(FromPyObject)] @@ -47,16 +48,18 @@ pub(crate) struct AuthoredGroup { } #[derive(FromPyObject)] -pub(crate) struct AuthoredPlatformCondition { +pub(crate) struct AuthoredSensorCondition { pub(crate) kind: String, pub(crate) kinds: Vec, + #[pyo3(default)] + pub(crate) job: Option, } #[derive(FromPyObject)] pub(crate) struct AuthoredSensor { pub(crate) name: String, pub(crate) description: String, - pub(crate) when: AuthoredPlatformCondition, + pub(crate) when: AuthoredSensorCondition, pub(crate) run: AuthoredJobReference, #[pyo3(from_py_with = authored_setting)] pub(crate) minimum_interval_seconds: Value, @@ -155,6 +158,16 @@ pub(crate) fn collect_declarations<'py>( )); } pending.push(target); + let condition = declaration.getattr("when")?; + if condition.getattr("kind")?.extract::()? == RUN_EVENT_CONDITION { + let source = condition.getattr("job")?; + if !source.is_instance(&job)? && !source.is_instance(&group)? { + return Err(PyTypeError::new_err( + "run_event job must be a Job or JobGroup object", + )); + } + pending.push(source); + } AuthoredDeclaration::Sensor(declaration.extract()?) } else if declaration.is_instance(&calendar)? { let target = declaration.getattr("job")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs index b787901fc2..0ce49660fa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -263,10 +263,14 @@ pub(crate) fn compile_sensors( if !sensor.targets.is_empty() && !sensor.targets.iter().any(|value| value == target) { continue; } + let mut condition = json!({"kind": sensor.when.kind, "kinds": sensor.when.kinds}); + if let Some(job) = sensor.when.job { + condition["job_key"] = json!(job.name); + } let definition: SensorDefinition = serde_json::from_value(json!({ "description": sensor.description, "job_key": sensor.run.name, - "when": {"kind": sensor.when.kind, "kinds": sensor.when.kinds}, + "when": condition, "minimum_interval_seconds": sensor.minimum_interval_seconds, "default_status": sensor.default_status, })) diff --git a/python-boundary.toml b/python-boundary.toml index c5d7d33fc7..b50c3c0fc6 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -25,7 +25,7 @@ note = "Public job and schedule authoring records only; selection resolution, va [[stays]] path = "orchestration_declarations/types.py" rule = 3 -note = "Public PlatformEventKind enum only, placed in types.py to satisfy enum ownership; no Python event processing or implementation allowance." +note = "Public PlatformEventKind and RunEventKind enums only, placed in types.py to satisfy enum ownership while retaining a zero implementation budget; no Python event processing allowance." [[stays]] path = "orchestration_declarations/main/compile_snapshot.py" diff --git a/src/sqlbuild/orchestration.py b/src/sqlbuild/orchestration.py index 0920f49d17..38fec4927a 100644 --- a/src/sqlbuild/orchestration.py +++ b/src/sqlbuild/orchestration.py @@ -7,6 +7,7 @@ Job, JobGroup, PlatformEventCondition, + RunEventCondition, Schedule, Sensor, ) @@ -14,9 +15,10 @@ from sqlbuild.orchestration_declarations.models import Job as job from sqlbuild.orchestration_declarations.models import JobGroup as job_group from sqlbuild.orchestration_declarations.models import PlatformEventCondition as platform_event +from sqlbuild.orchestration_declarations.models import RunEventCondition as run_event from sqlbuild.orchestration_declarations.models import Schedule as schedule from sqlbuild.orchestration_declarations.models import Sensor as sensor -from sqlbuild.orchestration_declarations.types import PlatformEventKind +from sqlbuild.orchestration_declarations.types import PlatformEventKind, RunEventKind __all__ = ( "CalendarSchedule", @@ -25,6 +27,8 @@ "JobGroup", "PlatformEventCondition", "PlatformEventKind", + "RunEventCondition", + "RunEventKind", "Schedule", "Sensor", "calendar_schedule", @@ -32,6 +36,7 @@ "job", "job_group", "platform_event", + "run_event", "schedule", "sensor", ) diff --git a/src/sqlbuild/orchestration_declarations/models.py b/src/sqlbuild/orchestration_declarations/models.py index 8086e34bf6..97a4106dd0 100644 --- a/src/sqlbuild/orchestration_declarations/models.py +++ b/src/sqlbuild/orchestration_declarations/models.py @@ -7,7 +7,7 @@ from datetime import datetime from typing import Literal -from sqlbuild.orchestration_declarations.types import PlatformEventKind +from sqlbuild.orchestration_declarations.types import PlatformEventKind, RunEventKind @dataclass(frozen=True, kw_only=True) @@ -90,13 +90,22 @@ class PlatformEventCondition: kind: Literal["platform_event"] = field(default="platform_event", init=False) +@dataclass(frozen=True, kw_only=True) +class RunEventCondition: + """Match a job/group's terminal outcomes in the sensor's location and target.""" + + job: Job | JobGroup + kinds: Sequence[RunEventKind] + kind: Literal["run_event"] = field(default="run_event", init=False) + + @dataclass(frozen=True, eq=False) class Sensor: """Start a job for each matching event; event identity deduplicates replayed requests.""" name: str description: str = field(kw_only=True) - when: PlatformEventCondition = field(kw_only=True) + when: PlatformEventCondition | RunEventCondition = field(kw_only=True) run: Job | JobGroup = field(kw_only=True) minimum_interval_seconds: int = field(default=30, kw_only=True) default_status: Literal["running", "stopped"] = field(default="running", kw_only=True) diff --git a/src/sqlbuild/orchestration_declarations/types.py b/src/sqlbuild/orchestration_declarations/types.py index 913d56da06..ac8ef3e001 100644 --- a/src/sqlbuild/orchestration_declarations/types.py +++ b/src/sqlbuild/orchestration_declarations/types.py @@ -12,3 +12,15 @@ class PlatformEventKind(StrEnum): CALENDAR_HORIZON_LOW = "calendar_horizon_low" LOCATION_BROKEN = "location_broken" LAUNCH_FAILED = "launch_failed" + + +class RunEventKind(StrEnum): + """COMPLETED selects every terminal outcome; it is not a run status.""" + + COMPLETED = "completed" + SUCCESS = "success" + WARN = "warn" + PARTIAL = "partial" + FAILED = "failed" + CANCELLED = "cancelled" + PRESUMED_FAILED = "presumed_failed" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index 87a99a43a4..c3169eea69 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -4,7 +4,7 @@ from dataclasses import dataclass, field -from sqlbuild.orchestration import Schedule +from sqlbuild.orchestration import CalendarSchedule, Job, JobGroup, Schedule, Sensor @dataclass(frozen=True) @@ -251,7 +251,7 @@ class SnapshotDiscoveryTestCase: description: str files: dict[str, str] expected_jobs: tuple[str, ...] - declarations: tuple[()] | None = None + declarations: tuple[Job | JobGroup | CalendarSchedule | Sensor, ...] | None = None expected_groups: dict[str, dict[str, list[str]]] = field(default_factory=dict) expected_sensors: dict[str, dict[str, object]] = field(default_factory=dict) expected_schedules: tuple[str, ...] = () diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index ae3edf17d5..f1f32da930 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -16,7 +16,7 @@ from datetime import UTC, datetime, timedelta from io import StringIO from pathlib import Path -from typing import Any +from typing import Any, cast from urllib.error import HTTPError from urllib.parse import quote from urllib.request import Request, urlopen @@ -387,10 +387,16 @@ def worker_result(worker: ScheduledWorker, test_case: WorkerExecutionTestCase) - "SELECT external_context FROM recall_invocations WHERE command = 'location'" ).fetchall() assert contexts, log.read_text() - assert all( - context is not None and json.loads(context)["run_id"] == run["run_id"] - for (context,) in contexts - ) + assert all(context is not None for (context,) in contexts) + expected_run_ids: set[str] = { + run_id + for (run_id,) in control.execute( + "SELECT run_id FROM sqb_runs WHERE kind<>'group' AND ended_at IS NOT NULL" + ).fetchall() + } + assert { + json.loads(cast(str, context))["run_id"] for (context,) in contexts + } == expected_run_ids assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) with duckdb.connect(str(project / "warehouse.duckdb"), read_only=True) as warehouse: assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py index 0812a7a2f5..c48b98bf0c 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py @@ -122,8 +122,13 @@ def test_given_retained_compiler_data_when_publishing_then_http_updates_atomical with urlopen(request, timeout=5) as response: triggers: list[dict[str, Any]] = json.load(response) by_name: dict[str, dict[str, Any]] = {trigger["name"]: trigger for trigger in triggers} - assert len(triggers) == 2 - assert set(by_name) == {"edge_snapshot_orders", "edge_platform_orders"} + assert len(triggers) == 4 + assert set(by_name) == { + "edge_snapshot_orders", + "edge_platform_orders", + "edge_job_outcomes", + "edge_group_outcomes", + } scheduled: dict[str, Any] = by_name["edge_snapshot_orders"] assert scheduled["status"] == "paused" assert scheduled["definition_json"]["starts_at"] == "2026-07-02T19:47:23.456789Z" @@ -135,6 +140,18 @@ def test_given_retained_compiler_data_when_publishing_then_http_updates_atomical "kind": "platform_event", "kinds": ["tick_overdue", "location_broken", "launch_failed"], } + assert by_name["edge_job_outcomes"]["definition_json"]["when"] == { + "kind": "run_event", + "job_key": "edge_snapshot_orders", + "kinds": ["warn", "partial"], + } + assert by_name["edge_group_outcomes"]["definition_json"]["when"] == { + "kind": "run_event", + "job_key": "edge_snapshot_group", + "kinds": ["completed"], + } + assert by_name["edge_job_outcomes"]["status"] == "paused" + assert by_name["edge_group_outcomes"]["status"] == "paused" @pytest.mark.parametrize( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index edae5b6127..8f46037730 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -487,6 +487,60 @@ def test_given_a_scheduled_job_when_published_then_an_isolated_worker_reports_re assert run["status"] == test_case.expected_status, worker.log.read_text() +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "group completion starts a real sensor worker", + "SELECT 59 AS amount", + "SELECT amount FROM orders", + "success", + [(59,)], + expected_origin="sensor", + files={ + "orchestration/orders.py": ( + "from sqlbuild.orchestration import RunEventKind, job, job_group, run_event, schedule, sensor\n" + "source = job('source_orders', select='orders', description='Source orders.')\n" + "source_group = job_group('source_group', description='Source group.', jobs=[source], " + "schedule=schedule(cron='0 0 1 1 *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" + "annual_orders = job('annual_orders', select='orders', description='Refresh orders.', settings={'reload': True})\n" + "on_group = sensor('on_group', description='Refresh after group completion.', " + "when=run_event(job=source_group, kinds=[RunEventKind.COMPLETED, RunEventKind.SUCCESS]), " + "run=annual_orders, minimum_interval_seconds=1)\n" + ) + }, + ) + ], + ids=lambda case: case.description, +) +def test_given_a_real_group_completion_when_a_run_sensor_matches_then_one_destination_worker_executes( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + destination: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_orders" + ) + assert destination["status"] == test_case.expected_status, worker.log.read_text() + result: dict[str, Any] = worker_result(worker, test_case) + assert result["run_id"] == destination["run_id"] + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT run_key,run_id FROM sqb_run_requests").fetchall() == [ + ( + control.execute( + "SELECT run_id FROM sqb_runs WHERE job_key='source_group'" + ).fetchone()[0], + destination["run_id"], + ) + ] + assert control.execute( + "SELECT job_key,status FROM sqb_runs ORDER BY job_key" + ).fetchall() == [ + ("annual_orders", "success"), + ("source_group", "success"), + ("source_orders", "success"), + ] + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 48ab859069..43cb006a6b 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -18,7 +18,16 @@ ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.orchestration import Job, compile_snapshot, job, schedule +from sqlbuild.orchestration import ( + Job, + RunEventKind, + compile_snapshot, + job, + job_group, + run_event, + schedule, + sensor, +) from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( SnapshotCatalogTestCase, SnapshotDeclarationRejectionTestCase, @@ -613,10 +622,66 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject }, "minimum_interval_seconds": 7, "default_status": "stopped", - } + }, + "edge_job_outcomes": { + "description": "Keep warning and partial outcomes distinct from success.", + "job_key": "edge_snapshot_inventory", + "when": { + "kind": "run_event", + "job_key": "edge_snapshot_orders", + "kinds": ["warn", "partial"], + }, + "minimum_interval_seconds": 30, + "default_status": "stopped", + }, + "edge_group_outcomes": { + "description": "Observe group completion including cancellation and presumed failure.", + "job_key": "edge_snapshot_inventory", + "when": { + "kind": "run_event", + "job_key": "edge_snapshot_group", + "kinds": ["completed"], + }, + "minimum_interval_seconds": 30, + "default_status": "stopped", + }, }, expected_schedules=("edge_snapshot_orders",), ), + SnapshotDiscoveryTestCase( + description="sensor-only roots collect source groups and destination jobs natively", + files={}, + declarations=( + sensor( + "on_orders", + description="Build inventory after group completion.", + when=run_event( + job=job_group( + "source_group", + description="Source orders.", + jobs=[job("source_orders", description="Orders.", select="orders")], + ), + kinds=[RunEventKind.COMPLETED], + ), + run=job("inventory", description="Inventory.", select="stg_orders"), + ), + ), + expected_jobs=("inventory", "source_orders"), + expected_groups={"source_group": {"source_orders": []}}, + expected_sensors={ + "on_orders": { + "description": "Build inventory after group completion.", + "job_key": "inventory", + "when": { + "kind": "run_event", + "job_key": "source_group", + "kinds": ["completed"], + }, + "minimum_interval_seconds": 30, + "default_status": "running", + } + }, + ), SnapshotDiscoveryTestCase( description="sensor-only roots preserve referenced jobs and filter targets", files={ From f00f20c6902e9b77b0d2d8870754cd6e5487251f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 06:37:23 +0000 Subject: [PATCH 52/59] feat: require successful executed nodes in run sensors --- .../edge_cases/orchestration/run_orders.py | 16 ++ .../orchestration/edge_cases/definitions.py | 6 +- .../src/sensors/_helpers/runs.rs | 56 ++++- .../src/sensors/constants.rs | 3 + .../sensors/main/complete_builtin_sensor.rs | 14 +- .../src/sensors/main/validate_sensors.rs | 19 +- .../sqlbuild-orchestration/src/sensors/mod.rs | 1 + .../src/sensors/models.rs | 2 + .../src/store/_helpers/migration.rs | 1 + .../src/store/executed_nodes.sql | 1 + .../src/store/main/migrate_store.rs | 5 +- .../tests/api/test_location_work.rs | 4 +- .../tests/sensors/helpers.rs | 44 ++-- .../tests/sensors/test_contracts.rs | 2 + .../tests/sensors/test_execution_bounds.rs | 176 ++++++++++++++ .../tests/sensors/test_runs.rs | 229 +++++++++++++++++- .../tests/sensors/test_types.rs | 10 + .../tests/snapshots/helpers.rs | 2 +- .../compiled_project/snapshot_declarations.rs | 1 + .../native_host/orchestration_declarations.rs | 2 + .../native_host/orchestration_triggers.rs | 16 ++ .../orchestration_declarations/models.py | 3 +- .../publication/test_calendar.py | 5 +- .../publication/test_publication.py | 1 + .../publication/test_worker.py | 2 +- .../test_orchestration_snapshot.py | 98 ++++++++ 26 files changed, 676 insertions(+), 43 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/run_orders.py create mode 100644 crates/sqlbuild-orchestration/src/sensors/constants.rs create mode 100644 crates/sqlbuild-orchestration/src/store/executed_nodes.sql create mode 100644 crates/sqlbuild-orchestration/tests/sensors/test_execution_bounds.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/run_orders.py b/benchmarks/duckshop/edge_cases/orchestration/run_orders.py new file mode 100644 index 0000000000..bf6d376187 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/run_orders.py @@ -0,0 +1,16 @@ +"""Freeze source-job selectors without executing user code in the scheduler.""" + +from sqlbuild.orchestration import RunEventKind, job, run_event, sensor + +source_orders = job("source_orders", description="Source orders.", select="orders") +inventory = job("inventory", description="Destination inventory.", select="stg_orders") +on_orders = sensor( + "on_orders", + description="Build inventory only after orders actually succeeds.", + when=run_event( + job=source_orders, + kinds=[RunEventKind.COMPLETED], + executed_successfully=("orders",), + ), + run=inventory, +) diff --git a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py index 8895ffef72..78b726a288 100644 --- a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py +++ b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py @@ -86,7 +86,11 @@ edge_group_outcomes = sensor( "edge_group_outcomes", description="Observe group completion including cancellation and presumed failure.", - when=run_event(job=edge_group, kinds=[RunEventKind.COMPLETED]), + when=run_event( + job=edge_group, + kinds=[RunEventKind.COMPLETED], + executed_successfully=("tag:edge_case",), + ), run=edge_inventory, default_status="stopped", ) diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/runs.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/runs.rs index 1ac8ccaa38..32b7e00e3c 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/runs.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/runs.rs @@ -5,6 +5,7 @@ use std::collections::BTreeSet; use uuid::Uuid; use crate::sensors::_helpers::requests::queue_sensor_run; +use crate::sensors::constants::EXECUTION_NODE_BATCH_SIZE; use crate::sensors::models::{ RunEventKind, SensorCursor, SensorEvaluationRequest, TerminalSensorRun, }; @@ -16,8 +17,8 @@ use crate::store::models::{ControlTransaction, Parameter}; pub(crate) async fn consume_run_events( transaction: ControlTransaction, evaluation: &SensorEvaluationRequest, - job_key: &str, - kinds: &BTreeSet, + (job_key, kinds): (&str, &BTreeSet), + executed_successfully: Option<&BTreeSet>, ) -> Result<(ControlTransaction, (SensorCursor, Vec)), ControlError> { let (mut transaction, rows) = query( transaction, @@ -33,12 +34,17 @@ pub(crate) async fn consume_run_events( ], ) .await?; + let runs: Vec = + rows.into_iter().map(decode_row).collect::>()?; + let mut eligible: BTreeSet = runs.iter().map(|run| run.run_id).collect(); + if let Some(nodes) = executed_successfully { + (transaction, eligible) = matching_executions(transaction, eligible, nodes).await?; + } let mut cursor = evaluation.cursor.clone(); let mut run_ids: Vec = Vec::new(); - for row in rows { - let run: TerminalSensorRun = decode_row(row)?; + for run in runs { cursor.event_storage_id = run.last_event_storage_id; - if kinds.iter().any(|kind| kind.matches(run.status)) { + if eligible.contains(&run.run_id) && kinds.iter().any(|kind| kind.matches(run.status)) { let run_id; (transaction, run_id) = queue_sensor_run(transaction, evaluation, &run.run_id.to_string()).await?; @@ -47,3 +53,43 @@ pub(crate) async fn consume_run_events( } Ok((transaction, (cursor, run_ids))) } + +/// Each chunk intersects the previous matches: every selected node must qualify. +async fn matching_executions( + mut transaction: ControlTransaction, + mut eligible: BTreeSet, + nodes: &BTreeSet, +) -> Result<(ControlTransaction, BTreeSet), ControlError> { + let nodes: Vec<&String> = nodes.iter().collect(); + for chunk in nodes.chunks(EXECUTION_NODE_BATCH_SIZE) { + if eligible.is_empty() { + break; + } + let mut parameters: Vec = eligible.iter().copied().map(Parameter::Id).collect(); + let run_parameters = (1..=parameters.len()) + .map(|index| format!("${index}")) + .collect::>() + .join(","); + let node_parameters = (parameters.len() + 1..=parameters.len() + chunk.len()) + .map(|index| format!("${index}")) + .collect::>() + .join(","); + parameters.extend(chunk.iter().map(|key| Parameter::Text((*key).to_string()))); + let rows; + (transaction, rows) = query(transaction, &format!( + "SELECT r.run_id,r.status,r.last_event_storage_id FROM sqb_runs r \ + WHERE r.run_id IN ({run_parameters}) AND \ + (SELECT COUNT(*) FROM sqb_run_nodes n WHERE n.run_id=r.run_id AND n.attempt=1 \ + AND n.node_key IN ({node_parameters}) AND n.status='success' AND n.ended_at IS NOT NULL \ + AND ((r.kind<>'group' AND n.started_at IS NOT NULL) OR \ + (r.kind='group' AND EXISTS(SELECT 1 FROM sqb_runs c \ + JOIN sqb_run_nodes cn ON cn.run_id=c.run_id AND cn.node_key=n.node_key AND cn.attempt=1 \ + WHERE c.parent_run_id=r.run_id AND c.location_id=r.location_id AND c.target=r.target \ + AND cn.status='success' AND cn.started_at IS NOT NULL AND cn.ended_at IS NOT NULL))))={}", + chunk.len()), parameters).await?; + let matches: Vec = + rows.into_iter().map(decode_row).collect::>()?; + eligible = matches.into_iter().map(|run| run.run_id).collect(); + } + Ok((transaction, eligible)) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/constants.rs b/crates/sqlbuild-orchestration/src/sensors/constants.rs new file mode 100644 index 0000000000..e2b63365a5 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/constants.rs @@ -0,0 +1,3 @@ +//! Bound query parameters independently of the size of an authored selection. + +pub(crate) const EXECUTION_NODE_BATCH_SIZE: usize = 500; diff --git a/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs b/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs index fc35a312c2..6a3640933e 100644 --- a/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs +++ b/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs @@ -51,8 +51,18 @@ pub async fn complete_builtin_sensor( SensorCondition::PlatformEvent { kinds } => { consume_platform_events(transaction, evaluation, kinds).await? } - SensorCondition::RunEvent { job_key, kinds } => { - consume_run_events(transaction, evaluation, job_key, kinds).await? + SensorCondition::RunEvent { + job_key, + kinds, + executed_successfully, + } => { + consume_run_events( + transaction, + evaluation, + (job_key, kinds), + executed_successfully.as_ref(), + ) + .await? } }; let count = run_ids.len(); diff --git a/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs b/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs index 60aee8f0ac..76b8bc2b75 100644 --- a/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs +++ b/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs @@ -8,10 +8,21 @@ pub(crate) fn validate_sensors(definitions: &Definitions) -> Result<(), ControlE for (name, sensor) in &definitions.sensors { let valid_condition = match &sensor.when { SensorCondition::PlatformEvent { kinds } => !kinds.is_empty(), - SensorCondition::RunEvent { job_key, kinds } => { + SensorCondition::RunEvent { + job_key, + kinds, + executed_successfully, + } => { !kinds.is_empty() - && (definitions.jobs.contains_key(job_key) - || definitions.groups.contains_key(job_key)) + && definitions + .jobs + .get(job_key) + .or_else(|| definitions.groups.get(job_key).map(|group| &group.job)) + .is_some_and(|source| { + executed_successfully.as_ref().is_none_or(|nodes| { + !nodes.is_empty() && nodes.is_subset(&source.selection) + }) + }) } }; if name.trim().is_empty() @@ -21,7 +32,7 @@ pub(crate) fn validate_sensors(definitions: &Definitions) -> Result<(), ControlE && !definitions.groups.contains_key(&sensor.job_key)) { return Err(ControlError::Invalid(format!( - "sensor {name} needs a description, a known job and nonempty event kinds" + "sensor {name} needs a description, a known job and nonempty event kinds; executed_successfully must select nonempty source job/group nodes" ))); } } diff --git a/crates/sqlbuild-orchestration/src/sensors/mod.rs b/crates/sqlbuild-orchestration/src/sensors/mod.rs index ca1039eddc..0f364e6e62 100644 --- a/crates/sqlbuild-orchestration/src/sensors/mod.rs +++ b/crates/sqlbuild-orchestration/src/sensors/mod.rs @@ -1,6 +1,7 @@ //! Sensor attempts and cursor commits are immutable facts, not mutable lifecycle rows. pub(crate) mod _helpers; +pub(crate) mod constants; pub mod main; pub mod models; diff --git a/crates/sqlbuild-orchestration/src/sensors/models.rs b/crates/sqlbuild-orchestration/src/sensors/models.rs index 9dff71a198..5784a28c19 100644 --- a/crates/sqlbuild-orchestration/src/sensors/models.rs +++ b/crates/sqlbuild-orchestration/src/sensors/models.rs @@ -38,6 +38,8 @@ pub enum SensorCondition { RunEvent { job_key: String, kinds: BTreeSet, + #[serde(default, skip_serializing_if = "Option::is_none")] + executed_successfully: Option>, }, } diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index 560317b6ac..58b8897399 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -12,6 +12,7 @@ pub(crate) const LOCATION_WORK: &str = include_str!("../location_work.sql"); pub(crate) const PLATFORM_SINK: &str = include_str!("../platform_sink.sql"); pub(crate) const SENSORS: &str = include_str!("../sensors.sql"); pub(crate) const RUN_SENSORS: &str = include_str!("../run_sensors.sql"); +pub(crate) const EXECUTED_NODES: &str = include_str!("../executed_nodes.sql"); pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) diff --git a/crates/sqlbuild-orchestration/src/store/executed_nodes.sql b/crates/sqlbuild-orchestration/src/store/executed_nodes.sql new file mode 100644 index 0000000000..4bfa512dad --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/executed_nodes.sql @@ -0,0 +1 @@ +CREATE INDEX sqb_runs_parent ON sqb_runs(parent_run_id) WHERE parent_run_id IS NOT NULL; diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index 59337ca99f..53a6d45481 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -4,8 +4,8 @@ use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; use crate::store::_helpers::migration::{ - LOCATION_INSTANCES, LOCATION_WORK, PLATFORM_SINK, RUN_SENSORS, SCHEMA, SENSORS, render, - verify_migrations, + EXECUTED_NODES, LOCATION_INSTANCES, LOCATION_WORK, PLATFORM_SINK, RUN_SENSORS, SCHEMA, SENSORS, + render, verify_migrations, }; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; @@ -34,6 +34,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { ("platform_sink", PLATFORM_SINK), ("sensors", SENSORS), ("run_sensors", RUN_SENSORS), + ("executed_nodes", EXECUTED_NODES), ]; let mut digests = vec![digest.clone()]; digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs index 19aa6d5e83..f52cad361a 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs @@ -46,9 +46,9 @@ fn given_outbound_work_when_claimed_retried_or_replaced_then_scope_and_immutable let store = &fixture.store; migrate_store(store).await.expect("fresh migration"); // Exercise both deployed predecessors, including checksums and repeated startup. - execute(store, "DROP INDEX sqb_runs_terminal_cursor; DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DELETE FROM sqb_schema_migrations WHERE version>=3").await; + execute(store, "DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DELETE FROM sqb_schema_migrations WHERE version>=3").await; migrate_store(store).await.expect("schema 2 upgrade"); - execute(store, "DROP INDEX sqb_runs_terminal_cursor; DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2").await; + execute(store, "DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2").await; migrate_store(store).await.expect("schema 1 upgrade"); migrate_store(store).await.expect("repeat migration"); let location_id = create_location(store, "orders", "test-admin") diff --git a/crates/sqlbuild-orchestration/tests/sensors/helpers.rs b/crates/sqlbuild-orchestration/tests/sensors/helpers.rs index 703b73d8e9..ff6089837f 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/helpers.rs @@ -145,6 +145,7 @@ pub(crate) async fn finish_run( job_key: &str, outcome: RunOutcome, nodes: [NodeOutcome; 2], + started: [bool; 2], ) -> Uuid { let run_id = Uuid::now_v7(); let context = AuditContext { @@ -168,6 +169,20 @@ pub(crate) async fn finish_run( ) .await .expect("request source run"); + finish_claimed_run(store, lease, scope, run_id, outcome, nodes, started).await; + run_id +} + +/// Execute only the frozen selection, including a from-failure rerun's reduced scope. +pub(crate) async fn finish_claimed_run( + store: &ControlStore, + lease: &SchedulerLease, + scope: (Uuid, &str), + run_id: Uuid, + outcome: RunOutcome, + nodes: [NodeOutcome; 2], + started: [bool; 2], +) { let limits = std::collections::BTreeMap::new(); advance_groups(store, lease, (&limits, scope)) .await @@ -181,23 +196,23 @@ pub(crate) async fn finish_run( run_id: claim.request.run_id, claim_id: claim.claim_id, }; - let changes = [ - RunChange::Started, - RunChange::NodeFinished { - node_key: "customers".into(), - outcome: nodes[0], - rows_affected: None, - error_code: None, - skipped_because_of: None, - }, - RunChange::NodeFinished { - node_key: "orders".into(), - outcome: nodes[1], + let mut changes = vec![RunChange::Started]; + for (index, key) in ["customers", "orders"] + .into_iter() + .enumerate() + .filter(|(_, key)| claim.request.selection.contains(*key)) + { + changes.extend(started[index].then_some(RunChange::NodeStarted { + node_key: key.into(), + })); + changes.push(RunChange::NodeFinished { + node_key: key.into(), + outcome: nodes[index], rows_affected: None, error_code: None, skipped_because_of: None, - }, - ]; + }); + } for change in changes { change_run( store, @@ -235,5 +250,4 @@ pub(crate) async fn finish_run( advance_groups(store, lease, (&limits, scope)) .await .expect("complete parent"); - run_id } diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs b/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs index b2e64838ac..81c48ae068 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs @@ -4,6 +4,8 @@ mod helpers; #[path = "test_delivery.rs"] mod test_delivery; +#[path = "test_execution_bounds.rs"] +mod test_execution_bounds; #[path = "test_recovery.rs"] mod test_recovery; #[path = "test_runs.rs"] diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_execution_bounds.rs b/crates/sqlbuild-orchestration/tests/sensors/test_execution_bounds.rs new file mode 100644 index 0000000000..3cc7469c46 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_execution_bounds.rs @@ -0,0 +1,176 @@ +//! A wide group forces multiple predicate chunks; the last node is not optional. + +use std::time::Duration; + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::runs::main::{ + advance_groups::advance_groups, change_run::change_run, claim_next_run::claim_next_run, + request_job_run::request_job_run, +}; +use crate::runs::models::{ + JobRunRequest, NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, +}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::sensors::main::{ + begin_sensor_evaluation::begin_sensor_evaluation, + complete_builtin_sensor::complete_builtin_sensor, +}; +use crate::sensors::tests::helpers::snapshot; +use crate::sensors::tests::test_types::SensorTestCase; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_wide_group_when_only_the_last_node_has_not_started_then_all_chunks_must_match_within_a_bounded_evaluation() + { + let test_cases = [ + SensorTestCase { + description: "SQLite wide executed-node predicate", + backend: Backend::Sqlite, + expected_runs: 1, + }, + SensorTestCase { + description: "PostgreSQL wide executed-node predicate", + backend: Backend::Postgres, + expected_runs: 1, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required backend"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + let node = input.nodes["orders"].clone(); + let selection: Vec = (0..501) + .map(|index| format!("model:orders_{index:04}")) + .collect(); + input + .nodes + .extend(selection.iter().map(|key| (key.clone(), node.clone()))); + input.documents.insert("jobs".into(), json!({ + "source": {"description": "Wide source", "selection": selection, "overlap": "allow"}, + "destination": {"description": "Destination", "selection": ["orders"], "overlap": "allow"} + })); + input.documents.insert("groups".into(), json!({ + "wide_orders": {"description": "Wide group", "selection": selection, "jobs": {"source": []}, "overlap": "allow"} + })); + input.documents.insert("triggers".into(), json!({ + "executed": {"description": "All source nodes must execute", "job_key": "destination", + "when": {"kind": "run_event", "job_key": "wide_orders", "kinds": ["completed"], "executed_successfully": selection}, + "minimum_interval_seconds": 1} + })); + publish_snapshot(store, &input) + .await + .expect("publish wide predicate"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = std::collections::BTreeMap::new(); + let scope = (location, "development"); + for include_last in [true, false] { + let parent = Uuid::now_v7(); + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + request_job_run( + store, + &context, + &JobRunRequest { + run_id: parent, + location_id: location, + target: "development".into(), + job_key: "wide_orders".into(), + reason: "Exercise wide execution evidence".into(), + priority: 10, + window: None, + }, + ) + .await + .expect("request"); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("admit parent"); + let claim = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim") + .expect("child"); + assert_eq!(claim.request.parent_run_id, Some(parent)); + let changes = std::iter::once(RunChange::Started) + .chain( + selection + .iter() + .filter(|key| include_last || key.as_str() != "model:orders_0500") + .map(|key| RunChange::NodeStarted { + node_key: key.clone(), + }), + ) + .chain(selection.iter().map(|key| RunChange::NodeFinished { + node_key: key.clone(), + outcome: NodeOutcome::Success, + rows_affected: None, + error_code: None, + skipped_because_of: None, + })) + .chain(std::iter::once(RunChange::Finished { + outcome: RunOutcome::Success, + reason: "All logical nodes succeed".into(), + exit_code: None, + })); + for change in changes { + change_run( + store, + RunAuthority::Worker { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change, + }, + ) + .await + .expect("real node evidence"); + } + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("finish parent"); + } + let trigger = list_triggers(store, location, "development") + .await + .expect("triggers") + .remove(0); + let evaluation = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("request evaluation") + .expect("due"); + let count = tokio::time::timeout( + Duration::from_secs(5), + complete_builtin_sensor(store, &lease, &evaluation), + ) + .await + .expect("wide predicate must not hold the write lock unboundedly") + .expect("complete"); + assert_eq!(count, test_case.expected_runs, "{}", test_case.description); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs index 00cdef84b8..7e3348ce3e 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs @@ -6,18 +6,20 @@ use std::time::Duration; use serde_json::json; use uuid::Uuid; +use crate::credentials::models::AuditContext; use crate::credentials::tests::location_helpers::create_location; use crate::declarations::main::list_triggers::list_triggers; use crate::declarations::models::TriggerChange; use crate::runs::main::list_runs::list_runs; -use crate::runs::models::{NodeOutcome, RunOutcome}; +use crate::runs::main::request_rerun::request_rerun; +use crate::runs::models::{NodeOutcome, RerunRequest, RunOutcome}; use crate::scheduler::tests::contracts::helpers::acquire_lease; use crate::sensors::main::{ begin_sensor_evaluation::begin_sensor_evaluation, complete_builtin_sensor::complete_builtin_sensor, }; -use crate::sensors::tests::helpers::{change, finish_run, snapshot}; -use crate::sensors::tests::test_types::SensorTestCase; +use crate::sensors::tests::helpers::{change, finish_claimed_run, finish_run, snapshot}; +use crate::sensors::tests::test_types::{SensorExecutionTestCase, SensorTestCase}; use crate::snapshots::main::publish_snapshot::publish_snapshot; use crate::store::main::{ begin::begin, commit::commit, migrate_store::migrate_store, query::query, @@ -36,12 +38,12 @@ fn given_terminal_jobs_and_groups_when_selectors_retry_then_outcomes_scope_and_r SensorTestCase { description: "SQLite terminal outcome sensor recovery", backend: Backend::Sqlite, - expected_runs: 16, + expected_runs: 106, }, SensorTestCase { description: "PostgreSQL terminal outcome sensor recovery", backend: Backend::Postgres, - expected_runs: 16, + expected_runs: 106, }, ]; for test_case in test_cases { @@ -86,6 +88,19 @@ fn given_terminal_jobs_and_groups_when_selectors_retry_then_outcomes_scope_and_r "minimum_interval_seconds": 1 })); } + for (name, kind, nodes) in [ + ("both_completed", "completed", vec!["customers", "orders"]), + ("customers_completed", "completed", vec!["customers"]), + ("customers_success", "success", vec!["customers"]), + ("customers_partial", "partial", vec!["customers"]), + ("orders_completed", "completed", vec!["orders"]), + ] { + sensors.insert(format!("{job_key}_executed_{name}"), json!({ + "description": "Require every selected node to actually succeed", "job_key": "inventory", + "when": {"kind": "run_event", "job_key": job_key, "kinds": [kind], "executed_successfully": nodes}, + "minimum_interval_seconds": 1 + })); + } } input.documents.insert("triggers".into(), json!(sensors)); publish_snapshot(store, &input) @@ -136,6 +151,7 @@ fn given_terminal_jobs_and_groups_when_selectors_retry_then_outcomes_scope_and_r "orders", outcome, nodes, + [false; 2], ) .await .to_string(); @@ -155,6 +171,7 @@ fn given_terminal_jobs_and_groups_when_selectors_retry_then_outcomes_scope_and_r "daily_orders", RunOutcome::Success, [NodeOutcome::Success; 2], + [false; 2], ) .await; let sources = list_runs(store, (location, "development"), 100) @@ -178,6 +195,199 @@ fn given_terminal_jobs_and_groups_when_selectors_retry_then_outcomes_scope_and_r .or_default() .insert(child.run_id.to_string()); } + let executions = [ + SensorExecutionTestCase { + description: "freshness-only success cannot borrow another execution's starts", + outcome: RunOutcome::Success, + nodes: [NodeOutcome::Success; 2], + started: [false; 2], + expected_selector: "success", + expected_filters: &[], + }, + SensorExecutionTestCase { + description: "all selected nodes must start, not merely any node", + outcome: RunOutcome::Success, + nodes: [NodeOutcome::Success; 2], + started: [true, false], + expected_selector: "success", + expected_filters: &["customers_completed", "customers_success"], + }, + SensorExecutionTestCase { + description: "both selected nodes actually succeed", + outcome: RunOutcome::Success, + nodes: [NodeOutcome::Success; 2], + started: [true; 2], + expected_selector: "success", + expected_filters: &[ + "both_completed", + "customers_completed", + "customers_success", + "orders_completed", + ], + }, + SensorExecutionTestCase { + description: "partial outcome is independent of selected-node success", + outcome: RunOutcome::Partial, + nodes: [NodeOutcome::Success, NodeOutcome::Failed], + started: [true; 2], + expected_selector: "partial", + expected_filters: &["customers_completed", "customers_partial"], + }, + SensorExecutionTestCase { + description: "warning is not successful execution", + outcome: RunOutcome::Warn, + nodes: [NodeOutcome::Warn, NodeOutcome::Success], + started: [true; 2], + expected_selector: "warn", + expected_filters: &["orders_completed"], + }, + SensorExecutionTestCase { + description: "start without success cannot borrow success from another run", + outcome: RunOutcome::Failed, + nodes: [NodeOutcome::Failed; 2], + started: [true; 2], + expected_selector: "failed", + expected_filters: &[], + }, + SensorExecutionTestCase { + description: "skipped selected node is not successful", + outcome: RunOutcome::Partial, + nodes: [NodeOutcome::Skipped, NodeOutcome::Success], + started: [false, true], + expected_selector: "partial", + expected_filters: &["orders_completed"], + }, + SensorExecutionTestCase { + description: "a partial run's freshness-only node still does not qualify", + outcome: RunOutcome::Partial, + nodes: [NodeOutcome::Success, NodeOutcome::Failed], + started: [false, true], + expected_selector: "partial", + expected_filters: &[], + }, + SensorExecutionTestCase { + description: "a genuinely started warning is not success", + outcome: RunOutcome::Warn, + nodes: [NodeOutcome::Warn, NodeOutcome::Success], + started: [true, false], + expected_selector: "warn", + expected_filters: &[], + }, + ]; + let mut source_ids: BTreeMap<(&str, &str), Uuid> = BTreeMap::new(); + for execution in executions { + for job_key in ["orders", "daily_orders"] { + let id = finish_run( + store, + &lease, + (location, "development"), + job_key, + execution.outcome, + execution.nodes, + execution.started, + ) + .await; + source_ids.insert((execution.description, job_key), id); + let transaction = begin(store, TransactionMode::Read) + .await + .expect("read lineage"); + let (transaction, children) = query( + transaction, + "SELECT run_id FROM sqb_runs WHERE parent_run_id=$1", + vec![Parameter::Id(id)], + ) + .await + .expect(execution.description); + commit(transaction).await.expect("read lineage committed"); + for (source, id) in std::iter::once((job_key, id.to_string())).chain( + children.into_iter().map(|row| { + ( + "orders", + row["run_id"].as_str().expect("child id").to_owned(), + ) + }), + ) { + for kind in ["completed", execution.expected_selector] { + expected + .entry(format!("{source}_{kind}")) + .or_default() + .insert(id.clone()); + } + for filter in execution.expected_filters { + expected + .entry(format!("{source}_executed_{filter}")) + .or_default() + .insert(id.clone()); + } + } + } + } + let retry = RerunRequest { + run_id: Uuid::now_v7(), + source_run_id: source_ids[&( + "partial outcome is independent of selected-node success", + "daily_orders", + )], + from_failure: true, + reason: "Retry only failed orders without borrowing earlier customer success" + .into(), + priority: 10, + }; + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + assert!( + request_rerun(store, &context, &retry) + .await + .expect("request rerun") + ); + assert!( + !request_rerun(store, &context, &retry) + .await + .expect("retry request") + ); + finish_claimed_run( + store, + &lease, + (location, "development"), + retry.run_id, + RunOutcome::Success, + [NodeOutcome::Success; 2], + [true; 2], + ) + .await; + let transaction = begin(store, TransactionMode::Read) + .await + .expect("read rerun lineage"); + let (transaction, children) = query( + transaction, + "SELECT run_id FROM sqb_runs WHERE parent_run_id=$1", + vec![Parameter::Id(retry.run_id)], + ) + .await + .expect("rerun child"); + commit(transaction).await.expect("read rerun committed"); + for (source, id) in std::iter::once(("daily_orders", retry.run_id.to_string())).chain( + children.into_iter().map(|row| { + ( + "orders", + row["run_id"].as_str().expect("child id").to_owned(), + ) + }), + ) { + for suffix in ["completed", "success", "executed_orders_completed"] { + expected + .entry(format!("{source}_{suffix}")) + .or_default() + .insert(id.clone()); + } + } + let sources = list_runs(store, (location, "development"), 500) + .await + .expect("all source executions"); // Same key in another target/location and a different job in this scope must not match. let mut other_input = input.clone(); other_input.location_id = other; @@ -197,6 +407,7 @@ fn given_terminal_jobs_and_groups_when_selectors_retry_then_outcomes_scope_and_r "orders", RunOutcome::Success, [NodeOutcome::Success; 2], + [false; 2], ) .await; } @@ -207,19 +418,23 @@ fn given_terminal_jobs_and_groups_when_selectors_retry_then_outcomes_scope_and_r "inventory", RunOutcome::Success, [NodeOutcome::Success; 2], + [false; 2], ) .await; // Recreate the exact pre-change index/migration state with populated run history. let transaction = begin(store, TransactionMode::Write) .await .expect("upgrade fixture"); + let (transaction, _) = query(transaction, "DROP INDEX sqb_runs_parent", vec![]) + .await + .expect("remove execution lookup index"); let (transaction, _) = query(transaction, "DROP INDEX sqb_runs_terminal_cursor", vec![]) .await .expect("remove new index"); let (transaction, _) = query( transaction, - "DELETE FROM sqb_schema_migrations WHERE name='run_sensors'", + "DELETE FROM sqb_schema_migrations WHERE version>=6", vec![], ) .await @@ -301,7 +516,7 @@ fn given_terminal_jobs_and_groups_when_selectors_retry_then_outcomes_scope_and_r .await .expect("replay deduplicates"); } - let after = list_runs(store, (location, "development"), 100) + let after = list_runs(store, (location, "development"), 500) .await .expect("queued sensors"); assert_eq!( diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_types.rs b/crates/sqlbuild-orchestration/tests/sensors/test_types.rs index b762f2f4c5..1b4e85322d 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_types.rs @@ -1,3 +1,4 @@ +use crate::runs::models::{NodeOutcome, RunOutcome}; use crate::store::tests::store_test_types::Backend; pub(crate) struct SensorTestCase { @@ -5,3 +6,12 @@ pub(crate) struct SensorTestCase { pub(crate) backend: Backend, pub(crate) expected_runs: usize, } + +pub(crate) struct SensorExecutionTestCase { + pub(crate) description: &'static str, + pub(crate) outcome: RunOutcome, + pub(crate) nodes: [NodeOutcome; 2], + pub(crate) started: [bool; 2], + pub(crate) expected_selector: &'static str, + pub(crate) expected_filters: &'static [&'static str], +} diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index 2996933768..4faee91009 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -243,7 +243,7 @@ pub(super) async fn sqlite_failed_publish_and_schema_floor() -> i64 { let connection = rusqlite::Connection::open(&path).expect("open fault injection connection"); connection .execute_batch( - "DROP INDEX sqb_runs_terminal_cursor; \ + "DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; \ DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; \ DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; \ DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; \ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index be7fa9ac3b..caab6c25e6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -203,6 +203,7 @@ pub(crate) fn compile_declarations( sensors, } = compile_sensors( &input.target, + graph, authored_sensors, compile_calendars( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs index 0d9a84cfd8..87db7962f8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs @@ -53,6 +53,8 @@ pub(crate) struct AuthoredSensorCondition { pub(crate) kinds: Vec, #[pyo3(default)] pub(crate) job: Option, + #[pyo3(default)] + pub(crate) executed_successfully: Option>, } #[derive(FromPyObject)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs index 0ce49660fa..f3ba7c955e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -10,6 +10,8 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods}; use serde_json::{Value, json}; +use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; +use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_cursor::algebra::main::parse_duration::parse_duration; use sqlbuild_host::entries::models::UserCodeHost; use sqlbuild_orchestration::declarations::models::{Definitions, Schedule, TriggerDefinition}; @@ -249,6 +251,7 @@ pub(crate) fn compile_calendars( pub(crate) fn compile_sensors( target: &str, + graph: &ProjectGraph, authored: Vec, mut definitions: Definitions, ) -> PyResult { @@ -267,6 +270,19 @@ pub(crate) fn compile_sensors( if let Some(job) = sensor.when.job { condition["job_key"] = json!(job.name); } + if let Some(selectors) = sensor.when.executed_successfully { + if selectors.is_empty() || selectors.iter().any(|value| value.trim().is_empty()) { + return Err(PyValueError::new_err( + "executed_successfully needs nonempty selectors", + )); + } + let nodes: BTreeSet = resolve_selectors(graph, &selectors, &[]) + .map_err(|error| PyValueError::new_err(error.message))? + .into_iter() + .map(|(kind, name)| format!("{kind}:{name}")) + .collect(); + condition["executed_successfully"] = json!(nodes); + } let definition: SensorDefinition = serde_json::from_value(json!({ "description": sensor.description, "job_key": sensor.run.name, diff --git a/src/sqlbuild/orchestration_declarations/models.py b/src/sqlbuild/orchestration_declarations/models.py index 97a4106dd0..077fde537f 100644 --- a/src/sqlbuild/orchestration_declarations/models.py +++ b/src/sqlbuild/orchestration_declarations/models.py @@ -92,10 +92,11 @@ class PlatformEventCondition: @dataclass(frozen=True, kw_only=True) class RunEventCondition: - """Match a job/group's terminal outcomes in the sensor's location and target.""" + """Match scoped terminal outcomes, optionally requiring genuinely executed successful nodes.""" job: Job | JobGroup kinds: Sequence[RunEventKind] + executed_successfully: Sequence[str] | None = None kind: Literal["run_event"] = field(default="run_event", init=False) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py index 7eb241a5fe..52fabe8463 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py @@ -147,7 +147,7 @@ def test_given_a_served_calendar_when_time_passes_and_a_provider_fails_then_refr "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" ).read_text(), "providers/fulfillment.py": ( - "import json, os\nfrom pathlib import Path\nfrom datetime import timedelta\n" + "import json, os\nfrom pathlib import Path\nfrom datetime import timedelta, timezone\n" "from sqlbuild.provider.classes.provider import Provider\n" "class Fulfillment(Provider):\n" ' """Upcoming synthetic fulfillment times."""\n' @@ -160,7 +160,8 @@ def test_given_a_served_calendar_when_time_passes_and_a_provider_fails_then_refr " self.record({'phase': 'evaluate', 'start': window.start.isoformat(), 'end': window.end.isoformat(), 'zone': window.start.tzinfo.key})\n" " data = json.loads(Path(os.environ['SQLBUILD_FULFILLMENT_DATA']).read_text())\n" " if data.get('fail'):\n raise ValueError('synthetic calendar unavailable')\n" - " return [window.start + timedelta(seconds=offset) for offset in data['offsets']]\n" + " start = window.start.astimezone(timezone.utc)\n" + " return [start + timedelta(seconds=offset) for offset in data['offsets']]\n" ), }, ) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py index c48b98bf0c..9b12212a01 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py @@ -149,6 +149,7 @@ def test_given_retained_compiler_data_when_publishing_then_http_updates_atomical "kind": "run_event", "job_key": "edge_snapshot_group", "kinds": ["completed"], + "executed_successfully": ["model:orders"], } assert by_name["edge_job_outcomes"]["status"] == "paused" assert by_name["edge_group_outcomes"]["status"] == "paused" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index 8f46037730..5e04e48389 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -505,7 +505,7 @@ def test_given_a_scheduled_job_when_published_then_an_isolated_worker_reports_re "schedule=schedule(cron='0 0 1 1 *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" "annual_orders = job('annual_orders', select='orders', description='Refresh orders.', settings={'reload': True})\n" "on_group = sensor('on_group', description='Refresh after group completion.', " - "when=run_event(job=source_group, kinds=[RunEventKind.COMPLETED, RunEventKind.SUCCESS]), " + "when=run_event(job=source_group, kinds=[RunEventKind.COMPLETED, RunEventKind.SUCCESS], executed_successfully=('orders',)), " "run=annual_orders, minimum_interval_seconds=1)\n" ) }, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 43cb006a6b..c4804832b3 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -641,6 +641,7 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject "kind": "run_event", "job_key": "edge_snapshot_group", "kinds": ["completed"], + "executed_successfully": ["model:orders"], }, "minimum_interval_seconds": 30, "default_status": "stopped", @@ -662,6 +663,7 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject jobs=[job("source_orders", description="Orders.", select="orders")], ), kinds=[RunEventKind.COMPLETED], + executed_successfully=("orders", "orders*"), ), run=job("inventory", description="Inventory.", select="stg_orders"), ), @@ -676,6 +678,30 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject "kind": "run_event", "job_key": "source_group", "kinds": ["completed"], + "executed_successfully": ["model:orders"], + }, + "minimum_interval_seconds": 30, + "default_status": "running", + } + }, + ), + SnapshotDiscoveryTestCase( + description="source-job execution selectors compile to frozen canonical keys", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" + ).read_text() + }, + expected_jobs=("inventory", "source_orders"), + expected_sensors={ + "on_orders": { + "description": "Build inventory only after orders actually succeeds.", + "job_key": "inventory", + "when": { + "kind": "run_event", + "job_key": "source_orders", + "kinds": ["completed"], + "executed_successfully": ["model:orders"], }, "minimum_interval_seconds": 30, "default_status": "running", @@ -858,6 +884,78 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects expected_error=ValueError, expected_message="unknown variant", ), + SnapshotDiscoveryRejectionTestCase( + description="explicit empty execution selectors cannot broaden matching", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" + ) + .read_text() + .replace('("orders",)', "()") + }, + expected_error=ValueError, + expected_message="executed_successfully needs nonempty selectors", + ), + SnapshotDiscoveryRejectionTestCase( + description="blank execution selectors cannot broaden matching", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" + ) + .read_text() + .replace('("orders",)', '(" ",)') + }, + expected_error=ValueError, + expected_message="executed_successfully needs nonempty selectors", + ), + SnapshotDiscoveryRejectionTestCase( + description="unresolved execution selectors fail closed", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" + ) + .read_text() + .replace('("orders",)', '("unknown_orders",)') + }, + expected_error=ValueError, + expected_message="unknown selector name 'unknown_orders'", + ), + SnapshotDiscoveryRejectionTestCase( + description="unmatched execution patterns cannot become an omitted filter", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" + ) + .read_text() + .replace('("orders",)', '("unknown_*",)') + }, + expected_error=ValueError, + expected_message="unknown selector pattern", + ), + SnapshotDiscoveryRejectionTestCase( + description="invalid execution selector syntax fails closed", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" + ) + .read_text() + .replace('("orders",)', '("tag:",)') + }, + expected_error=ValueError, + expected_message="has empty value after ':'", + ), + SnapshotDiscoveryRejectionTestCase( + description="execution selectors outside the source job fail closed", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" + ) + .read_text() + .replace('("orders",)', '("stg_orders",)') + }, + expected_error=ValueError, + expected_message="executed_successfully must select nonempty source job/group nodes", + ), SnapshotDiscoveryRejectionTestCase( description="sensor code defaults use running or stopped", files={ From 9b6e59e6ec5958c93dd9c409e1b36a2976d17b76 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 06:57:13 +0000 Subject: [PATCH 53/59] feat: trigger sensors for genuine successful node rebuilds Resolve authored selectors natively and consume immutable node finishes with earlier same-run and claim start evidence. Keep per-finish deduplication inside the fenced sensor transaction and exclude freshness-only results and group summaries. Extend only the existing enum declaration boundary for NodeEventKind, retaining a zero Python implementation budget. Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../edge_cases/orchestration/node_orders.py | 12 + .../declarations/main/snapshot_definitions.rs | 2 +- .../src/sensors/_helpers/conditions.rs | 37 ++ .../src/sensors/_helpers/mod.rs | 2 + .../src/sensors/_helpers/nodes.rs | 95 +++++ .../sensors/main/complete_builtin_sensor.rs | 24 +- .../src/sensors/main/validate_sensors.rs | 13 +- .../src/sensors/models.rs | 20 + .../src/store/_helpers/migration.rs | 1 + .../src/store/main/migrate_store.rs | 5 +- .../src/store/node_sensors.sql | 6 + .../tests/api/test_location_work.rs | 4 +- .../tests/sensors/test_contracts.rs | 2 + .../tests/sensors/test_nodes.rs | 376 ++++++++++++++++++ .../tests/sensors/test_runs.rs | 8 + .../tests/snapshots/helpers.rs | 3 +- .../native_host/orchestration_declarations.rs | 2 + .../native_host/orchestration_triggers.rs | 36 +- python-boundary.toml | 2 +- src/sqlbuild/orchestration.py | 7 +- .../orchestration_declarations/models.py | 13 +- .../orchestration_declarations/types.py | 6 + .../publication/test_worker.py | 59 +++ .../test_orchestration_snapshot.py | 46 +++ 24 files changed, 737 insertions(+), 44 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/node_orders.py create mode 100644 crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs create mode 100644 crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs create mode 100644 crates/sqlbuild-orchestration/src/store/node_sensors.sql create mode 100644 crates/sqlbuild-orchestration/tests/sensors/test_nodes.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/node_orders.py b/benchmarks/duckshop/edge_cases/orchestration/node_orders.py new file mode 100644 index 0000000000..ac8f6cb8fc --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/node_orders.py @@ -0,0 +1,12 @@ +"""Trigger on each real orders rebuild, not on freshness-only success.""" + +from sqlbuild.orchestration import NodeEventKind, job, node_event, sensor + +source_orders = job("source_orders", description="Source orders.", select="orders") +inventory = job("inventory", description="Destination inventory.", select="stg_orders") +on_orders = sensor( + "on_orders", + description="Build inventory for every genuine orders rebuild.", + when=node_event(select="orders orders*", kinds=[NodeEventKind.SUCCESS]), + run=inventory, +) diff --git a/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs b/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs index cbe7c7a152..0e288368d8 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs @@ -81,6 +81,6 @@ pub fn snapshot_definitions(input: &SnapshotInput) -> Result {} } } - validate_sensors(&definitions)?; + validate_sensors(&definitions, input)?; Ok(definitions) } diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs new file mode 100644 index 0000000000..6e2e9f5d1a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs @@ -0,0 +1,37 @@ +//! Each native condition consumes its bounded page inside the evaluation transaction. + +use uuid::Uuid; + +use crate::sensors::_helpers::nodes::consume_node_events; +use crate::sensors::_helpers::platform::consume_platform_events; +use crate::sensors::_helpers::runs::consume_run_events; +use crate::sensors::models::{SensorCondition, SensorCursor, SensorEvaluationRequest}; +use crate::store::errors::ControlError; +use crate::store::models::ControlTransaction; + +pub(crate) async fn consume_condition( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, +) -> Result<(ControlTransaction, (SensorCursor, Vec)), ControlError> { + match &evaluation.definition.when { + SensorCondition::PlatformEvent { kinds } => { + consume_platform_events(transaction, evaluation, kinds).await + } + SensorCondition::NodeEvent { selection, .. } => { + consume_node_events(transaction, evaluation, selection).await + } + SensorCondition::RunEvent { + job_key, + kinds, + executed_successfully, + } => { + consume_run_events( + transaction, + evaluation, + (job_key, kinds), + executed_successfully.as_ref(), + ) + .await + } + } +} diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs index bd9320484f..267996a120 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs @@ -1,5 +1,7 @@ pub(crate) mod attempts; +pub(crate) mod conditions; pub(crate) mod facts; +pub(crate) mod nodes; pub(crate) mod platform; pub(crate) mod requests; pub(crate) mod runs; diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs new file mode 100644 index 0000000000..c7f7a97b92 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs @@ -0,0 +1,95 @@ +//! Successful rebuilds require earlier immutable start evidence from the same run claim. + +use std::collections::BTreeSet; + +use uuid::Uuid; + +use crate::runs::models::{NodeOutcome, RunChange, RunChangeRequest}; +use crate::sensors::_helpers::requests::queue_sensor_run; +use crate::sensors::models::{NodeSensorEvent, SensorCursor, SensorEvaluationRequest}; +use crate::store::errors::ControlError; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn consume_node_events( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, + selection: &BTreeSet, +) -> Result<(ControlTransaction, (SensorCursor, Vec)), ControlError> { + let (mut transaction, rows) = query( + transaction, + "SELECT storage_id,event_id,run_id,payload FROM recall_events \ + WHERE producer='scheduler' AND event_type='run_node_finished' \ + AND invocation_id='scheduler/' || run_id AND storage_id>$1 \ + ORDER BY storage_id LIMIT 100", + vec![Parameter::Integer(evaluation.cursor.event_storage_id)], + ) + .await?; + let mut cursor = evaluation.cursor.clone(); + let mut run_ids: Vec = Vec::new(); + for row in rows { + let event: NodeSensorEvent = decode_row(row)?; + cursor.event_storage_id = event.storage_id; + let request: RunChangeRequest = serde_json::from_value(event.payload["change"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if request.run_id != event.run_id || request.event_id != event.event_id { + return Err(ControlError::Invalid("node event identity mismatch".into())); + } + let RunChange::NodeFinished { + node_key, + outcome: NodeOutcome::Success, + .. + } = &request.change + else { + continue; + }; + if !selection.contains(node_key) { + continue; + } + let started; + (transaction, started) = + prior_start(transaction, evaluation, &request, event.storage_id).await?; + if started { + let run_id; + (transaction, run_id) = + queue_sensor_run(transaction, evaluation, &event.event_id.to_string()).await?; + run_ids.push(run_id); + } + } + Ok((transaction, (cursor, run_ids))) +} + +async fn prior_start( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, + request: &RunChangeRequest, + storage_id: i64, +) -> Result<(ControlTransaction, bool), ControlError> { + let (RunChange::NodeFinished { node_key, .. }, Some(claim_id)) = + (&request.change, request.claim_id) + else { + return Ok((transaction, false)); + }; + let (transaction, rows) = query( + transaction, + "SELECT s.event_id FROM sqb_runs r JOIN recall_events s ON s.run_id=CAST(r.run_id AS TEXT) \ + WHERE s.invocation_id=$1 AND s.run_id=$2 AND s.producer='scheduler' \ + AND s.event_type='run_node_started' AND s.storage_id<$3 \ + AND (s.payload -> 'change' ->> 'claim_id')=$4 \ + AND (s.payload -> 'change' -> 'change' ->> 'node_key')=$5 \ + AND r.run_id=$6 AND r.location_id=$7 AND r.target=$8 AND r.kind<>'group' LIMIT 1", + vec![ + Parameter::Text(format!("scheduler/{}", request.run_id)), + Parameter::Text(request.run_id.to_string()), + Parameter::Integer(storage_id), + Parameter::Text(claim_id.to_string()), + Parameter::Text(node_key.clone()), + Parameter::Id(request.run_id), + Parameter::Id(evaluation.location_id), + Parameter::Text(evaluation.target.clone()), + ], + ) + .await?; + Ok((transaction, !rows.is_empty())) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs b/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs index 6a3640933e..8982ae3db5 100644 --- a/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs +++ b/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs @@ -6,11 +6,10 @@ use crate::declarations::models::TriggerStatus; use crate::scheduler::main::check_lease::check_lease; use crate::scheduler::models::SchedulerLease; use crate::sensors::_helpers::attempts::{defer_sensor, last_attempt, read_sensor}; +use crate::sensors::_helpers::conditions::consume_condition; use crate::sensors::_helpers::facts::{append_evaluation, commit_cursor}; -use crate::sensors::_helpers::platform::consume_platform_events; -use crate::sensors::_helpers::runs::consume_run_events; use crate::sensors::main::read_sensor_cursor::read_sensor_cursor; -use crate::sensors::models::{SensorCondition, SensorEvaluationOutcome, SensorEvaluationRequest}; +use crate::sensors::models::{SensorEvaluationOutcome, SensorEvaluationRequest}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; @@ -47,24 +46,7 @@ pub async fn complete_builtin_sensor( commit(transaction).await?; return Ok(0); } - let (transaction, (cursor, run_ids)) = match &evaluation.definition.when { - SensorCondition::PlatformEvent { kinds } => { - consume_platform_events(transaction, evaluation, kinds).await? - } - SensorCondition::RunEvent { - job_key, - kinds, - executed_successfully, - } => { - consume_run_events( - transaction, - evaluation, - (job_key, kinds), - executed_successfully.as_ref(), - ) - .await? - } - }; + let (transaction, (cursor, run_ids)) = consume_condition(transaction, evaluation).await?; let count = run_ids.len(); let (transaction, (event_id, storage_id)) = append_evaluation( transaction, diff --git a/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs b/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs index 76b8bc2b75..0459f426dc 100644 --- a/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs +++ b/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs @@ -2,12 +2,21 @@ use crate::declarations::models::Definitions; use crate::sensors::models::SensorCondition; +use crate::snapshots::models::SnapshotInput; use crate::store::errors::ControlError; -pub(crate) fn validate_sensors(definitions: &Definitions) -> Result<(), ControlError> { +pub(crate) fn validate_sensors( + definitions: &Definitions, + input: &SnapshotInput, +) -> Result<(), ControlError> { for (name, sensor) in &definitions.sensors { let valid_condition = match &sensor.when { SensorCondition::PlatformEvent { kinds } => !kinds.is_empty(), + SensorCondition::NodeEvent { selection, kinds } => { + !kinds.is_empty() + && !selection.is_empty() + && selection.iter().all(|key| input.nodes.contains_key(key)) + } SensorCondition::RunEvent { job_key, kinds, @@ -32,7 +41,7 @@ pub(crate) fn validate_sensors(definitions: &Definitions) -> Result<(), ControlE && !definitions.groups.contains_key(&sensor.job_key)) { return Err(ControlError::Invalid(format!( - "sensor {name} needs a description, a known job and nonempty event kinds; executed_successfully must select nonempty source job/group nodes" + "sensor {name} needs a description, a known job and nonempty event kinds; executed_successfully must select nonempty source job/group nodes; node_event must select nonempty snapshot nodes" ))); } } diff --git a/crates/sqlbuild-orchestration/src/sensors/models.rs b/crates/sqlbuild-orchestration/src/sensors/models.rs index 5784a28c19..5fa409329d 100644 --- a/crates/sqlbuild-orchestration/src/sensors/models.rs +++ b/crates/sqlbuild-orchestration/src/sensors/models.rs @@ -41,6 +41,18 @@ pub enum SensorCondition { #[serde(default, skip_serializing_if = "Option::is_none")] executed_successfully: Option>, }, + NodeEvent { + selection: BTreeSet, + kinds: BTreeSet, + }, +} + +#[derive( + Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, utoipa::ToSchema, +)] +#[serde(rename_all = "snake_case")] +pub enum NodeEventKind { + Success, } #[derive( @@ -80,6 +92,14 @@ pub(crate) struct TerminalSensorRun { pub(crate) last_event_storage_id: i64, } +#[derive(Debug, Deserialize)] +pub(crate) struct NodeSensorEvent { + pub(crate) event_id: Uuid, + pub(crate) run_id: Uuid, + pub(crate) storage_id: i64, + pub(crate) payload: serde_json::Value, +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(deny_unknown_fields)] pub struct SensorDefinition { diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index 58b8897399..b2c9b5fbf4 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -13,6 +13,7 @@ pub(crate) const PLATFORM_SINK: &str = include_str!("../platform_sink.sql"); pub(crate) const SENSORS: &str = include_str!("../sensors.sql"); pub(crate) const RUN_SENSORS: &str = include_str!("../run_sensors.sql"); pub(crate) const EXECUTED_NODES: &str = include_str!("../executed_nodes.sql"); +pub(crate) const NODE_SENSORS: &str = include_str!("../node_sensors.sql"); pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index 53a6d45481..6190e2798a 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -4,8 +4,8 @@ use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; use crate::store::_helpers::migration::{ - EXECUTED_NODES, LOCATION_INSTANCES, LOCATION_WORK, PLATFORM_SINK, RUN_SENSORS, SCHEMA, SENSORS, - render, verify_migrations, + EXECUTED_NODES, LOCATION_INSTANCES, LOCATION_WORK, NODE_SENSORS, PLATFORM_SINK, RUN_SENSORS, + SCHEMA, SENSORS, render, verify_migrations, }; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; @@ -35,6 +35,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { ("sensors", SENSORS), ("run_sensors", RUN_SENSORS), ("executed_nodes", EXECUTED_NODES), + ("node_sensors", NODE_SENSORS), ]; let mut digests = vec![digest.clone()]; digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); diff --git a/crates/sqlbuild-orchestration/src/store/node_sensors.sql b/crates/sqlbuild-orchestration/src/store/node_sensors.sql new file mode 100644 index 0000000000..372ccf6b4c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/node_sensors.sql @@ -0,0 +1,6 @@ +CREATE INDEX sqb_node_finished_cursor ON recall_events(storage_id) +WHERE producer='scheduler' AND event_type='run_node_finished'; +CREATE INDEX sqb_node_started_identity ON recall_events( + invocation_id, (payload -> 'change' ->> 'claim_id'), + (payload -> 'change' -> 'change' ->> 'node_key'), storage_id +) WHERE producer='scheduler' AND event_type='run_node_started'; diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs index f52cad361a..bc54a92560 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs @@ -46,9 +46,9 @@ fn given_outbound_work_when_claimed_retried_or_replaced_then_scope_and_immutable let store = &fixture.store; migrate_store(store).await.expect("fresh migration"); // Exercise both deployed predecessors, including checksums and repeated startup. - execute(store, "DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DELETE FROM sqb_schema_migrations WHERE version>=3").await; + execute(store, "DROP INDEX sqb_node_finished_cursor; DROP INDEX sqb_node_started_identity; DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DELETE FROM sqb_schema_migrations WHERE version>=3").await; migrate_store(store).await.expect("schema 2 upgrade"); - execute(store, "DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2").await; + execute(store, "DROP INDEX sqb_node_finished_cursor; DROP INDEX sqb_node_started_identity; DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2").await; migrate_store(store).await.expect("schema 1 upgrade"); migrate_store(store).await.expect("repeat migration"); let location_id = create_location(store, "orders", "test-admin") diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs b/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs index 81c48ae068..e6429a8404 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs @@ -6,6 +6,8 @@ mod helpers; mod test_delivery; #[path = "test_execution_bounds.rs"] mod test_execution_bounds; +#[path = "test_nodes.rs"] +mod test_nodes; #[path = "test_recovery.rs"] mod test_recovery; #[path = "test_runs.rs"] diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_nodes.rs b/crates/sqlbuild-orchestration/tests/sensors/test_nodes.rs new file mode 100644 index 0000000000..bc2556174c --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_nodes.rs @@ -0,0 +1,376 @@ +//! Real lifecycle facts exercise any-run rebuilds, early delivery and paged atomic recovery. + +use std::collections::BTreeMap; +use std::time::Duration; + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::declarations::models::TriggerChange; +use crate::runs::main::{ + change_run::change_run, claim_next_run::claim_next_run, list_runs::list_runs, + request_job_run::request_job_run, +}; +use crate::runs::models::{ + JobRunRequest, NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, +}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::sensors::main::{ + begin_sensor_evaluation::begin_sensor_evaluation, + complete_builtin_sensor::complete_builtin_sensor, +}; +use crate::sensors::tests::helpers::{change, finish_run, snapshot}; +use crate::sensors::tests::test_types::SensorTestCase; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_any_run_rebuilds_when_consuming_then_only_genuine_scoped_successes_queue_once_per_finish() +{ + let test_cases = [ + SensorTestCase { + description: "SQLite any-run node successes", + backend: Backend::Sqlite, + expected_runs: 8, + }, + SensorTestCase { + description: "PostgreSQL any-run node successes", + backend: Backend::Postgres, + expected_runs: 8, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required store"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + migrate_store(store).await.expect("warm migration"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "products", "test-admin") + .await + .expect("other location"); + let mut input = snapshot(location); + input.documents.insert("jobs".into(), json!({ + "orders": {"description": "Source orders", "selection": ["orders", "customers"], "overlap": "allow"}, + "inventory": {"description": "Sensor destination", "selection": ["orders", "customers"], "overlap": "allow"} + })); + input.documents.insert("groups".into(), json!({ + "daily_orders": {"description": "Source group", "selection": ["orders", "customers"], "jobs": {"orders": []}, "overlap": "allow"} + })); + input.documents.insert("triggers".into(), json!({ + "rebuilt_orders": {"description": "Each genuine rebuild", "job_key": "inventory", + "when": {"kind": "node_event", "selection": ["orders", "customers"], "kinds": ["success"]}, + "minimum_interval_seconds": 1} + })); + publish_snapshot(store, &input).await.expect("publish"); + let mut isolated = input.clone(); + isolated.location_id = other; + publish_snapshot(store, &isolated) + .await + .expect("other location snapshot"); + isolated.location_id = location; + isolated.target = "staging".into(); + publish_snapshot(store, &isolated) + .await + .expect("other target snapshot"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let trigger = list_triggers(store, location, "development") + .await + .expect("triggers") + .remove(0); + for (outcome, nodes, started) in [ + (RunOutcome::Success, [NodeOutcome::Success; 2], [true; 2]), + (RunOutcome::Success, [NodeOutcome::Success; 2], [false; 2]), + ( + RunOutcome::Warn, + [NodeOutcome::Warn, NodeOutcome::Success], + [true; 2], + ), + (RunOutcome::Failed, [NodeOutcome::Failed; 2], [true; 2]), + ( + RunOutcome::Partial, + [NodeOutcome::Skipped, NodeOutcome::Success], + [false, true], + ), + ( + RunOutcome::Cancelled, + [NodeOutcome::Cancelled; 2], + [true; 2], + ), + ] { + finish_run( + store, + &lease, + (location, "development"), + "orders", + outcome, + nodes, + started, + ) + .await; + } + for scope in [(other, "development"), (location, "staging")] { + finish_run( + store, + &lease, + scope, + "orders", + RunOutcome::Success, + [NodeOutcome::Success; 2], + [true; 2], + ) + .await; + } + finish_run( + store, + &lease, + (location, "development"), + "daily_orders", + RunOutcome::Success, + [NodeOutcome::Success; 2], + [true; 2], + ) + .await; + // Page past nonmatches without borrowing starts from earlier real runs. + for _ in 0..46 { + finish_run( + store, + &lease, + (location, "development"), + "orders", + RunOutcome::Success, + [NodeOutcome::Success; 2], + [false; 2], + ) + .await; + } + let run_id = Uuid::now_v7(); + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + request_job_run( + store, + &context, + &JobRunRequest { + run_id, + location_id: location, + target: "development".into(), + job_key: "orders".into(), + reason: "Deliver before whole-run completion".into(), + priority: 10, + window: None, + }, + ) + .await + .expect("request running source"); + let claim = claim_next_run( + store, + &lease, + &BTreeMap::new(), + Some((location, "development")), + ) + .await + .expect("claim") + .expect("ready"); + assert_eq!(claim.request.run_id, run_id); + let authority = RunAuthority::Worker { + run_id, + claim_id: claim.claim_id, + }; + let changes = [ + RunChange::Started, + RunChange::NodeStarted { + node_key: "orders".into(), + }, + RunChange::NodeFinished { + node_key: "orders".into(), + outcome: NodeOutcome::Success, + rows_affected: None, + error_code: None, + skipped_because_of: None, + }, + ]; + for change in changes { + let request = RunChangeRequest { + event_id: Uuid::now_v7(), + run_id, + claim_id: Some(claim.claim_id), + change, + }; + let mut wrong = request.clone(); + wrong.claim_id = Some(Uuid::now_v7()); + assert!( + change_run(store, authority, &wrong).await.is_err(), + "another claim cannot supply start/finish evidence" + ); + assert!( + change_run(store, authority, &request) + .await + .expect("worker fact") + ); + assert!( + !change_run(store, authority, &request) + .await + .expect("duplicate worker fact") + ); + } + let pending = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("request evaluation") + .expect("due"); + let replica = second_store(store).await; + let fault = TransactionFault { + description: "node cursor rollback", + table: "sqb_sensor_cursor_commits", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'injected node cursor failure')", + postgres_body: "RAISE EXCEPTION 'injected node cursor failure'", + }; + install_fault(store, &fault).await; + assert!( + complete_builtin_sensor(store, &lease, &pending) + .await + .is_err() + ); + remove_fault(store, &fault).await; + assert!( + list_runs(store, (location, "development"), 1000) + .await + .expect("rollback") + .iter() + .all(|run| run.job_key.as_deref() != Some("inventory")) + ); + assert_eq!( + begin_sensor_evaluation(&replica, &lease, trigger.trigger_id) + .await + .expect("recover pending"), + Some(pending.clone()) + ); + assert_eq!( + tokio::time::timeout( + Duration::from_secs(5), + complete_builtin_sensor(&replica, &lease, &pending), + ) + .await + .expect("bounded 100-event page") + .expect("first page"), + 6 + ); + assert_eq!( + complete_builtin_sensor(store, &lease, &pending) + .await + .expect("duplicate completion"), + 0 + ); + tokio::time::sleep(Duration::from_secs(1)).await; + let next = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("second evaluation") + .expect("due"); + assert!(next.cursor.event_storage_id > 0); + assert_eq!( + complete_builtin_sensor(store, &lease, &next) + .await + .expect("second page"), + 1, + "the other selected node is still pending" + ); + for change in [ + RunChange::NodeStarted { + node_key: "customers".into(), + }, + RunChange::NodeFinished { + node_key: "customers".into(), + outcome: NodeOutcome::Success, + rows_affected: None, + error_code: None, + skipped_because_of: None, + }, + ] { + change_run( + store, + authority, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id, + claim_id: Some(claim.claim_id), + change, + }, + ) + .await + .expect("later node in the same run"); + } + tokio::time::sleep(Duration::from_secs(1)).await; + let later = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("later evaluation") + .expect("due"); + assert_eq!( + complete_builtin_sensor(store, &lease, &later) + .await + .expect("later finish"), + 1, + "distinct successful finishes in the same run must not collapse" + ); + let runs = list_runs(store, (location, "development"), 1000) + .await + .expect("runs"); + assert_eq!( + runs.iter() + .filter(|run| run.job_key.as_deref() == Some("inventory")) + .count(), + test_case.expected_runs, + "{}", + test_case.description + ); + assert_eq!( + runs.iter() + .find(|run| run.run_id == run_id) + .expect("source retained") + .status, + crate::runs::models::RunStatus::Running + ); + change(store, trigger.trigger_id, TriggerChange::ResetCursor).await; + for _ in 0..2 { + tokio::time::sleep(Duration::from_secs(1)).await; + let replay = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("replay") + .expect("due"); + complete_builtin_sensor(store, &lease, &replay) + .await + .expect("deduplicated replay"); + } + assert_eq!( + list_runs(store, (location, "development"), 1000) + .await + .expect("replayed runs") + .iter() + .filter(|run| run.job_key.as_deref() == Some("inventory")) + .count(), + test_case.expected_runs + ); + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs index 7e3348ce3e..545f2373f9 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs @@ -425,6 +425,14 @@ fn given_terminal_jobs_and_groups_when_selectors_retry_then_outcomes_scope_and_r let transaction = begin(store, TransactionMode::Write) .await .expect("upgrade fixture"); + let (transaction, _) = + query(transaction, "DROP INDEX sqb_node_finished_cursor", vec![]) + .await + .expect("remove node finish cursor index"); + let (transaction, _) = + query(transaction, "DROP INDEX sqb_node_started_identity", vec![]) + .await + .expect("remove node start evidence index"); let (transaction, _) = query(transaction, "DROP INDEX sqb_runs_parent", vec![]) .await .expect("remove execution lookup index"); diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index 4faee91009..3464b72c6f 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -243,7 +243,8 @@ pub(super) async fn sqlite_failed_publish_and_schema_floor() -> i64 { let connection = rusqlite::Connection::open(&path).expect("open fault injection connection"); connection .execute_batch( - "DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; \ + "DROP INDEX sqb_node_finished_cursor; DROP INDEX sqb_node_started_identity; \ + DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; \ DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; \ DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; \ DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; \ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs index 87db7962f8..453a9f5145 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs @@ -55,6 +55,8 @@ pub(crate) struct AuthoredSensorCondition { pub(crate) job: Option, #[pyo3(default)] pub(crate) executed_successfully: Option>, + #[pyo3(default)] + pub(crate) select: Option, } #[derive(FromPyObject)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs index f3ba7c955e..03f6bb4e77 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -270,18 +270,13 @@ pub(crate) fn compile_sensors( if let Some(job) = sensor.when.job { condition["job_key"] = json!(job.name); } + if let Some(select) = sensor.when.select { + let selectors: Vec = select.split_whitespace().map(str::to_owned).collect(); + condition["selection"] = sensor_nodes(graph, &selectors, "node_event")?; + } if let Some(selectors) = sensor.when.executed_successfully { - if selectors.is_empty() || selectors.iter().any(|value| value.trim().is_empty()) { - return Err(PyValueError::new_err( - "executed_successfully needs nonempty selectors", - )); - } - let nodes: BTreeSet = resolve_selectors(graph, &selectors, &[]) - .map_err(|error| PyValueError::new_err(error.message))? - .into_iter() - .map(|(kind, name)| format!("{kind}:{name}")) - .collect(); - condition["executed_successfully"] = json!(nodes); + condition["executed_successfully"] = + sensor_nodes(graph, &selectors, "executed_successfully")?; } let definition: SensorDefinition = serde_json::from_value(json!({ "description": sensor.description, @@ -295,3 +290,22 @@ pub(crate) fn compile_sensors( } Ok(definitions) } + +fn sensor_nodes(graph: &ProjectGraph, selectors: &[String], field: &str) -> PyResult { + if selectors.is_empty() || selectors.iter().any(|value| value.trim().is_empty()) { + return Err(PyValueError::new_err(format!( + "{field} needs nonempty selectors" + ))); + } + let nodes: BTreeSet = resolve_selectors(graph, selectors, &[]) + .map_err(|error| PyValueError::new_err(error.message))? + .into_iter() + .map(|(kind, name)| format!("{kind}:{name}")) + .collect(); + if nodes.is_empty() { + return Err(PyValueError::new_err(format!( + "{field} needs nonempty selectors" + ))); + } + Ok(json!(nodes)) +} diff --git a/python-boundary.toml b/python-boundary.toml index b50c3c0fc6..159acb56e3 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -25,7 +25,7 @@ note = "Public job and schedule authoring records only; selection resolution, va [[stays]] path = "orchestration_declarations/types.py" rule = 3 -note = "Public PlatformEventKind and RunEventKind enums only, placed in types.py to satisfy enum ownership while retaining a zero implementation budget; no Python event processing allowance." +note = "Public PlatformEventKind, RunEventKind and NodeEventKind enums only, placed in types.py to satisfy enum ownership while retaining a zero implementation budget; no Python event processing allowance." [[stays]] path = "orchestration_declarations/main/compile_snapshot.py" diff --git a/src/sqlbuild/orchestration.py b/src/sqlbuild/orchestration.py index 38fec4927a..3532cf5f7d 100644 --- a/src/sqlbuild/orchestration.py +++ b/src/sqlbuild/orchestration.py @@ -6,6 +6,7 @@ CalendarWindow, Job, JobGroup, + NodeEventCondition, PlatformEventCondition, RunEventCondition, Schedule, @@ -14,17 +15,20 @@ from sqlbuild.orchestration_declarations.models import CalendarSchedule as calendar_schedule from sqlbuild.orchestration_declarations.models import Job as job from sqlbuild.orchestration_declarations.models import JobGroup as job_group +from sqlbuild.orchestration_declarations.models import NodeEventCondition as node_event from sqlbuild.orchestration_declarations.models import PlatformEventCondition as platform_event from sqlbuild.orchestration_declarations.models import RunEventCondition as run_event from sqlbuild.orchestration_declarations.models import Schedule as schedule from sqlbuild.orchestration_declarations.models import Sensor as sensor -from sqlbuild.orchestration_declarations.types import PlatformEventKind, RunEventKind +from sqlbuild.orchestration_declarations.types import NodeEventKind, PlatformEventKind, RunEventKind __all__ = ( "CalendarSchedule", "CalendarWindow", "Job", "JobGroup", + "NodeEventCondition", + "NodeEventKind", "PlatformEventCondition", "PlatformEventKind", "RunEventCondition", @@ -35,6 +39,7 @@ "compile_snapshot", "job", "job_group", + "node_event", "platform_event", "run_event", "schedule", diff --git a/src/sqlbuild/orchestration_declarations/models.py b/src/sqlbuild/orchestration_declarations/models.py index 077fde537f..8f4a44a9b8 100644 --- a/src/sqlbuild/orchestration_declarations/models.py +++ b/src/sqlbuild/orchestration_declarations/models.py @@ -7,7 +7,7 @@ from datetime import datetime from typing import Literal -from sqlbuild.orchestration_declarations.types import PlatformEventKind, RunEventKind +from sqlbuild.orchestration_declarations.types import NodeEventKind, PlatformEventKind, RunEventKind @dataclass(frozen=True, kw_only=True) @@ -100,13 +100,22 @@ class RunEventCondition: kind: Literal["run_event"] = field(default="run_event", init=False) +@dataclass(frozen=True, kw_only=True) +class NodeEventCondition: + """Match each successful rebuild in any run within the sensor's location and target.""" + + select: str + kinds: Sequence[NodeEventKind] + kind: Literal["node_event"] = field(default="node_event", init=False) + + @dataclass(frozen=True, eq=False) class Sensor: """Start a job for each matching event; event identity deduplicates replayed requests.""" name: str description: str = field(kw_only=True) - when: PlatformEventCondition | RunEventCondition = field(kw_only=True) + when: PlatformEventCondition | RunEventCondition | NodeEventCondition = field(kw_only=True) run: Job | JobGroup = field(kw_only=True) minimum_interval_seconds: int = field(default=30, kw_only=True) default_status: Literal["running", "stopped"] = field(default="running", kw_only=True) diff --git a/src/sqlbuild/orchestration_declarations/types.py b/src/sqlbuild/orchestration_declarations/types.py index ac8ef3e001..4635074185 100644 --- a/src/sqlbuild/orchestration_declarations/types.py +++ b/src/sqlbuild/orchestration_declarations/types.py @@ -24,3 +24,9 @@ class RunEventKind(StrEnum): FAILED = "failed" CANCELLED = "cancelled" PRESUMED_FAILED = "presumed_failed" + + +class NodeEventKind(StrEnum): + """SUCCESS requires a genuine rebuild, not freshness-only completion.""" + + SUCCESS = "success" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index 5e04e48389..391f828b68 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -541,6 +541,65 @@ def test_given_a_real_group_completion_when_a_run_sensor_matches_then_one_destin ] +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "group child node sensor", + "SELECT 67 AS amount", + "SELECT amount FROM orders", + "success", + [(67,)], + expected_origin="sensor", + files={ + "models/stg_orders.sql": "MODEL (description 'Staging orders.');\nSELECT 71 AS amount\n", + "orchestration/orders.py": ( + "from sqlbuild.orchestration import NodeEventKind, job, job_group, node_event, schedule, sensor\n" + "source = job('source_orders', select='orders', description='Source orders.')\n" + "source_group = job_group('source_group', description='Source group.', jobs=[source], " + "schedule=schedule(cron='0 0 1 1 *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" + "annual_orders = job('annual_orders', select='stg_orders', description='Refresh staging.', settings={'reload': True})\n" + "on_node = sensor('on_node', description='React to actual orders rebuilds.', " + "when=node_event(select='orders', kinds=[NodeEventKind.SUCCESS]), " + "run=annual_orders, minimum_interval_seconds=1)\n" + ), + }, + ) + ], + ids=lambda case: case.description, +) +def test_given_a_real_group_child_rebuild_when_a_node_sensor_matches_then_only_the_child_finish_queues_a_worker( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + destination: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_orders" + ) + assert destination["status"] == test_case.expected_status, worker.log.read_text() + with sqlite3.connect(worker.control_store) as control: + assert control.execute( + "SELECT job_key,status FROM sqb_runs ORDER BY job_key" + ).fetchall() == [ + ("annual_orders", "success"), + ("source_group", "success"), + ("source_orders", "success"), + ] + assert control.execute( + "SELECT q.run_key,q.run_id FROM sqb_run_requests q" + ).fetchall() == [ + ( + control.execute( + "SELECT e.event_id FROM recall_events e JOIN sqb_runs r ON r.run_id=e.run_id " + "WHERE r.job_key='source_orders' AND e.event_type='run_node_finished'" + ).fetchone()[0], + destination["run_id"], + ) + ] + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index c4804832b3..ded89951dd 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -685,6 +685,28 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject } }, ), + SnapshotDiscoveryTestCase( + description="any-run node selectors freeze canonical keys without a source job reference", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/node_orders.py" + ).read_text() + }, + expected_jobs=("inventory", "source_orders"), + expected_sensors={ + "on_orders": { + "description": "Build inventory for every genuine orders rebuild.", + "job_key": "inventory", + "when": { + "kind": "node_event", + "selection": ["model:orders"], + "kinds": ["success"], + }, + "minimum_interval_seconds": 30, + "default_status": "running", + } + }, + ), SnapshotDiscoveryTestCase( description="source-job execution selectors compile to frozen canonical keys", files={ @@ -884,6 +906,30 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects expected_error=ValueError, expected_message="unknown variant", ), + SnapshotDiscoveryRejectionTestCase( + description="empty node selectors cannot broaden matching to all nodes", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/node_orders.py" + ) + .read_text() + .replace('select="orders orders*"', 'select=" "') + }, + expected_error=ValueError, + expected_message="node_event needs nonempty selectors", + ), + SnapshotDiscoveryRejectionTestCase( + description="node event kinds do not imply unapproved failure transitions", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/node_orders.py" + ) + .read_text() + .replace("kinds=[NodeEventKind.SUCCESS]", "kinds=['failed']") + }, + expected_error=ValueError, + expected_message="unknown variant", + ), SnapshotDiscoveryRejectionTestCase( description="explicit empty execution selectors cannot broaden matching", files={ From 8b4572fb75ec92413e496df23e3b389260841a74 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 09:59:18 +0000 Subject: [PATCH 54/59] feat: execute custom polling sensors in isolated locations Commit proposed cursors and deduplicated requests only through fenced atomic completion. Defer pending evaluations fairly, preserve the public authoring imports, and isolate the approved mutable context declaration with a structural scope guard. Adopt Fensu 0.28.3 for declaration-role recognition. Preserve optional bare-command dispatch and confine adversarial terminal width to the location child process. Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../edge_cases/orchestration/custom_orders.py | 15 + .../src/native_commands/constants.rs | 3 + .../native_commands/main/_location_sensor.rs | 76 +++ .../native_commands/main/_location_work.rs | 32 +- .../src/native_commands/main/mod.rs | 1 + crates/sqlbuild-orchestration/src/api/mod.rs | 2 +- .../src/platform/_helpers/location_work.rs | 17 - .../src/platform/main/claim_location_work.rs | 13 +- .../platform/main/complete_location_work.rs | 30 +- .../src/platform/main/location_work.rs | 4 +- .../src/platform/main/mod.rs | 1 + .../src/platform/main/read_location_work.rs | 24 + .../platform/main/request_location_work.rs | 11 +- .../src/platform/models.rs | 6 + .../src/scheduler/_helpers/drain_sensors.rs | 8 +- .../src/scheduler/tests.rs | 2 + .../src/scheduler/tests/test_types.rs | 9 + .../src/sensors/_helpers/completion.rs | 77 +++ .../src/sensors/_helpers/conditions.rs | 3 + .../src/sensors/_helpers/custom.rs | 104 ++++ .../src/sensors/_helpers/mod.rs | 2 + .../src/sensors/_helpers/platform.rs | 1 + .../src/sensors/constants.rs | 2 + .../sensors/main/begin_sensor_evaluation.rs | 2 + .../sensors/main/complete_builtin_sensor.rs | 59 +-- .../sensors/main/complete_custom_sensor.rs | 42 ++ .../src/sensors/main/mod.rs | 1 + .../src/sensors/main/validate_sensors.rs | 4 + .../sqlbuild-orchestration/src/sensors/mod.rs | 4 +- .../src/sensors/models.rs | 22 + .../tests/api/test_contracts.rs | 2 +- .../tests/scheduler/test_sensor_fairness.rs | 128 +++++ .../tests/sensors/test_contracts.rs | 4 +- .../tests/sensors/test_custom.rs | 443 ++++++++++++++++++ .../tests/sensors/test_types.rs | 7 + .../src/bindings/_helpers/cli/entry.rs | 10 +- .../compiled_project/snapshot_declarations.rs | 4 +- .../_helpers/location_command/runtime.rs | 68 ++- .../_helpers/native_host/constants.rs | 3 + .../src/bindings/_helpers/native_host/mod.rs | 1 + .../native_host/orchestration_declarations.rs | 7 +- .../native_host/orchestration_environment.rs | 60 ++- .../native_host/orchestration_sensors.rs | 132 ++++++ .../native_host/orchestration_triggers.rs | 49 +- fensu.toml | 6 + pyproject.toml | 2 +- python-boundary.toml | 7 +- src/sqlbuild/orchestration.py | 28 +- .../main/compile_snapshot.py | 7 +- .../models/__init__.py | 0 .../{models.py => models/authoring.py} | 24 +- .../models/sensor_context.py | 12 + .../compiled_project/publication/helpers.py | 31 +- .../publication/test_custom_sensor.py | 160 +++++++ .../publication/test_reload.py | 7 +- .../test_orchestration_snapshot.py | 36 ++ uv.lock | 30 +- 57 files changed, 1679 insertions(+), 166 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/custom_orders.py create mode 100644 crates/sqlbuild-cli/src/native_commands/main/_location_sensor.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/read_location_work.rs create mode 100644 crates/sqlbuild-orchestration/src/sensors/_helpers/completion.rs create mode 100644 crates/sqlbuild-orchestration/src/sensors/_helpers/custom.rs create mode 100644 crates/sqlbuild-orchestration/src/sensors/main/complete_custom_sensor.rs create mode 100644 crates/sqlbuild-orchestration/tests/scheduler/test_sensor_fairness.rs create mode 100644 crates/sqlbuild-orchestration/tests/sensors/test_custom.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs create mode 100644 src/sqlbuild/orchestration_declarations/models/__init__.py rename src/sqlbuild/orchestration_declarations/{models.py => models/authoring.py} (87%) create mode 100644 src/sqlbuild/orchestration_declarations/models/sensor_context.py create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py diff --git a/benchmarks/duckshop/edge_cases/orchestration/custom_orders.py b/benchmarks/duckshop/edge_cases/orchestration/custom_orders.py new file mode 100644 index 0000000000..f448f2b85f --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/custom_orders.py @@ -0,0 +1,15 @@ +"""Polling callbacks return durable run keys and keep their cursor in the native transaction.""" + +from sqlbuild.orchestration import job, sensor + +orders = job("annual_orders", description="Build orders after source inspection.", select="orders") + + +@sensor( + "poll_orders", description="Poll fulfillment readiness.", run=orders, minimum_interval_seconds=1 +) +def poll_orders(ctx, fulfillment): + def poll(): + return fulfillment.poll(ctx) + + return poll() diff --git a/crates/sqlbuild-cli/src/native_commands/constants.rs b/crates/sqlbuild-cli/src/native_commands/constants.rs index 74c0358e56..65784a5e9b 100644 --- a/crates/sqlbuild-cli/src/native_commands/constants.rs +++ b/crates/sqlbuild-cli/src/native_commands/constants.rs @@ -90,3 +90,6 @@ pub(crate) const EXIT_FLUSH_FAILURE: &str = "Exception ignored in: <_io.TextIOWrapper name='' mode='w' encoding='utf-8'>"; pub(crate) const EXIT_FLUSH_FAILURE_STATUS: i32 = 120; pub(crate) const UNCAUGHT_EXCEPTION_STATUS: i32 = 1; +/// Reserve half the location work claim for acknowledgement retries after killing a callback. +pub(crate) const SENSOR_EVALUATION_SECONDS: u64 = 30; +pub(crate) const SENSOR_RESULT_BYTES: u64 = 1_048_576; diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_sensor.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_sensor.rs new file mode 100644 index 0000000000..86c4585092 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_sensor.rs @@ -0,0 +1,76 @@ +//! A callback gets its own interpreter; timeout and shutdown reap it before reporting a result. + +use std::future::Future; +use std::path::Path; +use std::time::Duration; + +use sqlbuild_orchestration::sensors::models::{CustomSensorOutcome, SensorEvaluationRequest}; +use sqlbuild_orchestration::store::errors::ControlError; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::process::Child; + +use crate::native_commands::_helpers::execution::location_runtime_child; +use crate::native_commands::constants::{SENSOR_EVALUATION_SECONDS, SENSOR_RESULT_BYTES}; +use crate::native_commands::models::LocationServe; + +pub(super) async fn evaluate_location_sensor( + options: &LocationServe, + script: &Path, + evaluation: &SensorEvaluationRequest, + stopped: impl Future>, +) -> Result, ControlError> { + let request = + serde_json::to_vec(evaluation).map_err(|error| ControlError::Invalid(error.to_string()))?; + let mut child = match location_runtime_child(options, script) { + Ok(child) => child, + Err(_) => return Ok(Some(CustomSensorOutcome::Failed)), + }; + tokio::pin!(stopped); + let outcome = tokio::select! { + result = &mut stopped => { + child.kill().await?; + result?; + return Ok(None); + }, + result = tokio::time::timeout(Duration::from_secs(SENSOR_EVALUATION_SECONDS), exchange_result(&mut child, request)) => { + match result { + Ok(Ok((_, outcome))) => outcome, + Ok(Err(_)) => CustomSensorOutcome::Failed, + Err(_) => CustomSensorOutcome::TimedOut, + } + }, + }; + if child.try_wait()?.is_none() { + child.kill().await?; + } + Ok(Some(outcome)) +} + +/// A bounded reply is accepted only after its interpreter exits successfully. +async fn exchange_result( + child: &mut Child, + request: Vec, +) -> Result<(&mut Child, CustomSensorOutcome), ControlError> { + let mut stdin = child + .stdin + .take() + .ok_or_else(|| ControlError::Invalid("sensor input is unavailable".into()))?; + let stdout = child + .stdout + .take() + .ok_or_else(|| ControlError::Invalid("sensor output is unavailable".into()))?; + stdin.write_all(&request).await?; + stdin.write_all(b"\n").await?; + stdin.shutdown().await?; + drop(stdin); + let mut bytes: Vec = Vec::new(); + let mut stdout = stdout.take(SENSOR_RESULT_BYTES + 1); + stdout.read_to_end(&mut bytes).await?; + if bytes.len() as u64 > SENSOR_RESULT_BYTES || !child.wait().await?.success() { + return Ok((child, CustomSensorOutcome::Failed)); + } + Ok(( + child, + serde_json::from_slice(&bytes).unwrap_or(CustomSensorOutcome::Failed), + )) +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs index 281b8bb686..abf575eb77 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs @@ -16,6 +16,7 @@ use crate::argv::models::Stream; use crate::native_commands::_helpers::execution::check_location_connection; use crate::native_commands::_helpers::paths::location_inputs; use crate::native_commands::_helpers::terminal::write_output; +use crate::native_commands::main::_location_sensor::evaluate_location_sensor; use crate::native_commands::models::{LocationServe, LocationWatch}; /// Retry the exact completed result after uncertain delivery; never rerun it to obtain a new result. @@ -44,11 +45,14 @@ pub(super) async fn run_location_work( .to_std() .unwrap_or_default(); let mut files: Vec = Vec::new(); - let outcome = if generation == Some(claim.generation) { - drop(write_output(&[( - Stream::Stderr, - "Checking requested location connection...\n", - )])); + let outcome = if generation == Some(claim.generation) + && location_inputs(&options.project_dir, &watch.rules, &watch.local_files) == watch.observed + { + let message = match &claim.request.work { + LocationWork::CheckConnection {} => "Checking requested location connection...\n", + LocationWork::EvaluateSensor { .. } => "Evaluating requested custom sensor...\n", + }; + drop(write_output(&[(Stream::Stderr, message)])); let mut shutting_down = false; let expires = async { tokio::select! { @@ -57,11 +61,21 @@ pub(super) async fn run_location_work( } }; let checked = match claim.request.work { - LocationWork::CheckConnection {} => { - check_location_connection(options, script, expires).await? + LocationWork::CheckConnection {} => check_location_connection(options, script, expires) + .await? + .map(|(healthy, files)| (LocationWorkOutcome::Connection { healthy }, files)), + LocationWork::EvaluateSensor { evaluation } => { + evaluate_location_sensor(options, script, &evaluation, expires) + .await? + .map(|outcome| { + ( + LocationWorkOutcome::SensorEvaluation { outcome }, + Vec::new(), + ) + }) } }; - let Some((healthy, local_files)) = checked else { + let Some((outcome, local_files)) = checked else { drop(write_output(&[( Stream::Stderr, "Location work stopped before completion; no result reported.\n", @@ -76,7 +90,7 @@ pub(super) async fn run_location_work( let mut excluded = watch.local_files.clone(); excluded.extend(files.clone()); if location_inputs(&options.project_dir, &watch.rules, &excluded) == expected { - LocationWorkOutcome::Connection { healthy } + outcome } else { LocationWorkOutcome::Superseded {} } diff --git a/crates/sqlbuild-cli/src/native_commands/main/mod.rs b/crates/sqlbuild-cli/src/native_commands/main/mod.rs index a1333f51a6..af7c252502 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/mod.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/mod.rs @@ -1,5 +1,6 @@ pub(super) mod _cost_report_options; pub(super) mod _location_compiler; +pub(super) mod _location_sensor; pub(super) mod _location_work; pub(super) mod _run_admin; pub(super) mod _run_control; diff --git a/crates/sqlbuild-orchestration/src/api/mod.rs b/crates/sqlbuild-orchestration/src/api/mod.rs index cb776dbca6..524d320a8c 100644 --- a/crates/sqlbuild-orchestration/src/api/mod.rs +++ b/crates/sqlbuild-orchestration/src/api/mod.rs @@ -8,4 +8,4 @@ pub mod models; #[cfg(test)] #[path = "../../tests/api/test_contracts.rs"] -mod tests; +pub(crate) mod tests; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/location_work.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/location_work.rs index c523d62210..8e2d429a5a 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/location_work.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/location_work.rs @@ -2,27 +2,10 @@ use uuid::Uuid; -use crate::platform::models::StoredLocationWork; use crate::store::errors::ControlError; -use crate::store::main::decode_row::decode_row; use crate::store::main::query::query; use crate::store::models::{ControlTransaction, Parameter}; -pub(crate) async fn read_work( - transaction: ControlTransaction, - request_id: Uuid, -) -> Result<(ControlTransaction, Option), ControlError> { - let (transaction, rows) = query(transaction, - "SELECT w.request_json, w.requested_by, w.requested_at, w.expires_at, w.generation, \ - w.claim_id, w.session_id, w.claimed_until, r.result_json, r.completed_at \ - FROM sqb_location_work w LEFT JOIN sqb_location_work_results r ON r.request_id=w.request_id \ - WHERE w.request_id=$1", vec![Parameter::Id(request_id)]).await?; - Ok(( - transaction, - rows.into_iter().next().map(decode_row).transpose()?, - )) -} - pub(crate) async fn require_instance( transaction: ControlTransaction, (location_id, session_id, target): (Uuid, Uuid, &str), diff --git a/crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs index fd24042528..7a36796c37 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs @@ -3,7 +3,8 @@ use chrono::Duration; use uuid::Uuid; -use crate::platform::_helpers::location_work::{read_work, require_instance}; +use crate::platform::_helpers::location_work::require_instance; +use crate::platform::main::read_location_work::read_location_work; use crate::platform::models::{LocationWorkClaim, LocationWorkPoll}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; @@ -26,6 +27,14 @@ pub async fn claim_location_work( "SELECT w.request_id FROM sqb_location_work w WHERE w.location_id=$1 AND w.target=$2 \ AND w.expires_at>$3 AND NOT EXISTS (SELECT 1 FROM sqb_location_work_results r WHERE r.request_id=w.request_id) \ AND (w.claimed_until IS NULL OR w.claimed_until<=$3 OR w.session_id=$4) \ + AND (w.request_json->'work'->>'kind'='check_connection' OR EXISTS ( \ + SELECT 1 FROM sqb_sensor_evaluation_events e JOIN sqb_triggers t ON t.trigger_id=e.trigger_id \ + JOIN sqb_trigger_state s ON s.trigger_id=t.trigger_id JOIN sqb_scheduler_lease l ON l.name='scheduler' \ + WHERE e.evaluation_id=w.request_id AND e.phase='requested' AND s.status='running' \ + AND t.snapshot_id=w.snapshot_id AND t.removed_in_snapshot_id IS NULL AND l.expires_at>$3 \ + AND l.epoch=CAST(e.fact_json->'request'->>'epoch' AS BIGINT) \ + AND NOT EXISTS (SELECT 1 FROM sqb_sensor_evaluation_events f \ + WHERE f.evaluation_id=e.evaluation_id AND f.phase<>'requested'))) \ ORDER BY w.requested_at, w.request_id LIMIT 1", vec![Parameter::Id(location_id), Parameter::Text(poll.target.clone()), Parameter::Time(now), Parameter::Id(poll.session_id)]).await?; let Some(row) = rows.first() else { @@ -34,7 +43,7 @@ pub async fn claim_location_work( }; let request_id = serde_json::from_value(row["request_id"].clone()) .map_err(|error| ControlError::Invalid(format!("invalid work identity: {error}")))?; - let (transaction, stored) = read_work(transaction, request_id).await?; + let (transaction, stored) = read_location_work(transaction, request_id).await?; let stored = stored.ok_or_else(|| ControlError::Invalid("location work disappeared".into()))?; let retained = stored.session_id == Some(poll.session_id) && stored.claimed_until.is_some_and(|until| until > now); diff --git a/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs index 0f1888ed8b..a044885fe4 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs @@ -3,8 +3,11 @@ use serde_json::json; use uuid::Uuid; -use crate::platform::_helpers::location_work::{read_work, require_instance}; -use crate::platform::models::LocationWorkResult; +use crate::platform::_helpers::location_work::require_instance; +use crate::platform::main::read_location_work::read_location_work; +use crate::platform::models::{LocationWork, LocationWorkOutcome, LocationWorkResult}; +use crate::sensors::constants::MAX_SENSOR_RUN_REQUESTS; +use crate::sensors::models::CustomSensorOutcome; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; @@ -17,7 +20,7 @@ pub async fn complete_location_work( result: &LocationWorkResult, ) -> Result { let transaction = begin(store, TransactionMode::Write).await?; - let (transaction, stored) = read_work(transaction, request_id).await?; + let (transaction, stored) = read_location_work(transaction, request_id).await?; let stored = stored .filter(|stored| stored.record.request_json.location_id == location_id) .ok_or_else(|| ControlError::Invalid("work is not assigned to this location".into()))?; @@ -48,6 +51,27 @@ pub async fn complete_location_work( { return Err(ControlError::Fenced); } + let valid = match (&stored.record.request_json.work, &result.outcome) { + (_, LocationWorkOutcome::Superseded {}) + | (LocationWork::CheckConnection {}, LocationWorkOutcome::Connection { .. }) => true, + ( + LocationWork::EvaluateSensor { .. }, + LocationWorkOutcome::SensorEvaluation { outcome }, + ) => match outcome { + CustomSensorOutcome::Completed { run_keys, .. } => { + run_keys.len() <= MAX_SENSOR_RUN_REQUESTS + && run_keys.iter().all(|key| !key.trim().is_empty()) + } + CustomSensorOutcome::Skipped { reason, .. } => !reason.trim().is_empty(), + CustomSensorOutcome::Failed | CustomSensorOutcome::TimedOut => true, + }, + _ => false, + }; + if !valid { + return Err(ControlError::Invalid( + "location result does not match its work".into(), + )); + } let now = transaction.now; let (transaction, _) = query(transaction, "INSERT INTO sqb_location_work_results (request_id,result_json,completed_at) VALUES ($1,$2,$3)", diff --git a/crates/sqlbuild-orchestration/src/platform/main/location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/location_work.rs index f71f848d79..8a8783c56f 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/location_work.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/location_work.rs @@ -2,7 +2,7 @@ use uuid::Uuid; -use crate::platform::_helpers::location_work::read_work; +use crate::platform::main::read_location_work::read_location_work; use crate::platform::models::LocationWorkRecord; use crate::store::errors::ControlError; use crate::store::main::begin::begin; @@ -14,7 +14,7 @@ pub async fn location_work( request_id: Uuid, ) -> Result, ControlError> { let transaction = begin(store, TransactionMode::Read).await?; - let (transaction, record) = read_work(transaction, request_id).await?; + let (transaction, record) = read_location_work(transaction, request_id).await?; commit(transaction).await?; Ok(record.map(|stored| stored.record)) } diff --git a/crates/sqlbuild-orchestration/src/platform/main/mod.rs b/crates/sqlbuild-orchestration/src/platform/main/mod.rs index 850f9e7c04..a782c64402 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/mod.rs @@ -4,6 +4,7 @@ pub mod connect_location_work; pub mod connect_platform_webhook; pub mod location_health; pub mod location_work; +pub(crate) mod read_location_work; pub(crate) mod read_platform_events; pub mod report_location_health; pub mod report_remote_location_health; diff --git a/crates/sqlbuild-orchestration/src/platform/main/read_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/read_location_work.rs new file mode 100644 index 0000000000..8401a196a0 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/read_location_work.rs @@ -0,0 +1,24 @@ +//! Read durable work inside a caller's existing transaction, including fenced sensor completion. + +use uuid::Uuid; + +use crate::platform::models::StoredLocationWork; +use crate::store::errors::ControlError; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn read_location_work( + transaction: ControlTransaction, + request_id: Uuid, +) -> Result<(ControlTransaction, Option), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT w.request_json, w.requested_by, w.requested_at, w.expires_at, w.generation, \ + w.claim_id, w.session_id, w.claimed_until, r.result_json, r.completed_at \ + FROM sqb_location_work w LEFT JOIN sqb_location_work_results r ON r.request_id=w.request_id \ + WHERE w.request_id=$1", vec![Parameter::Id(request_id)]).await?; + Ok(( + transaction, + rows.into_iter().next().map(decode_row).transpose()?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs index d03b92dce4..df0c73f23d 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs @@ -6,8 +6,8 @@ use uuid::Uuid; use crate::credentials::main::append_audit::append_audit; use crate::credentials::models::AuditContext; -use crate::platform::_helpers::location_work::read_work; -use crate::platform::models::LocationWorkRequest; +use crate::platform::main::read_location_work::read_location_work; +use crate::platform::models::{LocationWork, LocationWorkRequest}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; @@ -19,6 +19,11 @@ pub async fn request_location_work( context: &AuditContext, request: &LocationWorkRequest, ) -> Result { + if matches!(request.work, LocationWork::EvaluateSensor { .. }) { + return Err(ControlError::Invalid( + "sensor work must originate from a fenced evaluation".into(), + )); + } if request.target.trim().is_empty() || request.reason.trim().is_empty() || context.actor.trim().is_empty() @@ -28,7 +33,7 @@ pub async fn request_location_work( )); } let transaction = begin(store, TransactionMode::Write).await?; - let (transaction, previous) = read_work(transaction, request.request_id).await?; + let (transaction, previous) = read_location_work(transaction, request.request_id).await?; if let Some(previous) = previous { if previous.record.request_json != *request || previous.record.requested_by != context.actor { diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index 7ab2762ac9..fab9c20eab 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -6,6 +6,7 @@ use uuid::Uuid; use crate::api::models::ControlClient; use crate::declarations::models::TriggerDefinition; +use crate::sensors::models::{CustomSensorOutcome, SensorEvaluationRequest}; /// One pooled outbound connection; the location never receives a store handle. #[derive(Clone, Debug)] @@ -45,6 +46,10 @@ pub struct LocationHealthObservation { #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum LocationWork { CheckConnection {}, + EvaluateSensor { + #[schema(value_type = Object)] + evaluation: Box, + }, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] @@ -78,6 +83,7 @@ pub struct LocationWorkClaim { #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum LocationWorkOutcome { Connection { healthy: bool }, + SensorEvaluation { outcome: CustomSensorOutcome }, Superseded {}, } diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_sensors.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_sensors.rs index 2456383857..964ee871b6 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_sensors.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_sensors.rs @@ -4,6 +4,8 @@ use crate::scheduler::main::renew_lease::renew_lease; use crate::scheduler::models::SchedulerLease; use crate::sensors::main::begin_sensor_evaluation::begin_sensor_evaluation; use crate::sensors::main::complete_builtin_sensor::complete_builtin_sensor; +use crate::sensors::main::complete_custom_sensor::complete_custom_sensor; +use crate::sensors::models::SensorCondition; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; @@ -27,7 +29,11 @@ pub(crate) async fn drain_sensors( let trigger_id = serde_json::from_value(row["trigger_id"].clone()) .map_err(|error| ControlError::Invalid(error.to_string()))?; if let Some(request) = begin_sensor_evaluation(store, &lease, trigger_id).await? { - complete_builtin_sensor(store, &lease, &request).await?; + if matches!(request.definition.when, SensorCondition::Custom { .. }) { + complete_custom_sensor(store, &lease, &request).await?; + } else { + complete_builtin_sensor(store, &lease, &request).await?; + } evaluated += 1; } lease = renew_lease(store, &lease).await?; diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests.rs b/crates/sqlbuild-orchestration/src/scheduler/tests.rs index 927f476ce9..3a3124f551 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/tests.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/tests.rs @@ -6,6 +6,8 @@ mod helpers; mod test_backlog; #[path = "tests/test_cron.rs"] mod test_cron; +#[path = "../../tests/scheduler/test_sensor_fairness.rs"] +mod test_sensor_fairness; #[path = "tests/test_simulation.rs"] mod test_simulation; #[path = "tests/test_ticks.rs"] diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/test_types.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/test_types.rs index ee92508cab..ab3d443e4f 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/tests/test_types.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/test_types.rs @@ -1,6 +1,15 @@ //! Inputs make cursor, catch-up, and overlap decisions observably different. use crate::scheduler::models::{CatchUp, Overlap}; +use crate::store::tests::store_test_types::Backend; + +pub(super) struct SensorFairnessTestCase { + pub(super) description: &'static str, + pub(super) backend: Backend, + pub(super) pending: usize, + pub(super) builtin_status: &'static str, + pub(super) expected_completed: usize, +} #[derive(Debug)] pub(super) struct TickTestCase { diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/completion.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/completion.rs new file mode 100644 index 0000000000..20964ded97 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/completion.rs @@ -0,0 +1,77 @@ +//! Every evaluation result uses the same lease, snapshot, cursor and terminal boundary. + +use chrono::TimeDelta; + +use crate::declarations::models::TriggerStatus; +use crate::scheduler::main::check_lease::check_lease; +use crate::scheduler::models::SchedulerLease; +use crate::sensors::_helpers::attempts::{defer_sensor, last_attempt, read_sensor}; +use crate::sensors::_helpers::facts::{append_evaluation, commit_cursor}; +use crate::sensors::main::read_sensor_cursor::read_sensor_cursor; +use crate::sensors::models::{SensorEvaluationOutcome, SensorEvaluationRequest}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{ControlStore, ControlTransaction, TransactionMode}; + +pub(crate) async fn pending_evaluation( + store: &ControlStore, + lease: &SchedulerLease, + evaluation: &SensorEvaluationRequest, +) -> Result, ControlError> { + let transaction = check_lease(begin(store, TransactionMode::Write).await?, lease).await?; + if evaluation.epoch != lease.epoch { + return Err(ControlError::Fenced); + } + let (transaction, previous) = last_attempt(transaction, evaluation.trigger_id).await?; + let Some((previous, terminal)) = previous else { + return Err(ControlError::Fenced); + }; + if previous.request != *evaluation { + return Err(ControlError::Fenced); + } + if terminal { + commit(transaction).await?; + return Ok(None); + } + let (transaction, sensor) = read_sensor(transaction, evaluation.trigger_id).await?; + let (transaction, cursor) = read_sensor_cursor(transaction, evaluation.trigger_id).await?; + if sensor.is_none_or(|sensor| { + sensor.status != TriggerStatus::Running || sensor.snapshot_id != evaluation.snapshot_id + }) || cursor != evaluation.cursor + { + let (transaction, _) = + append_evaluation(transaction, evaluation, SensorEvaluationOutcome::Abandoned).await?; + commit(transaction).await?; + return Ok(None); + } + Ok(Some(transaction)) +} + +pub(crate) async fn finish_evaluation( + transaction: ControlTransaction, + lease: &SchedulerLease, + evaluation: &SensorEvaluationRequest, + outcome: SensorEvaluationOutcome, +) -> Result<(), ControlError> { + let cursor = match &outcome { + SensorEvaluationOutcome::Completed { cursor, .. } => Some(cursor.clone()), + _ => None, + }; + let (mut transaction, (event_id, storage_id)) = + append_evaluation(transaction, evaluation, outcome).await?; + if let Some(cursor) = cursor { + transaction = commit_cursor( + transaction, + (evaluation.trigger_id, event_id, storage_id), + &cursor, + ) + .await?; + } + let next = evaluation.requested_at + + TimeDelta::seconds(i64::from( + evaluation.definition.minimum_interval_seconds.get(), + )); + let transaction = defer_sensor(transaction, evaluation.trigger_id, next).await?; + commit(check_lease(transaction, lease).await?).await +} diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs index 6e2e9f5d1a..2e4243170e 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs @@ -14,6 +14,9 @@ pub(crate) async fn consume_condition( evaluation: &SensorEvaluationRequest, ) -> Result<(ControlTransaction, (SensorCursor, Vec)), ControlError> { match &evaluation.definition.when { + SensorCondition::Custom { .. } => Err(ControlError::Invalid( + "custom sensors require a location result".into(), + )), SensorCondition::PlatformEvent { kinds } => { consume_platform_events(transaction, evaluation, kinds).await } diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/custom.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/custom.rs new file mode 100644 index 0000000000..c2e65bb427 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/custom.rs @@ -0,0 +1,104 @@ +//! Custom callbacks cross only the existing location-scoped outbound-work boundary. + +use std::collections::BTreeSet; + +use chrono::TimeDelta; +use serde_json::json; +use uuid::Uuid; + +use crate::platform::main::read_location_work::read_location_work; +use crate::platform::models::{LocationWork, LocationWorkOutcome, LocationWorkRequest}; +use crate::sensors::_helpers::requests::queue_sensor_run; +use crate::sensors::models::{ + CustomSensorOutcome, SensorCondition, SensorEvaluationOutcome, SensorEvaluationRequest, +}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn request_evaluation_work( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, +) -> Result { + if !matches!(evaluation.definition.when, SensorCondition::Custom { .. }) { + return Ok(transaction); + } + let request = LocationWorkRequest { + request_id: evaluation.evaluation_id, + location_id: evaluation.location_id, + target: evaluation.target.clone(), + reason: "Evaluate custom sensor".into(), + work: LocationWork::EvaluateSensor { + evaluation: Box::new(evaluation.clone()), + }, + }; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_location_work \ + (request_id,location_id,target,snapshot_id,generation,request_json,requested_by,requested_at,expires_at) \ + SELECT $1,$2,$3,snapshot_id,generation,$4,'system:sensor',$5,$6 FROM sqb_snapshots WHERE snapshot_id=$7", + vec![Parameter::Id(request.request_id), Parameter::Id(request.location_id), Parameter::Text(request.target.clone()), + Parameter::Json(json!(request)), Parameter::Time(evaluation.requested_at), + Parameter::Time(evaluation.requested_at + TimeDelta::minutes(5)), Parameter::Id(evaluation.snapshot_id)]).await?; + Ok(transaction) +} + +pub(crate) async fn read_evaluation_result( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, +) -> Result<(ControlTransaction, Option), ControlError> { + let (transaction, work) = read_location_work(transaction, evaluation.evaluation_id).await?; + let work = work.ok_or_else(|| ControlError::Invalid("sensor has no location work".into()))?; + if work.record.result_json.is_none() { + let expired = work.record.expires_at <= transaction.now; + return Ok(( + transaction, + expired.then_some(SensorEvaluationOutcome::Abandoned), + )); + } + match work.record.result_json.map(|result| result.outcome) { + Some(LocationWorkOutcome::SensorEvaluation { outcome }) => { + apply_result(transaction, evaluation, outcome).await + } + Some(LocationWorkOutcome::Superseded {}) => { + Ok((transaction, Some(SensorEvaluationOutcome::Abandoned))) + } + _ => Err(ControlError::Invalid( + "sensor has an invalid location result".into(), + )), + } +} + +async fn apply_result( + mut transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, + outcome: CustomSensorOutcome, +) -> Result<(ControlTransaction, Option), ControlError> { + let (custom, run_keys) = match outcome { + CustomSensorOutcome::Completed { cursor, run_keys } => (cursor, run_keys), + CustomSensorOutcome::Skipped { cursor, .. } => (cursor, Vec::new()), + CustomSensorOutcome::Failed => { + return Ok(( + transaction, + Some(SensorEvaluationOutcome::Failed { + code: "custom_sensor_failed".into(), + }), + )); + } + CustomSensorOutcome::TimedOut => { + return Ok((transaction, Some(SensorEvaluationOutcome::TimedOut))); + } + }; + let mut cursor = evaluation.cursor.clone(); + cursor.custom = custom; + let mut run_ids: Vec = Vec::new(); + let run_keys: BTreeSet = run_keys.into_iter().collect(); + for key in run_keys { + let run_id; + (transaction, run_id) = queue_sensor_run(transaction, evaluation, &key).await?; + run_ids.push(run_id); + } + Ok(( + transaction, + Some(SensorEvaluationOutcome::Completed { cursor, run_ids }), + )) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs index 267996a120..720f46154e 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs @@ -1,5 +1,7 @@ pub(crate) mod attempts; +pub(crate) mod completion; pub(crate) mod conditions; +pub(crate) mod custom; pub(crate) mod facts; pub(crate) mod nodes; pub(crate) mod platform; diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs index 208470a995..6ce610bf03 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs @@ -20,6 +20,7 @@ pub(crate) async fn consume_platform_events( event_storage_id: events .last() .map_or(evaluation.cursor.event_storage_id, |event| event.storage_id), + ..evaluation.cursor.clone() }; let mut run_ids: Vec = Vec::new(); for event in events diff --git a/crates/sqlbuild-orchestration/src/sensors/constants.rs b/crates/sqlbuild-orchestration/src/sensors/constants.rs index e2b63365a5..c74670d166 100644 --- a/crates/sqlbuild-orchestration/src/sensors/constants.rs +++ b/crates/sqlbuild-orchestration/src/sensors/constants.rs @@ -1,3 +1,5 @@ //! Bound query parameters independently of the size of an authored selection. pub(crate) const EXECUTION_NODE_BATCH_SIZE: usize = 500; +pub(crate) const CUSTOM_SENSOR_RECHECK_SECONDS: i64 = 1; +pub const MAX_SENSOR_RUN_REQUESTS: usize = 100; diff --git a/crates/sqlbuild-orchestration/src/sensors/main/begin_sensor_evaluation.rs b/crates/sqlbuild-orchestration/src/sensors/main/begin_sensor_evaluation.rs index 45b82e4cbb..db7c36bb49 100644 --- a/crates/sqlbuild-orchestration/src/sensors/main/begin_sensor_evaluation.rs +++ b/crates/sqlbuild-orchestration/src/sensors/main/begin_sensor_evaluation.rs @@ -8,6 +8,7 @@ use crate::declarations::models::TriggerStatus; use crate::scheduler::main::check_lease::check_lease; use crate::scheduler::models::SchedulerLease; use crate::sensors::_helpers::attempts::{defer_sensor, last_attempt, read_sensor}; +use crate::sensors::_helpers::custom::request_evaluation_work; use crate::sensors::_helpers::facts::append_evaluation; use crate::sensors::main::read_sensor_cursor::read_sensor_cursor; use crate::sensors::models::{SensorEvaluationOutcome, SensorEvaluationRequest}; @@ -75,6 +76,7 @@ pub async fn begin_sensor_evaluation( }; let (transaction, _) = append_evaluation(transaction, &request, SensorEvaluationOutcome::Requested).await?; + let transaction = request_evaluation_work(transaction, &request).await?; commit(check_lease(transaction, lease).await?).await?; Ok(Some(request)) } diff --git a/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs b/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs index 8982ae3db5..5fbd35834d 100644 --- a/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs +++ b/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs @@ -1,73 +1,28 @@ //! Cursor, outcome, deduplicated requests and queued runs commit together. -use chrono::TimeDelta; - -use crate::declarations::models::TriggerStatus; -use crate::scheduler::main::check_lease::check_lease; use crate::scheduler::models::SchedulerLease; -use crate::sensors::_helpers::attempts::{defer_sensor, last_attempt, read_sensor}; +use crate::sensors::_helpers::completion::{finish_evaluation, pending_evaluation}; use crate::sensors::_helpers::conditions::consume_condition; -use crate::sensors::_helpers::facts::{append_evaluation, commit_cursor}; -use crate::sensors::main::read_sensor_cursor::read_sensor_cursor; use crate::sensors::models::{SensorEvaluationOutcome, SensorEvaluationRequest}; use crate::store::errors::ControlError; -use crate::store::main::begin::begin; -use crate::store::main::commit::commit; -use crate::store::models::{ControlStore, TransactionMode}; +use crate::store::models::ControlStore; pub async fn complete_builtin_sensor( store: &ControlStore, lease: &SchedulerLease, evaluation: &SensorEvaluationRequest, ) -> Result { - let transaction = check_lease(begin(store, TransactionMode::Write).await?, lease).await?; - if evaluation.epoch != lease.epoch { - return Err(ControlError::Fenced); - } - let (transaction, previous) = last_attempt(transaction, evaluation.trigger_id).await?; - let Some((previous, terminal)) = previous else { - return Err(ControlError::Fenced); - }; - if previous.request != *evaluation { - return Err(ControlError::Fenced); - } - if terminal { - commit(transaction).await?; + let Some(transaction) = pending_evaluation(store, lease, evaluation).await? else { return Ok(0); - } - let (transaction, sensor) = read_sensor(transaction, evaluation.trigger_id).await?; - let (transaction, cursor) = read_sensor_cursor(transaction, evaluation.trigger_id).await?; - if sensor.is_none_or(|sensor| { - sensor.status != TriggerStatus::Running || sensor.snapshot_id != evaluation.snapshot_id - }) || cursor != evaluation.cursor - { - let (transaction, _) = - append_evaluation(transaction, evaluation, SensorEvaluationOutcome::Abandoned).await?; - commit(transaction).await?; - return Ok(0); - } + }; let (transaction, (cursor, run_ids)) = consume_condition(transaction, evaluation).await?; let count = run_ids.len(); - let (transaction, (event_id, storage_id)) = append_evaluation( + finish_evaluation( transaction, + lease, evaluation, - SensorEvaluationOutcome::Completed { - cursor: cursor.clone(), - run_ids, - }, - ) - .await?; - let transaction = commit_cursor( - transaction, - (evaluation.trigger_id, event_id, storage_id), - &cursor, + SensorEvaluationOutcome::Completed { cursor, run_ids }, ) .await?; - let next = evaluation.requested_at - + TimeDelta::seconds(i64::from( - evaluation.definition.minimum_interval_seconds.get(), - )); - let transaction = defer_sensor(transaction, evaluation.trigger_id, next).await?; - commit(check_lease(transaction, lease).await?).await?; Ok(count) } diff --git a/crates/sqlbuild-orchestration/src/sensors/main/complete_custom_sensor.rs b/crates/sqlbuild-orchestration/src/sensors/main/complete_custom_sensor.rs new file mode 100644 index 0000000000..98d23dd900 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/main/complete_custom_sensor.rs @@ -0,0 +1,42 @@ +//! A location result cannot queue runs or advance a cursor until a fenced scheduler commits it. + +use chrono::TimeDelta; + +use crate::scheduler::main::check_lease::check_lease; +use crate::scheduler::models::SchedulerLease; +use crate::sensors::_helpers::attempts::defer_sensor; +use crate::sensors::_helpers::completion::{finish_evaluation, pending_evaluation}; +use crate::sensors::_helpers::custom::read_evaluation_result; +use crate::sensors::constants::CUSTOM_SENSOR_RECHECK_SECONDS; +use crate::sensors::models::{SensorCondition, SensorEvaluationOutcome, SensorEvaluationRequest}; +use crate::store::errors::ControlError; +use crate::store::main::commit::commit; +use crate::store::models::ControlStore; + +pub async fn complete_custom_sensor( + store: &ControlStore, + lease: &SchedulerLease, + evaluation: &SensorEvaluationRequest, +) -> Result { + if !matches!(evaluation.definition.when, SensorCondition::Custom { .. }) { + return Err(ControlError::Invalid( + "custom sensor evaluation required".into(), + )); + } + let Some(transaction) = pending_evaluation(store, lease, evaluation).await? else { + return Ok(0); + }; + let (transaction, outcome) = read_evaluation_result(transaction, evaluation).await?; + let Some(outcome) = outcome else { + let next = transaction.now + TimeDelta::seconds(CUSTOM_SENSOR_RECHECK_SECONDS); + let transaction = defer_sensor(transaction, evaluation.trigger_id, next).await?; + commit(check_lease(transaction, lease).await?).await?; + return Ok(0); + }; + let count = match &outcome { + SensorEvaluationOutcome::Completed { run_ids, .. } => run_ids.len(), + _ => 0, + }; + finish_evaluation(transaction, lease, evaluation, outcome).await?; + Ok(count) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/main/mod.rs b/crates/sqlbuild-orchestration/src/sensors/main/mod.rs index a57c8e3058..e819eb77e1 100644 --- a/crates/sqlbuild-orchestration/src/sensors/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/sensors/main/mod.rs @@ -1,5 +1,6 @@ pub mod begin_sensor_evaluation; pub mod complete_builtin_sensor; +pub mod complete_custom_sensor; pub(crate) mod publish_sensors; pub(crate) mod read_sensor_cursor; pub(crate) mod reset_sensor_cursor; diff --git a/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs b/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs index 0459f426dc..da488cc75d 100644 --- a/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs +++ b/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs @@ -11,6 +11,10 @@ pub(crate) fn validate_sensors( ) -> Result<(), ControlError> { for (name, sensor) in &definitions.sensors { let valid_condition = match &sensor.when { + SensorCondition::Custom { + function, + code_hash, + } => !function.trim().is_empty() && !code_hash.is_empty(), SensorCondition::PlatformEvent { kinds } => !kinds.is_empty(), SensorCondition::NodeEvent { selection, kinds } => { !kinds.is_empty() diff --git a/crates/sqlbuild-orchestration/src/sensors/mod.rs b/crates/sqlbuild-orchestration/src/sensors/mod.rs index 0f364e6e62..9580a2dd16 100644 --- a/crates/sqlbuild-orchestration/src/sensors/mod.rs +++ b/crates/sqlbuild-orchestration/src/sensors/mod.rs @@ -1,10 +1,10 @@ //! Sensor attempts and cursor commits are immutable facts, not mutable lifecycle rows. pub(crate) mod _helpers; -pub(crate) mod constants; +pub mod constants; pub mod main; pub mod models; #[cfg(test)] #[path = "../../tests/sensors/test_contracts.rs"] -mod tests; +pub(crate) mod tests; diff --git a/crates/sqlbuild-orchestration/src/sensors/models.rs b/crates/sqlbuild-orchestration/src/sensors/models.rs index 5fa409329d..38fbfced25 100644 --- a/crates/sqlbuild-orchestration/src/sensors/models.rs +++ b/crates/sqlbuild-orchestration/src/sensors/models.rs @@ -45,6 +45,10 @@ pub enum SensorCondition { selection: BTreeSet, kinds: BTreeSet, }, + Custom { + function: String, + code_hash: String, + }, } #[derive( @@ -117,6 +121,24 @@ pub struct SensorDefinition { #[serde(deny_unknown_fields)] pub struct SensorCursor { pub event_storage_id: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub custom: Option, +} + +/// Location-owned Python returns data; only the fenced scheduler may apply it. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "status", rename_all = "snake_case", deny_unknown_fields)] +pub enum CustomSensorOutcome { + Completed { + cursor: Option, + run_keys: Vec, + }, + Skipped { + cursor: Option, + reason: String, + }, + Failed, + TimedOut, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index d141d28f37..722e726650 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -1,7 +1,7 @@ //! HTTP contracts compiled within the API owner. #[path = "helpers.rs"] -mod helpers; +pub(crate) mod helpers; #[path = "output/helpers.rs"] mod output_helpers; #[path = "recall/helpers.rs"] diff --git a/crates/sqlbuild-orchestration/tests/scheduler/test_sensor_fairness.rs b/crates/sqlbuild-orchestration/tests/scheduler/test_sensor_fairness.rs new file mode 100644 index 0000000000..5259d9ee0b --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/test_sensor_fairness.rs @@ -0,0 +1,128 @@ +//! Unavailable callback locations cannot monopolize the shared due-sensor page. + +use chrono::{TimeDelta, Utc}; +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::scheduler::main::_scheduler_cycle::scheduler_cycle; +use crate::scheduler::models::SchedulerState; +use crate::scheduler::tests::test_types::SensorFairnessTestCase; +use crate::sensors::models::SensorEvaluationOutcome; +use crate::sensors::tests::helpers::{evaluations, snapshot}; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::migrate_store::migrate_store; +use crate::store::main::query::query; +use crate::store::models::{Parameter, TransactionMode}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_pending_custom_sensors_when_scheduling_then_rechecks_wait_and_other_locations_progress() { + let test_cases = [ + SensorFairnessTestCase { + description: "SQLite pending callback waits", + backend: Backend::Sqlite, + pending: 1, + builtin_status: "paused", + expected_completed: 0, + }, + SensorFairnessTestCase { + description: "PostgreSQL pending callback waits", + backend: Backend::Postgres, + pending: 1, + builtin_status: "paused", + expected_completed: 0, + }, + SensorFairnessTestCase { + description: "SQLite unavailable callback page permits builtin progress", + backend: Backend::Sqlite, + pending: 100, + builtin_status: "running", + expected_completed: 1, + }, + SensorFairnessTestCase { + description: "PostgreSQL unavailable callback page permits builtin progress", + backend: Backend::Postgres, + pending: 100, + builtin_status: "running", + expected_completed: 1, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate fairness store"); + let custom = create_location(store, "fulfillment", "test-admin") + .await + .expect("callback location"); + let builtin = create_location(store, "reporting", "test-admin") + .await + .expect("builtin location"); + let mut input = snapshot(custom); + input.documents.insert("triggers".into(), json!({})); + for index in 0..test_case.pending { + input.documents.get_mut("triggers").expect("triggers")[format!("orders_{index}")] = json!({ + "description": "Poll fulfillment", "job_key": "orders", "when": {"kind": "custom", "function": "orders_ready", "code_hash": "orders_v1"}, + "minimum_interval_seconds": 1 + }); + } + publish_snapshot(store, &input) + .await + .expect("publish unavailable callbacks"); + publish_snapshot(store, &snapshot(builtin)) + .await + .expect("publish independent builtin"); + let transaction = begin(store, TransactionMode::Write) + .await + .expect("set due order"); + let (transaction, _) = query(transaction, + "UPDATE sqb_trigger_state SET next_due_at=$1 WHERE trigger_id IN (SELECT trigger_id FROM sqb_triggers WHERE location_id=$2)", + vec![Parameter::Time(Utc::now() - TimeDelta::hours(2)), Parameter::Id(custom)]).await.expect("callbacks first"); + let (transaction, _) = query(transaction, + "UPDATE sqb_trigger_state SET next_due_at=$1,status=$2 WHERE trigger_id IN (SELECT trigger_id FROM sqb_triggers WHERE location_id=$3)", + vec![Parameter::Time(Utc::now() - TimeDelta::hours(1)), Parameter::Text(test_case.builtin_status.into()), Parameter::Id(builtin)]).await.expect("builtin last"); + commit(transaction).await.expect("commit due order"); + let (state, first) = scheduler_cycle(store, SchedulerState::new(Uuid::now_v7())) + .await + .expect("pending page"); + assert_eq!( + first.evaluated, test_case.pending, + "{}", + test_case.description + ); + assert_eq!( + first.wait_for.is_zero(), + test_case.expected_completed == 1, + "{}: only genuinely overdue unprocessed work should require an immediate retry", + test_case.description + ); + scheduler_cycle(store, state).await.expect("next page"); + let facts = evaluations(store).await; + assert_eq!( + facts + .iter() + .filter(|fact| fact.outcome == SensorEvaluationOutcome::Requested) + .count(), + test_case.pending + test_case.expected_completed, + "{}", + test_case.description + ); + assert_eq!( + facts + .iter() + .filter(|fact| fact.outcome != SensorEvaluationOutcome::Requested) + .count(), + test_case.expected_completed, + "{}: builtin at another location completes without waiting for unavailable callbacks", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs b/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs index e6429a8404..84e211d230 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs @@ -1,7 +1,9 @@ //! Real stores prove atomic sensor consumption and recovery across scheduler ownership changes. #[path = "helpers.rs"] -mod helpers; +pub(crate) mod helpers; +#[path = "test_custom.rs"] +mod test_custom; #[path = "test_delivery.rs"] mod test_delivery; #[path = "test_execution_bounds.rs"] diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_custom.rs b/crates/sqlbuild-orchestration/tests/sensors/test_custom.rs new file mode 100644 index 0000000000..e3407c4419 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_custom.rs @@ -0,0 +1,443 @@ +//! The real outbound HTTP boundary returns data, never scheduler authority. + +use std::time::Duration; + +use chrono::{TimeDelta, Utc}; +use serde_json::json; +use uuid::Uuid; + +use crate::api::tests::helpers::{start_api, stop_api}; +use crate::credentials::main::issue_token::issue_token; +use crate::credentials::models::{AuditContext, CredentialScope}; +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::platform::main::connect_location_work::connect_location_work; +use crate::platform::main::report_location_health::report_location_health; +use crate::platform::models::{ + LocationHealth, LocationHealthReport, LocationWork, LocationWorkOutcome, LocationWorkPoll, + LocationWorkResult, +}; +use crate::runs::main::list_runs::list_runs; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::sensors::main::begin_sensor_evaluation::begin_sensor_evaluation; +use crate::sensors::main::complete_custom_sensor::complete_custom_sensor; +use crate::sensors::models::{CustomSensorOutcome, SensorEvaluationOutcome}; +use crate::sensors::tests::helpers::{evaluations, snapshot}; +use crate::sensors::tests::test_types::{CustomSensorResultCase, SensorTestCase}; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::lease_helpers::expire_lease; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_custom_sensor_work_when_results_retry_then_cursor_runs_and_outcome_commit_atomically() { + let test_cases = [ + SensorTestCase { + description: "SQLite custom sensor protocol", + backend: Backend::Sqlite, + expected_runs: 2, + }, + SensorTestCase { + description: "PostgreSQL custom sensor protocol", + backend: Backend::Postgres, + expected_runs: 2, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + input.documents.get_mut("triggers").expect("triggers")["health_orders"]["when"] = + json!({"kind": "custom", "function": "orders_ready", "code_hash": "orders_v1"}); + publish_snapshot(store, &input) + .await + .expect("publish custom sensor data"); + let trigger = list_triggers(store, location, "development") + .await + .expect("triggers") + .remove(0); + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let token = issue_token( + store, + &context, + CredentialScope::Location { + location_id: location, + }, + Utc::now() + TimeDelta::hours(1), + ) + .await + .expect("scoped token"); + let api = start_api(store).await; + let client = + connect_location_work((&api.url, &token.bearer), location).expect("client"); + let poll = LocationWorkPoll { + session_id: Uuid::now_v7(), + target: "development".into(), + }; + report_location_health( + store, + location, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id: poll.session_id, + target: poll.target.clone(), + health: LocationHealth::Heartbeat {}, + }, + ) + .await + .expect("register location instance"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let point = TransactionFault { + description: "request and outbound work are atomic", + table: "sqb_location_work", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'injected sensor request failure')", + postgres_body: "RAISE EXCEPTION 'injected sensor request failure'", + }; + install_fault(store, &point).await; + assert!( + begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .is_err() + ); + remove_fault(store, &point).await; + assert!( + evaluations(store).await.is_empty(), + "request fact rolled back with work" + ); + let requested = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("begin") + .expect("due"); + assert_eq!( + complete_custom_sensor(store, &lease, &requested) + .await + .expect("still waiting"), + 0 + ); + assert_eq!( + evaluations(store).await.len(), + 1, + "no invented terminal phase while waiting" + ); + let claim = client.poll(&poll).await.expect("poll").expect("work"); + assert_eq!(claim.request.request_id, requested.evaluation_id); + assert_eq!( + claim.request.work, + LocationWork::EvaluateSensor { + evaluation: Box::new(requested.clone()) + } + ); + assert_eq!( + client.poll(&poll).await.expect("lost poll response"), + Some(claim.clone()) + ); + let wrong = LocationWorkResult { + claim_id: claim.claim_id, + session_id: poll.session_id, + outcome: LocationWorkOutcome::Connection { healthy: true }, + }; + assert!( + client + .complete(requested.evaluation_id, &wrong) + .await + .is_err(), + "wrong result kind" + ); + let empty_key = LocationWorkResult { + outcome: LocationWorkOutcome::SensorEvaluation { + outcome: CustomSensorOutcome::Completed { + cursor: Some("unsafe".into()), + run_keys: vec![" ".into()], + }, + }, + ..wrong.clone() + }; + assert!( + client + .complete(requested.evaluation_id, &empty_key) + .await + .is_err(), + "empty keys fail closed" + ); + let result = LocationWorkResult { + outcome: LocationWorkOutcome::SensorEvaluation { + outcome: CustomSensorOutcome::Completed { + cursor: Some("orders/page-2".into()), + run_keys: vec!["orders-2".into(), "orders-1".into(), "orders-2".into()], + }, + }, + ..wrong + }; + assert!( + client + .complete(requested.evaluation_id, &result) + .await + .expect("durable location result") + ); + assert!( + !client + .complete(requested.evaluation_id, &result) + .await + .expect("uncertain acknowledgement retry") + ); + assert!( + list_runs(store, (location, "development"), 100) + .await + .expect("runs") + .is_empty(), + "location result alone has no queue authority" + ); + for table in [ + "sqb_runs", + "sqb_run_requests", + "sqb_sensor_evaluation_events", + "sqb_sensor_cursor_commits", + ] { + let point = TransactionFault { + description: "custom sensor atomic publication", + table, + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'injected sensor failure')", + postgres_body: "RAISE EXCEPTION 'injected sensor failure'", + }; + install_fault(store, &point).await; + assert!( + complete_custom_sensor(store, &lease, &requested) + .await + .is_err() + ); + remove_fault(store, &point).await; + assert!( + list_runs(store, (location, "development"), 100) + .await + .expect("rollback runs") + .is_empty() + ); + assert_eq!( + evaluations(store).await.len(), + 1, + "outcome rolled back with cursor and queue" + ); + assert_eq!( + begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("unchanged cursor"), + Some(requested.clone()) + ); + } + let replica = second_store(store).await; + assert_eq!( + complete_custom_sensor(&replica, &lease, &requested) + .await + .expect("replica commit"), + test_case.expected_runs + ); + assert_eq!( + complete_custom_sensor(store, &lease, &requested) + .await + .expect("retry commit"), + 0 + ); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("dedup runs") + .len(), + test_case.expected_runs, + "{}", + test_case.description + ); + let facts = evaluations(store).await; + let SensorEvaluationOutcome::Completed { cursor, run_ids } = + &facts.last().expect("completed").outcome + else { + panic!("completed phase required"); + }; + assert_eq!(cursor.custom.as_deref(), Some("orders/page-2")); + assert_eq!(cursor.event_storage_id, requested.cursor.event_storage_id); + assert_eq!(run_ids.len(), test_case.expected_runs); + for result_case in [ + CustomSensorResultCase { + outcome: CustomSensorOutcome::Completed { + cursor: Some("orders/page-2".into()), + run_keys: vec!["orders-2".into(), "orders-1".into()], + }, + expected_phase: "completed", + expected_cursor: Some("orders/page-2"), + }, + CustomSensorResultCase { + outcome: CustomSensorOutcome::Failed, + expected_phase: "failed", + expected_cursor: None, + }, + CustomSensorResultCase { + outcome: CustomSensorOutcome::TimedOut, + expected_phase: "timed_out", + expected_cursor: None, + }, + CustomSensorResultCase { + outcome: CustomSensorOutcome::Skipped { + cursor: Some("orders/page-3".into()), + reason: "No new orders".into(), + }, + expected_phase: "completed", + expected_cursor: Some("orders/page-3"), + }, + ] { + tokio::time::sleep(Duration::from_secs(1)).await; + let pending = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("next attempt") + .expect("due"); + assert_eq!( + pending.cursor.custom.as_deref(), + Some("orders/page-2"), + "failed and timeout leave cursor unchanged" + ); + let claim = client + .poll(&poll) + .await + .expect("next poll") + .expect("next work"); + assert_eq!(claim.request.request_id, pending.evaluation_id); + let result = LocationWorkResult { + claim_id: claim.claim_id, + session_id: poll.session_id, + outcome: LocationWorkOutcome::SensorEvaluation { + outcome: result_case.outcome, + }, + }; + assert!( + client + .complete(pending.evaluation_id, &result) + .await + .expect("complete callback") + ); + complete_custom_sensor(store, &lease, &pending) + .await + .expect("apply phase"); + let facts = evaluations(store).await; + let outcome = &facts.last().expect("terminal evaluation phase").outcome; + assert_eq!(outcome.phase(), result_case.expected_phase); + let payload = serde_json::to_value(outcome).expect("outcome payload"); + assert_eq!( + payload["cursor"]["custom"].as_str(), + result_case.expected_cursor + ); + } + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("still deduped") + .len(), + test_case.expected_runs + ); + tokio::time::sleep(Duration::from_secs(1)).await; + let stale = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("request before takeover") + .expect("due"); + let stale_claim = client + .poll(&poll) + .await + .expect("claim before takeover") + .expect("work"); + assert_eq!(stale_claim.request.request_id, stale.evaluation_id); + expire_lease(store).await; + assert!( + client + .poll(&poll) + .await + .expect("no expired-epoch dispatch") + .is_none() + ); + assert!(matches!( + complete_custom_sensor(store, &lease, &stale).await, + Err(ControlError::Fenced) + )); + let successor = acquire_lease(&replica, Uuid::now_v7()) + .await + .expect("takeover") + .expect("leader"); + let late = LocationWorkResult { + claim_id: stale_claim.claim_id, + session_id: poll.session_id, + outcome: LocationWorkOutcome::SensorEvaluation { + outcome: CustomSensorOutcome::Completed { + cursor: Some("uncommitted/stale".into()), + run_keys: vec!["must-not-queue".into()], + }, + }, + }; + assert!( + client + .complete(stale.evaluation_id, &late) + .await + .expect("late location result") + ); + assert!(matches!( + complete_custom_sensor(&replica, &successor, &stale).await, + Err(ControlError::Fenced) + )); + tokio::time::sleep(Duration::from_secs(1)).await; + let recovered = begin_sensor_evaluation(&replica, &successor, trigger.trigger_id) + .await + .expect("recover abandoned callback") + .expect("due after minimum interval"); + assert_eq!(recovered.cursor.custom.as_deref(), Some("orders/page-3")); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("no stale queued request") + .len(), + test_case.expected_runs + ); + assert_eq!( + evaluations(store) + .await + .iter() + .filter(|fact| fact.request.evaluation_id == stale.evaluation_id + && fact.outcome == SensorEvaluationOutcome::Abandoned) + .count(), + 1 + ); + assert_eq!( + client + .poll(&poll) + .await + .expect("new epoch work") + .expect("new claim") + .request + .request_id, + recovered.evaluation_id, + "retired evaluation must not block or execute before current work" + ); + close_store(replica).await; + stop_api(api).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_types.rs b/crates/sqlbuild-orchestration/tests/sensors/test_types.rs index 1b4e85322d..af27d0d35e 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_types.rs @@ -1,4 +1,5 @@ use crate::runs::models::{NodeOutcome, RunOutcome}; +use crate::sensors::models::CustomSensorOutcome; use crate::store::tests::store_test_types::Backend; pub(crate) struct SensorTestCase { @@ -7,6 +8,12 @@ pub(crate) struct SensorTestCase { pub(crate) expected_runs: usize, } +pub(crate) struct CustomSensorResultCase { + pub(crate) outcome: CustomSensorOutcome, + pub(crate) expected_phase: &'static str, + pub(crate) expected_cursor: Option<&'static str>, +} + pub(crate) struct SensorExecutionTestCase { pub(crate) description: &'static str, pub(crate) outcome: RunOutcome, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index 4b4c372cee..fd3e005f42 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -775,12 +775,14 @@ fn parsed_invocation<'py>( let namespace = args.getattr("__dict__")?.cast_into::()?; let command: Option = namespace .get_item("command")? - .map(|value| value.extract()) - .transpose()?; + .map(|value| value.extract::>()) + .transpose()? + .flatten(); let location_command: Option = namespace .get_item("location_command")? - .map(|value| value.extract()) - .transpose()?; + .map(|value| value.extract::>()) + .transpose()? + .flatten(); if command.as_deref() == Some("location") && location_command.as_deref() == Some("runtime") { return within(&host::redirect_stdout_to_stderr(py)?, |_| { execute_parsed_invocation(args, handlers, identity, use_color) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index caab6c25e6..c04d8a5435 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -202,8 +202,8 @@ pub(crate) fn compile_declarations( triggers, sensors, } = compile_sensors( - &input.target, - graph, + py, + (project_dir, input.target.as_str(), graph), authored_sensors, compile_calendars( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs index 6a0505dbda..7a5f85c971 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs @@ -3,12 +3,14 @@ use std::io::{self, Write}; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyResult, Python}; +use pyo3::prelude::{Bound, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_orchestration::sensors::models::{CustomSensorOutcome, SensorEvaluationRequest}; use crate::bindings::_helpers::boundary::python_calls::within; use crate::bindings::_helpers::location_command::command::{location_arguments, publish_location}; use crate::bindings::_helpers::native_host::host::{redirect_stdout_to_stderr, write_console}; +use crate::bindings::_helpers::native_host::orchestration_sensors::evaluate_sensor; pub(super) fn run_runtime(py: Python<'_>, options: Bound<'_, PyDict>) -> PyResult { options.set_item("action", "publish")?; @@ -22,23 +24,57 @@ pub(super) fn run_runtime(py: Python<'_>, options: Bound<'_, PyDict>) -> PyResul let Some(line) = line else { break; }; - let generation: i64 = serde_json::from_str(&line) - .map_err(|_| PyValueError::new_err("runtime expects a JSON generation per line"))?; - options.set_item("generation", generation)?; - let arguments = location_arguments(py, Some(options.clone()))?; - let record = match publish_location(py, &arguments) { - Ok(record) => Some(record), - Err(error) => { - write_console( - py, - "stderr", - &format!("Location compilation/publication failed: {error}\n"), - )?; - None + let request: serde_json::Value = serde_json::from_str(&line).map_err(|_| { + PyValueError::new_err( + "runtime expects a JSON generation or sensor evaluation per line", + ) + })?; + let output = if let Some(generation) = request.as_i64() { + options.set_item("generation", generation)?; + let arguments = location_arguments(py, Some(options.clone()))?; + let record = match publish_location(py, &arguments) { + Ok(record) => Some(record), + Err(error) => { + write_console( + py, + "stderr", + &format!("Location compilation/publication failed: {error}\n"), + )?; + None + } + }; + serde_json::to_string(&record) + } else { + let evaluation: SensorEvaluationRequest = + serde_json::from_value(request).map_err(|_| { + PyValueError::new_err("runtime received an invalid sensor evaluation") + })?; + let arguments = location_arguments(py, Some(options.clone()))?; + if arguments.value("location_id")?.extract::()? + != evaluation.location_id.to_string() + || arguments.value("selected_target")?.extract::()? != evaluation.target + { + return Err(PyValueError::new_err( + "sensor work belongs to another location or target", + )); } + let directory = arguments + .value("project_dir")? + .extract::()?; + let outcome = match evaluate_sensor(py, &directory, &evaluation) { + Ok(outcome) => outcome, + Err(_) => { + write_console( + py, + "stderr", + "Custom sensor evaluation failed. No cursor or run requests committed.\n", + )?; + CustomSensorOutcome::Failed + } + }; + serde_json::to_string(&outcome) }; - let output = serde_json::to_string(&record) - .map_err(|error| PyValueError::new_err(error.to_string()))?; + let output = output.map_err(|error| PyValueError::new_err(error.to_string()))?; py.detach(|| -> io::Result<()> { let mut stdout = io::stdout().lock(); writeln!(stdout, "{output}")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs index b7e704b8c0..035588e5e9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs @@ -4,3 +4,6 @@ pub(crate) const CALENDAR_WINDOW_PARAMETER: &str = "window"; pub(crate) const RUN_EVENT_CONDITION: &str = "run_event"; + +/// The sensor context is supplied by the host, not a provider. +pub(crate) const SENSOR_CONTEXT_PARAMETER: &str = "ctx"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index ad5619410e..cb882f735f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -7,6 +7,7 @@ pub(crate) mod host; pub(crate) mod model_lookup; pub(crate) mod orchestration_declarations; pub(crate) mod orchestration_environment; +pub(crate) mod orchestration_sensors; pub(crate) mod orchestration_triggers; pub(crate) mod registration; pub(crate) mod results; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs index 453a9f5145..12796bb6fe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs @@ -63,7 +63,8 @@ pub(crate) struct AuthoredSensorCondition { pub(crate) struct AuthoredSensor { pub(crate) name: String, pub(crate) description: String, - pub(crate) when: AuthoredSensorCondition, + pub(crate) when: Option, + pub(crate) function: Option>, pub(crate) run: AuthoredJobReference, #[pyo3(from_py_with = authored_setting)] pub(crate) minimum_interval_seconds: Value, @@ -163,7 +164,9 @@ pub(crate) fn collect_declarations<'py>( } pending.push(target); let condition = declaration.getattr("when")?; - if condition.getattr("kind")?.extract::()? == RUN_EVENT_CONDITION { + if !condition.is_none() + && condition.getattr("kind")?.extract::()? == RUN_EVENT_CONDITION + { let source = condition.getattr("job")?; if !source.is_instance(&job)? && !source.is_instance(&group)? { return Err(PyTypeError::new_err( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_environment.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_environment.rs index 5b49bcaa07..26cfa2bf67 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_environment.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_environment.rs @@ -2,8 +2,12 @@ use std::path::Path; -use pyo3::prelude::{PyAnyMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sha2::{Digest, Sha256}; +use sqlbuild_discovery::constants::{ORCHESTRATION_DIRECTORY, PYTHON_FILE_SUFFIX}; +use sqlbuild_discovery::tree::main::rglob::rglob; +use sqlbuild_discovery::tree::models::ProjectTree; /// A lockfile binds declared dependencies; without one, bind installed distribution versions. pub(crate) fn environment_hash(py: Python<'_>, project_dir: &Path) -> PyResult { @@ -48,3 +52,57 @@ pub(crate) fn environment_hash(py: Python<'_>, project_dir: &Path) -> PyResult PyResult { + let tree = ProjectTree::new(project_dir); + let files = rglob(&tree, ORCHESTRATION_DIRECTORY, |entry| { + !entry.is_dir && entry.name.ends_with(PYTHON_FILE_SUFFIX) + }) + .map_err(|error| { + pyo3::exceptions::PyValueError::new_err(format!( + "sensor source discovery failed: {error:?}" + )) + })?; + let mut hash = Sha256::new(); + for file in files { + let bytes = std::fs::read(tree.absolute(&file))?; + hash.update(file.len().to_le_bytes()); + hash.update(file.as_bytes()); + hash.update(bytes.len().to_le_bytes()); + hash.update(bytes); + } + Ok(format!("{:x}", hash.finalize())) +} + +/// Include the callable's code even for public in-memory declaration compilation. +pub(crate) fn callback_code_hash(function: &Bound<'_, PyAny>, sources: &str) -> PyResult { + let bytes: Vec = function + .py() + .import("marshal")? + .call_method1("dumps", (portable_code(function.getattr("__code__")?)?,))? + .extract()?; + let mut hash = Sha256::new(); + hash.update(sources.as_bytes()); + hash.update(bytes); + Ok(format!("{:x}", hash.finalize())) +} + +/// Code objects retain absolute import paths, including inside nested functions. +fn portable_code(code: Bound<'_, PyAny>) -> PyResult> { + let py = code.py(); + let code_type = py.import("types")?.getattr("CodeType")?; + let mut constants: Vec> = Vec::new(); + for value in code.getattr("co_consts")?.try_iter()? { + let value = value?; + constants.push(if value.is_instance(&code_type)? { + portable_code(value)? + } else { + value + }); + } + let changes = PyDict::new(py); + changes.set_item("co_filename", "")?; + changes.set_item("co_consts", PyTuple::new(py, constants)?)?; + code.call_method("replace", (), Some(&changes)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs new file mode 100644 index 0000000000..e6c05e8861 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs @@ -0,0 +1,132 @@ +//! User callbacks and provider lifetimes belong only to the isolated location interpreter. + +use std::path::Path; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods}; +use sqlbuild_host::entries::models::UserCodeHost; +use sqlbuild_orchestration::sensors::models::{ + CustomSensorOutcome, SensorCondition, SensorEvaluationRequest, +}; + +use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, within}; +use crate::bindings::_helpers::compiled_project::snapshot_declarations::discover_declarations; +use crate::bindings::_helpers::discovery::imported_modules::{provider_by_name, provider_usages}; +use crate::bindings::_helpers::discovery::imported_providers::discover_provider_classes; +use crate::bindings::_helpers::native_host::constants::SENSOR_CONTEXT_PARAMETER; +use crate::bindings::_helpers::native_host::host::entry; +use crate::bindings::_helpers::native_host::orchestration_environment::{ + callback_code_hash, orchestration_code_hash, +}; + +pub(crate) fn evaluate_sensor( + py: Python<'_>, + project_dir: &Path, + evaluation: &SensorEvaluationRequest, +) -> PyResult { + let SensorCondition::Custom { + function: name, + code_hash, + } = &evaluation.definition.when + else { + return Err(PyValueError::new_err("custom sensor work required")); + }; + let public = py.import("sqlbuild.orchestration")?; + let sensor_class = public.getattr("Sensor")?; + let mut matches: Vec> = Vec::new(); + for declaration in discover_declarations(py, project_dir)? { + if declaration.is_instance(&sensor_class)? + && declaration.getattr("name")?.extract::()? == *name + { + matches.push(declaration); + } + } + let [sensor] = matches.as_slice() else { + return Err(PyValueError::new_err( + "custom sensor declaration must resolve uniquely", + )); + }; + let targets: Vec = sensor.getattr("targets")?.extract()?; + if !sensor.getattr("when")?.is_none() + || (!targets.is_empty() && !targets.contains(&evaluation.target)) + || sensor + .getattr("run")? + .getattr("name")? + .extract::()? + != evaluation.definition.job_key + { + return Err(PyValueError::new_err("custom sensor definition changed")); + } + let function = sensor.getattr("function")?; + let sources = py.detach(|| orchestration_code_hash(project_dir))?; + if callback_code_hash(&function, &sources)? != *code_hash { + return Err(PyValueError::new_err("custom sensor code changed")); + } + let fields = PyDict::new(py); + fields.set_item("cursor", &evaluation.cursor.custom)?; + fields.set_item("target", &evaluation.target)?; + fields.set_item("evaluation_id", evaluation.evaluation_id.to_string())?; + let context = public.getattr("SensorContext")?.call((), Some(&fields))?; + let providers = + discover_provider_classes(imported(py, "pathlib", "Path")?.call1((project_dir,))?)?; + let by_name = provider_by_name(&providers)?; + let session = call_with( + &entry(py, UserCodeHost::BuildProviderSession)?, + &[ + ("discovered_providers", providers), + ("allow_shared", PyBool::new(py, false).to_owned().into_any()), + ], + )?; + within(&session, |_| { + let arguments = PyDict::new(py); + arguments.set_item("ctx", &context)?; + for usage in provider_usages(&function, &by_name)?.try_iter()? { + let usage = usage?; + if usage.getattr("parameter_name")?.extract::()? != SENSOR_CONTEXT_PARAMETER { + arguments.set_item( + usage.getattr("parameter_name")?, + session.call_method1("get", (usage.getattr("provider_name")?,))?, + )?; + } + } + let result = function.call((), Some(&arguments))?; + if result.is_instance(&public.getattr("SkipReason")?)? { + let reason: String = result.getattr("reason")?.extract()?; + if reason.trim().is_empty() { + return Err(PyValueError::new_err("SkipReason needs a nonempty reason")); + } + return Ok(CustomSensorOutcome::Skipped { + cursor: context.getattr("cursor")?.extract()?, + reason, + }); + } + let request_class = public.getattr("RunRequest")?; + let mut run_keys: Vec = Vec::new(); + for request in result + .try_iter()? + .take(sqlbuild_orchestration::sensors::constants::MAX_SENSOR_RUN_REQUESTS + 1) + { + let request = request?; + if !request.is_instance(&request_class)? { + return Err(PyValueError::new_err( + "custom sensors return RunRequest records or SkipReason", + )); + } + let key: String = request.getattr("run_key")?.extract()?; + if key.trim().is_empty() { + return Err(PyValueError::new_err("RunRequest needs a nonempty run_key")); + } + run_keys.push(key); + } + if run_keys.len() > sqlbuild_orchestration::sensors::constants::MAX_SENSOR_RUN_REQUESTS { + return Err(PyValueError::new_err( + "custom sensor exceeds the bounded run-request page", + )); + } + Ok(CustomSensorOutcome::Completed { + cursor: context.getattr("cursor")?.extract()?, + run_keys, + }) + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs index 03f6bb4e77..256be90f0c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -26,6 +26,9 @@ use crate::bindings::_helpers::native_host::host::entry; use crate::bindings::_helpers::native_host::orchestration_declarations::{ AuthoredSensor, authored_setting, }; +use crate::bindings::_helpers::native_host::orchestration_environment::{ + callback_code_hash, orchestration_code_hash, +}; #[derive(FromPyObject)] struct AuthoredControls { @@ -250,11 +253,16 @@ pub(crate) fn compile_calendars( } pub(crate) fn compile_sensors( - target: &str, - graph: &ProjectGraph, + py: Python<'_>, + (project_dir, target, graph): (&Path, &str, &ProjectGraph), authored: Vec, mut definitions: Definitions, ) -> PyResult { + let sources = if authored.iter().any(|sensor| sensor.function.is_some()) { + orchestration_code_hash(project_dir)? + } else { + String::new() + }; let mut names: BTreeSet = BTreeSet::new(); for sensor in authored { if !names.insert(sensor.name.clone()) || definitions.triggers.contains_key(&sensor.name) { @@ -266,18 +274,31 @@ pub(crate) fn compile_sensors( if !sensor.targets.is_empty() && !sensor.targets.iter().any(|value| value == target) { continue; } - let mut condition = json!({"kind": sensor.when.kind, "kinds": sensor.when.kinds}); - if let Some(job) = sensor.when.job { - condition["job_key"] = json!(job.name); - } - if let Some(select) = sensor.when.select { - let selectors: Vec = select.split_whitespace().map(str::to_owned).collect(); - condition["selection"] = sensor_nodes(graph, &selectors, "node_event")?; - } - if let Some(selectors) = sensor.when.executed_successfully { - condition["executed_successfully"] = - sensor_nodes(graph, &selectors, "executed_successfully")?; - } + let condition = match (sensor.when, sensor.function) { + (Some(when), None) => { + let mut condition = json!({"kind": when.kind, "kinds": when.kinds}); + if let Some(job) = when.job { + condition["job_key"] = json!(job.name); + } + if let Some(select) = when.select { + let selectors: Vec = + select.split_whitespace().map(str::to_owned).collect(); + condition["selection"] = sensor_nodes(graph, &selectors, "node_event")?; + } + if let Some(selectors) = when.executed_successfully { + condition["executed_successfully"] = + sensor_nodes(graph, &selectors, "executed_successfully")?; + } + condition + } + (None, Some(function)) => json!({"kind": "custom", "function": sensor.name, + "code_hash": callback_code_hash(function.bind(py), &sources)?}), + _ => { + return Err(PyValueError::new_err( + "sensor needs either a built-in condition or a decorated polling callback; callbacks with when are not supported yet", + )); + } + }; let definition: SensorDefinition = serde_json::from_value(json!({ "description": sensor.description, "job_key": sensor.run.name, diff --git a/fensu.toml b/fensu.toml index 8c5126e9c4..16d15444db 100644 --- a/fensu.toml +++ b/fensu.toml @@ -2655,6 +2655,7 @@ module_layers = [ "model_config=front", "native_host=infra", "native_host/model_lookup=project", + "native_host/orchestration_sensors=commands", "native_host/orchestration_triggers=project", "native_threads=infra", "output_capture=output", @@ -2750,3 +2751,8 @@ rule = "FFS120" path = "src/sqlbuild/executor/scheduling/classes/recovering_connection_pool.py" symbols = ["RecoveringConnectionPool.put"] reason = "The override keeps queue.Queue.put's positional protocol, which every pool consumer calls." + +[[targets.python.rule_exceptions]] +rule = "FFS201" +path = "src/sqlbuild/orchestration_declarations/models/sensor_context.py" +reason = "Only SensorContext lives here: invocation-local ctx.cursor assignment proposes progress, committed solely by native fenced completion. The package regression guards this declaration-only scope; class-symbol exceptions are unsupported." diff --git a/pyproject.toml b/pyproject.toml index c29137b916..daf4fa4c8f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,7 @@ dev = [ "pytest-cov>=7.0.0", "pytest-xdist>=3.8.0", "ruff>=0.12.0", - "fensu>=0.28.1", + "fensu>=0.28.3", "testcontainers>=4.8.1", "ty>=0.0.1a19", ] diff --git a/python-boundary.toml b/python-boundary.toml index 159acb56e3..5829d6322f 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -18,10 +18,15 @@ rule = 3 note = "Public orchestration facade: re-exports only." [[stays]] -path = "orchestration_declarations/models.py" +path = "orchestration_declarations/models/authoring.py" rule = 3 note = "Public job and schedule authoring records only; selection resolution, validation and scheduling stay native." +[[stays]] +path = "orchestration_declarations/models/sensor_context.py" +rule = 3 +note = "Only the mutable SensorContext authoring declaration required by ctx.cursor assignment; no Python runtime allowance. Native fenced completion validates and commits proposed progress." + [[stays]] path = "orchestration_declarations/types.py" rule = 3 diff --git a/src/sqlbuild/orchestration.py b/src/sqlbuild/orchestration.py index 3532cf5f7d..568ce36709 100644 --- a/src/sqlbuild/orchestration.py +++ b/src/sqlbuild/orchestration.py @@ -1,7 +1,7 @@ """Public authoring and snapshot compilation API for orchestration declarations.""" from sqlbuild.orchestration_declarations.main.compile_snapshot import compile_snapshot -from sqlbuild.orchestration_declarations.models import ( +from sqlbuild.orchestration_declarations.models.authoring import ( CalendarSchedule, CalendarWindow, Job, @@ -9,17 +9,24 @@ NodeEventCondition, PlatformEventCondition, RunEventCondition, + RunRequest, Schedule, Sensor, + SkipReason, ) -from sqlbuild.orchestration_declarations.models import CalendarSchedule as calendar_schedule -from sqlbuild.orchestration_declarations.models import Job as job -from sqlbuild.orchestration_declarations.models import JobGroup as job_group -from sqlbuild.orchestration_declarations.models import NodeEventCondition as node_event -from sqlbuild.orchestration_declarations.models import PlatformEventCondition as platform_event -from sqlbuild.orchestration_declarations.models import RunEventCondition as run_event -from sqlbuild.orchestration_declarations.models import Schedule as schedule -from sqlbuild.orchestration_declarations.models import Sensor as sensor +from sqlbuild.orchestration_declarations.models.authoring import ( + CalendarSchedule as calendar_schedule, +) +from sqlbuild.orchestration_declarations.models.authoring import Job as job +from sqlbuild.orchestration_declarations.models.authoring import JobGroup as job_group +from sqlbuild.orchestration_declarations.models.authoring import NodeEventCondition as node_event +from sqlbuild.orchestration_declarations.models.authoring import ( + PlatformEventCondition as platform_event, +) +from sqlbuild.orchestration_declarations.models.authoring import RunEventCondition as run_event +from sqlbuild.orchestration_declarations.models.authoring import Schedule as schedule +from sqlbuild.orchestration_declarations.models.authoring import Sensor as sensor +from sqlbuild.orchestration_declarations.models.sensor_context import SensorContext from sqlbuild.orchestration_declarations.types import NodeEventKind, PlatformEventKind, RunEventKind __all__ = ( @@ -33,8 +40,11 @@ "PlatformEventKind", "RunEventCondition", "RunEventKind", + "RunRequest", "Schedule", "Sensor", + "SensorContext", + "SkipReason", "calendar_schedule", "compile_snapshot", "job", diff --git a/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py b/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py index cc7f6c83f2..0f9fc8b052 100644 --- a/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py +++ b/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py @@ -7,7 +7,12 @@ from sqlbuild._native import NativeOrchestrationSnapshot, build_orchestration_snapshot from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.orchestration_declarations.models import CalendarSchedule, Job, JobGroup, Sensor +from sqlbuild.orchestration_declarations.models.authoring import ( + CalendarSchedule, + Job, + JobGroup, + Sensor, +) def compile_snapshot( diff --git a/src/sqlbuild/orchestration_declarations/models/__init__.py b/src/sqlbuild/orchestration_declarations/models/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/sqlbuild/orchestration_declarations/models.py b/src/sqlbuild/orchestration_declarations/models/authoring.py similarity index 87% rename from src/sqlbuild/orchestration_declarations/models.py rename to src/sqlbuild/orchestration_declarations/models/authoring.py index 8f4a44a9b8..fd188208ed 100644 --- a/src/sqlbuild/orchestration_declarations/models.py +++ b/src/sqlbuild/orchestration_declarations/models/authoring.py @@ -115,8 +115,30 @@ class Sensor: name: str description: str = field(kw_only=True) - when: PlatformEventCondition | RunEventCondition | NodeEventCondition = field(kw_only=True) + when: PlatformEventCondition | RunEventCondition | NodeEventCondition | None = field( + default=None, kw_only=True + ) run: Job | JobGroup = field(kw_only=True) minimum_interval_seconds: int = field(default=30, kw_only=True) default_status: Literal["running", "stopped"] = field(default="running", kw_only=True) targets: tuple[str, ...] = field(default=(), kw_only=True) + function: Callable[..., Iterable[RunRequest] | SkipReason] | None = field( + default=None, repr=False, kw_only=True + ) + + def __call__(self, function: Callable[..., Iterable[RunRequest] | SkipReason]) -> Sensor: + return replace(self, function=function) + + +@dataclass(frozen=True, kw_only=True) +class RunRequest: + """A sensor-local key deduplicates requests across evaluations and retries.""" + + run_key: str + + +@dataclass(frozen=True) +class SkipReason: + """Complete an evaluation without starting a run.""" + + reason: str diff --git a/src/sqlbuild/orchestration_declarations/models/sensor_context.py b/src/sqlbuild/orchestration_declarations/models/sensor_context.py new file mode 100644 index 0000000000..648ce981db --- /dev/null +++ b/src/sqlbuild/orchestration_declarations/models/sensor_context.py @@ -0,0 +1,12 @@ +"""Invocation-local context; native fenced completion alone commits a proposed cursor.""" + +from dataclasses import dataclass + + +@dataclass(kw_only=True) +class SensorContext: + """Cursor assignment proposes progress; failure or timeout retains the committed cursor.""" + + cursor: str | None + target: str + evaluation_id: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index f1f32da930..8f5aca9f03 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -47,11 +47,13 @@ def write(self, text: str) -> int: @contextmanager def location_process( - arguments: list[str], log: Path, publications: Path + arguments: list[str], log: Path, publications: Path, *, env: dict[str, str] | None = None ) -> Iterator[subprocess.Popen[bytes]]: """Own and reap the supervisor even when an assertion interrupts a reload.""" with log.open("ab") as stderr, publications.open("ab") as stdout: - process: subprocess.Popen[bytes] = subprocess.Popen(arguments, stderr=stderr, stdout=stdout) + process: subprocess.Popen[bytes] = subprocess.Popen( + arguments, stderr=stderr, stdout=stdout, env=env + ) try: yield process finally: @@ -563,3 +565,28 @@ def claimed_worker(directory: Path, api: SnapshotApi, publication: dict[str, Any ), ) return ClaimedWorker(run_id=run_id, claim_id=claim_id, bearer=bearer) + + +def wait_for_sensor_phase( + store: Path, phase: str, *, empty_runs: bool = False, timeout: float = 45 +) -> dict[str, Any]: + """Wait for real scheduler facts, optionally requiring a completed skip.""" + deadline: float = time.monotonic() + timeout + row: tuple[str] | None = None + while row is None: + assert time.monotonic() < deadline, f"Sensor did not record {phase}" + with sqlite3.connect(store) as control: + row = control.execute( + "SELECT fact_json FROM sqb_sensor_evaluation_events WHERE phase=? " + "AND (? = FALSE OR fact_json->'outcome'->'run_ids'='[]') " + "ORDER BY storage_id DESC LIMIT 1", + (phase, empty_runs), + ).fetchone() + time.sleep(0.05) + return json.loads(row[0]) + + +def sensor_calls(path: Path, fields: dict[str, Any]) -> list[dict[str, Any]]: + """Select recorded callback invocations without inferring execution from declarations.""" + records: list[dict[str, Any]] = [json.loads(line) for line in path.read_text().splitlines()] + return list(filter(lambda record: fields.items() <= record.items(), records)) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py new file mode 100644 index 0000000000..1c6fbcdc4c --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py @@ -0,0 +1,160 @@ +"""A real location callback times out, restarts and queues a real DuckDB worker.""" + +import ast +import json +import os +import sqlite3 +import sys +from pathlib import Path +from typing import Any, cast + +import duckdb +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + WorkerExecutionTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + location_process, + scheduled_worker, + sensor_calls, + wait_for_generation, + wait_for_run, + wait_for_sensor_phase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "polling fulfillment sensor", + "SELECT 83 AS amount", + "SELECT amount FROM orders", + "success", + [(83,)], + expected_origin="sensor", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/custom_orders.py" + ).read_text(), + "providers/fulfillment.py": ( + "import json, os, time\nfrom pathlib import Path\n" + "from sqlbuild.orchestration import RunRequest, SkipReason\n" + "from sqlbuild.provider.classes.provider import Provider\n" + "class Fulfillment(Provider):\n" + ' """Synthetic fulfillment readiness."""\n' + " def record(self, value):\n" + " with Path(os.environ['SQLBUILD_SENSOR_CALLS']).open('a') as out:\n" + " out.write(json.dumps(value | {'pid': os.getpid()}) + '\\n')\n" + " def setup(self, ctx):\n self.record({'phase': 'setup'})\n" + " def teardown(self):\n self.record({'phase': 'teardown'})\n" + " def poll(self, ctx):\n" + " mode = json.loads(Path(os.environ['SQLBUILD_FULFILLMENT_DATA']).read_text())['mode']\n" + " self.record({'phase': 'evaluate', 'cursor': ctx.cursor, 'evaluation_id': ctx.evaluation_id, 'target': ctx.target, 'mode': mode})\n" + " if mode == 'failed':\n" + " ctx.cursor = 'uncommitted'\n" + " raise ValueError('synthetic source unavailable')\n" + " if mode == 'timeout':\n time.sleep(90)\n" + " if ctx.cursor:\n return SkipReason('No new orders')\n" + " ctx.target = 'not-an-authority-input'\n" + " ctx.evaluation_id = 'not-an-authority-input'\n" + " ctx.cursor = 'orders/page-1'\n" + " return [RunRequest(run_key='orders-1'), RunRequest(run_key='orders-1')]\n" + ), + }, + ) + ], + ids=lambda case: case.description, +) +def test_given_a_custom_polling_sensor_when_a_callback_fails_or_times_out_then_a_fresh_process_retries_without_advancing_its_cursor( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + context_module: ast.Module = ast.parse( + Path("src/sqlbuild/orchestration_declarations/models/sensor_context.py").read_text() + ) + assert [type(node).__name__ for node in context_module.body] == [ + "Expr", + "ImportFrom", + "ClassDef", + ] + declaration: ast.ClassDef = cast(ast.ClassDef, context_module.body[-1]) + assert declaration.name == "SensorContext" + assert [type(node).__name__ for node in declaration.body] == [ + "Expr", + "AnnAssign", + "AnnAssign", + "AnnAssign", + ] + data: Path = tmp_path / "fulfillment.json" + calls: Path = tmp_path / "sensor-calls.jsonl" + data.write_text(json.dumps({"mode": "failed"})) + monkeypatch.setenv("SQLBUILD_FULFILLMENT_DATA", str(data)) + monkeypatch.setenv("SQLBUILD_SENSOR_CALLS", str(calls)) + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", worker.api.bearer) + monkeypatch.setenv("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://must-not-open") + log: Path = tmp_path / "location.log" + with location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(worker.project), + "location", + "serve", + "--server", + worker.api.url, + "--location-id", + worker.api.location_id, + "--target", + "development", + ], + log, + tmp_path / "publications.jsonl", + ) as location: + _ = wait_for_generation(worker.api, "development", 2, location, log) + failed: dict[str, Any] = wait_for_sensor_phase(worker.control_store, "failed") + assert failed["request"]["cursor"] == {"event_storage_id": 0} + data.write_text(json.dumps({"mode": "timeout"})) + timed_out: dict[str, Any] = wait_for_sensor_phase(worker.control_store, "timed_out") + assert timed_out["request"]["cursor"] == failed["request"]["cursor"] + timed_call: dict[str, Any] = sensor_calls( + calls, {"evaluation_id": timed_out["request"]["evaluation_id"]} + )[0] + with pytest.raises(ProcessLookupError): + os.kill(timed_call["pid"], 0) + data.write_text(json.dumps({"mode": "ready"})) + destination: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_orders" + ) + assert destination["status"] == test_case.expected_status, log.read_text() + # The next successful evaluation skips but retains its committed cursor. + skipped: dict[str, Any] = wait_for_sensor_phase( + worker.control_store, "completed", empty_runs=True + ) + assert skipped["outcome"]["cursor"]["custom"] == "orders/page-1" + evaluated: list[dict[str, Any]] = sensor_calls(calls, {"phase": "evaluate"}) + assert {record["target"] for record in evaluated} == {"development"} + assert len({record["pid"] for record in evaluated}) == len(evaluated) + assert { + record["cursor"] + for record in sensor_calls(calls, {"phase": "evaluate", "mode": "failed"}) + } == {None} + assert { + record["cursor"] + for record in sensor_calls(calls, {"phase": "evaluate", "mode": "timeout"}) + } == {None} + assert location.poll() is None and worker.process.poll() is None + with sqlite3.connect(worker.control_store) as control: + assert control.execute( + "SELECT run_key,run_id FROM sqb_run_requests" + ).fetchall() == [("orders-1", destination["run_id"])] + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + with duckdb.connect( + str(worker.project / "warehouse.duckdb"), read_only=True + ) as warehouse: + assert ( + warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + ) + assert not (worker.project / ".sqlbuild/control.sqlite3").exists() + assert worker.api.bearer not in log.read_text() diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py index abc6cdea92..b8e7303b67 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py @@ -77,7 +77,8 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ with snapshot_api(tmp_path / "control") as api: monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", api.bearer) # A presentation-only width fallback must not bypass the native-only supervisor. - monkeypatch.setenv("COLUMNS", "999999999999999999999999999999") + # Keep the adversarial width out of pytest's own terminal reporter. + environment: dict[str, str] = os.environ | {"COLUMNS": "999999999999999999999999999999"} arguments: list[str] = [ str(Path(sys.executable).with_name("sqb")), "--project-dir", @@ -93,7 +94,7 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ ] log: Path = project / "location.log" publications: Path = project / "publications.jsonl" - with location_process(arguments, log, publications) as process: + with location_process(arguments, log, publications, env=environment) as process: first: dict[str, Any] = wait_for_generation(api, test_case.target, 1, process, log) first_interpreter: int = int(compiler_pid.read_text()) heartbeat: dict[str, Any] = wait_for_location_health( @@ -182,7 +183,7 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ ) process.terminate() assert process.wait(timeout=5) == test_case.expected_stop_code - with location_process(arguments, log, publications) as process: + with location_process(arguments, log, publications, env=environment) as process: restarted: dict[str, Any] = wait_for_generation( api, test_case.target, test_case.expected_generation, process, log ) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index ded89951dd..6f6cc7a182 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -123,6 +123,16 @@ }, frozenset({"model", "source"}), ), + SnapshotCatalogTestCase( + "DuckShop polling callback with nested code is relocatable", + TYPE_PROOF_FILES + | { + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/custom_orders.py" + ).read_text(), + }, + frozenset({"model"}), + ), SnapshotCatalogTestCase( "DuckShop inherited seed and function namespaces", TYPE_PROOF_FILES @@ -801,6 +811,32 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects expected_error=TypeError, expected_message="group members must be Job objects", ), + SnapshotDiscoveryRejectionTestCase( + description="a polling sensor needs an actual callback", + files={ + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, sensor\n" + "orders = job('orders', description='Build orders.', select='orders')\n" + "poll = sensor('poll', description='Poll orders.', run=orders)\n" + ), + }, + expected_error=ValueError, + expected_message="either a built-in condition or a decorated polling callback", + ), + SnapshotDiscoveryRejectionTestCase( + description="callbacks with event conditions remain explicitly unsupported", + files={ + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, sensor, platform_event, PlatformEventKind\n" + "orders = job('orders', description='Build orders.', select='orders')\n" + "@sensor('poll', description='Poll orders.', run=orders, " + "when=platform_event(kinds=[PlatformEventKind.TICK_OVERDUE]))\n" + "def poll(ctx):\n return []\n" + ), + }, + expected_error=ValueError, + expected_message="callbacks with when are not supported yet", + ), SnapshotDiscoveryRejectionTestCase( description="explicit group cycles cannot publish", files={ diff --git a/uv.lock b/uv.lock index 17225a2936..d0a6aa78ca 100644 --- a/uv.lock +++ b/uv.lock @@ -1343,31 +1343,31 @@ wheels = [ [[package]] name = "fensu" -version = "0.28.1" +version = "0.28.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "fensu-cli" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ab/c7/baf72530c3a67a6d26957658475d32e11adbeb2dbb452425fd5d4af1f575/fensu-0.28.1.tar.gz", hash = "sha256:b81119f0301580367799b77312d5fd82f3ce5a14613b5b709537bca4c070b912", size = 539035, upload-time = "2026-10-31T19:33:41.927Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8f/3d/d5b4fb0ad09426dae015af2817ba8416d2ea974a0c4a6ebea63112f90b1b/fensu-0.28.3.tar.gz", hash = "sha256:a7fc87cb60eaef964b4e980d19bd6de9c95cecc75c019d8b0058e02b6751f4b5", size = 539031, upload-time = "2026-11-01T09:41:23.715Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e8/02/8288dc1ecb853941750d25c2b1effc6fe68d9eeeb188f3d289823731cf69/fensu-0.28.1-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:bf1d49d2c800e7c8acdba031c12e6ea1a38544f45c0b75fbe8c35369b63ee98b", size = 4154268, upload-time = "2026-10-31T19:33:24.239Z" }, - { url = "https://files.pythonhosted.org/packages/4c/70/f21686ef9b89f70c84ea3a94dcdbda47839263496c36aa962c15c2eb093e/fensu-0.28.1-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:7e7564b2c4ffb9a031659b3bd82368be70deb97c3f4f620ad05ddfb5f8086e9c", size = 4004303, upload-time = "2026-10-31T19:33:25.807Z" }, - { url = "https://files.pythonhosted.org/packages/3c/50/558ac8fadf943aad5a8a5ba500a3f63e0f5245fd1f66480b6cd8590ce599/fensu-0.28.1-cp312-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:70da92dad5e5dfbe927bfe870893fe8dc612f28e72780de32014ca42703ba7db", size = 4230255, upload-time = "2026-10-31T19:33:27.649Z" }, - { url = "https://files.pythonhosted.org/packages/5b/5f/b7659a2d6489293a6c16cafc5f8c500b5d5de22536767f468147bed813e3/fensu-0.28.1-cp312-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:207a911faee0e14f40005060281ea624b6360f7f35cf6440a02dd0870c4f0bfc", size = 4307330, upload-time = "2026-10-31T19:33:29.361Z" }, - { url = "https://files.pythonhosted.org/packages/69/a6/aa98ad24bb27e3c829c9675c2b753f5d793fb5f546e8b266ade494d6e044/fensu-0.28.1-cp312-abi3-win_amd64.whl", hash = "sha256:88089637c5a52a2e074dd7b29fb76d4e35d1d9d28e8ef4398be30c3bed7d8071", size = 3977891, upload-time = "2026-10-31T19:33:30.84Z" }, + { url = "https://files.pythonhosted.org/packages/2f/a7/9493fbcf3158ae009651d19c7289855e8ef571dfaf5c7c77154a8a6a4a21/fensu-0.28.3-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:e3e3896f173d2bb676fe04defdd4ebdf4db38aee3ee6e57d7a5a297df32d7c5c", size = 4154274, upload-time = "2026-11-01T09:41:03.614Z" }, + { url = "https://files.pythonhosted.org/packages/a2/71/20fdb7147e268c5dbd9eae0fc9dd99641e416f9e75e8fd067ac089f6db78/fensu-0.28.3-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:d74db7466b2ddd533413d2daf4c704aa4acbcae61e2a0edba2564a4910666707", size = 4004309, upload-time = "2026-11-01T09:41:05.719Z" }, + { url = "https://files.pythonhosted.org/packages/d0/4c/40b01d8e46ff3464eb503529e00ad23eddd2ffbd605282618672d0aff45f/fensu-0.28.3-cp312-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:da5843aa40c7d6398a95b340b3c60d4360b5feddac34960663ee84aec09df8ef", size = 4230261, upload-time = "2026-11-01T09:41:07.481Z" }, + { url = "https://files.pythonhosted.org/packages/e7/7a/243cccd83531c3c86c660b993e77a7014c65d1d3f20c07e45b067dba81cf/fensu-0.28.3-cp312-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:52e1623e614f101c5ada2eda59548ff441b7efe4228813e69bc0f0ef9b3fbca6", size = 4307334, upload-time = "2026-11-01T09:41:09.938Z" }, + { url = "https://files.pythonhosted.org/packages/88/a3/38dbb50d5079a1a5fcaf62bcdd582c12a53609c575a747cb2461e83ddfcf/fensu-0.28.3-cp312-abi3-win_amd64.whl", hash = "sha256:a39c407725ea4d92123414bdbe9f80d5db0b7a412af49082a87378203c9ecb04", size = 3977904, upload-time = "2026-11-01T09:41:11.672Z" }, ] [[package]] name = "fensu-cli" -version = "0.28.1" +version = "0.28.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6d/da/a4ef081ab0ec23cb719dfbce2747c11ee3832a2dda2aaef1f7722548d1e6/fensu_cli-0.28.1.tar.gz", hash = "sha256:4857b2ef850d35bae19462cd6e26fdc0b33c00d97f13f20bb5b00b5c879ef2e9", size = 765839, upload-time = "2026-10-31T19:33:43.051Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/17/3d0953c33387caa0a6becc9bb02210d4a020eaaf73ef07bdc0c8f8952562/fensu_cli-0.28.3.tar.gz", hash = "sha256:3ba38bedc88a9e6db8563f638aa6ba8649a3e5750de176e22a7b35561d14bd9e", size = 766571, upload-time = "2026-11-01T09:41:25.271Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/33/03/1c5a56b3d233360d1fb699ff7a92cee005faf8f8fda8c92a7688cc0be167/fensu_cli-0.28.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:67b0bc97cdaeb08e633e68336b1d82b3e598d615b54b9b6f840772929cf6929e", size = 7975688, upload-time = "2026-10-31T19:33:32.569Z" }, - { url = "https://files.pythonhosted.org/packages/f2/a7/d90a388b17a89d9e34238e16e03ce306331c212d215f02ff4c43d6d93189/fensu_cli-0.28.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e6a9d97525aea16e079b773b7141a1167bd799bc07ff8bfc720b8f311c91d20b", size = 7492140, upload-time = "2026-10-31T19:33:34.634Z" }, - { url = "https://files.pythonhosted.org/packages/62/f3/4a5247e312d4725ead4ab37d0076371b731a8858c5819cfcceb1d6ce0b69/fensu_cli-0.28.1-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b8fd61d84ffd2be3c023335e374cabc72eac74dfe5cd90103bde52eba6c40da5", size = 7860157, upload-time = "2026-10-31T19:33:36.407Z" }, - { url = "https://files.pythonhosted.org/packages/d4/71/a954282a9f3f372fe77c942ff6acc8cc2f797b7da82b3b40a6c69715b668/fensu_cli-0.28.1-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1115a28579b917d6d8c0010e99488318e4cd264236642b5081bb79e8378bdc15", size = 8323299, upload-time = "2026-10-31T19:33:38.357Z" }, - { url = "https://files.pythonhosted.org/packages/35/cb/aa860d0afcb17a7931ab14dcab061d234143caef12e2791d17f27d0816c3/fensu_cli-0.28.1-py3-none-win_amd64.whl", hash = "sha256:4294965dcf4c317b66e4e489c9b9b38e8f8d82000c59bf9f3275fe887a6b45d8", size = 7705331, upload-time = "2026-10-31T19:33:40.314Z" }, + { url = "https://files.pythonhosted.org/packages/ff/bb/09753cc0584c21af1d76eca7c9a6ea5e2d876d71c079c5505dda93742f3d/fensu_cli-0.28.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:5179b032074f4ff5097950c0c0b063fd872cffcd23a4bf8a89c71ab0d3d8074b", size = 7963143, upload-time = "2026-11-01T09:41:13.908Z" }, + { url = "https://files.pythonhosted.org/packages/47/59/663fb61d56172bfb053375e8623a23db389b746052fb00dd087cf6c1a76b/fensu_cli-0.28.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:41077b9ec6e2eb46405af809d1010b26519c8928d4264137ca4c6d47ce81d2e1", size = 7479791, upload-time = "2026-11-01T09:41:16.253Z" }, + { url = "https://files.pythonhosted.org/packages/f7/7c/1a03bd25bc2a231ca03478ba3dfd8f21c909eae564c3df751a8982222beb/fensu_cli-0.28.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ac929d598046846c18eae62c8b75a9a49e9b6bbcadb46bd413a5da96ef6453e2", size = 7854377, upload-time = "2026-11-01T09:41:18.109Z" }, + { url = "https://files.pythonhosted.org/packages/17/5e/8395e4bd4e9e4d282a0949e42e721101127fb30f45cb71d6e5af05329051/fensu_cli-0.28.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9ce453889008381184a416acaf274ffa4a46c7cb96ad4397de264816bb10626a", size = 8316670, upload-time = "2026-11-01T09:41:20.045Z" }, + { url = "https://files.pythonhosted.org/packages/89/40/6e3831cd39b16f163916b36c03010754c1cfb58583fd737517bd544751d5/fensu_cli-0.28.3-py3-none-win_amd64.whl", hash = "sha256:66ec08187a6dcc43392244ba4b7349ee29e8da63fbe7110114468f07a000d577", size = 7710605, upload-time = "2026-11-01T09:41:22.073Z" }, ] [[package]] @@ -4346,7 +4346,7 @@ dev = [ { name = "dbt-snowflake", specifier = "==1.12.0b1" }, { name = "dbt-sqlserver", specifier = ">=1.9.0,<1.11" }, { name = "dlt", extras = ["duckdb", "filesystem", "sql-database"], specifier = ">=1.28.0,<2" }, - { name = "fensu", specifier = ">=0.28.1" }, + { name = "fensu", specifier = ">=0.28.3" }, { name = "ingestr", specifier = ">=1.1.73" }, { name = "jsonschema", specifier = ">=4.23.0" }, { name = "maturin", specifier = ">=1.9,<2" }, From bdf3f558bbc8e6bada286aa8d02c82025b3fb2b1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 09:59:27 +0000 Subject: [PATCH 55/59] feat: persist warehouse observation identities and read history Add nullable deterministic event IDs to fingerprint and source-freshness writers without rewriting predecessor rows or changing latest-state reads. Preserve source identities under location, target and state-table scoping; use fixed legacy fields only when the source ID is absent. Keep warehouse history reads genuinely read-only and bounded. Fence atomic ingestion pages to the current location session and generation, and preserve transient retries during populated fingerprint upgrades. Document the identity contract separately from unfinished automatic synchronization. Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- Cargo.lock | 1 + .../orchestration/legacy_warehouse_state.sql | 22 ++ crates/README.md | 25 ++ .../src/fingerprint_storage/constants.rs | 3 +- .../src/fingerprint_storage/main/_event_id.rs | 20 ++ .../main/create_table_sql.rs | 10 +- .../fingerprint_storage/main/insert_sql.rs | 7 +- .../src/fingerprint_storage/main/mod.rs | 1 + .../fingerprint_storage/tests/test_storage.rs | 5 +- crates/sqlbuild-orchestration/Cargo.toml | 1 + .../src/api/_helpers/mod.rs | 1 + .../src/api/_helpers/warehouse_metadata.rs | 30 ++ .../src/api/main/serve_api.rs | 6 +- .../sqlbuild-orchestration/src/api/models.rs | 5 +- crates/sqlbuild-orchestration/src/lib.rs | 1 + .../src/platform/_helpers/mod.rs | 1 - .../src/platform/_helpers/work_client.rs | 21 ++ .../src/platform/main/claim_location_work.rs | 2 +- .../platform/main/complete_location_work.rs | 2 +- .../src/platform/main/mod.rs | 1 + .../require_instance.rs} | 2 +- .../src/store/_helpers/migration.rs | 1 + .../src/store/main/migrate_store.rs | 3 +- .../src/store/warehouse_freshness.sql | 10 + .../src/warehouse_metadata/constants.rs | 3 + .../main/ingest_source_freshness.rs | 103 ++++++ .../src/warehouse_metadata/main/mod.rs | 1 + .../src/warehouse_metadata/mod.rs | 5 + .../src/warehouse_metadata/models.rs | 56 ++++ .../tests/api/test_contracts.rs | 2 + .../tests/api/test_location_work.rs | 2 + .../tests/api/test_types.rs | 6 + .../tests/api/test_warehouse_metadata.rs | 306 ++++++++++++++++++ .../tests/sensors/test_runs.rs | 3 + .../tests/snapshots/helpers.rs | 3 +- .../_helpers/exported_constants/constants.rs | 9 +- .../_helpers/fingerprints/storage_io.rs | 73 +++-- .../_helpers/freshness/publication.rs | 5 + .../bindings/_helpers/freshness/state_io.rs | 34 +- .../_helpers/location_command/runtime.rs | 20 ++ .../src/bindings/_helpers/native_host/mod.rs | 2 + .../native_host/state_column_evolution.rs | 85 +++++ .../native_host/warehouse_metadata.rs | 205 ++++++++++++ .../bindings/_helpers/records/constants.rs | 2 + .../bindings/_helpers/state_rows/freshness.rs | 38 ++- .../_helpers/state_rows/state_tables.rs | 10 +- .../sqlbuild-state/src/freshness/constants.rs | 5 +- .../src/freshness/main/event_id.rs | 22 ++ .../sqlbuild-state/src/freshness/main/mod.rs | 2 + .../src/freshness/main/read_history_sql.rs | 112 +++++++ crates/sqlbuild-state/src/freshness/models.rs | 15 + fensu.toml | 1 + src/sqlbuild/_native.pyi | 9 + .../contract/classes/duckdb_backed_adapter.py | 4 +- .../sqlserver/classes/sqlserver_adapter.py | 4 +- .../executor/clone/_helpers/fingerprinting.py | 2 + .../executor/diff/_helpers/query_artifacts.py | 2 + .../executor/functions/main/_execute.py | 2 + .../python_nodes/_helpers/fingerprinting.py | 2 + .../executor/review/_helpers/marker.py | 2 + .../run/_helpers/reuse/fingerprinting.py | 2 + .../executor/seed/_helpers/fingerprinting.py | 2 + .../publication/_test_types.py | 11 + .../compiled_project/publication/helpers.py | 56 +++- .../publication/test_warehouse_metadata.py | 293 +++++++++++++++++ 65 files changed, 1643 insertions(+), 59 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_state.sql create mode 100644 crates/sqlbuild-analysis/src/fingerprint_storage/main/_event_id.rs create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/warehouse_metadata.rs rename crates/sqlbuild-orchestration/src/platform/{_helpers/location_work.rs => main/require_instance.rs} (90%) create mode 100644 crates/sqlbuild-orchestration/src/store/warehouse_freshness.sql create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/native_host/state_column_evolution.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs create mode 100644 crates/sqlbuild-state/src/freshness/main/event_id.rs create mode 100644 crates/sqlbuild-state/src/freshness/main/read_history_sql.rs create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py diff --git a/Cargo.lock b/Cargo.lock index 8a5982d252..37cab0cc6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2597,6 +2597,7 @@ dependencies = [ "serde", "serde_json", "sqlbuild-recall", + "sqlbuild-state", "sqlx-core", "sqlx-postgres", "subtle", diff --git a/benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_state.sql b/benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_state.sql new file mode 100644 index 0000000000..b135ecff51 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_state.sql @@ -0,0 +1,22 @@ +-- Populated predecessor schemas for additive event-ID and read-only history-sync checks. +CREATE TABLE main._sqlbuild_fingerprints ( + node_type VARCHAR NOT NULL, node_name VARCHAR NOT NULL, + target_database VARCHAR, target_schema VARCHAR, target_name VARCHAR, + run_id VARCHAR NOT NULL, definition_hash VARCHAR NOT NULL, version_hash VARCHAR NOT NULL, + schema_fingerprint VARCHAR NOT NULL, definition_b64 VARCHAR NOT NULL, + metadata_json_b64 VARCHAR NOT NULL, ts TIMESTAMP NOT NULL +); +INSERT INTO main._sqlbuild_fingerprints VALUES ( + 'model', 'orders', NULL, 'main', 'orders', 'legacy-run', 'legacy-definition', + 'legacy-version', 'legacy-schema', 'U0VMRUNUIDEgQVMgb3JkZXJfaWQ=', 'e30=', '2026-01-01 10:00:00' +); +CREATE TABLE main._sqlbuild_source_freshness ( + source_name VARCHAR NOT NULL, target_database VARCHAR, target_schema VARCHAR, + target_name VARCHAR, run_id VARCHAR NOT NULL, strategy VARCHAR NOT NULL, + value_kind VARCHAR NOT NULL, data_version VARCHAR, data_version_hash VARCHAR NOT NULL, + observed_at TIMESTAMP NOT NULL +); +INSERT INTO main._sqlbuild_source_freshness VALUES ( + 'raw.orders', NULL, 'main', 'orders', 'legacy-run', 'max_column', 'integer', + '7', 'legacy-value', '2026-01-01 10:00:00' +); diff --git a/crates/README.md b/crates/README.md index 37c7b39166..41952bc041 100644 --- a/crates/README.md +++ b/crates/README.md @@ -107,6 +107,31 @@ the goldens in `tests/goldens/compiler`, with stage captures and full discovery, analysis coverage required. CI also runs it on Python 3.13 and 3.14, and compiles and builds a playground project on Windows. +## Warehouse event identity + +Fingerprint and source-freshness writers persist deterministic `event_id` values in nullable text +columns. Existing tables evolve additively; no predecessor rows are rewritten. Latest-state readers +retain their existing projections. IDs are lowercase SHA-256 hex over compact UTF-8 JSON arrays, +whose fixed namespace is independent of the package version: + +| Table | Array fields, in order | +|---|---| +| `_sqlbuild_fingerprints` | `"sqlbuild.fingerprint.event.v1"`, node kind, node name, target database, target schema, target name, run ID, version hash | +| `_sqlbuild_source_freshness` | `"sqlbuild.source-freshness.event.v1"`, source name, target database, target schema, target name, run ID, observation time, strategy, value kind, value hash | + +Nullable identity fields remain JSON null, distinct from empty strings. Freshness observation time +is normalized to UTC ISO text with six fractional digits and a `+00:00` offset. Retrying the same +observation retains its ID; another run or observation is distinct. Definitions, display values, +and future storage columns do not alter these identities. + +The location history reader is read-only, including local database connections: it neither creates +databases nor evolves tables. Source-freshness ingestion preserves warehouse IDs, scoped by location, +target and physical state table. Only missing or NULL IDs use the fixed legacy observation tuple, +never a hash of the evolving full row. Bounded pages commit atomically under the current location +session and snapshot generation, and repeated pages are idempotent. These are read/ingestion +primitives; automatic polling, watermark/overlap management and deletion reconciliation are separate +location-sync responsibilities, not implemented by the page reader. + ## Working on the crates - Tests live next to the module they cover; run them for one crate with diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/constants.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/constants.rs index 791dcc6822..59e8a2328c 100644 --- a/crates/sqlbuild-analysis/src/fingerprint_storage/constants.rs +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/constants.rs @@ -12,8 +12,9 @@ pub const COLUMN_SCHEMA_FINGERPRINT: &str = "schema_fingerprint"; pub const COLUMN_DEFINITION_B64: &str = "definition_b64"; pub const COLUMN_METADATA_JSON_B64: &str = "metadata_json_b64"; pub const COLUMN_TIMESTAMP: &str = "ts"; +pub const COLUMN_EVENT_ID: &str = "event_id"; -/// Every stored column, in table order. +/// Legacy columns used by latest-row readers, including readers of tables without an event ID. pub const FINGERPRINT_COLUMNS: [&str; 12] = [ COLUMN_NODE_TYPE, COLUMN_NODE_NAME, diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/main/_event_id.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/main/_event_id.rs new file mode 100644 index 0000000000..6ef1ccee43 --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/main/_event_id.rs @@ -0,0 +1,20 @@ +//! Fixed writer identity excludes timestamps, definitions and future storage columns. + +use sha2::{Digest, Sha256}; + +use crate::fingerprint_storage::models::StoredFingerprint; + +/// Stable SHA-256 of the versioned JSON tuple named by the fingerprint storage contract. +pub(super) fn event_id(row: &StoredFingerprint) -> String { + let identity = serde_json::json!([ + "sqlbuild.fingerprint.event.v1", + row.node_type, + row.node_name, + row.target_database, + row.target_schema, + row.target_name, + row.run_id, + row.version_hash, + ]); + format!("{:x}", Sha256::digest(identity.to_string().as_bytes())) +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/main/create_table_sql.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/main/create_table_sql.rs index 7c0a00587e..db3a126811 100644 --- a/crates/sqlbuild-analysis/src/fingerprint_storage/main/create_table_sql.rs +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/main/create_table_sql.rs @@ -1,9 +1,10 @@ //! Fingerprint table DDL. use crate::fingerprint_storage::constants::{ - COLUMN_DEFINITION_B64, COLUMN_DEFINITION_HASH, COLUMN_METADATA_JSON_B64, COLUMN_NODE_NAME, - COLUMN_NODE_TYPE, COLUMN_RUN_ID, COLUMN_SCHEMA_FINGERPRINT, COLUMN_TARGET_DATABASE, - COLUMN_TARGET_NAME, COLUMN_TARGET_SCHEMA, COLUMN_TIMESTAMP, COLUMN_VERSION_HASH, + COLUMN_DEFINITION_B64, COLUMN_DEFINITION_HASH, COLUMN_EVENT_ID, COLUMN_METADATA_JSON_B64, + COLUMN_NODE_NAME, COLUMN_NODE_TYPE, COLUMN_RUN_ID, COLUMN_SCHEMA_FINGERPRINT, + COLUMN_TARGET_DATABASE, COLUMN_TARGET_NAME, COLUMN_TARGET_SCHEMA, COLUMN_TIMESTAMP, + COLUMN_VERSION_HASH, }; /// `CREATE TABLE IF NOT EXISTS` for the fingerprint table, with the adapter's column types. @@ -21,6 +22,7 @@ pub fn create_table_sql(qualified_name: &str, string_type: &str, timestamp_type: {COLUMN_SCHEMA_FINGERPRINT} {string_type} NOT NULL, \ {COLUMN_DEFINITION_B64} {string_type} NOT NULL, \ {COLUMN_METADATA_JSON_B64} {string_type} NOT NULL, \ - {COLUMN_TIMESTAMP} {timestamp_type} NOT NULL)" + {COLUMN_TIMESTAMP} {timestamp_type} NOT NULL, \ + {COLUMN_EVENT_ID} {string_type})" ) } diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/main/insert_sql.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/main/insert_sql.rs index 25e54bf696..dbd785c797 100644 --- a/crates/sqlbuild-analysis/src/fingerprint_storage/main/insert_sql.rs +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/main/insert_sql.rs @@ -3,7 +3,9 @@ use crate::fingerprint_storage::_helpers::base64::encode; use crate::fingerprint_storage::_helpers::literals::{optional_string_literal, string_literal}; use crate::fingerprint_storage::_helpers::table_columns::column_list; +use crate::fingerprint_storage::constants::COLUMN_EVENT_ID; use crate::fingerprint_storage::errors::FingerprintStorageError; +use crate::fingerprint_storage::main::_event_id::event_id; use crate::fingerprint_storage::models::StoredFingerprint; /// One multi-row `INSERT` for a non-empty batch. @@ -16,14 +18,14 @@ pub fn insert_sql( } let rows: Vec = fingerprints.iter().map(row_values).collect(); Ok(format!( - "INSERT INTO {qualified_name} ({}) VALUES {}", + "INSERT INTO {qualified_name} ({}, {COLUMN_EVENT_ID}) VALUES {}", column_list(), rows.join(", ") )) } fn row_values(fingerprint: &StoredFingerprint) -> String { - let values: [String; 12] = [ + let values: [String; 13] = [ string_literal(&fingerprint.node_type), string_literal(&fingerprint.node_name), optional_string_literal(fingerprint.target_database.as_deref()), @@ -36,6 +38,7 @@ fn row_values(fingerprint: &StoredFingerprint) -> String { format!("'{}'", encode(fingerprint.definition.as_bytes())), format!("'{}'", encode(fingerprint.metadata_json.as_bytes())), format!("'{}'", fingerprint.ts), + string_literal(&event_id(fingerprint)), ]; format!("({})", values.join(", ")) } diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/main/mod.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/main/mod.rs index 54c1f22e6f..ab782386e7 100644 --- a/crates/sqlbuild-analysis/src/fingerprint_storage/main/mod.rs +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/main/mod.rs @@ -1,3 +1,4 @@ +mod _event_id; pub mod create_table_sql; pub mod decode_storage; pub mod insert_batch_lengths; diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/tests/test_storage.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/tests/test_storage.rs index 59c72c1dd6..5595b45108 100644 --- a/crates/sqlbuild-analysis/src/fingerprint_storage/tests/test_storage.rs +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/tests/test_storage.rs @@ -142,8 +142,9 @@ fn given_rows_when_inserting_then_literals_are_escaped_and_definitions_encoded() expected_sql: Ok( "INSERT INTO main.t (node_type, node_name, target_database, target_schema, \ target_name, run_id, definition_hash, version_hash, schema_fingerprint, \ - definition_b64, metadata_json_b64, ts) VALUES ('model', 'o''rders', NULL, \ - 'main', 'orders', 'run', 'd', '', 's', 'YWI=', 'e30=', '2026-01-01T00:00:00')", + definition_b64, metadata_json_b64, ts, event_id) VALUES ('model', 'o''rders', NULL, \ + 'main', 'orders', 'run', 'd', '', 's', 'YWI=', 'e30=', '2026-01-01T00:00:00', \ + 'b9397954c803e7f99f6a233c2985be772c7ddf2c6f234197d77bc227ae175cbd')", ), }, InsertTestCase { diff --git a/crates/sqlbuild-orchestration/Cargo.toml b/crates/sqlbuild-orchestration/Cargo.toml index 4e31942af6..24f97c2b34 100644 --- a/crates/sqlbuild-orchestration/Cargo.toml +++ b/crates/sqlbuild-orchestration/Cargo.toml @@ -20,6 +20,7 @@ semver.workspace = true serde.workspace = true serde_json.workspace = true sqlbuild-recall.workspace = true +sqlbuild-state.workspace = true sqlx-core.workspace = true sqlx-postgres.workspace = true subtle.workspace = true diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs index 996c7546bb..94872aa28d 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs @@ -6,3 +6,4 @@ pub(crate) mod run_reads; pub(crate) mod runs; pub(crate) mod snapshots; pub(crate) mod trigger_reads; +pub(crate) mod warehouse_metadata; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/warehouse_metadata.rs b/crates/sqlbuild-orchestration/src/api/_helpers/warehouse_metadata.rs new file mode 100644 index 0000000000..5ddb1f2001 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/warehouse_metadata.rs @@ -0,0 +1,30 @@ +//! An authenticated location can mirror only its own registered reader's target. + +use axum::extract::{Path, State}; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::_helpers::authentication::require_location; +use crate::api::errors::ApiError; +use crate::api::models::MutationResponse; +use crate::credentials::models::TokenIdentity; +use crate::store::models::ControlStore; +use crate::warehouse_metadata::main::ingest_source_freshness::ingest_source_freshness; +use crate::warehouse_metadata::models::SourceFreshnessBatch; + +#[utoipa::path(post, path = "/api/locations/{location_id}/targets/{target}/metadata/source-freshness", + params(("location_id" = Uuid, Path), ("target" = String, Path)), request_body = SourceFreshnessBatch, + responses((status = 200, body = MutationResponse), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized")), + security(("bearer" = [])))] +pub(crate) async fn report( + State(store): State, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, + Json(batch): Json, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + Ok(Json(MutationResponse { + applied: ingest_source_freshness(&store, (location_id, &target), &batch).await?, + })) +} diff --git a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs index 37918480f4..dd000306a9 100644 --- a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs +++ b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs @@ -6,7 +6,7 @@ use axum::{Json, Router, middleware}; use crate::api::_helpers::authentication::authenticate; use crate::api::_helpers::{ - location_health, location_work, run_reads, runs, snapshots, trigger_reads, + location_health, location_work, run_reads, runs, snapshots, trigger_reads, warehouse_metadata, }; use crate::api::constants::BODY_BYTES; use crate::api::main::openapi_document::openapi_document; @@ -33,6 +33,10 @@ pub async fn serve_api( "/api/locations/{location_id}/health", get(location_health::recent).post(location_health::report), ) + .route( + "/api/locations/{location_id}/targets/{target}/metadata/source-freshness", + post(warehouse_metadata::report), + ) .route( "/api/locations/{location_id}/targets/{target}/snapshots", get(snapshots::current).post(snapshots::publish), diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index 06471e75a9..9668db92c1 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize}; use crate::api::_helpers::{ - location_health, location_work, run_reads, runs, snapshots, trigger_reads, + location_health, location_work, run_reads, runs, snapshots, trigger_reads, warehouse_metadata, }; #[derive(Debug, utoipa::OpenApi)] @@ -27,7 +27,8 @@ use crate::api::_helpers::{ run_reads::recall, trigger_reads::current, trigger_reads::events, - trigger_reads::ticks + trigger_reads::ticks, + warehouse_metadata::report ))] pub(crate) struct ControlApi; diff --git a/crates/sqlbuild-orchestration/src/lib.rs b/crates/sqlbuild-orchestration/src/lib.rs index cc435f1701..3f1421053e 100644 --- a/crates/sqlbuild-orchestration/src/lib.rs +++ b/crates/sqlbuild-orchestration/src/lib.rs @@ -11,4 +11,5 @@ pub mod scheduler; pub mod sensors; pub mod snapshots; pub mod store; +pub mod warehouse_metadata; pub mod worker; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs index 574aaa4680..fddc0b0e5a 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs @@ -1,6 +1,5 @@ pub(crate) mod liveness; pub(crate) mod location_heartbeat; -pub(crate) mod location_work; pub(crate) mod observations; pub(crate) mod watchdog; pub(crate) mod webhook_delivery; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs index fe1e104915..5c1351ac68 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs @@ -8,8 +8,29 @@ use crate::api::models::MutationResponse; use crate::platform::models::{ LocationWorkClaim, LocationWorkClient, LocationWorkPoll, LocationWorkResult, }; +use crate::warehouse_metadata::models::SourceFreshnessBatch; impl LocationWorkClient { + /// Metadata shares the location's scoped pool, not the scheduler's execution authority. + pub async fn sync_source_freshness( + &self, + target: &str, + batch: &SourceFreshnessBatch, + ) -> Result { + let endpoint = self.connection.endpoint(&[ + "api", + "locations", + &self.location_id.to_string(), + "targets", + target, + "metadata", + "source-freshness", + ])?; + let response: MutationResponse = + client_response(self.connection.client.post(endpoint).json(batch)).await?; + Ok(response.applied) + } + pub async fn poll( &self, poll: &LocationWorkPoll, diff --git a/crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs index 7a36796c37..8d071b1626 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs @@ -3,8 +3,8 @@ use chrono::Duration; use uuid::Uuid; -use crate::platform::_helpers::location_work::require_instance; use crate::platform::main::read_location_work::read_location_work; +use crate::platform::main::require_instance::require_instance; use crate::platform::models::{LocationWorkClaim, LocationWorkPoll}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; diff --git a/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs index a044885fe4..cc10fab99c 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs @@ -3,8 +3,8 @@ use serde_json::json; use uuid::Uuid; -use crate::platform::_helpers::location_work::require_instance; use crate::platform::main::read_location_work::read_location_work; +use crate::platform::main::require_instance::require_instance; use crate::platform::models::{LocationWork, LocationWorkOutcome, LocationWorkResult}; use crate::sensors::constants::MAX_SENSOR_RUN_REQUESTS; use crate::sensors::models::CustomSensorOutcome; diff --git a/crates/sqlbuild-orchestration/src/platform/main/mod.rs b/crates/sqlbuild-orchestration/src/platform/main/mod.rs index a782c64402..8124f49518 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/mod.rs @@ -9,5 +9,6 @@ pub(crate) mod read_platform_events; pub mod report_location_health; pub mod report_remote_location_health; pub mod request_location_work; +pub(crate) mod require_instance; pub mod run_platform_sink; pub mod run_watchdog; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/require_instance.rs similarity index 90% rename from crates/sqlbuild-orchestration/src/platform/_helpers/location_work.rs rename to crates/sqlbuild-orchestration/src/platform/main/require_instance.rs index 8e2d429a5a..9d64b6068f 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/location_work.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/require_instance.rs @@ -1,4 +1,4 @@ -//! Request retries and completion share the same immutable result reader and instance scope. +//! Named work and warehouse observations share the registered location/target boundary. use uuid::Uuid; diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index b2c9b5fbf4..6199e096da 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -14,6 +14,7 @@ pub(crate) const SENSORS: &str = include_str!("../sensors.sql"); pub(crate) const RUN_SENSORS: &str = include_str!("../run_sensors.sql"); pub(crate) const EXECUTED_NODES: &str = include_str!("../executed_nodes.sql"); pub(crate) const NODE_SENSORS: &str = include_str!("../node_sensors.sql"); +pub(crate) const WAREHOUSE_FRESHNESS: &str = include_str!("../warehouse_freshness.sql"); pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index 6190e2798a..f2237bd093 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -5,7 +5,7 @@ use uuid::Uuid; use crate::store::_helpers::migration::{ EXECUTED_NODES, LOCATION_INSTANCES, LOCATION_WORK, NODE_SENSORS, PLATFORM_SINK, RUN_SENSORS, - SCHEMA, SENSORS, render, verify_migrations, + SCHEMA, SENSORS, WAREHOUSE_FRESHNESS, render, verify_migrations, }; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; @@ -36,6 +36,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { ("run_sensors", RUN_SENSORS), ("executed_nodes", EXECUTED_NODES), ("node_sensors", NODE_SENSORS), + ("warehouse_freshness", WAREHOUSE_FRESHNESS), ]; let mut digests = vec![digest.clone()]; digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); diff --git a/crates/sqlbuild-orchestration/src/store/warehouse_freshness.sql b/crates/sqlbuild-orchestration/src/store/warehouse_freshness.sql new file mode 100644 index 0000000000..5cdd1500e6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/warehouse_freshness.sql @@ -0,0 +1,10 @@ +CREATE TABLE sqb_warehouse_freshness ( + event_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + source_event_id TEXT NOT NULL, + observed_at {time} NOT NULL, + record_json {json} NOT NULL, + received_at {time} NOT NULL +); +CREATE INDEX sqb_warehouse_freshness_scope ON sqb_warehouse_freshness (location_id, target, observed_at, event_id); diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs new file mode 100644 index 0000000000..05ee34c05a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs @@ -0,0 +1,3 @@ +//! Limits bound each authenticated warehouse observation transaction. + +pub const MAX_FRESHNESS_RECORDS: usize = 100; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs new file mode 100644 index 0000000000..8f170f2fce --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs @@ -0,0 +1,103 @@ +//! Persist source identities; only legacy rows use a fixed observation-key fallback. + +use serde_json::json; +use sqlbuild_state::freshness::main::event_id::event_id as legacy_event_id; +use sqlbuild_state::freshness::models::FreshnessEventIdentity; +use uuid::Uuid; + +use crate::platform::main::require_instance::require_instance; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::warehouse_metadata::constants::MAX_FRESHNESS_RECORDS; +use crate::warehouse_metadata::models::SourceFreshnessBatch; + +pub async fn ingest_source_freshness( + store: &ControlStore, + (location_id, target): (Uuid, &str), + batch: &SourceFreshnessBatch, +) -> Result { + let invalid_identity = batch + .records + .iter() + .flat_map(|record| { + [ + &record.state_schema, + &record.source_name, + &record.run_id, + &record.strategy, + &record.value_kind, + &record.data_version_hash, + ] + }) + .any(|value| value.trim().is_empty()); + let invalid_source_id = batch.records.iter().any(|record| { + record + .event_id + .as_ref() + .is_some_and(|value| value.trim().is_empty()) + }); + if batch.generation < 1 + || batch.records.len() > MAX_FRESHNESS_RECORDS + || invalid_identity + || invalid_source_id + { + return Err(ControlError::Invalid(format!( + "source freshness sync needs a positive generation, at most {MAX_FRESHNESS_RECORDS} records and nonempty state identities", + ))); + } + let transaction = require_instance( + begin(store, TransactionMode::Write).await?, + (location_id, batch.session_id, target), + ) + .await?; + let (mut transaction, rows) = query( + transaction, + "SELECT s.snapshot_id FROM sqb_current_snapshots c JOIN sqb_snapshots s ON s.snapshot_id=c.snapshot_id \ + WHERE c.location_id=$1 AND c.target=$2 AND s.generation=$3", + vec![Parameter::Id(location_id), Parameter::Text(target.into()), Parameter::Integer(batch.generation)], + ).await?; + if rows.is_empty() { + return Err(ControlError::Fenced); + } + let mut applied = false; + for record in &batch.records { + let source_event_id = record.event_id.clone().unwrap_or_else(|| { + legacy_event_id(&FreshnessEventIdentity { + source_name: &record.source_name, + target_database: record.target_database.as_deref(), + target_schema: record.target_schema.as_deref(), + target_name: record.target_name.as_deref(), + run_id: &record.run_id, + strategy: &record.strategy, + value_kind: &record.value_kind, + data_version_hash: &record.data_version_hash, + observed_at: &record + .observed_at + .format("%Y-%m-%dT%H:%M:%S%.6f+00:00") + .to_string(), + }) + }); + let identity = serde_json::to_vec(&( + target, + &record.state_database, + &record.state_schema, + "_sqlbuild_source_freshness", + &source_event_id, + )) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let event_id = Uuid::new_v5(&location_id, &identity); + let received_at = transaction.now; + let inserted; + (transaction, inserted) = query(transaction, + "INSERT INTO sqb_warehouse_freshness (event_id,location_id,target,source_event_id,observed_at,record_json,received_at) \ + VALUES ($1,$2,$3,$4,$5,$6,$7) ON CONFLICT (event_id) DO NOTHING RETURNING event_id", + vec![Parameter::Id(event_id), Parameter::Id(location_id), Parameter::Text(target.into()), + Parameter::Text(source_event_id), Parameter::Time(record.observed_at), Parameter::Json(json!(record)), Parameter::Time(received_at)]).await?; + applied |= !inserted.is_empty(); + } + commit(transaction).await?; + Ok(applied) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs new file mode 100644 index 0000000000..0ff12b7aff --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs @@ -0,0 +1 @@ +pub mod ingest_source_freshness; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/mod.rs new file mode 100644 index 0000000000..69fb999ac1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/mod.rs @@ -0,0 +1,5 @@ +//! Warehouse-owned observations mirrored by authenticated code locations, never execution state. + +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs new file mode 100644 index 0000000000..388464b8ab --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -0,0 +1,56 @@ +//! Existing source-freshness state rows, with their physical state-table provenance. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +/// Preserve observations, not an inferred latest version, age status or run outcome. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseFreshnessRecord { + pub event_id: Option, + pub state_database: Option, + pub state_schema: String, + pub source_name: String, + pub target_database: Option, + pub target_schema: Option, + pub target_name: Option, + pub run_id: String, + pub strategy: String, + pub value_kind: String, + pub data_version: Option, + pub data_version_hash: String, + pub observed_at: DateTime, +} + +/// A bounded page from one registered reader of a published location/target generation. +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct SourceFreshnessBatch { + pub session_id: Uuid, + pub generation: i64, + pub records: Vec, +} + +/// Physical namespace of SQLBuild's source-freshness state table, never a source query. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WarehouseStateLocation { + pub database: Option, + pub schema: String, +} + +/// Private interpreter input; cursor values are data and receive adapter literal rendering. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SourceFreshnessRead { + pub location: WarehouseStateLocation, + pub after: Option<[Option; 11]>, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SourceFreshnessPage { + pub records: Vec, + pub next_after: Option<[Option; 11]>, +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index 722e726650..2faba15f57 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -16,6 +16,8 @@ mod test_location_work; mod test_reads; #[path = "test_types.rs"] mod test_types; +#[path = "test_warehouse_metadata.rs"] +mod test_warehouse_metadata; #[path = "test_worker.rs"] mod test_worker; #[path = "work/helpers.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs index bc54a92560..a89d419bcd 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs @@ -46,8 +46,10 @@ fn given_outbound_work_when_claimed_retried_or_replaced_then_scope_and_immutable let store = &fixture.store; migrate_store(store).await.expect("fresh migration"); // Exercise both deployed predecessors, including checksums and repeated startup. + execute(store, "DROP TABLE sqb_warehouse_freshness").await; execute(store, "DROP INDEX sqb_node_finished_cursor; DROP INDEX sqb_node_started_identity; DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DELETE FROM sqb_schema_migrations WHERE version>=3").await; migrate_store(store).await.expect("schema 2 upgrade"); + execute(store, "DROP TABLE sqb_warehouse_freshness").await; execute(store, "DROP INDEX sqb_node_finished_cursor; DROP INDEX sqb_node_started_identity; DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2").await; migrate_store(store).await.expect("schema 1 upgrade"); migrate_store(store).await.expect("repeat migration"); diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index 133e475a53..71c81b400d 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -44,3 +44,9 @@ pub(crate) struct LocationWorkTestCase { pub(crate) backend: Backend, pub(crate) expected_generation: i64, } + +pub(crate) struct WarehouseMetadataTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_observations: usize, +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs new file mode 100644 index 0000000000..efc31663b5 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs @@ -0,0 +1,306 @@ +//! A scoped HTTP reader mirrors immutable warehouse observations, including external runs. + +use chrono::{Duration, Utc}; +use reqwest::StatusCode; +use serde_json::json; +use uuid::Uuid; + +use crate::api::tests::helpers::{start_api, stop_api}; +use crate::api::tests::test_types::WarehouseMetadataTestCase; +use crate::credentials::main::issue_token::issue_token; +use crate::credentials::models::{AuditContext, CredentialScope}; +use crate::credentials::tests::location_helpers::create_location; +use crate::platform::main::connect_location_work::connect_location_work; +use crate::platform::main::report_location_health::report_location_health; +use crate::platform::models::{LocationHealth, LocationHealthReport}; +use crate::runs::tests::helpers::snapshot; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::TransactionMode; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; +use crate::warehouse_metadata::models::SourceFreshnessBatch; + +#[test] +fn given_warehouse_observations_when_pages_retry_then_scope_history_and_atomicity_are_preserved() { + let test_cases = [ + WarehouseMetadataTestCase { + description: "SQLite warehouse history", + backend: Backend::Sqlite, + expected_observations: 4, + }, + WarehouseMetadataTestCase { + description: "PostgreSQL warehouse history", + backend: Backend::Postgres, + expected_observations: 4, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "inventory", "test-admin") + .await + .expect("other location"); + let input = snapshot(location); + publish_snapshot(store, &input).await.expect("snapshot"); + let session_id = Uuid::now_v7(); + report_location_health( + store, + location, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id, + target: "development".into(), + health: LocationHealth::Heartbeat {}, + }, + ) + .await + .expect("registered reader"); + let token = issue_token( + store, + &AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }, + CredentialScope::Location { + location_id: location, + }, + Utc::now() + Duration::hours(1), + ) + .await + .expect("token"); + let api = start_api(store).await; + let client = reqwest::Client::new(); + let url = format!( + "{}/api/locations/{location}/targets/development/metadata/source-freshness", + api.url + ); + let record = json!({ + "event_id": "warehouse-observation-1", + "state_database": null, "state_schema": "main", "source_name": "raw_orders", + "target_database": null, "target_schema": "main", "target_name": "orders", + "run_id": "external-command-1", "strategy": "max_column", "value_kind": "integer", + "data_version": "18446744073709551617", "data_version_hash": "same-data-version", + "observed_at": "2026-01-01T10:00:00Z", + }); + let mut later = record.clone(); + later["event_id"] = json!("warehouse-observation-2"); + later["observed_at"] = json!("2026-01-01T11:00:00Z"); + let mut other_run = record.clone(); + other_run + .as_object_mut() + .expect("record") + .remove("event_id"); + other_run["run_id"] = json!("external-command-2"); + let mut distinct = record.clone(); + distinct["event_id"] = json!("warehouse-observation-3"); + let mut page = vec![record.clone(); 100]; + page[1] = later; + page[2] = other_run.clone(); + page[3] = distinct; + // Both missing and explicitly NULL identities use the same fixed legacy fallback. + other_run["event_id"] = json!(null); + page[4] = other_run; + let batch = json!({"session_id": session_id, "generation": 1, "records": page}); + let outbound = connect_location_work((&api.url, &token.bearer), location) + .expect("pooled native location client"); + let typed: SourceFreshnessBatch = serde_json::from_value(batch.clone()).expect("page"); + for expected_applied in [true, false] { + assert_eq!( + outbound + .sync_source_freshness("development", &typed) + .await + .expect("native client sync"), + expected_applied + ); + } + let mut equivalent = batch.clone(); + equivalent["records"] = json!([record]); + equivalent["records"][0]["observed_at"] = json!("2026-01-01T11:00:00+01:00"); + assert_eq!( + client + .post(&url) + .bearer_auth(&token.bearer) + .json(&equivalent) + .send() + .await + .expect("equivalent timestamp") + .json::() + .await + .expect("reply"), + json!({"applied": false}) + ); + let mut stale = batch.clone(); + stale["generation"] = json!(2); + let mut invalid = batch.clone(); + invalid["records"][1]["source_name"] = json!(" "); + let mut invalid_id = batch.clone(); + invalid_id["records"][0]["event_id"] = json!(" "); + let mut oversized = batch.clone(); + oversized["records"] = json!(vec![record.clone(); 101]); + let mut unknown = batch.clone(); + unknown["records"][0]["status"] = json!("success"); + let foreign = url.replace(&location.to_string(), &other.to_string()); + let other_target = url.replace("/development/", "/production/"); + for (endpoint, body, expected) in [ + (&url, stale, StatusCode::CONFLICT), + (&url, invalid, StatusCode::BAD_REQUEST), + (&url, invalid_id, StatusCode::BAD_REQUEST), + (&url, oversized, StatusCode::BAD_REQUEST), + (&url, unknown, StatusCode::UNPROCESSABLE_ENTITY), + (&foreign, batch.clone(), StatusCode::FORBIDDEN), + (&other_target, batch.clone(), StatusCode::BAD_REQUEST), + ] { + assert_eq!( + client + .post(endpoint) + .bearer_auth(&token.bearer) + .json(&body) + .send() + .await + .expect("rejected page") + .status(), + expected + ); + } + assert_eq!( + client + .post(&url) + .body("invalid") + .send() + .await + .expect("anonymous") + .status(), + StatusCode::UNAUTHORIZED + ); + let fault = TransactionFault { + description: "warehouse page commits all records or none", + table: "sqb_warehouse_freshness", + operation: "INSERT", + condition: "WHEN NEW.record_json->>'source_name'='inventory'", + sqlite_body: "SELECT RAISE(ABORT, 'injected metadata failure')", + postgres_body: "RAISE EXCEPTION 'injected metadata failure'", + }; + install_fault(store, &fault).await; + let mut new_record = record.clone(); + new_record["state_schema"] = json!("reporting"); + let mut inventory = new_record.clone(); + inventory["event_id"] = json!("inventory-observation-1"); + inventory["source_name"] = json!("inventory"); + let mut additional = batch.clone(); + additional["records"] = json!([new_record, inventory]); + assert_eq!( + client + .post(&url) + .bearer_auth(&token.bearer) + .json(&additional) + .send() + .await + .expect("faulted page") + .status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + remove_fault(store, &fault).await; + let (transaction, rows) = query( + begin(store, TransactionMode::Read).await.expect("read"), + "SELECT record_json FROM sqb_warehouse_freshness ORDER BY observed_at,event_id", + vec![], + ) + .await + .expect("mirrored history"); + commit(transaction).await.expect("close read"); + assert_eq!( + rows.len(), + test_case.expected_observations, + "{}: no observation collapsed by data version and no partial page committed", + test_case.description + ); + for row in rows { + assert_eq!(row["record_json"]["data_version"], "18446744073709551617"); + assert_eq!(row["record_json"]["state_schema"], "main"); + } + assert_eq!( + client + .post(&url) + .bearer_auth(&token.bearer) + .json(&additional) + .send() + .await + .expect("retry after rollback") + .json::() + .await + .expect("reply"), + json!({"applied": true}) + ); + migrate_store(store) + .await + .expect("warm migration preserves observations"); + let (transaction, rows) = query( + begin(store, TransactionMode::Read).await.expect("read"), + "SELECT COUNT(*) AS n FROM sqb_warehouse_freshness", + vec![], + ) + .await + .expect("retained history"); + commit(transaction).await.expect("close read"); + assert_eq!(rows[0]["n"], 6); + // The same warehouse rows in another target are separate observations, not replays. + let preview_session = Uuid::now_v7(); + let mut preview = input.clone(); + preview.target = "preview".into(); + publish_snapshot(store, &preview) + .await + .expect("preview snapshot"); + report_location_health( + store, + location, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id: preview_session, + target: "preview".into(), + health: LocationHealth::Heartbeat {}, + }, + ) + .await + .expect("preview reader"); + let mut preview_batch = batch.clone(); + preview_batch["session_id"] = json!(preview_session); + let preview_url = url.replace("/development/", "/preview/"); + assert_eq!( + client + .post(&preview_url) + .bearer_auth(&token.bearer) + .json(&preview_batch) + .send() + .await + .expect("separate target") + .json::() + .await + .expect("reply"), + json!({"applied": true}) + ); + let (transaction, rows) = query(begin(store, TransactionMode::Read).await.expect("read"), + "SELECT target, COUNT(*) AS n FROM sqb_warehouse_freshness GROUP BY target ORDER BY target", vec![]).await.expect("isolated history"); + commit(transaction).await.expect("close read"); + assert_eq!( + serde_json::to_value(rows).expect("rows"), + json!([{"target": "development", "n": 6}, {"target": "preview", "n": 4}]) + ); + stop_api(api).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs index 545f2373f9..44a6f6f588 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs @@ -425,6 +425,9 @@ fn given_terminal_jobs_and_groups_when_selectors_retry_then_outcomes_scope_and_r let transaction = begin(store, TransactionMode::Write) .await .expect("upgrade fixture"); + let (transaction, _) = query(transaction, "DROP TABLE sqb_warehouse_freshness", vec![]) + .await + .expect("remove subsequent warehouse mirror migration"); let (transaction, _) = query(transaction, "DROP INDEX sqb_node_finished_cursor", vec![]) .await diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index 3464b72c6f..2badd5ab9b 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -243,7 +243,8 @@ pub(super) async fn sqlite_failed_publish_and_schema_floor() -> i64 { let connection = rusqlite::Connection::open(&path).expect("open fault injection connection"); connection .execute_batch( - "DROP INDEX sqb_node_finished_cursor; DROP INDEX sqb_node_started_identity; \ + "DROP TABLE sqb_warehouse_freshness; \ + DROP INDEX sqb_node_finished_cursor; DROP INDEX sqb_node_started_identity; \ DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; \ DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; \ DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; \ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs index 7e4ed5ab92..6a71912e04 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs @@ -3,10 +3,10 @@ use fensu_policy::lifecycle::constants::CUSTOM_HOST_PROTOCOL_VERSION; use sqlbuild_analysis::contracts::constants::CONTRACT_ENFORCED; use sqlbuild_analysis::fingerprint_storage::constants::{ - COLUMN_DEFINITION_B64, COLUMN_DEFINITION_HASH, COLUMN_METADATA_JSON_B64, COLUMN_NODE_NAME, - COLUMN_NODE_TYPE, COLUMN_RUN_ID, COLUMN_SCHEMA_FINGERPRINT, COLUMN_TARGET_DATABASE, - COLUMN_TARGET_NAME, COLUMN_TARGET_SCHEMA, COLUMN_TIMESTAMP, COLUMN_VERSION_HASH, - NODE_TYPE_MODEL, NODE_TYPE_SEED, + COLUMN_DEFINITION_B64, COLUMN_DEFINITION_HASH, COLUMN_EVENT_ID, COLUMN_METADATA_JSON_B64, + COLUMN_NODE_NAME, COLUMN_NODE_TYPE, COLUMN_RUN_ID, COLUMN_SCHEMA_FINGERPRINT, + COLUMN_TARGET_DATABASE, COLUMN_TARGET_NAME, COLUMN_TARGET_SCHEMA, COLUMN_TIMESTAMP, + COLUMN_VERSION_HASH, NODE_TYPE_MODEL, NODE_TYPE_SEED, }; use sqlbuild_analysis::planner_changes::constants::{ AUDIT_GATE_METADATA_KEY, MIGRATION_FINGERPRINT_METADATA_KEY, @@ -99,6 +99,7 @@ pub(crate) const EXPORTED_TEXTS: &[(&str, &str)] = &[ ("COLUMN_DEFINITION_B64", COLUMN_DEFINITION_B64), ("COLUMN_METADATA_JSON_B64", COLUMN_METADATA_JSON_B64), ("COLUMN_TIMESTAMP", COLUMN_TIMESTAMP), + ("COLUMN_EVENT_ID", COLUMN_EVENT_ID), ("NODE_TYPE_MODEL", NODE_TYPE_MODEL), ("NODE_TYPE_SEED", NODE_TYPE_SEED), ("AUDIT_GATE_METADATA_KEY", AUDIT_GATE_METADATA_KEY), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs index 3a92672f90..b4612c1c7a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs @@ -24,6 +24,7 @@ use crate::bindings::_helpers::fingerprints::storage_sql::{ use crate::bindings::_helpers::records::construct::new_record; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::native_host::state_column_evolution::ensure_event_id_column; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; @@ -97,8 +98,13 @@ fn create_sql( )) } -/// The table DDL, adapter-rendered when the adapter overrides it, then any index DDL. -fn setup_sqls(location: &Location<'_, '_>, writer: &Writer<'_, '_>) -> PyResult> { +/// Create the table/indexes, then add the nullable identity column to a pre-existing table. +fn setup<'py>( + location: &Location<'_, 'py>, + writer: &Writer<'_, 'py>, + request: &Request<'_, 'py>, + describe: &dyn Fn(&PyErr) -> PyResult>, +) -> PyResult<()> { let py: Python<'_> = writer.execute.py(); let mut sqls: Vec = vec![match writer.render_create_table_sql { Some(render) => render @@ -118,7 +124,31 @@ fn setup_sqls(location: &Location<'_, '_>, writer: &Writer<'_, '_>) -> PyResult< sqls.push(sql?.extract()?); } } - Ok(sqls) + let database = location.database.into_pyobject(py)?.into_any(); + let schema = location.schema.into_pyobject(py)?.into_any(); + retry_write( + py, + || { + for sql in &sqls { + execute_sql(writer, sql)?; + } + ensure_event_id_column( + (writer.connection, writer.execute), + (&database, &schema), + (FINGERPRINT_TABLE_NAME, &qualified_table_name(location)?), + ( + &request + .optional("read_columns")? + .unwrap_or_else(|| py.None().into_bound(py)), + &request + .optional("render_add_columns")? + .unwrap_or_else(|| py.None().into_bound(py)), + writer.render_framework_type, + ), + ) + }, + describe, + ) } fn execute_sql<'py>(writer: &Writer<'_, 'py>, sql: &str) -> PyResult> { @@ -156,22 +186,14 @@ fn error_details<'py>(py: Python<'py>, error: &PyErr) -> PyResult( - writer: &Writer<'_, 'py>, - sqls: &[String], +/// Retry a complete warehouse operation after transient exceptions within the write budget. +fn retry_write<'py>( + py: Python<'py>, + write: impl Fn() -> PyResult<()>, describe: &dyn Fn(&PyErr) -> PyResult>, ) -> PyResult<()> { - let py: Python<'py> = writer.execute.py(); for attempt in 0..WRITE_ATTEMPTS { - let mut failure: Option = None; - for sql in sqls { - if let Err(error) = execute_sql(writer, sql) { - failure = Some(error); - break; - } - } - let Some(error) = failure else { + let Err(error) = write() else { return Ok(()); }; if attempt + 1 == WRITE_ATTEMPTS || !error.is_instance_of::(py) { @@ -376,16 +398,16 @@ fn write_fingerprint<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult return Err(error), } } - let mut sqls: Vec = setup_sqls(&location, &writer)?; - sqls.push(insert); - execute_with_retry(&writer, &sqls, &|error| { + let describe = |error: &PyErr| { let mut details: Vec> = vec![ native_field(&fingerprint, "node_type")?, native_field(&fingerprint, "node_name")?, ]; details.extend(error_details(py, error)?); Ok((SINGLE_RETRY_MESSAGE, details)) - })?; + }; + setup(&location, &writer, &request, &describe)?; + retry_write(py, || execute_sql(&writer, &insert).map(|_| ()), &describe)?; Ok(true) } @@ -420,8 +442,7 @@ fn write_fingerprints<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult<()> let on_progress: Option> = present(&request, "on_progress")?; let parts: WriteParts<'py> = WriteParts::read(&request)?; let (location, writer) = (parts.location(), parts.writer()); - let setup: Vec = setup_sqls(&location, &writer)?; - execute_with_retry(&writer, &setup, &|error| { + setup(&location, &writer, &request, &|error| { batch_warning(&fingerprints, error) })?; let batches = fingerprint_insert_batches( @@ -434,9 +455,11 @@ fn write_fingerprints<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult<()> let mut completed: usize = 0; for (batch, insert_sql) in batches { let members: Vec> = batch.iter().collect(); - execute_with_retry(&writer, &[insert_sql], &|error| { - batch_warning(&members, error) - })?; + retry_write( + py, + || execute_sql(&writer, &insert_sql).map(|_| ()), + &|error| batch_warning(&members, error), + )?; completed += members.len(); if let Some(progress) = &on_progress { let keywords: Bound<'py, PyDict> = PyDict::new(py); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs index ab91f568ae..598be15e62 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs @@ -177,6 +177,11 @@ fn renderers<'py>(adapter: &Bound<'py, PyAny>) -> PyResult> { "render_create_index_sqls", native_field(adapter, "render_create_source_freshness_index_sqls")?, ), + ("read_columns", native_field(adapter, "get_columns")?), + ( + "render_add_columns", + native_field(adapter, "render_add_columns")?, + ), ], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs index 292bada80f..4a004cbbda 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs @@ -4,7 +4,10 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME; -use sqlbuild_state::freshness::constants::{REQUIRED_STATE_COLUMNS, SOURCE_FRESHNESS_COLUMN_TYPES}; +use sqlbuild_state::freshness::constants::{ + REQUIRED_STATE_COLUMNS, SOURCE_FRESHNESS_COLUMN_TYPES, SOURCE_FRESHNESS_EVENT_ID, + STRING_STATE_VALUE_TYPE, +}; use sqlbuild_state::freshness::main::read_latest_sql::read_latest_sql; use sqlbuild_state::freshness::main::relevant_sources_sql::relevant_sources_sql; @@ -13,6 +16,7 @@ use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; use crate::bindings::_helpers::freshness::support::{ INPUT_ERROR, exception_text, freshness_error, optional_str, presumed_utc_datetime, text, }; +use crate::bindings::_helpers::native_host::state_column_evolution::ensure_event_id_column; use crate::bindings::_helpers::records::construct::new_record_with; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; @@ -82,12 +86,20 @@ fn create_table_sql<'py>( ) -> PyResult> { let py: Python<'py> = database.py(); let qualified_name: String = qualified_table_name(render_qualified_name, (database, schema))?; - let columns: Bound<'py, PyAny> = - PyTuple::new(py, SOURCE_FRESHNESS_COLUMN_TYPES.map(|(column, _)| column))?.into_any(); + let columns: Bound<'py, PyAny> = PyTuple::new( + py, + SOURCE_FRESHNESS_COLUMN_TYPES + .iter() + .map(|(column, _)| *column) + .chain([SOURCE_FRESHNESS_EVENT_ID]) + .collect::>(), + )? + .into_any(); let column_types: Bound<'py, PyDict> = PyDict::new(py); for (column, declared) in SOURCE_FRESHNESS_COLUMN_TYPES { column_types.set_item(column, declared)?; } + column_types.set_item(SOURCE_FRESHNESS_EVENT_ID, STRING_STATE_VALUE_TYPE)?; let column_types: Bound<'py, PyAny> = column_types.into_any(); let required: Bound<'py, PyAny> = PyFrozenSet::new(py, REQUIRED_STATE_COLUMNS)?.into_any(); let sql: String = state_table_create_sql( @@ -180,7 +192,7 @@ freshness table to regenerate source freshness state." ) } -fn row_record<'py>(row: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn row_record<'py>(row: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = row.py(); let raw_observed_at: Bound<'py, PyAny> = row.get_item(9)?; let observed_at: Bound<'py, PyAny> = if date_time_typed(&raw_observed_at)? { @@ -306,6 +318,20 @@ pub(crate) fn write_source_freshness_records<'py>( call_with(&render_create, &location())? }; execute_sql((execute, connection), create_sql)?; + let qualified = qualified_table_name( + &native_field(renderers, "render_qualified_name")?, + (database, schema), + )?; + ensure_event_id_column( + (connection, execute), + (database, schema), + (SOURCE_FRESHNESS_TABLE_NAME, &qualified), + ( + &native_field(renderers, "read_columns")?, + &native_field(renderers, "render_add_columns")?, + &native_field(renderers, "render_framework_type")?, + ), + )?; let render_indexes: Bound<'py, PyAny> = native_field(renderers, "render_create_index_sqls")?; if !render_indexes.is_none() { for index_sql in call_with(&render_indexes, &location())?.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs index 7a5f85c971..1ca8f40504 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs @@ -6,11 +6,13 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_orchestration::sensors::models::{CustomSensorOutcome, SensorEvaluationRequest}; +use sqlbuild_orchestration::warehouse_metadata::models::SourceFreshnessRead; use crate::bindings::_helpers::boundary::python_calls::within; use crate::bindings::_helpers::location_command::command::{location_arguments, publish_location}; use crate::bindings::_helpers::native_host::host::{redirect_stdout_to_stderr, write_console}; use crate::bindings::_helpers::native_host::orchestration_sensors::evaluate_sensor; +use crate::bindings::_helpers::native_host::warehouse_metadata::read_source_freshness; pub(super) fn run_runtime(py: Python<'_>, options: Bound<'_, PyDict>) -> PyResult { options.set_item("action", "publish")?; @@ -44,6 +46,24 @@ pub(super) fn run_runtime(py: Python<'_>, options: Bound<'_, PyDict>) -> PyResul } }; serde_json::to_string(&record) + } else if request.get("source_freshness").is_some() { + let metadata: SourceFreshnessRead = + serde_json::from_value(request["source_freshness"].clone()).map_err(|_| { + PyValueError::new_err("runtime received an invalid metadata read") + })?; + let arguments = location_arguments(py, Some(options.clone()))?; + let page = match read_source_freshness(py, &arguments, &metadata) { + Ok(page) => Some(page), + Err(_) => { + write_console( + py, + "stderr", + "Warehouse metadata read failed. No page delivered.\n", + )?; + None + } + }; + serde_json::to_string(&page) } else { let evaluation: SensorEvaluationRequest = serde_json::from_value(request).map_err(|_| { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index cb882f735f..248bf8cc95 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -13,3 +13,5 @@ pub(crate) mod registration; pub(crate) mod results; pub(crate) mod sqlite_connection; pub(crate) mod sqlite_cursor; +pub(crate) mod state_column_evolution; +pub(crate) mod warehouse_metadata; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/state_column_evolution.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/state_column_evolution.rs new file mode 100644 index 0000000000..bfb47dcba2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/state_column_evolution.rs @@ -0,0 +1,85 @@ +//! Warehouse catalog and nullable-column DDL adapter edge for persisted event identity. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyString, PyTuple}; +use sqlbuild_host::entries::models::AdapterHost; + +use crate::bindings::_helpers::boundary::python_calls::call_with; +use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::records::fields::field; + +/// Missing callbacks retain the existing renderer-only API for already-current/new schemas. +pub(crate) fn ensure_event_id_column<'py>( + (connection, execute): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (database, schema): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (table, qualified): (&str, &str), + (read_columns, render_add, render_type): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), +) -> PyResult<()> { + if read_columns.is_none() && render_add.is_none() { + return Ok(()); + } + if read_columns.is_none() || render_add.is_none() { + return Err(PyValueError::new_err( + "state column evolution requires both read_columns and render_add_columns", + )); + } + let py = connection.py(); + let scope = [ + ("connection", connection.clone()), + ("database", database.clone()), + ("schema", schema.clone()), + ("name", PyString::new(py, table).into_any()), + ]; + if event_id_present(read_columns, &scope)? { + return Ok(()); + } + let string_type = render_type.call1((field( + &host::entry(py, AdapterHost::FrameworkType)?, + "STRING", + )?,))?; + let column = call_with( + &host::entry(py, AdapterHost::ColumnInfo)?, + &[ + ("name", PyString::new(py, "event_id").into_any()), + ("type", string_type), + ], + )?; + let statements = call_with( + render_add, + &[ + ("destination", PyString::new(py, qualified).into_any()), + ("columns", PyTuple::new(py, [column])?.into_any()), + ], + )?; + for statement in statements.try_iter()? { + if let Err(error) = call_with( + execute, + &[("connection", connection.clone()), ("sql", statement?)], + ) && !event_id_present(read_columns, &scope).unwrap_or(false) + { + return Err(error); + } + } + Ok(()) +} + +/// Concurrent additions are harmless only once the authoritative catalog confirms the column. +fn event_id_present( + read_columns: &Bound<'_, PyAny>, + scope: &[(&str, Bound<'_, PyAny>)], +) -> PyResult { + for column in call_with(read_columns, scope)?.try_iter()? { + if field(&column?, "name")? + .extract::()? + .eq_ignore_ascii_case("event_id") + { + return Ok(true); + } + } + Ok(false) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs new file mode 100644 index 0000000000..7be0ce4f6d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs @@ -0,0 +1,205 @@ +//! Read-only warehouse adapter edge; never compile declarations or write warehouse state. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; +use serde_json::{Value, json}; +use sqlbuild_config::command_connection::constants::{ + DUCKDB_ADAPTER, DUCKDB_MEMORY_DATABASE, LOCAL_DATABASE_PATH_KEYS, +}; +use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME; +use sqlbuild_core::models::SqlValue; +use sqlbuild_orchestration::warehouse_metadata::constants::MAX_FRESHNESS_RECORDS; +use sqlbuild_orchestration::warehouse_metadata::models::{ + SourceFreshnessPage, SourceFreshnessRead, WarehouseFreshnessRecord, +}; +use sqlbuild_state::freshness::constants::{OBSERVED_AT_COLUMN, SOURCE_FRESHNESS_COLUMN_TYPES}; +use sqlbuild_state::freshness::main::read_history_sql::read_history_sql; + +use crate::bindings::_helpers::authored_values::sql_values::{SqlValueClasses, sql_value}; +use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::command_runtime::connection::project_connection_config; +use crate::bindings::_helpers::freshness::state_io::{qualified_table_name, row_record}; +use crate::bindings::_helpers::query_command::project::{ + ProjectAdapter, ProjectInputs, project_adapter, +}; +use crate::bindings::_helpers::records::fields::field; + +pub(crate) fn read_source_freshness( + py: Python<'_>, + arguments: &KeywordArguments<'_>, + request: &SourceFreshnessRead, +) -> PyResult { + if request.location.schema.trim().is_empty() { + return Err(PyValueError::new_err( + "metadata state schema must not be blank", + )); + } + let ProjectAdapter { + project_dir, + discovered, + adapter, + } = project_adapter( + py, + Some(arguments.value("project_dir")?), + ProjectInputs::Configuration, + )?; + let connection_config = project_connection_config( + &discovered, + (&project_dir, &arguments.value("selected_target")?), + (&py.None().into_bound(py), true), + )?; + let adapter_name: String = adapter.getattr("adapter_name")?.extract()?; + if let Some((_, path_key)) = LOCAL_DATABASE_PATH_KEYS + .iter() + .find(|(name, _)| *name == adapter_name) + { + let config = connection_config.cast::()?; + let path: Option = config + .get_item(path_key)? + .map(|path| path.extract()) + .transpose()?; + if path + .as_deref() + .is_some_and(|path| path != DUCKDB_MEMORY_DATABASE) + { + config.set_item("read_only", true)?; + } + if adapter_name == DUCKDB_ADAPTER + && let Some(attachments) = config.get_item("attach")? + { + for attachment in attachments.try_iter()? { + attachment?.set_item("read_only", true)?; + } + } + } + let connection = adapter.call_method1("connect", (connection_config,))?; + let read = read_page(py, (&adapter, &connection), request); + let closed = adapter.call_method1("close", (&connection,)); + let page = read?; + closed?; + Ok(page) +} + +fn read_page( + py: Python<'_>, + (adapter, connection): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + request: &SourceFreshnessRead, +) -> PyResult { + let database = request.location.database.as_ref().map_or_else( + || py.None().into_bound(py), + |value| PyString::new(py, value).into_any(), + ); + let schema = PyString::new(py, &request.location.schema).into_any(); + let name = PyString::new(py, SOURCE_FRESHNESS_TABLE_NAME).into_any(); + let exists = call_with( + &adapter.getattr("relation_exists")?, + &[ + ("connection", connection.clone()), + ("database", database.clone()), + ("schema", schema.clone()), + ("name", name), + ], + )? + .is_truthy()?; + if !exists { + return Ok(SourceFreshnessPage { + records: Vec::new(), + next_after: None, + }); + } + let columns = call_with( + &adapter.getattr("get_columns")?, + &[ + ("connection", connection.clone()), + ("database", database.clone()), + ("schema", schema.clone()), + ( + "name", + PyString::new(py, SOURCE_FRESHNESS_TABLE_NAME).into_any(), + ), + ], + )?; + let mut has_event_id = false; + for column in columns.try_iter()? { + has_event_id |= field(&column?, "name")? + .extract::()? + .eq_ignore_ascii_case("event_id"); + } + let mut literals: [Option; 11] = Default::default(); + if let Some(after) = &request.after { + let classes = SqlValueClasses::load(py)?; + for (output, value) in literals.iter_mut().zip(after) { + if let Some(value) = value { + let value = sql_value(&SqlValue::String(value.clone()), &classes)?; + *output = Some( + call_with( + &adapter.getattr("render_typed_scalar")?, + &[("value", value)], + )? + .extract()?, + ); + } + } + } + let qualified = qualified_table_name( + &adapter.getattr("render_qualified_name")?, + (&database, &schema), + )?; + let adapter_name = adapter.getattr("adapter_name")?.extract::()?; + let sql = read_history_sql( + &qualified, + request.after.as_ref().map(|_| &literals), + (MAX_FRESHNESS_RECORDS, &adapter_name, has_event_id), + ) + .map_err(PyValueError::new_err)?; + let result = call_with( + &adapter.getattr("query")?, + &[ + ("connection", connection.clone()), + ("sql", PyString::new(py, &sql).into_any()), + ( + "limit", + pyo3::IntoPyObjectExt::into_bound_py_any(MAX_FRESHNESS_RECORDS, py)?, + ), + ], + )?; + let mut records: Vec = Vec::new(); + let mut next_after = None; + for row in field(&result, "rows")?.try_iter()? { + let row = row?; + let record = row_record(&row)?; + let mut cursor: [Option; 11] = Default::default(); + let mut value = json!({"state_database": request.location.database, "state_schema": request.location.schema}); + for (index, (name, _)) in SOURCE_FRESHNESS_COLUMN_TYPES.iter().enumerate() { + let raw = row.get_item(index)?; + cursor[index] = if raw.is_none() { + None + } else { + Some(raw.str()?.extract()?) + }; + let item = field(&record, name)?; + value[*name] = if item.is_none() { + Value::Null + } else if *name == OBSERVED_AT_COLUMN { + json!(item.call_method0("isoformat")?.extract::()?) + } else { + json!(item.extract::()?) + }; + } + cursor[10] = row.get_item(10)?.extract::>()?; + value["event_id"] = json!(cursor[10]); + records.push( + serde_json::from_value::(value) + .map_err(|error| PyValueError::new_err(error.to_string()))?, + ); + next_after = Some(cursor); + } + if records.len() < MAX_FRESHNESS_RECORDS { + next_after = None; + } + Ok(SourceFreshnessPage { + records, + next_after, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index 6a1c04bac6..3fdd402cc8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -1367,6 +1367,8 @@ const SOURCE_FRESHNESS_RENDERERS: RecordSpec = RecordSpec { FieldSpec::required("render_insert_records_sql"), FieldSpec::new("render_create_table_sql", FieldDefault::None), FieldSpec::new("render_create_index_sqls", FieldDefault::None), + FieldSpec::new("read_columns", FieldDefault::None), + FieldSpec::new("render_add_columns", FieldDefault::None), ], }; const SOURCE_FRESHNESS_OBSERVATION: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs index 2a93fd2855..b02904a131 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs @@ -14,7 +14,9 @@ use sqlbuild_state::freshness::main::age_status::age_status; use sqlbuild_state::freshness::main::data_version_hash::data_version_hash; use sqlbuild_state::freshness::main::normalized_data_version::normalized_data_version; use sqlbuild_state::freshness::main::records_equivalent::records_equivalent; -use sqlbuild_state::freshness::models::{AgeStatus, ComparableRecord, DataVersion}; +use sqlbuild_state::freshness::models::{ + AgeStatus, ComparableRecord, DataVersion, FreshnessEventIdentity, +}; use sqlbuild_state::freshness::types::TimestampParser; use crate::bindings::_helpers::boundary::python_calls::imported; @@ -31,6 +33,40 @@ const TIMESTAMP_KIND: &str = "timestamp"; const INTEGER_KIND: &str = "integer"; const STRING_KIND: &str = "string"; +/// Normalize timestamp offsets and precision before computing a persisted observation identity. +pub(crate) fn record_event_id(record: &Bound<'_, PyAny>) -> PyResult { + let observed_at = presumed_utc(&native_field(record, "observed_at")?)?; + let observed_at = format!( + "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}.{:06}+00:00", + observed_at.date.year, + observed_at.date.month, + observed_at.date.day, + observed_at.hour, + observed_at.minute, + observed_at.second, + observed_at.microsecond, + ); + Ok(sqlbuild_state::freshness::main::event_id::event_id( + &FreshnessEventIdentity { + source_name: &native_field(record, "source_name")?.extract::()?, + target_database: native_field(record, "target_database")? + .extract::>()? + .as_deref(), + target_schema: native_field(record, "target_schema")? + .extract::>()? + .as_deref(), + target_name: native_field(record, "target_name")? + .extract::>()? + .as_deref(), + run_id: &native_field(record, "run_id")?.extract::()?, + strategy: &native_field(record, "strategy")?.extract::()?, + value_kind: &native_field(record, "value_kind")?.extract::()?, + data_version_hash: &native_field(record, "data_version_hash")?.extract::()?, + observed_at: &observed_at, + }, + )) +} + fn raised(py: Python<'_>, class: NativeAttr, message: String) -> PyErr { match registry::native(py, class).and_then(|class| class.call1((message,))) { Ok(error) => PyErr::from_value(error), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs index 40a1d39bf6..9f7aca6b06 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs @@ -9,7 +9,8 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME; use sqlbuild_host::entries::models::AdapterHost; use sqlbuild_state::freshness::constants::{ - SOURCE_FRESHNESS_COLUMN_TYPES, STRING_STATE_VALUE_TYPE, TEXT_TIMESTAMP_STATE_VALUE_TYPE, + SOURCE_FRESHNESS_COLUMN_TYPES, SOURCE_FRESHNESS_EVENT_ID, STRING_STATE_VALUE_TYPE, + TEXT_TIMESTAMP_STATE_VALUE_TYPE, }; use sqlbuild_state::state_tables::main::create_table_sql::create_table_sql; use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; @@ -20,6 +21,7 @@ use crate::bindings::_helpers::boundary::python_calls::call_with; use crate::bindings::_helpers::contracts::state_literals::render_state_sql_literal; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::state_rows::freshness::record_event_id; const MISSING_SCHEMA_MESSAGE: &str = "source freshness table requires a target schema"; /// Each state value type a state table may declare and the framework type that renders it. @@ -121,18 +123,20 @@ fn render_insert_source_freshness_records_sql<'py>( let mut rows: Vec> = Vec::new(); for record in records.try_iter()? { let record: Bound<'py, PyAny> = record?; - let row: Vec = SOURCE_FRESHNESS_COLUMN_TYPES + let mut row: Vec = SOURCE_FRESHNESS_COLUMN_TYPES .iter() .map(|(column, declared)| { render_state_sql_literal(py, &native_field(&record, column)?, declared, None) }) .collect::>()?; + row.push(format!("'{}'", record_event_id(&record)?)); rows.push(row); } - let names: Vec<&str> = SOURCE_FRESHNESS_COLUMN_TYPES + let mut names: Vec<&str> = SOURCE_FRESHNESS_COLUMN_TYPES .iter() .map(|(column, _)| *column) .collect(); + names.push(SOURCE_FRESHNESS_EVENT_ID); Ok(values_insert_sql( &table.extract::()?, &names, diff --git a/crates/sqlbuild-state/src/freshness/constants.rs b/crates/sqlbuild-state/src/freshness/constants.rs index 8083997c83..4bec5268c8 100644 --- a/crates/sqlbuild-state/src/freshness/constants.rs +++ b/crates/sqlbuild-state/src/freshness/constants.rs @@ -41,7 +41,7 @@ pub const REQUIRED_STATE_COLUMNS: [&str; 6] = [ pub const STRING_STATE_VALUE_TYPE: &str = "string"; /// State value type of a timestamp stored as ISO text. pub const TEXT_TIMESTAMP_STATE_VALUE_TYPE: &str = "text_timestamp"; -/// Source freshness state table columns in table order, with the state value type each declares. +/// Legacy latest-reader columns; the nullable event ID is intentionally not required by reads. pub const SOURCE_FRESHNESS_COLUMN_TYPES: [(&str, &str); 10] = [ ("source_name", STRING_STATE_VALUE_TYPE), ("target_database", STRING_STATE_VALUE_TYPE), @@ -54,5 +54,8 @@ pub const SOURCE_FRESHNESS_COLUMN_TYPES: [(&str, &str); 10] = [ ("data_version_hash", STRING_STATE_VALUE_TYPE), ("observed_at", TEXT_TIMESTAMP_STATE_VALUE_TYPE), ]; +pub const SOURCE_FRESHNESS_EVENT_ID: &str = "event_id"; +pub const OBSERVED_AT_COLUMN: &str = "observed_at"; +pub(crate) const SQLSERVER_ADAPTER: &str = "sqlserver"; pub(crate) const PHYSICAL_TABLE_SOURCE_FRAGMENT: &str = "requires a physical table source"; pub(crate) const INCOMPLETE_CONFIGURATION_FRAGMENT: &str = "incomplete"; diff --git a/crates/sqlbuild-state/src/freshness/main/event_id.rs b/crates/sqlbuild-state/src/freshness/main/event_id.rs new file mode 100644 index 0000000000..72bdeab470 --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/main/event_id.rs @@ -0,0 +1,22 @@ +//! Observation identity excludes mutable payload and future storage columns. + +use sha2::{Digest, Sha256}; + +use crate::freshness::models::FreshnessEventIdentity; + +/// `observed_at` is UTC ISO text with six fractional digits and a `+00:00` offset. +pub fn event_id(row: &FreshnessEventIdentity<'_>) -> String { + let identity = serde_json::json!([ + "sqlbuild.source-freshness.event.v1", + row.source_name, + row.target_database, + row.target_schema, + row.target_name, + row.run_id, + row.observed_at, + row.strategy, + row.value_kind, + row.data_version_hash, + ]); + format!("{:x}", Sha256::digest(identity.to_string().as_bytes())) +} diff --git a/crates/sqlbuild-state/src/freshness/main/mod.rs b/crates/sqlbuild-state/src/freshness/main/mod.rs index 9ec47ed663..5a1085a0e2 100644 --- a/crates/sqlbuild-state/src/freshness/main/mod.rs +++ b/crates/sqlbuild-state/src/freshness/main/mod.rs @@ -1,7 +1,9 @@ pub mod age_status; pub mod configuration_error; pub mod data_version_hash; +pub mod event_id; pub mod normalized_data_version; +pub mod read_history_sql; pub mod read_latest_sql; pub mod records_equivalent; pub mod relevant_sources_sql; diff --git a/crates/sqlbuild-state/src/freshness/main/read_history_sql.rs b/crates/sqlbuild-state/src/freshness/main/read_history_sql.rs new file mode 100644 index 0000000000..ce7df1f246 --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/main/read_history_sql.rs @@ -0,0 +1,112 @@ +//! Keyset pages preserve distinct observations without rereading or ranking all preceding rows. + +use crate::freshness::constants::{ + OBSERVED_AT_COLUMN, SOURCE_FRESHNESS_COLUMN_TYPES, SOURCE_FRESHNESS_EVENT_ID, SQLSERVER_ADAPTER, +}; + +/// Cursor values are already rendered by the adapter; `None` sorts before every SQL value. +pub fn read_history_sql( + qualified_name: &str, + after: Option<&[Option; 11]>, + (limit, adapter, has_event_id): (usize, &str, bool), +) -> Result { + let mut columns: Vec<&str> = SOURCE_FRESHNESS_COLUMN_TYPES + .iter() + .map(|(name, _)| *name) + .collect(); + let mut projection = columns.join(", "); + projection.push_str(if has_event_id { + ", event_id" + } else { + ", NULL AS event_id" + }); + columns.push(if has_event_id { + SOURCE_FRESHNESS_EVENT_ID + } else { + "NULL" + }); + let ordered = columns + .iter() + .map(|column| { + if *column == OBSERVED_AT_COLUMN { + Ok((*column).to_owned()) + } else { + binary(column, adapter) + } + }) + .collect::, _>>()?; + let mut greater: Vec = Vec::new(); + let mut equal: Vec = Vec::new(); + if let Some(after) = after { + for ((column, ordered), value) in columns.iter().zip(&ordered).zip(after) { + let literal = value + .as_ref() + .map(|value| { + if *column == OBSERVED_AT_COLUMN { + Ok(value.clone()) + } else { + binary(value, adapter) + } + }) + .transpose()?; + let comparison = literal.as_ref().map_or_else( + || format!("{column} IS NOT NULL"), + |literal| format!("{ordered} > {literal}"), + ); + let prefix = if equal.is_empty() { + String::new() + } else { + format!("{} AND ", equal.join(" AND ")) + }; + greater.push(format!("({prefix}{comparison})")); + equal.push(literal.as_ref().map_or_else( + || format!("{column} IS NULL"), + |literal| format!("{ordered} = {literal}"), + )); + } + } + let filter = if greater.is_empty() { + String::new() + } else { + format!(" WHERE {}", greater.join(" OR ")) + }; + let order: Vec = columns + .iter() + .zip(&ordered) + .map(|(column, ordered)| format!("CASE WHEN {column} IS NULL THEN 0 ELSE 1 END, {ordered}")) + .collect(); + let top = adapter == SQLSERVER_ADAPTER; + let head = if top { + format!("TOP {limit} ") + } else { + String::new() + }; + let tail = if top { + String::new() + } else { + format!(" LIMIT {limit}") + }; + Ok(format!( + "SELECT {head}{projection} FROM {qualified_name}{filter} ORDER BY {}{tail}", + order.join(", ") + )) +} + +/// Session and column collations must not merge case, accents or trailing spaces at page boundaries. +fn binary(value: &str, adapter: &str) -> Result { + Ok(match adapter { + "duckdb" | "motherduck" => format!("encode({value})"), + "sqlite" => format!("CAST({value} AS BLOB)"), + "postgres" => format!("convert_to({value}, 'UTF8')"), + "snowflake" => format!("TO_BINARY({value}, 'UTF-8')"), + "bigquery" => format!("CAST({value} AS BYTES)"), + "databricks" => format!("CAST({value} AS BINARY)"), + "sqlserver" => format!("CONVERT(VARBINARY(MAX), CONVERT(NVARCHAR(MAX), {value}))"), + "clickhouse" => value.to_owned(), + _ => { + return Err(format!( + "source freshness history does not support adapter '{adapter}'" + )); + } + }) +} diff --git a/crates/sqlbuild-state/src/freshness/models.rs b/crates/sqlbuild-state/src/freshness/models.rs index dbe1341a71..9f7cbfb958 100644 --- a/crates/sqlbuild-state/src/freshness/models.rs +++ b/crates/sqlbuild-state/src/freshness/models.rs @@ -2,6 +2,21 @@ use sqlbuild_cursor::algebra::models::LocalDateTime; +/// Stable identity fields shared by warehouse writers and legacy history ingestion. +#[derive(Debug)] +pub struct FreshnessEventIdentity<'a> { + pub source_name: &'a str, + pub target_database: Option<&'a str>, + pub target_schema: Option<&'a str>, + pub target_name: Option<&'a str>, + pub run_id: &'a str, + pub strategy: &'a str, + pub value_kind: &'a str, + pub data_version_hash: &'a str, + /// UTC ISO text with six fractional digits and a `+00:00` offset. + pub observed_at: &'a str, +} + /// An observed data version of one value kind. #[derive(Clone, Debug, PartialEq, Eq)] pub enum DataVersion { diff --git a/fensu.toml b/fensu.toml index 16d15444db..5a13ca223d 100644 --- a/fensu.toml +++ b/fensu.toml @@ -2657,6 +2657,7 @@ module_layers = [ "native_host/model_lookup=project", "native_host/orchestration_sensors=commands", "native_host/orchestration_triggers=project", + "native_host/warehouse_metadata=commands", "native_threads=infra", "output_capture=output", "plan_command=commands", diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 84509d6ae7..31832558f1 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -199,6 +199,7 @@ COLUMN_SCHEMA_FINGERPRINT: str COLUMN_DEFINITION_B64: str COLUMN_METADATA_JSON_B64: str COLUMN_TIMESTAMP: str +COLUMN_EVENT_ID: str NODE_TYPE_MODEL: str NODE_TYPE_SEED: str AUDIT_GATE_METADATA_KEY: str @@ -4187,6 +4188,8 @@ def write_fingerprint( render_create_table_sql: Callable[..., str] | None = None, render_create_index_sqls: Callable[..., tuple[str, ...]] | None = None, ensure_table: bool = True, + read_columns: Callable[..., tuple[ColumnInfo, ...]] | None = None, + render_add_columns: Callable[..., tuple[str, ...]] | None = None, ) -> bool: ... def write_fingerprints( *, @@ -4200,6 +4203,8 @@ def write_fingerprints( render_create_table_sql: Callable[..., str] | None = None, render_create_index_sqls: Callable[..., tuple[str, ...]] | None = None, on_progress: FingerprintWriteProgress | None = None, + read_columns: Callable[..., tuple[ColumnInfo, ...]] | None = None, + render_add_columns: Callable[..., tuple[str, ...]] | None = None, ) -> None: ... def select_latest_fingerprints( *, @@ -7838,6 +7843,8 @@ class SourceFreshnessRenderers(NativeRecord): render_insert_records_sql: Callable[..., str] render_create_table_sql: Callable[..., str] | None render_create_index_sqls: Callable[..., tuple[str, ...]] | None + read_columns: Callable[..., tuple[ColumnInfo, ...]] | None + render_add_columns: Callable[..., tuple[str, ...]] | None def __init__( self, render_qualified_name: Callable[..., str | None], @@ -7845,6 +7852,8 @@ class SourceFreshnessRenderers(NativeRecord): render_insert_records_sql: Callable[..., str], render_create_table_sql: Callable[..., str] | None = None, render_create_index_sqls: Callable[..., tuple[str, ...]] | None = None, + read_columns: Callable[..., tuple[ColumnInfo, ...]] | None = None, + render_add_columns: Callable[..., tuple[str, ...]] | None = None, ) -> None: ... class SourceFreshnessObservation(NativeRecord): diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index d376a6a228..f6d52b95c4 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -1080,7 +1080,9 @@ def connect(self, config: dict[str, Any]) -> Any: import duckdb database: str = str(config.get("database", ":memory:")) - raw_connection: duckdb.DuckDBPyConnection = duckdb.connect(database=database) + raw_connection: duckdb.DuckDBPyConnection = duckdb.connect( + database=database, read_only=bool(config.get("read_only", False)) + ) return self._configured_connection(raw_connection=raw_connection, config=config) def connect_local_replay(self, database: Path) -> Any: diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index 2f200b531d..25d13ae0f1 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -340,6 +340,7 @@ def render_create_fingerprint_table_sql( from sqlbuild._native import ( COLUMN_DEFINITION_B64, COLUMN_DEFINITION_HASH, + COLUMN_EVENT_ID, COLUMN_METADATA_JSON_B64, COLUMN_NODE_NAME, COLUMN_NODE_TYPE, @@ -373,7 +374,8 @@ def render_create_fingerprint_table_sql( f"{COLUMN_SCHEMA_FINGERPRINT} NVARCHAR(450) NOT NULL, " f"{COLUMN_DEFINITION_B64} NVARCHAR(MAX) NOT NULL, " f"{COLUMN_METADATA_JSON_B64} NVARCHAR(MAX) NOT NULL, " - f"{COLUMN_TIMESTAMP} DATETIME2 NOT NULL" + f"{COLUMN_TIMESTAMP} DATETIME2 NOT NULL, " + f"{COLUMN_EVENT_ID} NVARCHAR(MAX)" f")" ) return self._create_table_if_missing_sql( diff --git a/src/sqlbuild/executor/clone/_helpers/fingerprinting.py b/src/sqlbuild/executor/clone/_helpers/fingerprinting.py index 5c2abd241e..0378d85973 100644 --- a/src/sqlbuild/executor/clone/_helpers/fingerprinting.py +++ b/src/sqlbuild/executor/clone/_helpers/fingerprinting.py @@ -205,6 +205,8 @@ def _on_group_progress( fingerprints=group, render_qualified_name=adapter.render_qualified_name, render_framework_type=adapter.render_framework_type, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, render_create_table_sql=adapter.render_create_fingerprint_table_sql, render_create_index_sqls=adapter.render_create_fingerprint_index_sqls, on_progress=_on_group_progress, diff --git a/src/sqlbuild/executor/diff/_helpers/query_artifacts.py b/src/sqlbuild/executor/diff/_helpers/query_artifacts.py index d8d0c9f088..f4559e8027 100644 --- a/src/sqlbuild/executor/diff/_helpers/query_artifacts.py +++ b/src/sqlbuild/executor/diff/_helpers/query_artifacts.py @@ -118,6 +118,8 @@ def materialize_query_diff_artifact( ), render_qualified_name=adapter.render_qualified_name, render_framework_type=adapter.render_framework_type, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, render_create_table_sql=adapter.render_create_fingerprint_table_sql, render_create_index_sqls=adapter.render_create_fingerprint_index_sqls, ) diff --git a/src/sqlbuild/executor/functions/main/_execute.py b/src/sqlbuild/executor/functions/main/_execute.py index a95ec0cd3b..c7f935e162 100644 --- a/src/sqlbuild/executor/functions/main/_execute.py +++ b/src/sqlbuild/executor/functions/main/_execute.py @@ -182,6 +182,8 @@ def _try_write_function_fingerprint( fingerprint=fingerprint, render_qualified_name=adapter.render_qualified_name, render_framework_type=adapter.render_framework_type, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, render_create_table_sql=adapter.render_create_fingerprint_table_sql, render_create_index_sqls=adapter.render_create_fingerprint_index_sqls, ) diff --git a/src/sqlbuild/executor/python_nodes/_helpers/fingerprinting.py b/src/sqlbuild/executor/python_nodes/_helpers/fingerprinting.py index edc76da34c..310f994670 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/fingerprinting.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/fingerprinting.py @@ -55,6 +55,8 @@ def try_write_python_node_identity_fingerprint( ), render_qualified_name=adapter.render_qualified_name, render_framework_type=adapter.render_framework_type, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, render_create_table_sql=adapter.render_create_fingerprint_table_sql, render_create_index_sqls=adapter.render_create_fingerprint_index_sqls, ) diff --git a/src/sqlbuild/executor/review/_helpers/marker.py b/src/sqlbuild/executor/review/_helpers/marker.py index ee87afbfbf..1f2f23e173 100644 --- a/src/sqlbuild/executor/review/_helpers/marker.py +++ b/src/sqlbuild/executor/review/_helpers/marker.py @@ -52,6 +52,8 @@ def write_review_marker( ), render_qualified_name=adapter.render_qualified_name, render_framework_type=adapter.render_framework_type, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, render_create_table_sql=adapter.render_create_fingerprint_table_sql, render_create_index_sqls=adapter.render_create_fingerprint_index_sqls, ) diff --git a/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py b/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py index e084b6ab5d..065fe30123 100644 --- a/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py +++ b/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py @@ -97,6 +97,8 @@ def try_write_fingerprint( fingerprint=fingerprint, render_qualified_name=adapter.render_qualified_name, render_framework_type=adapter.render_framework_type, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, render_create_table_sql=adapter.render_create_fingerprint_table_sql, render_create_index_sqls=adapter.render_create_fingerprint_index_sqls, ) diff --git a/src/sqlbuild/executor/seed/_helpers/fingerprinting.py b/src/sqlbuild/executor/seed/_helpers/fingerprinting.py index a103b8bb6b..02a44cc063 100644 --- a/src/sqlbuild/executor/seed/_helpers/fingerprinting.py +++ b/src/sqlbuild/executor/seed/_helpers/fingerprinting.py @@ -56,6 +56,8 @@ def try_write_seed_fingerprint( fingerprint=fingerprint, render_qualified_name=adapter.render_qualified_name, render_framework_type=adapter.render_framework_type, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, render_create_table_sql=adapter.render_create_fingerprint_table_sql, render_create_index_sqls=adapter.render_create_fingerprint_index_sqls, ) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index ee942f38ab..681b669497 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -1,5 +1,6 @@ """Publication cases and isolated server credentials.""" +from collections.abc import Callable from dataclasses import dataclass, field from pathlib import Path from subprocess import Popen @@ -143,3 +144,13 @@ class CalendarRejectionTestCase: class CalendarRefreshTestCase: description: str expected_timezone: str + + +@dataclass(frozen=True) +class WarehouseHistoryTestCase: + description: str + adapter_name: str + expected_page_lengths: tuple[int, ...] + connection_key: str + journal_modes: tuple[str, ...] + append_fingerprints: Callable[..., None] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index 8f5aca9f03..16170f6d72 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -12,7 +12,7 @@ import sys import time from collections.abc import Callable, Iterator -from contextlib import AbstractContextManager, contextmanager, nullcontext, suppress +from contextlib import AbstractContextManager, closing, contextmanager, nullcontext, suppress from datetime import UTC, datetime, timedelta from io import StringIO from pathlib import Path @@ -25,6 +25,7 @@ import duckdb import pytest +from sqlbuild._native import Fingerprint, write_fingerprint from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( ClaimedWorker, ScheduledWorker, @@ -590,3 +591,56 @@ def sensor_calls(path: Path, fields: dict[str, Any]) -> list[dict[str, Any]]: """Select recorded callback invocations without inferring execution from declarations.""" records: list[dict[str, Any]] = [json.loads(line) for line in path.read_text().splitlines()] return list(filter(lambda record: fields.items() <= record.items(), records)) + + +def append_retried_fingerprints( + *, + adapter: Any, + connection: Any, + database_path: Path, + fingerprint: Fingerprint, + read_columns: Callable[..., Any] | None = None, +) -> None: + """Retry the same observation without substituting its identity or live catalog.""" + del database_path + for _attempt in range(2): + _ = write_fingerprint( + connection=connection, + execute=adapter.execute, + database=None, + schema="main", + fingerprint=fingerprint, + ensure_table=False, + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + read_columns=read_columns or adapter.get_columns, + render_add_columns=adapter.render_add_columns, + render_create_table_sql=adapter.render_create_fingerprint_table_sql, + ) + + +def append_contended_fingerprints( + *, adapter: Any, connection: Any, database_path: Path, fingerprint: Fingerprint +) -> None: + """Release a real SQLite write lock only after the first ALTER has failed.""" + with closing(sqlite3.connect(database_path)) as blocker: + _ = adapter.execute(connection=connection, sql="PRAGMA busy_timeout=1") + catalog_actions: Iterator[Callable[[], Any]] = iter( + (lambda: blocker.execute("BEGIN IMMEDIATE"), blocker.commit) + ) + + def read_columns(**scope: Any) -> Any: + columns: Any = adapter.get_columns(**scope) + next(catalog_actions, lambda: None)() + return columns + + try: + append_retried_fingerprints( + adapter=adapter, + connection=connection, + database_path=database_path, + fingerprint=fingerprint, + read_columns=read_columns, + ) + finally: + blocker.rollback() diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py new file mode 100644 index 0000000000..df565a66a6 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py @@ -0,0 +1,293 @@ +"""Real warehouse writers, populated predecessors and the isolated read-only history protocol.""" + +import hashlib +import json +import os +import sqlite3 +import subprocess +import sys +from contextlib import closing +from datetime import UTC, datetime, timedelta, timezone +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild._native import ( + Fingerprint, + SourceFreshnessRecord, + SourceFreshnessRenderers, + read_latest_fingerprints, + read_latest_source_freshness, + write_source_freshness_records, +) +from sqlbuild.adapter.discovery.main.resolve_adapter import resolve_adapter +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + WarehouseHistoryTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + append_contended_fingerprints, + append_retried_fingerprints, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + + +@pytest.mark.parametrize( + "test_case", + [ + WarehouseHistoryTestCase( + "DuckDB typed timestamps", + "duckdb", + (100, 6), + "database", + (), + append_retried_fingerprints, + ), + WarehouseHistoryTestCase( + "SQLite timestamp text", + "sqlite", + (100, 6), + "path", + ("delete",), + append_contended_fingerprints, + ), + ], + ids=lambda case: case.description, +) +def test_given_populated_predecessor_tables_when_writers_add_ids_then_history_preserves_new_legacy_and_retried_observations( + test_case: WarehouseHistoryTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + connection_key: str = test_case.connection_key + write_files( + project_dir=project, + files={ + "sqlbuild_project.toml": ( + f'name = "orders"\nadapter = "{test_case.adapter_name}"\n' + f'[connections.local]\n{connection_key} = "warehouse.db"\n' + '[targets.development]\nschema = "main"\n' + ), + "orchestration/orders.py": 'raise AssertionError("metadata must not compile user declarations")\n', + }, + ) + adapter: Any = resolve_adapter(adapter_name=test_case.adapter_name) + connection: Any = adapter.connect({connection_key: str(project / "warehouse.db")}) + legacy_sql: str = Path( + "benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_state.sql" + ).read_text() + for statement in legacy_sql.rstrip().removesuffix(";").split(";"): + _ = adapter.execute(connection=connection, sql=statement) + adapter.close(connection) + for journal_mode in test_case.journal_modes: + with closing(sqlite3.connect(project / "warehouse.db")) as predecessor: + assert predecessor.execute("PRAGMA journal_mode=DELETE").fetchone() == (journal_mode,) + + def page(after: list[str | None] | None = None) -> Any: + result: subprocess.CompletedProcess[str] = subprocess.run( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "runtime", + "--server", + "http://127.0.0.1:1", + "--location-id", + "00000000-0000-0000-0000-000000000001", + "--target", + "development", + ], + input=json.dumps( + { + "source_freshness": { + "location": {"database": None, "schema": "main"}, + "after": after, + } + } + ) + + "\n", + text=True, + capture_output=True, + timeout=20, + env=os.environ | {"SQLBUILD_CONTROL_DATABASE_URL": "unsupported://must-not-open"}, + ) + assert result.returncode == 0, result.stderr + return json.loads(result.stdout) + + old: dict[str, Any] = page() + assert len(old["records"]) == 1 and old["records"][0]["event_id"] is None + for journal_mode in test_case.journal_modes: + with closing(sqlite3.connect(project / "warehouse.db")) as predecessor: + assert predecessor.execute("PRAGMA journal_mode").fetchone() == (journal_mode,) + connection = adapter.connect({connection_key: str(project / "warehouse.db")}) + # Reading history never upgrades the warehouse or imports project declarations. + assert "event_id" not in { + column.name + for column in adapter.get_columns( + connection=connection, database=None, schema="main", name="_sqlbuild_source_freshness" + ) + } + latest: Any = read_latest_fingerprints( + connection=connection, + execute=adapter.execute, + table_exists=True, + database=None, + schema="main", + render_qualified_name=adapter.render_qualified_name, + render_read_latest_sql=adapter.render_read_latest_fingerprints_sql, + ) + assert latest.fingerprints["orders"].run_id == "legacy-run" + fingerprint: Fingerprint = Fingerprint( + node_type="model", + node_name="orders", + target_database=None, + target_schema="main", + target_name="orders", + run_id="external-run", + definition_hash="definition", + version_hash="version", + schema_fingerprint="schema", + definition="SELECT 2 AS order_id", + ts=datetime(2026, 2, 1, tzinfo=UTC), + ) + test_case.append_fingerprints( + adapter=adapter, + connection=connection, + database_path=project / "warehouse.db", + fingerprint=fingerprint, + ) + expected_fingerprint_id: str = hashlib.sha256( + json.dumps( + [ + "sqlbuild.fingerprint.event.v1", + "model", + "orders", + None, + "main", + "orders", + "external-run", + "version", + ], + ensure_ascii=False, + separators=(",", ":"), + ).encode() + ).hexdigest() + assert adapter.query( + connection=connection, + sql="SELECT event_id FROM main._sqlbuild_fingerprints ORDER BY ts", + limit=10, + ).rows == ( + (None,), + (expected_fingerprint_id,), + (expected_fingerprint_id,), + ) + records: tuple[SourceFreshnessRecord, ...] = tuple( + SourceFreshnessRecord( + source_name=("Orders", "orders", "orders ", "ordérs")[index % 4], + target_database=("", None)[index % 2], + target_schema="main", + target_name="orders", + run_id=f"external-{index}", + strategy="max_column", + value_kind="string", + data_version=("o'rders 🧾", None, None)[index % 3], + data_version_hash=f"value-{index}", + observed_at=datetime(2026, 2, 1, microsecond=index, tzinfo=UTC), + ) + for index in range(104) + ) + renderers: SourceFreshnessRenderers = SourceFreshnessRenderers( + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + render_insert_records_sql=adapter.render_insert_source_freshness_records_sql, + render_create_table_sql=adapter.render_create_source_freshness_table_sql, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, + ) + for batch in (records, records[:1]): + write_source_freshness_records( + connection=connection, + execute=adapter.execute, + database=None, + schema="main", + records=batch, + renderers=renderers, + ) + expected_freshness_id: str = hashlib.sha256( + json.dumps( + [ + "sqlbuild.source-freshness.event.v1", + "Orders", + "", + "main", + "orders", + "external-0", + "2026-02-01T00:00:00.000000+00:00", + "max_column", + "string", + "value-0", + ], + ensure_ascii=False, + separators=(",", ":"), + ).encode() + ).hexdigest() + assert adapter.query( + connection=connection, + sql="SELECT event_id FROM main._sqlbuild_source_freshness WHERE run_id='external-0'", + limit=10, + ).rows == ( + (expected_freshness_id,), + (expected_freshness_id,), + ) + # An equivalent offset denotes the same observation, not another event. + equivalent: SourceFreshnessRecord = SourceFreshnessRecord( + source_name="Orders", + target_database="", + target_schema="main", + target_name="orders", + run_id="external-0", + strategy="max_column", + value_kind="string", + data_version="o'rders 🧾", + data_version_hash="value-0", + observed_at=datetime(2026, 2, 1, 1, tzinfo=timezone(timedelta(hours=1))), + ) + assert expected_freshness_id in adapter.render_insert_source_freshness_records_sql( + database=None, schema="main", records=(equivalent,) + ) + current: Any = read_latest_source_freshness( + connection=connection, + execute=adapter.execute, + table_exists=True, + database=None, + schema="main", + render_qualified_name=adapter.render_qualified_name, + render_read_latest_sql=adapter.render_read_latest_source_freshness_sql, + ) + assert {row.run_id for row in current.records.values()} == { + "legacy-run", + "external-100", + "external-101", + "external-102", + "external-103", + } + adapter.close(connection) + pages: list[dict[str, Any]] = [page()] + while pages[-1]["next_after"] is not None: + pages.append(page(pages[-1]["next_after"])) + assert len(pages) < 4 + assert tuple(len(item["records"]) for item in pages) == test_case.expected_page_lengths + imported: list[dict[str, Any]] = [] + for item in pages: + imported.extend(item["records"]) + event_ids: set[str | None] = {row["event_id"] for row in imported} + assert len(event_ids - {None}) == 104 + assert sum(row["event_id"] is None for row in imported) == 1 + assert {row["run_id"] for row in imported} == {"legacy-run"} | { + f"external-{index}" for index in range(104) + } + assert {row["target_database"] for row in imported} == {None, ""} + assert not (project / ".sqlbuild/control.sqlite3").exists() + (project / "warehouse.db").rename(project / "retained-warehouse.db") + assert page() is None + assert not (project / "warehouse.db").exists() From c77b500be656ac64aa78ea3ce411034b2fd10946 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 10:23:24 +0000 Subject: [PATCH 56/59] test: name fingerprint columns in stale-row fixture Preserve the legacy stale-row planning regression after additive warehouse schema evolution. Leave the nullable event identity absent and keep every query-change assertion unchanged. Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../cli/commands/main/build/test_query_change_tracking.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_change_tracking.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_change_tracking.py index 4fe9306891..ebb83d9c96 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_change_tracking.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_change_tracking.py @@ -146,7 +146,10 @@ def test_given_unchanged_project_when_planning_after_build_then_models_are_not_q try: stale_row: tuple[object, ...] = tuple(fingerprint_rows[0]) connection.execute( - "INSERT INTO main._sqlbuild_fingerprints VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "INSERT INTO main._sqlbuild_fingerprints " + "(node_type, node_name, target_database, target_schema, target_name, run_id, " + "definition_hash, version_hash, schema_fingerprint, definition_b64, " + "metadata_json_b64, ts) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", ( stale_row[0], stale_row[1], From 69a6aa18b0d80e726d350d8f3ff9cf75d0f6f4f6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 10:23:25 +0000 Subject: [PATCH 57/59] feat: bound freshness history by observation time Keep an inclusive observation-time bound separate from row-key pagination. Admit legacy SQLite timestamp formats through a date-only SQL superset, enforce the exact decoded UTC bound, and retain continuation after an empty filtered page. Exercise populated predecessor schemas, offset and microsecond boundaries, retries and both local warehouse backends without changing latest readers or adding producer policy. Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- crates/README.md | 11 +++-- .../src/warehouse_metadata/models.rs | 2 + .../native_host/warehouse_metadata.rs | 40 ++++++++++++++++--- .../sqlbuild-state/src/freshness/constants.rs | 1 + .../src/freshness/main/read_history_sql.rs | 12 +++++- .../publication/test_warehouse_metadata.py | 38 +++++++++++++++++- 6 files changed, 92 insertions(+), 12 deletions(-) diff --git a/crates/README.md b/crates/README.md index 41952bc041..03c551e8d2 100644 --- a/crates/README.md +++ b/crates/README.md @@ -128,9 +128,14 @@ The location history reader is read-only, including local database connections: databases nor evolves tables. Source-freshness ingestion preserves warehouse IDs, scoped by location, target and physical state table. Only missing or NULL IDs use the fixed legacy observation tuple, never a hash of the evolving full row. Bounded pages commit atomically under the current location -session and snapshot generation, and repeated pages are idempotent. These are read/ingestion -primitives; automatic polling, watermark/overlap management and deletion reconciliation are separate -location-sync responsibilities, not implemented by the page reader. +session and snapshot generation, and repeated pages are idempotent. A history read can hold an +inclusive observation-time lower bound fixed across pages, independently of the row-key cursor. +SQLite's SQL prefilter includes the boundary day so legacy space-separated timestamps and variable +fractional precision remain readable; decoded UTC timestamps enforce the exact bound. Other adapters +prefilter at whole-second precision. Continuation follows the raw row count: even an empty decoded +page can have more rows to inspect. These are read/ingestion primitives; automatic polling, +watermark/overlap management and deletion reconciliation are separate location-sync responsibilities, +not implemented by the page reader. ## Working on the crates diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index 388464b8ab..13ac7c9600 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -46,6 +46,8 @@ pub struct WarehouseStateLocation { pub struct SourceFreshnessRead { pub location: WarehouseStateLocation, pub after: Option<[Option; 11]>, + /// Inclusive time bound, held fixed across pages; independent of the row-key cursor. + pub observed_since: Option>, } #[derive(Clone, Debug, Serialize, Deserialize)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs index 7be0ce4f6d..3422a7d6f1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs @@ -13,7 +13,9 @@ use sqlbuild_orchestration::warehouse_metadata::constants::MAX_FRESHNESS_RECORDS use sqlbuild_orchestration::warehouse_metadata::models::{ SourceFreshnessPage, SourceFreshnessRead, WarehouseFreshnessRecord, }; -use sqlbuild_state::freshness::constants::{OBSERVED_AT_COLUMN, SOURCE_FRESHNESS_COLUMN_TYPES}; +use sqlbuild_state::freshness::constants::{ + OBSERVED_AT_COLUMN, SOURCE_FRESHNESS_COLUMN_TYPES, SQLITE_ADAPTER, +}; use sqlbuild_state::freshness::main::read_history_sql::read_history_sql; use crate::bindings::_helpers::authored_values::sql_values::{SqlValueClasses, sql_value}; @@ -147,9 +149,29 @@ fn read_page( (&database, &schema), )?; let adapter_name = adapter.getattr("adapter_name")?.extract::()?; + let observed_since = request + .observed_since + .map(|since| { + let text = since + .format(if adapter_name == SQLITE_ADAPTER { + "%Y-%m-%d" + } else { + "%Y-%m-%dT%H:%M:%S+00:00" + }) + .to_string(); + let classes = SqlValueClasses::load(py)?; + let value = sql_value(&SqlValue::String(text), &classes)?; + call_with( + &adapter.getattr("render_typed_scalar")?, + &[("value", value)], + )? + .extract::() + }) + .transpose()?; let sql = read_history_sql( &qualified, request.after.as_ref().map(|_| &literals), + observed_since.as_deref(), (MAX_FRESHNESS_RECORDS, &adapter_name, has_event_id), ) .map_err(PyValueError::new_err)?; @@ -166,7 +188,9 @@ fn read_page( )?; let mut records: Vec = Vec::new(); let mut next_after = None; + let mut read_count = 0; for row in field(&result, "rows")?.try_iter()? { + read_count += 1; let row = row?; let record = row_record(&row)?; let mut cursor: [Option; 11] = Default::default(); @@ -189,13 +213,17 @@ fn read_page( } cursor[10] = row.get_item(10)?.extract::>()?; value["event_id"] = json!(cursor[10]); - records.push( - serde_json::from_value::(value) - .map_err(|error| PyValueError::new_err(error.to_string()))?, - ); + let record = serde_json::from_value::(value) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + if request + .observed_since + .is_none_or(|since| record.observed_at >= since) + { + records.push(record); + } next_after = Some(cursor); } - if records.len() < MAX_FRESHNESS_RECORDS { + if read_count < MAX_FRESHNESS_RECORDS { next_after = None; } Ok(SourceFreshnessPage { diff --git a/crates/sqlbuild-state/src/freshness/constants.rs b/crates/sqlbuild-state/src/freshness/constants.rs index 4bec5268c8..0dc528ac8c 100644 --- a/crates/sqlbuild-state/src/freshness/constants.rs +++ b/crates/sqlbuild-state/src/freshness/constants.rs @@ -56,6 +56,7 @@ pub const SOURCE_FRESHNESS_COLUMN_TYPES: [(&str, &str); 10] = [ ]; pub const SOURCE_FRESHNESS_EVENT_ID: &str = "event_id"; pub const OBSERVED_AT_COLUMN: &str = "observed_at"; +pub const SQLITE_ADAPTER: &str = "sqlite"; pub(crate) const SQLSERVER_ADAPTER: &str = "sqlserver"; pub(crate) const PHYSICAL_TABLE_SOURCE_FRAGMENT: &str = "requires a physical table source"; pub(crate) const INCOMPLETE_CONFIGURATION_FRAGMENT: &str = "incomplete"; diff --git a/crates/sqlbuild-state/src/freshness/main/read_history_sql.rs b/crates/sqlbuild-state/src/freshness/main/read_history_sql.rs index ce7df1f246..e894646829 100644 --- a/crates/sqlbuild-state/src/freshness/main/read_history_sql.rs +++ b/crates/sqlbuild-state/src/freshness/main/read_history_sql.rs @@ -8,6 +8,7 @@ use crate::freshness::constants::{ pub fn read_history_sql( qualified_name: &str, after: Option<&[Option; 11]>, + observed_since: Option<&str>, (limit, adapter, has_event_id): (usize, &str, bool), ) -> Result { let mut columns: Vec<&str> = SOURCE_FRESHNESS_COLUMN_TYPES @@ -65,10 +66,17 @@ pub fn read_history_sql( )); } } - let filter = if greater.is_empty() { + let mut filters: Vec = Vec::new(); + if !greater.is_empty() { + filters.push(format!("({})", greater.join(" OR "))); + } + if let Some(since) = observed_since { + filters.push(format!("{OBSERVED_AT_COLUMN} >= {since}")); + } + let filter = if filters.is_empty() { String::new() } else { - format!(" WHERE {}", greater.join(" OR ")) + format!(" WHERE {}", filters.join(" AND ")) }; let order: Vec = columns .iter() diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py index df565a66a6..beaa693a20 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py @@ -82,7 +82,7 @@ def test_given_populated_predecessor_tables_when_writers_add_ids_then_history_pr with closing(sqlite3.connect(project / "warehouse.db")) as predecessor: assert predecessor.execute("PRAGMA journal_mode=DELETE").fetchone() == (journal_mode,) - def page(after: list[str | None] | None = None) -> Any: + def page(after: list[str | None] | None = None, observed_since: str | None = None) -> Any: result: subprocess.CompletedProcess[str] = subprocess.run( [ str(Path(sys.executable).with_name("sqb")), @@ -102,6 +102,7 @@ def page(after: list[str | None] | None = None) -> Any: "source_freshness": { "location": {"database": None, "schema": "main"}, "after": after, + "observed_since": observed_since, } } ) @@ -116,6 +117,11 @@ def page(after: list[str | None] | None = None) -> Any: old: dict[str, Any] = page() assert len(old["records"]) == 1 and old["records"][0]["event_id"] is None + assert page(observed_since="2026-01-01T09:00:00Z") == old + assert page(observed_since="2026-01-01T10:00:00.000001Z") == { + "records": [], + "next_after": None, + } for journal_mode in test_case.journal_modes: with closing(sqlite3.connect(project / "warehouse.db")) as predecessor: assert predecessor.execute("PRAGMA journal_mode").fetchone() == (journal_mode,) @@ -287,6 +293,36 @@ def page(after: list[str | None] | None = None) -> Any: f"external-{index}" for index in range(104) } assert {row["target_database"] for row in imported} == {None, ""} + # The time bound is inclusive and independent of source-name/keyset ordering. + # SQL may read a superset within the boundary day, so a short decoded page + # must retain its continuation if the raw query filled the page. + observed_since: str = "2026-02-01T01:00:00.000050+01:00" + recent: list[dict[str, Any]] = [page(observed_since=observed_since)] + while recent[-1]["next_after"] is not None: + recent.append(page(recent[-1]["next_after"], observed_since)) + assert len(recent) < 4 + recent_records: list[dict[str, Any]] = [] + for item in recent: + recent_records.extend(item["records"]) + assert len(recent_records) == 54 + assert {row["run_id"] for row in recent_records} == { + f"external-{index}" for index in range(50, 104) + } + zero: dict[str, Any] = page(observed_since="2026-02-01T00:00:00Z") + assert ( + len(zero["records"]) + len(page(zero["next_after"], "2026-02-01T00:00:00Z")["records"]) + == 105 + ) + assert sum(row["run_id"] == "external-0" for row in zero["records"]) == 2 + # An empty decoded page can still have another bounded SQL page to inspect. + empty_since: str = "2026-02-01T00:00:00.000104Z" + empty: dict[str, Any] = page(observed_since=empty_since) + assert empty["records"] == [] and empty["next_after"] is not None + assert page(empty["next_after"], empty_since) == {"records": [], "next_after": None} + assert page(observed_since="2030-01-01T00:00:00Z") == { + "records": [], + "next_after": None, + } assert not (project / ".sqlbuild/control.sqlite3").exists() (project / "warehouse.db").rename(project / "retained-warehouse.db") assert page() is None From 9eb27532e2cf81022dbc7aa6bb9bfcb05cea4dc8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 10:33:00 +0000 Subject: [PATCH 58/59] refactor: share warehouse history SQL construction Keep history row ordering and time-bound semantics unchanged while giving append-only state tables one keyset SQL owner. Rebuilt DuckDB and SQLite history regressions pass unchanged. Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../native_host/warehouse_metadata.rs | 15 ++++++-- .../sqlbuild-state/src/freshness/constants.rs | 1 - .../sqlbuild-state/src/freshness/main/mod.rs | 1 - .../src/state_tables/constants.rs | 3 ++ .../src/state_tables/main/mod.rs | 1 + .../main/read_history_sql.rs | 37 +++++++++---------- crates/sqlbuild-state/src/state_tables/mod.rs | 1 + .../sqlbuild-state/src/state_tables/models.rs | 11 ++++++ 8 files changed, 45 insertions(+), 25 deletions(-) create mode 100644 crates/sqlbuild-state/src/state_tables/constants.rs rename crates/sqlbuild-state/src/{freshness => state_tables}/main/read_history_sql.rs (81%) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs index 3422a7d6f1..4958ffa439 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs @@ -16,7 +16,8 @@ use sqlbuild_orchestration::warehouse_metadata::models::{ use sqlbuild_state::freshness::constants::{ OBSERVED_AT_COLUMN, SOURCE_FRESHNESS_COLUMN_TYPES, SQLITE_ADAPTER, }; -use sqlbuild_state::freshness::main::read_history_sql::read_history_sql; +use sqlbuild_state::state_tables::main::read_history_sql::read_history_sql; +use sqlbuild_state::state_tables::models::HistoryRead; use crate::bindings::_helpers::authored_values::sql_values::{SqlValueClasses, sql_value}; use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with}; @@ -170,9 +171,15 @@ fn read_page( .transpose()?; let sql = read_history_sql( &qualified, - request.after.as_ref().map(|_| &literals), - observed_since.as_deref(), - (MAX_FRESHNESS_RECORDS, &adapter_name, has_event_id), + request.after.as_ref().map(|_| literals.as_slice()), + HistoryRead { + columns: &SOURCE_FRESHNESS_COLUMN_TYPES.map(|(name, _)| name), + timestamp_column: OBSERVED_AT_COLUMN, + observed_since: observed_since.as_deref(), + adapter: &adapter_name, + has_event_id, + limit: MAX_FRESHNESS_RECORDS, + }, ) .map_err(PyValueError::new_err)?; let result = call_with( diff --git a/crates/sqlbuild-state/src/freshness/constants.rs b/crates/sqlbuild-state/src/freshness/constants.rs index 0dc528ac8c..184ed9e83b 100644 --- a/crates/sqlbuild-state/src/freshness/constants.rs +++ b/crates/sqlbuild-state/src/freshness/constants.rs @@ -57,6 +57,5 @@ pub const SOURCE_FRESHNESS_COLUMN_TYPES: [(&str, &str); 10] = [ pub const SOURCE_FRESHNESS_EVENT_ID: &str = "event_id"; pub const OBSERVED_AT_COLUMN: &str = "observed_at"; pub const SQLITE_ADAPTER: &str = "sqlite"; -pub(crate) const SQLSERVER_ADAPTER: &str = "sqlserver"; pub(crate) const PHYSICAL_TABLE_SOURCE_FRAGMENT: &str = "requires a physical table source"; pub(crate) const INCOMPLETE_CONFIGURATION_FRAGMENT: &str = "incomplete"; diff --git a/crates/sqlbuild-state/src/freshness/main/mod.rs b/crates/sqlbuild-state/src/freshness/main/mod.rs index 5a1085a0e2..2e5f2dbe8f 100644 --- a/crates/sqlbuild-state/src/freshness/main/mod.rs +++ b/crates/sqlbuild-state/src/freshness/main/mod.rs @@ -3,7 +3,6 @@ pub mod configuration_error; pub mod data_version_hash; pub mod event_id; pub mod normalized_data_version; -pub mod read_history_sql; pub mod read_latest_sql; pub mod records_equivalent; pub mod relevant_sources_sql; diff --git a/crates/sqlbuild-state/src/state_tables/constants.rs b/crates/sqlbuild-state/src/state_tables/constants.rs new file mode 100644 index 0000000000..8bf6191df2 --- /dev/null +++ b/crates/sqlbuild-state/src/state_tables/constants.rs @@ -0,0 +1,3 @@ +//! Adapter decisions shared by append-only state history reads. + +pub(crate) const SQLSERVER_ADAPTER: &str = "sqlserver"; diff --git a/crates/sqlbuild-state/src/state_tables/main/mod.rs b/crates/sqlbuild-state/src/state_tables/main/mod.rs index 712abe33fc..6b46ba4e98 100644 --- a/crates/sqlbuild-state/src/state_tables/main/mod.rs +++ b/crates/sqlbuild-state/src/state_tables/main/mod.rs @@ -1,2 +1,3 @@ pub mod create_table_sql; +pub mod read_history_sql; pub mod values_insert_sql; diff --git a/crates/sqlbuild-state/src/freshness/main/read_history_sql.rs b/crates/sqlbuild-state/src/state_tables/main/read_history_sql.rs similarity index 81% rename from crates/sqlbuild-state/src/freshness/main/read_history_sql.rs rename to crates/sqlbuild-state/src/state_tables/main/read_history_sql.rs index e894646829..2d871377ba 100644 --- a/crates/sqlbuild-state/src/freshness/main/read_history_sql.rs +++ b/crates/sqlbuild-state/src/state_tables/main/read_history_sql.rs @@ -1,35 +1,34 @@ //! Keyset pages preserve distinct observations without rereading or ranking all preceding rows. -use crate::freshness::constants::{ - OBSERVED_AT_COLUMN, SOURCE_FRESHNESS_COLUMN_TYPES, SOURCE_FRESHNESS_EVENT_ID, SQLSERVER_ADAPTER, -}; +use crate::state_tables::constants::SQLSERVER_ADAPTER; +use crate::state_tables::models::HistoryRead; /// Cursor values are already rendered by the adapter; `None` sorts before every SQL value. pub fn read_history_sql( qualified_name: &str, - after: Option<&[Option; 11]>, - observed_since: Option<&str>, - (limit, adapter, has_event_id): (usize, &str, bool), + after: Option<&[Option]>, + read: HistoryRead<'_>, ) -> Result { - let mut columns: Vec<&str> = SOURCE_FRESHNESS_COLUMN_TYPES - .iter() - .map(|(name, _)| *name) - .collect(); + let HistoryRead { + columns, + timestamp_column, + observed_since, + adapter, + has_event_id, + limit, + } = read; + let mut columns = columns.to_vec(); let mut projection = columns.join(", "); projection.push_str(if has_event_id { ", event_id" } else { ", NULL AS event_id" }); - columns.push(if has_event_id { - SOURCE_FRESHNESS_EVENT_ID - } else { - "NULL" - }); + columns.push(if has_event_id { "event_id" } else { "NULL" }); let ordered = columns .iter() .map(|column| { - if *column == OBSERVED_AT_COLUMN { + if *column == timestamp_column { Ok((*column).to_owned()) } else { binary(column, adapter) @@ -43,7 +42,7 @@ pub fn read_history_sql( let literal = value .as_ref() .map(|value| { - if *column == OBSERVED_AT_COLUMN { + if *column == timestamp_column { Ok(value.clone()) } else { binary(value, adapter) @@ -71,7 +70,7 @@ pub fn read_history_sql( filters.push(format!("({})", greater.join(" OR "))); } if let Some(since) = observed_since { - filters.push(format!("{OBSERVED_AT_COLUMN} >= {since}")); + filters.push(format!("{timestamp_column} >= {since}")); } let filter = if filters.is_empty() { String::new() @@ -113,7 +112,7 @@ fn binary(value: &str, adapter: &str) -> Result { "clickhouse" => value.to_owned(), _ => { return Err(format!( - "source freshness history does not support adapter '{adapter}'" + "warehouse history does not support adapter '{adapter}'" )); } }) diff --git a/crates/sqlbuild-state/src/state_tables/mod.rs b/crates/sqlbuild-state/src/state_tables/mod.rs index 1a70a48981..042268f15e 100644 --- a/crates/sqlbuild-state/src/state_tables/mod.rs +++ b/crates/sqlbuild-state/src/state_tables/mod.rs @@ -1,3 +1,4 @@ +pub mod constants; pub mod main; pub mod models; diff --git a/crates/sqlbuild-state/src/state_tables/models.rs b/crates/sqlbuild-state/src/state_tables/models.rs index 7a343e134d..8c8a1f5792 100644 --- a/crates/sqlbuild-state/src/state_tables/models.rs +++ b/crates/sqlbuild-state/src/state_tables/models.rs @@ -7,3 +7,14 @@ pub struct TableColumn { pub rendered_type: String, pub required: bool, } + +/// A bounded legacy-compatible history projection, with its optional source event ID. +#[derive(Clone, Copy, Debug)] +pub struct HistoryRead<'a> { + pub columns: &'a [&'a str], + pub timestamp_column: &'a str, + pub observed_since: Option<&'a str>, + pub adapter: &'a str, + pub has_event_id: bool, + pub limit: usize, +} From dff273e8933a504aa02d9aedf96d4053f4a27787 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 10:51:39 +0000 Subject: [PATCH 59/59] feat: read bounded fingerprint history from locations Read fingerprint observations through the isolated location protocol without changing latest readers or writing warehouse state. Preserve source IDs and predecessor rows, decode definitions through the existing reader, and keep raw keyset cursors independent of exact UTC bounds. Cover populated schemas, retries, mixed identities, Unicode storage and SQLite offsets across a date boundary. --- .../legacy_warehouse_fingerprint_offsets.sql | 10 + crates/README.md | 22 +- .../src/warehouse_metadata/constants.rs | 2 +- .../main/ingest_source_freshness.rs | 6 +- .../src/warehouse_metadata/models.rs | 52 ++++- .../_helpers/fingerprints/latest_rows.rs | 2 +- .../_helpers/location_command/runtime.rs | 14 +- .../native_host/warehouse_metadata.rs | 197 ++++++++++------ .../publication/_test_types.py | 12 + .../compiled_project/publication/helpers.py | 64 ++++++ .../publication/test_warehouse_metadata.py | 212 +++++++++++++----- 11 files changed, 452 insertions(+), 141 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_fingerprint_offsets.sql diff --git a/benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_fingerprint_offsets.sql b/benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_fingerprint_offsets.sql new file mode 100644 index 0000000000..fe7ba68f98 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_fingerprint_offsets.sql @@ -0,0 +1,10 @@ +-- SQLite predecessor text can carry the writer's offset, across the UTC date boundary. +-- Load after legacy_warehouse_state.sql, before any writer adds event_id. +INSERT INTO main._sqlbuild_fingerprints ( + node_type, node_name, target_database, target_schema, target_name, run_id, + definition_hash, version_hash, schema_fingerprint, definition_b64, metadata_json_b64, ts +) VALUES ( + 'model', 'offset_orders', NULL, 'main', 'offset_orders', 'offset-run', + 'legacy-definition', 'legacy-version', 'legacy-schema', + 'U0VMRUNUIDEgQVMgb3JkZXJfaWQ=', 'e30=', '2025-12-31T23:00:00-11:00' +); diff --git a/crates/README.md b/crates/README.md index 03c551e8d2..e1727686ae 100644 --- a/crates/README.md +++ b/crates/README.md @@ -124,16 +124,20 @@ is normalized to UTC ISO text with six fractional digits and a `+00:00` offset. observation retains its ID; another run or observation is distinct. Definitions, display values, and future storage columns do not alter these identities. -The location history reader is read-only, including local database connections: it neither creates -databases nor evolves tables. Source-freshness ingestion preserves warehouse IDs, scoped by location, -target and physical state table. Only missing or NULL IDs use the fixed legacy observation tuple, -never a hash of the evolving full row. Bounded pages commit atomically under the current location -session and snapshot generation, and repeated pages are idempotent. A history read can hold an +The location history reader pages fingerprint and source-freshness observations without selecting +only the latest row. It is read-only, including local database connections: it neither creates +databases nor evolves tables. Fingerprint definitions and metadata use the existing storage decoder; +raw storage values remain the keyset cursor, separate from decoded UTC timestamps and text. +Source-freshness ingestion preserves warehouse IDs, scoped by location, target and physical state +table. Only missing or NULL IDs use the fixed legacy observation tuple, never a hash of the evolving +full row. Its bounded pages commit atomically under the current location session and snapshot +generation, and repeated pages are idempotent. A history read can hold an inclusive observation-time lower bound fixed across pages, independently of the row-key cursor. -SQLite's SQL prefilter includes the boundary day so legacy space-separated timestamps and variable -fractional precision remain readable; decoded UTC timestamps enforce the exact bound. Other adapters -prefilter at whole-second precision. Continuation follows the raw row count: even an empty decoded -page can have more rows to inspect. These are read/ingestion primitives; automatic polling, +SQLite's SQL prefilter includes the boundary day and its predecessor so legacy separators, variable +fractional precision and writer offsets crossing a UTC date remain readable; decoded UTC timestamps +enforce the exact bound. Other adapters prefilter at whole-second precision. Continuation follows +the raw row count: even an empty decoded page can have more rows to inspect. Fingerprint ingestion +and per-hash definition storage remain separate from this read-only transport. Automatic polling, watermark/overlap management and deletion reconciliation are separate location-sync responsibilities, not implemented by the page reader. diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs index 05ee34c05a..2615174d34 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs @@ -1,3 +1,3 @@ //! Limits bound each authenticated warehouse observation transaction. -pub const MAX_FRESHNESS_RECORDS: usize = 100; +pub const MAX_HISTORY_RECORDS: usize = 100; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs index 8f170f2fce..a734b0440b 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs @@ -11,7 +11,7 @@ use crate::store::main::begin::begin; use crate::store::main::commit::commit; use crate::store::main::query::query; use crate::store::models::{ControlStore, Parameter, TransactionMode}; -use crate::warehouse_metadata::constants::MAX_FRESHNESS_RECORDS; +use crate::warehouse_metadata::constants::MAX_HISTORY_RECORDS; use crate::warehouse_metadata::models::SourceFreshnessBatch; pub async fn ingest_source_freshness( @@ -40,12 +40,12 @@ pub async fn ingest_source_freshness( .is_some_and(|value| value.trim().is_empty()) }); if batch.generation < 1 - || batch.records.len() > MAX_FRESHNESS_RECORDS + || batch.records.len() > MAX_HISTORY_RECORDS || invalid_identity || invalid_source_id { return Err(ControlError::Invalid(format!( - "source freshness sync needs a positive generation, at most {MAX_FRESHNESS_RECORDS} records and nonempty state identities", + "source freshness sync needs a positive generation, at most {MAX_HISTORY_RECORDS} records and nonempty state identities", ))); } let transaction = require_instance( diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index 13ac7c9600..3a59cbc391 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -1,4 +1,4 @@ -//! Existing source-freshness state rows, with their physical state-table provenance. +//! Existing warehouse state rows, with their physical state-table provenance. use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; @@ -50,9 +50,53 @@ pub struct SourceFreshnessRead { pub observed_since: Option>, } +/// Fingerprints remain history here; latest-row selection belongs to the existing planner reader. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WarehouseFingerprintRecord { + pub event_id: Option, + pub state_database: Option, + pub state_schema: String, + pub node_type: String, + pub node_name: String, + pub target_database: Option, + pub target_schema: Option, + pub target_name: Option, + pub run_id: String, + pub definition_hash: String, + pub version_hash: String, + pub schema_fingerprint: String, + pub definition: String, + pub metadata_json: String, + pub ts: DateTime, +} + #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] -pub struct SourceFreshnessPage { - pub records: Vec, - pub next_after: Option<[Option; 11]>, +pub struct FingerprintRead { + pub location: WarehouseStateLocation, + pub after: Option<[Option; 13]>, + pub observed_since: Option>, +} + +/// The private interpreter accepts only these two known state tables, never arbitrary SQL. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WarehouseHistoryRead { + SourceFreshness(SourceFreshnessRead), + Fingerprints(FingerprintRead), +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(untagged)] +pub enum WarehouseHistoryRecord { + SourceFreshness(WarehouseFreshnessRecord), + Fingerprint(WarehouseFingerprintRecord), +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WarehouseHistoryPage { + pub records: Vec, + pub next_after: Option>>, } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs index e977d6a155..3347752271 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs @@ -130,7 +130,7 @@ fn borrowed(identities: &[(String, String)]) -> Vec> { .collect() } -fn row_fingerprint<'py>( +pub(crate) fn row_fingerprint<'py>( fingerprint_class: &Bound<'py, PyAny>, from_iso: &Bound<'py, PyAny>, row: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs index 1ca8f40504..bc75c07fdd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs @@ -6,13 +6,13 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_orchestration::sensors::models::{CustomSensorOutcome, SensorEvaluationRequest}; -use sqlbuild_orchestration::warehouse_metadata::models::SourceFreshnessRead; +use sqlbuild_orchestration::warehouse_metadata::models::WarehouseHistoryRead; use crate::bindings::_helpers::boundary::python_calls::within; use crate::bindings::_helpers::location_command::command::{location_arguments, publish_location}; use crate::bindings::_helpers::native_host::host::{redirect_stdout_to_stderr, write_console}; use crate::bindings::_helpers::native_host::orchestration_sensors::evaluate_sensor; -use crate::bindings::_helpers::native_host::warehouse_metadata::read_source_freshness; +use crate::bindings::_helpers::native_host::warehouse_metadata::read_history; pub(super) fn run_runtime(py: Python<'_>, options: Bound<'_, PyDict>) -> PyResult { options.set_item("action", "publish")?; @@ -46,13 +46,15 @@ pub(super) fn run_runtime(py: Python<'_>, options: Bound<'_, PyDict>) -> PyResul } }; serde_json::to_string(&record) - } else if request.get("source_freshness").is_some() { - let metadata: SourceFreshnessRead = - serde_json::from_value(request["source_freshness"].clone()).map_err(|_| { + } else if request.get("source_freshness").is_some() + || request.get("fingerprints").is_some() + { + let metadata: WarehouseHistoryRead = + serde_json::from_value(request).map_err(|_| { PyValueError::new_err("runtime received an invalid metadata read") })?; let arguments = location_arguments(py, Some(options.clone()))?; - let page = match read_source_freshness(py, &arguments, &metadata) { + let page = match read_history(py, &arguments, &metadata) { Ok(page) => Some(page), Err(_) => { write_console( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs index 4958ffa439..d07490aeb7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs @@ -4,14 +4,15 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString}; use serde_json::{Value, json}; +use sqlbuild_analysis::fingerprint_storage::constants::{COLUMN_TIMESTAMP, FINGERPRINT_COLUMNS}; use sqlbuild_config::command_connection::constants::{ DUCKDB_ADAPTER, DUCKDB_MEMORY_DATABASE, LOCAL_DATABASE_PATH_KEYS, }; -use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME; +use sqlbuild_core::constants::{FINGERPRINT_TABLE_NAME, SOURCE_FRESHNESS_TABLE_NAME}; use sqlbuild_core::models::SqlValue; -use sqlbuild_orchestration::warehouse_metadata::constants::MAX_FRESHNESS_RECORDS; +use sqlbuild_orchestration::warehouse_metadata::constants::MAX_HISTORY_RECORDS; use sqlbuild_orchestration::warehouse_metadata::models::{ - SourceFreshnessPage, SourceFreshnessRead, WarehouseFreshnessRecord, + WarehouseHistoryPage, WarehouseHistoryRead, WarehouseHistoryRecord, }; use sqlbuild_state::freshness::constants::{ OBSERVED_AT_COLUMN, SOURCE_FRESHNESS_COLUMN_TYPES, SQLITE_ADAPTER, @@ -22,18 +23,25 @@ use sqlbuild_state::state_tables::models::HistoryRead; use crate::bindings::_helpers::authored_values::sql_values::{SqlValueClasses, sql_value}; use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::command_runtime::connection::project_connection_config; -use crate::bindings::_helpers::freshness::state_io::{qualified_table_name, row_record}; +use crate::bindings::_helpers::fingerprints::latest_rows::row_fingerprint; +use crate::bindings::_helpers::freshness::state_io::row_record; +use crate::bindings::_helpers::freshness::support::presumed_utc_datetime; use crate::bindings::_helpers::query_command::project::{ ProjectAdapter, ProjectInputs, project_adapter, }; use crate::bindings::_helpers::records::fields::field; +use crate::bindings::_helpers::records::{construct::record_class, ids::RecordId}; -pub(crate) fn read_source_freshness( +pub(crate) fn read_history( py: Python<'_>, arguments: &KeywordArguments<'_>, - request: &SourceFreshnessRead, -) -> PyResult { - if request.location.schema.trim().is_empty() { + request: &WarehouseHistoryRead, +) -> PyResult { + let location = match request { + WarehouseHistoryRead::SourceFreshness(read) => &read.location, + WarehouseHistoryRead::Fingerprints(read) => &read.location, + }; + if location.schema.trim().is_empty() { return Err(PyValueError::new_err( "metadata state schema must not be blank", )); @@ -87,50 +95,66 @@ pub(crate) fn read_source_freshness( fn read_page( py: Python<'_>, (adapter, connection): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), - request: &SourceFreshnessRead, -) -> PyResult { - let database = request.location.database.as_ref().map_or_else( + request: &WarehouseHistoryRead, +) -> PyResult { + let freshness_columns = SOURCE_FRESHNESS_COLUMN_TYPES.map(|(name, _)| name); + let (location, after, since, table, columns, timestamp) = match request { + WarehouseHistoryRead::SourceFreshness(read) => ( + &read.location, + read.after.as_ref().map(|cursor| cursor.as_slice()), + read.observed_since, + SOURCE_FRESHNESS_TABLE_NAME, + freshness_columns.as_slice(), + OBSERVED_AT_COLUMN, + ), + WarehouseHistoryRead::Fingerprints(read) => ( + &read.location, + read.after.as_ref().map(|cursor| cursor.as_slice()), + read.observed_since, + FINGERPRINT_TABLE_NAME, + FINGERPRINT_COLUMNS.as_slice(), + COLUMN_TIMESTAMP, + ), + }; + let database = location.database.as_ref().map_or_else( || py.None().into_bound(py), |value| PyString::new(py, value).into_any(), ); - let schema = PyString::new(py, &request.location.schema).into_any(); - let name = PyString::new(py, SOURCE_FRESHNESS_TABLE_NAME).into_any(); + let schema = PyString::new(py, &location.schema).into_any(); + let name = PyString::new(py, table).into_any(); let exists = call_with( &adapter.getattr("relation_exists")?, &[ ("connection", connection.clone()), ("database", database.clone()), ("schema", schema.clone()), - ("name", name), + ("name", name.clone()), ], )? .is_truthy()?; if !exists { - return Ok(SourceFreshnessPage { + return Ok(WarehouseHistoryPage { records: Vec::new(), next_after: None, }); } - let columns = call_with( + let catalog_columns = call_with( &adapter.getattr("get_columns")?, &[ ("connection", connection.clone()), ("database", database.clone()), ("schema", schema.clone()), - ( - "name", - PyString::new(py, SOURCE_FRESHNESS_TABLE_NAME).into_any(), - ), + ("name", name.clone()), ], )?; let mut has_event_id = false; - for column in columns.try_iter()? { + for column in catalog_columns.try_iter()? { has_event_id |= field(&column?, "name")? .extract::()? .eq_ignore_ascii_case("event_id"); } - let mut literals: [Option; 11] = Default::default(); - if let Some(after) = &request.after { + let mut literals = vec![None; columns.len() + 1]; + if let Some(after) = after { let classes = SqlValueClasses::load(py)?; for (output, value) in literals.iter_mut().zip(after) { if let Some(value) = value { @@ -145,21 +169,24 @@ fn read_page( } } } - let qualified = qualified_table_name( + let qualified: String = call_with( &adapter.getattr("render_qualified_name")?, - (&database, &schema), - )?; + &[("database", database), ("schema", schema), ("name", name)], + )? + .extract()?; let adapter_name = adapter.getattr("adapter_name")?.extract::()?; - let observed_since = request - .observed_since + let observed_since = since .map(|since| { - let text = since - .format(if adapter_name == SQLITE_ADAPTER { - "%Y-%m-%d" - } else { - "%Y-%m-%dT%H:%M:%S+00:00" - }) - .to_string(); + let text = if adapter_name == SQLITE_ADAPTER { + since + .date_naive() + .pred_opt() + .unwrap_or(since.date_naive()) + .format("%Y-%m-%d") + .to_string() + } else { + since.format("%Y-%m-%dT%H:%M:%S+00:00").to_string() + }; let classes = SqlValueClasses::load(py)?; let value = sql_value(&SqlValue::String(text), &classes)?; call_with( @@ -171,14 +198,14 @@ fn read_page( .transpose()?; let sql = read_history_sql( &qualified, - request.after.as_ref().map(|_| literals.as_slice()), + after.map(|_| literals.as_slice()), HistoryRead { - columns: &SOURCE_FRESHNESS_COLUMN_TYPES.map(|(name, _)| name), - timestamp_column: OBSERVED_AT_COLUMN, + columns, + timestamp_column: timestamp, observed_since: observed_since.as_deref(), adapter: &adapter_name, has_event_id, - limit: MAX_FRESHNESS_RECORDS, + limit: MAX_HISTORY_RECORDS, }, ) .map_err(PyValueError::new_err)?; @@ -189,52 +216,94 @@ fn read_page( ("sql", PyString::new(py, &sql).into_any()), ( "limit", - pyo3::IntoPyObjectExt::into_bound_py_any(MAX_FRESHNESS_RECORDS, py)?, + pyo3::IntoPyObjectExt::into_bound_py_any(MAX_HISTORY_RECORDS, py)?, ), ], )?; - let mut records: Vec = Vec::new(); + let mut records: Vec = Vec::new(); let mut next_after = None; let mut read_count = 0; for row in field(&result, "rows")?.try_iter()? { read_count += 1; let row = row?; - let record = row_record(&row)?; - let mut cursor: [Option; 11] = Default::default(); - let mut value = json!({"state_database": request.location.database, "state_schema": request.location.schema}); - for (index, (name, _)) in SOURCE_FRESHNESS_COLUMN_TYPES.iter().enumerate() { + let record = decode_record(py, (request, &row, &qualified), (columns, timestamp))?; + let mut cursor = vec![None; columns.len() + 1]; + for (index, value) in cursor.iter_mut().enumerate() { let raw = row.get_item(index)?; - cursor[index] = if raw.is_none() { + *value = if raw.is_none() { None } else { Some(raw.str()?.extract()?) }; - let item = field(&record, name)?; - value[*name] = if item.is_none() { - Value::Null - } else if *name == OBSERVED_AT_COLUMN { - json!(item.call_method0("isoformat")?.extract::()?) - } else { - json!(item.extract::()?) - }; } - cursor[10] = row.get_item(10)?.extract::>()?; - value["event_id"] = json!(cursor[10]); - let record = serde_json::from_value::(value) - .map_err(|error| PyValueError::new_err(error.to_string()))?; - if request - .observed_since - .is_none_or(|since| record.observed_at >= since) - { + let observed = match &record { + WarehouseHistoryRecord::SourceFreshness(record) => record.observed_at, + WarehouseHistoryRecord::Fingerprint(record) => record.ts, + }; + if since.is_none_or(|since| observed >= since) { records.push(record); } next_after = Some(cursor); } - if read_count < MAX_FRESHNESS_RECORDS { + if read_count < MAX_HISTORY_RECORDS { next_after = None; } - Ok(SourceFreshnessPage { + Ok(WarehouseHistoryPage { records, next_after, }) } + +fn decode_record( + py: Python<'_>, + (request, row, qualified): (&WarehouseHistoryRead, &Bound<'_, PyAny>, &str), + (columns, timestamp): (&[&str], &str), +) -> PyResult { + let (location, record) = match request { + WarehouseHistoryRead::SourceFreshness(read) => (&read.location, row_record(row)?), + WarehouseHistoryRead::Fingerprints(read) => ( + &read.location, + row_fingerprint( + &record_class(py, RecordId::Fingerprint)?, + &py.import("datetime")? + .getattr("datetime")? + .getattr("fromisoformat")?, + row, + qualified, + )?, + ), + }; + let mut value = json!({ + "state_database": location.database, + "state_schema": location.schema, + "event_id": row.get_item(columns.len())?.extract::>()?, + }); + for name in columns { + let name = match *name { + "definition_b64" => "definition", + "metadata_json_b64" => "metadata_json", + name => name, + }; + let item = field(&record, name)?; + value[name] = if item.is_none() { + Value::Null + } else if name == timestamp { + json!( + presumed_utc_datetime(&item)? + .call_method0("isoformat")? + .extract::()? + ) + } else { + json!(item.extract::()?) + }; + } + match request { + WarehouseHistoryRead::SourceFreshness(_) => { + serde_json::from_value(value).map(WarehouseHistoryRecord::SourceFreshness) + } + WarehouseHistoryRead::Fingerprints(_) => { + serde_json::from_value(value).map(WarehouseHistoryRecord::Fingerprint) + } + } + .map_err(|error| PyValueError::new_err(error.to_string())) +} diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 681b669497..20b540641f 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -2,6 +2,7 @@ from collections.abc import Callable from dataclasses import dataclass, field +from datetime import datetime from pathlib import Path from subprocess import Popen from typing import Any, Literal, NamedTuple @@ -154,3 +155,14 @@ class WarehouseHistoryTestCase: connection_key: str journal_modes: tuple[str, ...] append_fingerprints: Callable[..., None] + + +@dataclass(frozen=True) +class FingerprintHistoryTestCase: + description: str + adapter_name: str + connection_key: str + observed_at: datetime + journal_modes: tuple[str, ...] + fixture_paths: tuple[Path, ...] + expected_legacy_runs: tuple[str, ...] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index 16170f6d72..ae1526b8c8 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -26,6 +26,7 @@ import pytest from sqlbuild._native import Fingerprint, write_fingerprint +from sqlbuild.adapter.discovery.main.resolve_adapter import resolve_adapter from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( ClaimedWorker, ScheduledWorker, @@ -66,6 +67,69 @@ def location_process( _ = process.wait(timeout=5) +def open_legacy_warehouse(project: Path, adapter_name: str, connection_key: str) -> tuple[Any, Any]: + """The caller owns the returned real connection; declarations must never run during reads.""" + write_files( + project_dir=project, + files={ + "sqlbuild_project.toml": ( + f'name = "orders"\nadapter = "{adapter_name}"\n' + f'[connections.local]\n{connection_key} = "warehouse.db"\n' + '[targets.development]\nschema = "main"\n' + ), + "orchestration/orders.py": 'raise AssertionError("unexpected declaration import")\n', + }, + ) + adapter: Any = resolve_adapter(adapter_name=adapter_name) + connection: Any = adapter.connect({connection_key: str(project / "warehouse.db")}) + legacy_sql: str = Path( + "benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_state.sql" + ).read_text() + for statement in legacy_sql.rstrip().removesuffix(";").split(";"): + _ = adapter.execute(connection=connection, sql=statement) + return adapter, connection + + +def warehouse_history_page( + project: Path, + table: str, + after: list[str | None] | None = None, + observed_since: str | None = None, +) -> Any: + """Exercise the real isolated reader without a server, state database or declaration import.""" + result: subprocess.CompletedProcess[str] = subprocess.run( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "runtime", + "--server", + "http://127.0.0.1:1", + "--location-id", + "00000000-0000-0000-0000-000000000001", + "--target", + "development", + ], + input=json.dumps( + { + table: { + "location": {"database": None, "schema": "main"}, + "after": after, + "observed_since": observed_since, + } + } + ) + + "\n", + text=True, + capture_output=True, + timeout=20, + env=os.environ | {"SQLBUILD_CONTROL_DATABASE_URL": "unsupported://must-not-open"}, + ) + assert result.returncode == 0, result.stderr + return json.loads(result.stdout) + + def wait_for_generation( api: SnapshotApi, target: str, generation: int, process: subprocess.Popen[bytes], log: Path ) -> dict[str, Any]: diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py index beaa693a20..2b328d4c5f 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py @@ -2,10 +2,7 @@ import hashlib import json -import os import sqlite3 -import subprocess -import sys from contextlib import closing from datetime import UTC, datetime, timedelta, timezone from pathlib import Path @@ -19,17 +16,19 @@ SourceFreshnessRenderers, read_latest_fingerprints, read_latest_source_freshness, + write_fingerprints, write_source_freshness_records, ) -from sqlbuild.adapter.discovery.main.resolve_adapter import resolve_adapter from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + FingerprintHistoryTestCase, WarehouseHistoryTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( append_contended_fingerprints, append_retried_fingerprints, + open_legacy_warehouse, + warehouse_history_page, ) -from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files @pytest.mark.parametrize( @@ -59,61 +58,18 @@ def test_given_populated_predecessor_tables_when_writers_add_ids_then_history_pr ) -> None: project: Path = tmp_path / "project" connection_key: str = test_case.connection_key - write_files( - project_dir=project, - files={ - "sqlbuild_project.toml": ( - f'name = "orders"\nadapter = "{test_case.adapter_name}"\n' - f'[connections.local]\n{connection_key} = "warehouse.db"\n' - '[targets.development]\nschema = "main"\n' - ), - "orchestration/orders.py": 'raise AssertionError("metadata must not compile user declarations")\n', - }, + adapter: Any + connection: Any + adapter, connection = open_legacy_warehouse( + project, test_case.adapter_name, test_case.connection_key ) - adapter: Any = resolve_adapter(adapter_name=test_case.adapter_name) - connection: Any = adapter.connect({connection_key: str(project / "warehouse.db")}) - legacy_sql: str = Path( - "benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_state.sql" - ).read_text() - for statement in legacy_sql.rstrip().removesuffix(";").split(";"): - _ = adapter.execute(connection=connection, sql=statement) adapter.close(connection) for journal_mode in test_case.journal_modes: with closing(sqlite3.connect(project / "warehouse.db")) as predecessor: assert predecessor.execute("PRAGMA journal_mode=DELETE").fetchone() == (journal_mode,) def page(after: list[str | None] | None = None, observed_since: str | None = None) -> Any: - result: subprocess.CompletedProcess[str] = subprocess.run( - [ - str(Path(sys.executable).with_name("sqb")), - "--project-dir", - str(project), - "location", - "runtime", - "--server", - "http://127.0.0.1:1", - "--location-id", - "00000000-0000-0000-0000-000000000001", - "--target", - "development", - ], - input=json.dumps( - { - "source_freshness": { - "location": {"database": None, "schema": "main"}, - "after": after, - "observed_since": observed_since, - } - } - ) - + "\n", - text=True, - capture_output=True, - timeout=20, - env=os.environ | {"SQLBUILD_CONTROL_DATABASE_URL": "unsupported://must-not-open"}, - ) - assert result.returncode == 0, result.stderr - return json.loads(result.stdout) + return warehouse_history_page(project, "source_freshness", after, observed_since) old: dict[str, Any] = page() assert len(old["records"]) == 1 and old["records"][0]["event_id"] is None @@ -327,3 +283,153 @@ def page(after: list[str | None] | None = None, observed_since: str | None = Non (project / "warehouse.db").rename(project / "retained-warehouse.db") assert page() is None assert not (project / "warehouse.db").exists() + + +@pytest.mark.parametrize( + "test_case", + [ + FingerprintHistoryTestCase( + "DuckDB typed fingerprint timestamps", + "duckdb", + "database", + datetime(2026, 2, 1, tzinfo=UTC), + (), + (), + ("legacy-run",), + ), + FingerprintHistoryTestCase( + "SQLite offset fingerprint timestamps", + "sqlite", + "path", + datetime(2026, 1, 31, 23, tzinfo=timezone(timedelta(hours=-1))), + ("delete",), + ( + Path( + "benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_fingerprint_offsets.sql" + ), + ), + ("legacy-run", "offset-run"), + ), + ], + ids=lambda case: case.description, +) +def test_given_fingerprint_history_when_read_in_isolated_pages_then_legacy_retries_definitions_and_time_bounds_survive( + test_case: FingerprintHistoryTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + adapter: Any + connection: Any + adapter, connection = open_legacy_warehouse( + project, test_case.adapter_name, test_case.connection_key + ) + for fixture in test_case.fixture_paths: + _ = adapter.execute(connection=connection, sql=fixture.read_text()) + adapter.close(connection) + + def page(after: list[str | None] | None = None, observed_since: str | None = None) -> Any: + return warehouse_history_page(project, "fingerprints", after, observed_since) + + legacy: dict[str, Any] = page() + assert legacy["next_after"] is None + assert {row["run_id"] for row in legacy["records"]} == set(test_case.expected_legacy_runs) + assert legacy["records"][0]["event_id"] is None + assert legacy["records"][0]["definition"] == "SELECT 1 AS order_id" + assert legacy["records"][0]["metadata_json"] == "{}" + assert page(observed_since="2026-01-01T09:00:00Z") == legacy + assert page(observed_since="2026-01-01T10:00:00.000001Z") == { + "records": [], + "next_after": None, + } + connection = adapter.connect({test_case.connection_key: str(project / "warehouse.db")}) + assert "event_id" not in { + column.name + for column in adapter.get_columns( + connection=connection, database=None, schema="main", name="_sqlbuild_fingerprints" + ) + } + fingerprints: tuple[Fingerprint, ...] = tuple( + Fingerprint( + node_type=("model", "seed")[index % 2], + node_name=("Orders", "orders", "orders ", "ordérs", "o'rders 🧾")[index % 5], + target_database=(None, "")[index % 2], + target_schema="main", + target_name="orders", + run_id=f"external-{index}", + definition_hash="definition", + version_hash=f"version-{index}", + schema_fingerprint="schema", + definition="SELECT 'o''rders 🧾' AS order_note", + metadata_json='{"note":"orders 🧾"}', + ts=test_case.observed_at + timedelta(microseconds=index), + ) + for index in range(104) + ) + for batch in (fingerprints, fingerprints[:1]): + write_fingerprints( + connection=connection, + execute=adapter.execute, + database=None, + schema="main", + fingerprints=batch, + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + render_create_table_sql=adapter.render_create_fingerprint_table_sql, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, + ) + adapter.close(connection) + for journal_mode in test_case.journal_modes: + with closing(sqlite3.connect(project / "warehouse.db")) as predecessor: + assert predecessor.execute("PRAGMA journal_mode=DELETE").fetchone() == (journal_mode,) + pages: list[dict[str, Any]] = [page()] + while pages[-1]["next_after"] is not None: + assert len(pages[-1]["next_after"]) == 13 + pages.append(page(pages[-1]["next_after"])) + assert len(pages) < 4 + assert tuple(len(item["records"]) for item in pages) == ( + 100, + 5 + len(test_case.expected_legacy_runs), + ) + imported: list[dict[str, Any]] = [] + for item in pages: + imported.extend(item["records"]) + assert {row["run_id"] for row in imported} == set(test_case.expected_legacy_runs) | { + f"external-{index}" for index in range(104) + } + assert len({row["event_id"] for row in imported} - {None}) == 104 + assert sum(row["event_id"] is None for row in imported) == len(test_case.expected_legacy_runs) + assert {row["target_database"] for row in imported} == {None, ""} + first: list[dict[str, Any]] = list(filter(lambda row: row["run_id"] == "external-0", imported)) + assert len(first) == 2 and first[0] == first[1] + assert first[0]["definition"] == "SELECT 'o''rders 🧾' AS order_note" + assert first[0]["metadata_json"] == '{"note":"orders 🧾"}' + assert ( + first[0]["event_id"] + == hashlib.sha256( + b'["sqlbuild.fingerprint.event.v1","model","Orders",null,"main","orders","external-0","version-0"]' + ).hexdigest() + ) + assert datetime.fromisoformat(first[0]["ts"]) == datetime(2026, 2, 1, tzinfo=UTC) + observed_since: str = "2026-02-01T01:00:00.000050+01:00" + recent: list[dict[str, Any]] = [page(observed_since=observed_since)] + while recent[-1]["next_after"] is not None: + recent.append(page(recent[-1]["next_after"], observed_since)) + assert len(recent) < 4 + recent_records: list[dict[str, Any]] = [] + for item in recent: + recent_records.extend(item["records"]) + assert len(recent_records) == 54 + assert {row["run_id"] for row in recent_records} == { + f"external-{index}" for index in range(50, 104) + } + empty_since: str = "2026-02-01T00:00:00.000104Z" + empty: dict[str, Any] = page(observed_since=empty_since) + assert empty["records"] == [] and empty["next_after"] is not None + assert page(empty["next_after"], empty_since) == {"records": [], "next_after": None} + for journal_mode in test_case.journal_modes: + with closing(sqlite3.connect(project / "warehouse.db")) as predecessor: + assert predecessor.execute("PRAGMA journal_mode").fetchone() == (journal_mode,) + assert not (project / ".sqlbuild/control.sqlite3").exists() + (project / "warehouse.db").rename(project / "retained-warehouse.db") + assert page() is None + assert not (project / "warehouse.db").exists()