feat(server)!: unify resource options and add per-stream keys (#4615) Stream, topic and user settings followed separate paths. Updating one setting meant resending the name and every other field, nothing could return a setting to its default, and each SDK encoded options its own way per resource and transport. Encryption shared one data key across all streams, and a read could not tell which key a stream used. Route every create and update through one scoped catalog on the existing ResourceOptions map. Each key declares its scopes, kind, storage, mutability, default, visibility and criticality, and one admission path enforces it on every transport. UpdateResource carries only the keys that change and explicit resets, and metadata prepare resolves it into a per-scope internal operation so defaults are replicated, not recomputed. Names and user status stay indexed columns while the catalog validates them. Give every encrypted stream its own data key, supplied by the client or generated by the admitting node, and store it only inside an authenticated envelope wrapped under a key derived from the config key. Reads expose the key source and fingerprints, never key bytes. Rotating the config key replicates rewrapped envelopes and leaves message data untouched, and boot refuses envelopes the configured keys cannot open. Typed options replace raw maps in every SDK, with a raw escape hatch for keys newer than the SDK. Unknown critical state fails closed. Protocol 0.12.0 and snapshot format 13 move together because both layouts change. BREAKING CHANGE: clients and servers must run protocol 0.12.0, data directories need snapshot format 13, and UpdateStream, UpdateTopic and UpdateUser are replaced by UpdateResource.
diff --git a/.dockerignore b/.dockerignore index b4405ac..30d0a2b 100644 --- a/.dockerignore +++ b/.dockerignore
@@ -10,15 +10,21 @@ /helm /licenses /local_data +/local_state /performance_results /test_logs +**/.venv +**/node_modules +**/.gradle /foreign/swift/.build +/foreign/java/**/build /bdd/java/build /examples/java/**/build -/foreign/java/**/build /web/build -**/.gradle -**/node_modules +/foreign/csharp/**/bin +/foreign/csharp/**/obj +/foreign/cpp/bazel-* +/foreign/php/vendor # Selectively exclude scripts/ subdirs not needed in container builds. # scripts/ci/third-party-licenses.sh + render-node-licenses.mjs are # required by the convenience-binary Dockerfiles to render LICENSE-binary
diff --git a/.github/actions/cpp-bazel/pre-merge/action.yml b/.github/actions/cpp-bazel/pre-merge/action.yml index ebfc348..8913da1 100644 --- a/.github/actions/cpp-bazel/pre-merge/action.yml +++ b/.github/actions/cpp-bazel/pre-merge/action.yml
@@ -88,6 +88,8 @@ uses: ./.github/actions/utils/server-start with: bin: target/debug/iggy-server + encryption-enabled: "true" + encryption-key: "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=" - name: Test Rust bridge and C++ coverage if: inputs.task == 'test' @@ -104,6 +106,37 @@ pid-file: ${{ steps.iggy.outputs.pid_file }} log-file: ${{ steps.iggy.outputs.log_file }} + - name: Setup server on the shipped default + id: iggy-default + if: inputs.task == 'test' + uses: ./.github/actions/utils/server-start + with: + bin: target/debug/iggy-server + pid-file: ${{ runner.temp }}/iggy-server-default.pid + log-file: ${{ runner.temp }}/iggy-server-default.log + env: + IGGY_PATH: ${{ runner.temp }}/iggy-cpp-default-data + + # The resource options cases need a server key, so they run on the lane above. + - name: Test C++ e2e on the shipped default + if: inputs.task == 'test' + shell: bash + run: | + cd foreign/cpp + bazel test \ + --config=debug \ + --lockfile_mode=error \ + --strategy=TestRunner=standalone \ + --test_arg=--gtest_filter=-*ResourceOptions* \ + //:e2e + + - name: Stop server after shipped default tests + if: always() && inputs.task == 'test' + uses: ./.github/actions/utils/server-stop + with: + pid-file: ${{ steps.iggy-default.outputs.pid_file }} + log-file: ${{ steps.iggy-default.outputs.log_file }} + - name: Validate task if: inputs.task != 'lint' && inputs.task != 'build' && inputs.task != 'test' shell: bash
diff --git a/.github/actions/go/pre-merge/action.yml b/.github/actions/go/pre-merge/action.yml index ebf6406..a04736c 100644 --- a/.github/actions/go/pre-merge/action.yml +++ b/.github/actions/go/pre-merge/action.yml
@@ -129,6 +129,8 @@ uses: ./.github/actions/utils/server-start with: bin: target/debug/iggy-server + encryption-enabled: "true" + encryption-key: "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=" replica-id: "0" pid-file: ${{ runner.temp }}/iggy-go-e2e.pid log-file: ${{ runner.temp }}/iggy-go-e2e.log @@ -179,12 +181,45 @@ pid-file: ${{ steps.iggy.outputs.pid_file }} log-file: ${{ steps.iggy.outputs.log_file }} + - name: Start Iggy server (shipped default) + id: iggy-default + if: inputs.task == 'e2e' + uses: ./.github/actions/utils/server-start + with: + bin: target/debug/iggy-server + replica-id: "0" + pid-file: ${{ runner.temp }}/iggy-go-e2e-default.pid + log-file: ${{ runner.temp }}/iggy-go-e2e-default.log + wait-timeout-seconds: "90" + env: + IGGY_PATH: ${{ runner.temp }}/iggy-go-e2e-default-data + + # The resource options cases need a server key, so they run on the other lanes. + - name: Run e2e tests (shipped default) + shell: bash + if: inputs.task == 'e2e' + env: + IGGY_TCP_ADDRESS: 127.0.0.1:8090 + run: | + cd foreign/go + go test -v -race ./tests/... \ + -skip '^TestE2E_(StreamKeyProjection|KeylessStream|StreamRename|TopicReset|UserOptions)$' + + - name: Stop Iggy server (shipped default) + if: always() && inputs.task == 'e2e' + uses: ./.github/actions/utils/server-stop + with: + pid-file: ${{ steps.iggy-default.outputs.pid_file }} + log-file: ${{ steps.iggy-default.outputs.log_file }} + - name: Start Iggy server (TLS) id: iggy-tls if: inputs.task == 'e2e' uses: ./.github/actions/utils/server-start with: bin: target/debug/iggy-server + encryption-enabled: "true" + encryption-key: "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=" replica-id: "0" pid-file: ${{ runner.temp }}/iggy-go-e2e-tls.pid log-file: ${{ runner.temp }}/iggy-go-e2e-tls.log @@ -223,6 +258,8 @@ uses: ./.github/actions/utils/server-start with: bin: target/debug/iggy-server + encryption-enabled: "true" + encryption-key: "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=" replica-id: "0" pid-file: ${{ runner.temp }}/iggy-go-cluster-0.pid log-file: ${{ runner.temp }}/iggy-go-cluster-0.log @@ -237,6 +274,8 @@ uses: ./.github/actions/utils/server-start with: bin: target/debug/iggy-server + encryption-enabled: "true" + encryption-key: "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=" replica-id: "1" tcp_address: 127.0.0.1:8091 http_address: 127.0.0.1:3001
diff --git a/.github/actions/java-gradle/pre-merge/action.yml b/.github/actions/java-gradle/pre-merge/action.yml index 35920d0..060c3b4 100644 --- a/.github/actions/java-gradle/pre-merge/action.yml +++ b/.github/actions/java-gradle/pre-merge/action.yml
@@ -86,6 +86,8 @@ uses: ./.github/actions/utils/server-start with: bin: target/debug/iggy-server + encryption-enabled: "true" + encryption-key: "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=" wait-timeout-seconds: "90" env: IGGY_PATH: ${{ runner.temp }}/iggy-java-data @@ -156,6 +158,8 @@ pid-file: ${{ steps.iggy.outputs.pid_file }} log-file: ${{ steps.iggy.outputs.log_file }} + # No encryption key, the shipped default, so this lane also covers a server + # that keeps streams plaintext and refuses client stream keys. - name: Start Iggy server (TLS) id: iggy-tls if: inputs.task == 'test'
diff --git a/.github/actions/node-npm/pre-merge/action.yml b/.github/actions/node-npm/pre-merge/action.yml index 97f0eb8..1275ba8 100644 --- a/.github/actions/node-npm/pre-merge/action.yml +++ b/.github/actions/node-npm/pre-merge/action.yml
@@ -95,6 +95,8 @@ uses: ./.github/actions/utils/server-start with: bin: target/debug/iggy-server + encryption-enabled: "true" + encryption-key: "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=" wait-timeout-seconds: "90" - name: E2E tests @@ -120,12 +122,40 @@ path: ${{ steps.iggy.outputs.log_file }} if-no-files-found: ignore + - name: Start Iggy server (shipped default) + id: iggy-default + if: inputs.task == 'e2e' + uses: ./.github/actions/utils/server-start + with: + bin: target/debug/iggy-server + pid-file: ${{ runner.temp }}/iggy-server-default.pid + log-file: ${{ runner.temp }}/iggy-server-default.log + env: + IGGY_PATH: ${{ runner.temp }}/iggy-node-default-data + + # The resource options suites need a server key, so they run on the lanes above. + - name: E2E tests (shipped default) + if: inputs.task == 'e2e' + run: | + cd foreign/node + node --import @swc-node/register/esm-register --test --test-force-exit \ + $(ls src/e2e/*.e2e.ts | grep -v -e resource-options -e tls.system) + shell: bash + + - name: Stop Iggy server (shipped default) + if: always() && inputs.task == 'e2e' + uses: ./.github/actions/utils/server-stop + with: + pid-file: ${{ steps.iggy-default.outputs.pid_file }} + log-file: ${{ steps.iggy-default.outputs.log_file }} + - name: Start Iggy server (TLS) id: iggy-tls if: inputs.task == 'e2e' uses: ./.github/actions/utils/server-start with: bin: target/debug/iggy-server + encryption-key: "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=" pid-file: ${{ runner.temp }}/iggy-server-tls.pid log-file: ${{ runner.temp }}/iggy-server-tls.log env:
diff --git a/.github/actions/php/pre-merge/action.yml b/.github/actions/php/pre-merge/action.yml index 4f88da3..3b8ba14 100644 --- a/.github/actions/php/pre-merge/action.yml +++ b/.github/actions/php/pre-merge/action.yml
@@ -34,6 +34,7 @@ composer \ libclang-dev \ libhwloc-dev \ + libphp8.5-embed \ libssl-dev \ php8.5-cli \ php8.5-dev \ @@ -102,12 +103,13 @@ exit 1 fi raw_stubs="$(mktemp)" - cargo php stubs "$extension" -o "$raw_stubs" + # cargo-php loads the extension outside PHP, so expose PHP's allocator symbols. + LD_PRELOAD=/usr/lib/libphp8.5.so cargo php stubs "$extension" -o "$raw_stubs" { printf '<?php\n' awk 'NF { print "// " $0; next } { print "//" }' ASF_LICENSE.txt printf '\n' - tail -n +2 "$raw_stubs" + tail -n +2 "$raw_stubs" | sed -E 's/(public readonly \?string \$[A-Za-z_][A-Za-z0-9_]*) = null;/\1;/' } > foreign/php/iggy-php.stubs.php rm -f "$raw_stubs" git diff --exit-code -- foreign/php/iggy-php.stubs.php @@ -165,6 +167,8 @@ uses: ./.github/actions/utils/server-start with: bin: target/debug/iggy-server + encryption-enabled: "true" + encryption-key: "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=" - name: Run PHP SDK tests if: inputs.task == 'test' @@ -185,6 +189,8 @@ pid-file: ${{ steps.iggy.outputs.pid_file }} log-file: ${{ steps.iggy.outputs.log_file }} + # No encryption key, the shipped default, so this lane also covers a server + # that keeps streams plaintext and refuses client stream keys. - name: Start Iggy server (TLS) if: inputs.task == 'test' id: iggy-tls @@ -194,6 +200,7 @@ pid-file: ${{ runner.temp }}/iggy-server-tls.pid log-file: ${{ runner.temp }}/iggy-server-tls.log env: + IGGY_PATH: ${{ runner.temp }}/iggy-php-tls-data IGGY_TCP_TLS_ENABLED: "true" IGGY_TCP_TLS_CERT_FILE: core/certs/iggy_cert.pem IGGY_TCP_TLS_KEY_FILE: core/certs/iggy_key.pem
diff --git a/.github/actions/python-maturin/pre-merge/action.yml b/.github/actions/python-maturin/pre-merge/action.yml index 80ca908..f71d29a 100644 --- a/.github/actions/python-maturin/pre-merge/action.yml +++ b/.github/actions/python-maturin/pre-merge/action.yml
@@ -54,6 +54,8 @@ uses: ./.github/actions/utils/server-start with: bin: target/debug/iggy-server + encryption-enabled: "true" + encryption-key: "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=" - name: Setup Python uses: actions/setup-python@v6 @@ -214,6 +216,45 @@ pid-file: ${{ steps.iggy.outputs.pid_file }} log-file: ${{ steps.iggy.outputs.log_file }} + - name: Start Iggy server (shipped default) + if: inputs.task == 'test' + id: iggy-default + uses: ./.github/actions/utils/server-start + with: + bin: target/debug/iggy-server + pid-file: ${{ runner.temp }}/iggy-server-default.pid + log-file: ${{ runner.temp }}/iggy-server-default.log + env: + IGGY_PATH: ${{ runner.temp }}/iggy-python-default-data + + # The resource options cases need a server key, and the TLS cases start + # their own server, so both run on the lane above. + - name: Run Python integration tests (shipped default) + if: inputs.task == 'test' + run: | + cd foreign/python + tcp_port="${{ steps.iggy-default.outputs.tcp_address }}" + http_port="${{ steps.iggy-default.outputs.http_address }}" + IGGY_SERVER_HOST=127.0.0.1 \ + IGGY_SERVER_TCP_PORT="${tcp_port##*:}" \ + IGGY_SERVER_QUIC_PORT=8080 \ + IGGY_SERVER_HTTP_PORT="${http_port##*:}" \ + IGGY_SERVER_WS_PORT=8092 \ + uv run --no-sync pytest tests/ -v \ + --ignore=tests/test_resource_options.py \ + --ignore=tests/test_tls.py \ + --junitxml=../../reports/python-junit-default.xml \ + --tb=short \ + --capture=no + shell: bash + + - name: Stop Iggy server (shipped default) + if: always() && inputs.task == 'test' + uses: ./.github/actions/utils/server-stop + with: + pid-file: ${{ steps.iggy-default.outputs.pid_file }} + log-file: ${{ steps.iggy-default.outputs.log_file }} + - name: Generate coverage report if: inputs.task == 'test' run: |
diff --git a/.github/actions/swift/pre-merge/action.yml b/.github/actions/swift/pre-merge/action.yml index 077f49e..d06030f 100644 --- a/.github/actions/swift/pre-merge/action.yml +++ b/.github/actions/swift/pre-merge/action.yml
@@ -124,6 +124,8 @@ uses: ./.github/actions/utils/server-start with: bin: target/debug/iggy-server + encryption-enabled: "true" + encryption-key: "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=" replica-id: "0" pid-file: ${{ runner.temp }}/iggy-swift-e2e.pid log-file: ${{ runner.temp }}/iggy-swift-e2e.log @@ -150,12 +152,46 @@ pid-file: ${{ steps.iggy.outputs.pid_file }} log-file: ${{ steps.iggy.outputs.log_file }} + - name: Start Iggy server (shipped default) + id: iggy-default + if: inputs.task == 'e2e' + uses: ./.github/actions/utils/server-start + with: + bin: target/debug/iggy-server + replica-id: "0" + pid-file: ${{ runner.temp }}/iggy-swift-e2e-default.pid + log-file: ${{ runner.temp }}/iggy-swift-e2e-default.log + wait-timeout-seconds: "90" + env: + IGGY_PATH: ${{ runner.temp }}/iggy-swift-e2e-default-data + + # The resource options cases need a server key, so they run on the other lanes. + - name: Run e2e tests (shipped default) + shell: bash + if: inputs.task == 'e2e' + env: + IGGY_TCP_ADDRESS: 127.0.0.1:8090 + IGGY_ROOT_USERNAME: iggy + IGGY_ROOT_PASSWORD: iggy + run: | + cd foreign/swift + swift test --skip-build --filter IggyE2ETests --skip ResourceOptionsE2ETests + + - name: Stop Iggy server (shipped default) + if: always() && inputs.task == 'e2e' + uses: ./.github/actions/utils/server-stop + with: + pid-file: ${{ steps.iggy-default.outputs.pid_file }} + log-file: ${{ steps.iggy-default.outputs.log_file }} + - name: Start Iggy server (TLS) id: iggy-tls if: inputs.task == 'e2e' uses: ./.github/actions/utils/server-start with: bin: target/debug/iggy-server + encryption-enabled: "true" + encryption-key: "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=" replica-id: "0" pid-file: ${{ runner.temp }}/iggy-swift-e2e-tls.pid log-file: ${{ runner.temp }}/iggy-swift-e2e-tls.log @@ -197,6 +233,8 @@ uses: ./.github/actions/utils/server-start with: bin: target/debug/iggy-server + encryption-enabled: "true" + encryption-key: "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=" replica-id: "0" pid-file: ${{ runner.temp }}/iggy-swift-cluster-0.pid log-file: ${{ runner.temp }}/iggy-swift-cluster-0.log @@ -211,6 +249,8 @@ uses: ./.github/actions/utils/server-start with: bin: target/debug/iggy-server + encryption-enabled: "true" + encryption-key: "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=" replica-id: "1" tcp_address: 127.0.0.1:8091 http_address: 127.0.0.1:3001
diff --git a/.github/actions/utils/server-start/action.yml b/.github/actions/utils/server-start/action.yml index 31e6bfe..9638ed1 100644 --- a/.github/actions/utils/server-start/action.yml +++ b/.github/actions/utils/server-start/action.yml
@@ -53,6 +53,14 @@ description: "--replica-id <u8> value (leave empty to omit)" required: false default: "" + encryption-enabled: + description: "Give streams created without their own key a generated key, which the SDK server-generated key tests need" + required: false + default: "false" + encryption-key: + description: "Server encryption key (base64 of 32 bytes). Empty, the shipped default, starts without a key, so streams stay plaintext and client stream keys are refused. Lanes that test stream keys pass the fixed key shared with the BDD server." + required: false + default: "" outputs: pid: description: "PID of the background server process" @@ -116,6 +124,13 @@ export IGGY_HTTP_ADDRESS="${{ inputs.http_address }}" export IGGY_ROOT_USERNAME="iggy" export IGGY_ROOT_PASSWORD="iggy" + if [[ -n "${{ inputs.encryption-key }}" ]]; then + export IGGY_ENCRYPTION_KEY="${{ inputs.encryption-key }}" + fi + if [[ "${{ inputs.encryption-enabled }}" == "true" ]]; then + [[ -n "${{ inputs.encryption-key }}" ]] || { echo "::error::encryption-enabled requires encryption-key"; exit 1; } + export IGGY_ENCRYPTION_ENABLED="true" + fi CLI_ARGS=() if [[ -n "${{ inputs.replica-id }}" ]]; then CLI_ARGS+=(--replica-id "${{ inputs.replica-id }}") @@ -124,7 +139,7 @@ nohup "${{ inputs.bin }}" "${CLI_ARGS[@]}" >"$LOG_FILE" 2>&1 & echo $! > "$PID_FILE" echo "Started server PID $(cat "$PID_FILE") → logs: $LOG_FILE" - echo "Server configured with TCP=$IGGY_TCP_ADDRESS, HTTP=$IGGY_HTTP_ADDRESS" + echo "Server configured with TCP=$IGGY_TCP_ADDRESS, HTTP=$IGGY_HTTP_ADDRESS, encryption=${IGGY_ENCRYPTION_ENABLED:-false}" - name: Wait for readiness shell: bash
diff --git a/.github/workflows/coverage-baseline.yml b/.github/workflows/coverage-baseline.yml index aada7fa..47fa67e 100644 --- a/.github/workflows/coverage-baseline.yml +++ b/.github/workflows/coverage-baseline.yml
@@ -148,6 +148,8 @@ uses: ./.github/actions/utils/server-start with: bin: target/debug/iggy-server + encryption-enabled: "true" + encryption-key: "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=" - name: Run tests and generate coverage working-directory: foreign/java @@ -298,6 +300,8 @@ uses: ./.github/actions/utils/server-start with: bin: target/debug/iggy-server + encryption-enabled: "true" + encryption-key: "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=" - name: Setup Python uses: actions/setup-python@v7.0.0 @@ -447,6 +451,8 @@ uses: ./.github/actions/utils/server-start with: bin: target/debug/iggy-server + encryption-enabled: "true" + encryption-key: "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=" wait-timeout-seconds: "90" - name: Run unit tests with coverage @@ -499,6 +505,8 @@ uses: ./.github/actions/utils/server-start with: bin: target/debug/iggy-server + encryption-enabled: "true" + encryption-key: "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=" replica-id: "0" wait-timeout-seconds: "90"
diff --git a/.gitignore b/.gitignore index 2b6bf9c..2dfad02 100644 --- a/.gitignore +++ b/.gitignore
@@ -48,6 +48,8 @@ *.a *.lib *.out +*.pid +__pycache__/ go.work core/bench/dashboard/frontend/dist core/integration/tests/connectors/postgres/cdc_restart_connectors/*
diff --git a/.typos.toml b/.typos.toml index 64448b2..4203157 100644 --- a/.typos.toml +++ b/.typos.toml
@@ -162,5 +162,7 @@ "**/wiremock/__files/*.json", # Swift SDK golden vectors dumped from the Rust crates (hex payloads) "foreign/swift/Tests/IggyTests/Fixtures/golden.json", + # Resource options fixture copies in every SDK test tree (hex payloads) + "**/resource_options.json", "**/.python-version", ]
diff --git a/Cargo.lock b/Cargo.lock index ff5a829..b2155b6 100644 --- a/Cargo.lock +++ b/Cargo.lock
@@ -250,6 +250,15 @@ ] [[package]] +name = "aead-stream" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a901e5bcd15b4a9555a8f17a608d28ef92cf77bc4aa3b4a0c1a3fce1a9cc0bac" +dependencies = [ + "aead 0.6.1", +] + +[[package]] name = "aes" version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -269,6 +278,7 @@ "cipher 0.5.2", "cpubits", "cpufeatures 0.3.1", + "zeroize", ] [[package]] @@ -297,6 +307,22 @@ "ctr 0.10.1", "ctutils", "ghash 0.6.0", + "zeroize", +] + +[[package]] +name = "aes-gcm-siv" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c32325128cb24de2a2d14d92da7569dae7073f9e47334521d494747fa94d902" +dependencies = [ + "aead 0.6.1", + "aes 0.9.3", + "cipher 0.5.2", + "ctr 0.10.1", + "ctutils", + "polyval 0.7.3", + "zeroize", ] [[package]] @@ -1431,6 +1457,10 @@ "futures-util", "log", "pin-project-lite", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", "tungstenite 0.29.0", ] @@ -3203,7 +3233,7 @@ "compio-log", "compio-send-wrapper", "crossbeam-queue", - "flume 0.12.0", + "flume", "futures-util", "io-uring", "libc", @@ -3324,7 +3354,7 @@ "compio-log", "compio-net", "compio-runtime", - "flume 0.12.0", + "flume", "futures-util", "libc", "quinn-proto", @@ -5331,17 +5361,6 @@ [[package]] name = "flume" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" -dependencies = [ - "futures-core", - "futures-sink", - "spin", -] - -[[package]] -name = "flume" version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e139bc46ca777eb5efaf62df0ab8cc5fd400866427e56c68b22e414e53bd3be" @@ -5759,6 +5778,7 @@ checksum = "2eecf2d5dc9b66b732b97707a0210906b1d30523eb773193ab777c0c84b3e8d5" dependencies = [ "polyval 0.7.3", + "zeroize", ] [[package]] @@ -7069,7 +7089,7 @@ [[package]] name = "iggy" -version = "0.11.1-edge.1" +version = "0.11.1-edge.2" dependencies = [ "async-broadcast", "async-dropper", @@ -7078,7 +7098,7 @@ "bytemuck", "bytes", "dashmap", - "flume 0.12.0", + "flume", "futures", "futures-util", "iggy_binary_protocol", @@ -7092,6 +7112,7 @@ "reqwest-middleware", "reqwest-retry", "reqwest-tracing", + "ring", "rustls", "secrecy", "serde", @@ -7106,7 +7127,7 @@ [[package]] name = "iggy-bench" -version = "0.6.1-edge.1" +version = "0.6.1-edge.2" dependencies = [ "async-trait", "bench-report", @@ -7164,7 +7185,7 @@ [[package]] name = "iggy-cli" -version = "0.14.1-edge.1" +version = "0.14.1-edge.2" dependencies = [ "anyhow", "apple-native-keyring-store", @@ -7198,7 +7219,7 @@ [[package]] name = "iggy-connectors" -version = "0.5.1-edge.1" +version = "0.5.1-edge.2" dependencies = [ "apache-avro 0.22.0", "async-trait", @@ -7215,7 +7236,7 @@ "dotenvy", "figlet-rs", "figment", - "flume 0.12.0", + "flume", "futures", "iggy", "iggy_common", @@ -7298,7 +7319,7 @@ [[package]] name = "iggy-mcp" -version = "0.5.1-edge.1" +version = "0.5.1-edge.2" dependencies = [ "axum", "axum-server", @@ -7333,12 +7354,13 @@ [[package]] name = "iggy_binary_protocol" -version = "0.11.1-edge.1" +version = "0.11.1-edge.2" dependencies = [ "aligned-vec", "bytemuck", "bytes", "enumset", + "ring", "secrecy", "serde", "thiserror 2.0.21", @@ -7347,9 +7369,10 @@ [[package]] name = "iggy_common" -version = "0.11.1-edge.1" +version = "0.11.1-edge.2" dependencies = [ "aes-gcm 0.11.1", + "aes-gcm-siv", "async-broadcast", "async-trait", "base64 0.23.1", @@ -7361,6 +7384,7 @@ "clap", "comfy-table", "derive_more", + "hex", "humantime", "iggy_binary_protocol", "papaya", @@ -7378,6 +7402,7 @@ "ulid", "url", "uuid", + "xaes-256-gcm", ] [[package]] @@ -7666,7 +7691,8 @@ "humantime", "iggy_common", "iggy_connector_sdk", - "rumqttc", + "rumqttc-v4-next", + "rumqttc-v5-next", "rustls-native-certs", "rustls-pemfile", "secrecy", @@ -8120,7 +8146,8 @@ "reqwest-retry", "ring", "rmcp", - "rumqttc", + "rumqttc-v4-next", + "rumqttc-v5-next", "rust-s3", "secrecy", "serde", @@ -8532,7 +8559,7 @@ "backon", "cfg-if", "event-listener", - "flume 0.12.0", + "flume", "futures-core", "futures-io", "tracing", @@ -9410,6 +9437,16 @@ ] [[package]] +name = "mqttbytes-core-next" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "366b6ba2b4209ca4bc5ac731ccddf570d09831981eed07e5fbd63564cf0cf1aa" +dependencies = [ + "bytes", + "thiserror 2.0.21", +] + +[[package]] name = "multiversion_no_op" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -10872,6 +10909,7 @@ "cpubits", "cpufeatures 0.3.1", "universal-hash 0.6.1", + "zeroize", ] [[package]] @@ -12138,23 +12176,66 @@ ] [[package]] -name = "rumqttc" -version = "0.25.1" +name = "rumqttc-core-next" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0feff8d882bff0b2fddaf99355a10336d43dd3ed44204f85ece28cf9626ab519" +checksum = "249896ab27ed630590971738264baa8f722f18965d2e387c706c40a3c2a572cc" dependencies = [ - "bytes", - "fixedbitset", - "flume 0.11.1", - "futures-util", - "log", + "async-tungstenite", "rustls-native-certs", - "rustls-pemfile", - "rustls-webpki 0.102.8", + "rustls-pki-types", + "rustls-webpki", "thiserror 2.0.21", "tokio", "tokio-rustls", - "tokio-stream", +] + +[[package]] +name = "rumqttc-v4-next" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3767dde22a804ea806107d0f4331e33cb794d1b8468b11c86fb0c1366f07054" +dependencies = [ + "async-tungstenite", + "bytes", + "fixedbitset", + "flume", + "futures-util", + "log", + "mqttbytes-core-next", + "percent-encoding", + "rumqttc-core-next", + "rustls-native-certs", + "rustls-pki-types", + "rustls-webpki", + "thiserror 2.0.21", + "tokio", + "tokio-rustls", + "tokio-util", + "url", +] + +[[package]] +name = "rumqttc-v5-next" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dfa6ddcc7a7dd5688f9bf78d8f81cb94f367bce56c055d8d94cf81ecb0518bf" +dependencies = [ + "async-tungstenite", + "bytes", + "fixedbitset", + "flume", + "futures-util", + "log", + "mqttbytes-core-next", + "percent-encoding", + "rumqttc-core-next", + "rustls-native-certs", + "rustls-pki-types", + "rustls-webpki", + "thiserror 2.0.21", + "tokio", + "tokio-rustls", "tokio-util", "url", ] @@ -12328,7 +12409,7 @@ "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.103.15", + "rustls-webpki", "subtle", "zeroize", ] @@ -12345,7 +12426,7 @@ "rustls", "rustls-pki-types", "rustls-platform-verifier", - "rustls-webpki 0.103.15", + "rustls-webpki", ] [[package]] @@ -12393,7 +12474,7 @@ "rustls", "rustls-native-certs", "rustls-platform-verifier-android", - "rustls-webpki 0.103.15", + "rustls-webpki", "security-framework", "security-framework-sys", "webpki-root-certs", @@ -12408,17 +12489,6 @@ [[package]] name = "rustls-webpki" -version = "0.102.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" -dependencies = [ - "ring", - "rustls-pki-types", - "untrusted 0.9.0", -] - -[[package]] -name = "rustls-webpki" version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" @@ -12912,7 +12982,7 @@ [[package]] name = "server" -version = "0.9.1-edge.1" +version = "0.9.1-edge.2" dependencies = [ "ahash", "arc-swap", @@ -13607,7 +13677,7 @@ dependencies = [ "atoi", "chrono", - "flume 0.12.0", + "flume", "form_urlencoded", "futures-channel", "futures-core", @@ -16204,6 +16274,19 @@ ] [[package]] +name = "xaes-256-gcm" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc0e35bc669c7ce781caf52f51a1e672237825014c7a8cbb1292e194800d97fe" +dependencies = [ + "aead 0.6.1", + "aead-stream", + "aes 0.9.3", + "aes-gcm 0.11.1", + "cipher 0.5.2", +] + +[[package]] name = "xattr" version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index"
diff --git a/Cargo.toml b/Cargo.toml index 96a1a62..d93a388 100644 --- a/Cargo.toml +++ b/Cargo.toml
@@ -88,7 +88,8 @@ actix-cors = "0.7.2" actix-files = "0.7.0" actix-web = "4.16.0" -aes-gcm = "0.11.1" +aes-gcm = { version = "0.11.1", features = ["zeroize"] } +aes-gcm-siv = { version = "0.12.1", features = ["zeroize"] } ahash = { version = "0.8.12", features = ["serde"] } aligned-vec = "0.6.4" anyhow = "1.0.104" @@ -212,11 +213,11 @@ iceberg = "0.10.1" iceberg-catalog-rest = "0.10.1" iceberg-storage-opendal = "0.10.1" -iggy = { path = "core/sdk", version = "0.11.1-edge.1" } -iggy-cli = { path = "core/cli", version = "0.14.1-edge.1" } +iggy = { path = "core/sdk", version = "0.11.1-edge.2" } +iggy-cli = { path = "core/cli", version = "0.14.1-edge.2" } iggy-gateway-kafka = { path = "gateways/kafka" } -iggy_binary_protocol = { path = "core/binary_protocol", version = "0.11.1-edge.1" } -iggy_common = { path = "core/common", version = "0.11.1-edge.1" } +iggy_binary_protocol = { path = "core/binary_protocol", version = "0.11.1-edge.2" } +iggy_common = { path = "core/common", version = "0.11.1-edge.2" } iggy_connector_doris_sink = { path = "core/connectors/sinks/doris_sink" } iggy_connector_sdk = { path = "core/connectors/sdk", version = "0.5.1-edge.2" } indexmap = "2.14.2" @@ -304,7 +305,8 @@ rmp = "0.8.15" rmp-serde = "1.3.1" rolling-file = "0.2.0" -rumqttc = { version = "0.25.1", features = ["url"] } +rumqttc-v4 = { package = "rumqttc-v4-next", version = "0.34.0", features = ["url"] } +rumqttc-v5 = { package = "rumqttc-v5-next", version = "0.34.0", features = ["url"] } rust-embed = "8.13.0" rust-s3 = { version = "0.38.0", default-features = false, features = ["tokio-rustls-tls", "tags"] } rustls = { version = "0.23.46", features = ["ring"] } @@ -396,6 +398,7 @@ windows-native-keyring-store = "1.1.0" wiremock = "0.6" x509-parser = "0.18.1" +xaes-256-gcm = "0.1.0" yew = { version = "0.23", features = ["csr"] } yew-router = "0.20" zbus-secret-service-keyring-store = { version = "1.0.1", features = ["rt-async-io-crypto-rust"] }
diff --git a/README.md b/README.md index 07990be..a837c58 100644 --- a/README.md +++ b/README.md
@@ -104,7 +104,7 @@ - **TLS** support for all transport protocols (TCP, WebSocket, QUIC, HTTPS) - **[Connectors](https://github.com/apache/iggy/tree/master/core/connectors)** - sinks, sources and data transformations based on the **custom Rust plugins** - **[Model Context Protocol](https://github.com/apache/iggy/tree/master/core/ai/mcp)** - provide context to LLM with **MCP server** -- Optional server-side as well as client-side **data encryption** using AES-256-GCM +- Optional server-side as well as client-side **data encryption** using AES-256-GCM, server-side with a separate key per stream - Optional metadata support in the form of **message headers** - Server-side message compression is not supported yet. Topic compression values are reserved for future disk/network compression support; use message headers @@ -146,6 +146,8 @@ We also publish edge releases (versions with an `-edge.N` suffix, and `apache/iggy:edge`) for the SDKs and the Docker images. They track the latest changes, but are not guaranteed to be stable and are not recommended for production use. +This checkout moves the binary protocol to 0.12.0 and the metadata snapshot format to 13. Both are breaking: older clients and servers do not interoperate with it, and data directories written by older releases are refused. See [Upgrade recovery](core/server/README.md#upgrade-recovery). + --- ## Supported languages SDK @@ -294,6 +296,8 @@ The CLI exposes `--durability persisted` and `--consumer-offset-durability persisted` on `topic create`. Select either independently. The policy names describe completion guarantees and do not prescribe an I/O syscall. +Streams, topics and users carry options from one catalog, listed by `iggy options <scope>` and changed by one sparse update. Each encrypted stream has its own data key, wrapped under the server's `[encryption] key`, and a stream created without encryption stays plaintext for life. Rotating the server key is a two-phase change through `previous_keys` and `retired_kek_fingerprints`. See [Resource options](core/server/README.md#resource-options) and [Stream encryption](core/server/README.md#stream-encryption). + For the detailed documentation of the configuration file, please refer to the [configuration](https://iggy.apache.org/docs/server/configuration) section. ---
diff --git a/bdd/cpp/features/step_definitions/messaging_steps.cpp b/bdd/cpp/features/step_definitions/messaging_steps.cpp index ed89444..cfe5599 100644 --- a/bdd/cpp/features/step_definitions/messaging_steps.cpp +++ b/bdd/cpp/features/step_definitions/messaging_steps.cpp
@@ -46,7 +46,7 @@ REGEX_PARAM(std::string, name); cucumber::ScenarioScope<bdd::GlobalContext> context; - context->client->create_stream(name, error_info); + context->client->create_stream(name, {}, error_info); } THEN("^the stream should be created successfully$") {
diff --git a/bdd/cpp/features/step_definitions/resource_options_steps.cpp b/bdd/cpp/features/step_definitions/resource_options_steps.cpp new file mode 100644 index 0000000..598fab9 --- /dev/null +++ b/bdd/cpp/features/step_definitions/resource_options_steps.cpp
@@ -0,0 +1,475 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Unique step-class prefix so the generated CukeObject<n> symbols do not clash with the other +// step files when linked together (see background_steps.cpp for details). +#define CUKE_OBJECT_PREFIX IggyBddResourceOptions + +#include <gtest/gtest.h> + +#include <cucumber-cpp/autodetect.hpp> + +#include <cstddef> +#include <cstdint> +#include <limits> +#include <optional> +#include <stdexcept> +#include <string> +#include <utility> +#include <vector> + +#include "world.hpp" + +namespace { + +constexpr std::size_t kKeyLength = 32; +constexpr std::uint32_t kPartitionId = 0; +constexpr std::size_t kFingerprintHex = 16; + +constexpr iggy::StreamOptionKey kStreamOptionKeys[] = { + iggy::StreamOptionKey::Name, + iggy::StreamOptionKey::EncryptionKey, + iggy::StreamOptionKey::EncryptionKeySource, + iggy::StreamOptionKey::EncryptionKeyFingerprint, + iggy::StreamOptionKey::EncryptionKekFingerprint, +}; + +constexpr iggy::TopicOptionKey kTopicOptionKeys[] = { + iggy::TopicOptionKey::Name, + iggy::TopicOptionKey::CompressionAlgorithm, + iggy::TopicOptionKey::MessageExpiry, + iggy::TopicOptionKey::MaxTopicSize, + iggy::TopicOptionKey::SegmentSize, + iggy::TopicOptionKey::Durability, + iggy::TopicOptionKey::ConsumerOffsetDurability, + iggy::TopicOptionKey::PartitionResizePolicy, + iggy::TopicOptionKey::MessagesRequiredToSave, + iggy::TopicOptionKey::SizeOfMessagesRequiredToSave, + iggy::TopicOptionKey::PreallocateSegments, +}; + +iggy::IggyBlockingClient &Sdk(bdd::GlobalContext &context) { + if (!context.sdk) { + auto client = + iggy::IggyBlockingClient::Builder() + .WithServerAddress(bdd::required_env("IGGY_TCP_ADDRESS")) + .WithAutoLogin(bdd::required_env("IGGY_ROOT_USERNAME"), bdd::required_env("IGGY_ROOT_PASSWORD")) + .Build(); + client.Connect(); + context.sdk.emplace(std::move(client)); + } + return *context.sdk; +} + +template <typename Key, std::size_t Count> +Key KeyFromName(const Key (&keys)[Count], const std::string &name) { + for (const auto key : keys) { + if (iggy::to_string(key) == name) { + return key; + } + } + throw std::invalid_argument("Unknown option key " + name); +} + +iggy::UserStatus UserStatusFromName(const std::string &name) { + if (name == "active") { + return iggy::UserStatus::Active; + } + if (name == "inactive") { + return iggy::UserStatus::Inactive; + } + throw std::invalid_argument("Unknown user status " + name); +} + +iggy::Identifier StreamIdentifier(const bdd::GlobalContext &context) { + return iggy::Identifier::Numeric(context.last_stream_id.value()); +} + +iggy::Identifier TopicIdentifier(const bdd::GlobalContext &context) { + return iggy::Identifier::Numeric(context.last_topic_id.value()); +} + +iggy::StreamDetails FetchStream(bdd::GlobalContext &context) { + return Sdk(context).GetStream(StreamIdentifier(context)); +} + +iggy::TopicDetails FetchTopic(bdd::GlobalContext &context) { + return Sdk(context).GetTopic(StreamIdentifier(context), TopicIdentifier(context)); +} + +iggy::UserInfoDetails FetchUser(bdd::GlobalContext &context) { + return Sdk(context).GetUser(iggy::Identifier::Numeric(context.last_user_id.value())); +} + +void CreateStream(bdd::GlobalContext &context, + const std::string &name, + const std::optional<std::vector<std::uint8_t>> &key) { + iggy::StreamCreateOptions options; + if (key) { + options.SetEncryptionKey(*key); + } + const auto stream = Sdk(context).CreateStream(name, options); + context.last_stream_id = stream.Id(); + context.last_stream_key = key; +} + +void CreateTopic(bdd::GlobalContext &context, const std::string &name, iggy::TopicCreateOptions options) { + const auto topic = Sdk(context).CreateTopic(StreamIdentifier(context), name, options); + context.last_topic_id = topic.Id(); +} + +iggy::ffi::HeaderField StringField(const std::string &text) { + iggy::ffi::HeaderField field; + field.kind = static_cast<std::uint8_t>(iggy::ffi::HeaderKind::String); + field.value = bdd::to_payload(text); + return field; +} + +void SendMessage(bdd::GlobalContext &context, const std::string &payload, rust::Vec<iggy::ffi::HeaderEntry> headers) { + iggy::ffi::ErrorInfo error_info{}; + rust::Vec<iggy::ffi::IggyMessageToSend> messages; + messages.push_back(iggy::ffi::make_message(bdd::to_payload(payload), std::move(headers))); + context.client->send_messages(bdd::make_numeric_identifier(context.last_stream_id.value()), + bdd::make_numeric_identifier(context.last_topic_id.value()), "partition_id", + bdd::partition_id_bytes(kPartitionId), std::move(messages), error_info); +} + +iggy::ffi::IggyMessagePolled PollFirst(bdd::GlobalContext &context) { + iggy::ffi::ErrorInfo error_info{}; + auto polled = + context.client->poll_messages(bdd::make_numeric_identifier(context.last_stream_id.value()), + bdd::make_numeric_identifier(context.last_topic_id.value()), kPartitionId, + "consumer", bdd::make_numeric_identifier(1), "offset", 0, 1, false, error_info); + if (polled.messages.empty()) { + throw std::runtime_error("No message was polled"); + } + return std::move(polled.messages[0]); +} + +const iggy::OptionSpec &StreamOptionSpec(const bdd::GlobalContext &context, const std::string &key) { + for (const auto &spec : context.last_option_specs) { + if (spec.Key() == key) { + return spec; + } + } + throw std::runtime_error("Stream option " + key + " was not described"); +} + +void UpdateTopic(bdd::GlobalContext &context, const iggy::TopicUpdateOptions &options) { + Sdk(context).UpdateTopic(StreamIdentifier(context), TopicIdentifier(context), options); +} + +} // namespace + +WHEN("^I create a stream named \"([^\"]+)\" with an encryption key of 32 bytes valued ([0-9]+)$") { + REGEX_PARAM(std::string, name); + REGEX_PARAM(int, byte); + cucumber::ScenarioScope<bdd::GlobalContext> context; + + CreateStream(*context, name, std::vector<std::uint8_t>(kKeyLength, static_cast<std::uint8_t>(byte))); +} + +GIVEN("^a stream named \"([^\"]+)\" with an encryption key of 32 bytes valued ([0-9]+) exists$") { + REGEX_PARAM(std::string, name); + REGEX_PARAM(int, byte); + cucumber::ScenarioScope<bdd::GlobalContext> context; + + CreateStream(*context, name, std::vector<std::uint8_t>(kKeyLength, static_cast<std::uint8_t>(byte))); +} + +WHEN("^I create a stream named \"([^\"]+)\" without an encryption key$") { + REGEX_PARAM(std::string, name); + cucumber::ScenarioScope<bdd::GlobalContext> context; + + CreateStream(*context, name, std::nullopt); +} + +GIVEN("^a stream named \"([^\"]+)\" exists$") { + REGEX_PARAM(std::string, name); + cucumber::ScenarioScope<bdd::GlobalContext> context; + + CreateStream(*context, name, std::nullopt); +} + +THEN("^the stream should report encryption key source \"([^\"]+)\"$") { + REGEX_PARAM(std::string, source); + cucumber::ScenarioScope<bdd::GlobalContext> context; + + const auto stream = FetchStream(*context); + ASSERT_TRUE(stream.EncryptionKeySource().has_value()); + EXPECT_EQ(iggy::to_string(*stream.EncryptionKeySource()), source); +} + +THEN("^the stream encryption key fingerprint should match the key$") { + cucumber::ScenarioScope<bdd::GlobalContext> context; + + ASSERT_TRUE(context->last_stream_key.has_value()); + const auto stream = FetchStream(*context); + EXPECT_EQ(stream.EncryptionKeyFingerprint(), iggy::EncryptionKeyFingerprint(*context->last_stream_key)); +} + +THEN("^the stream should report an encryption key fingerprint$") { + cucumber::ScenarioScope<bdd::GlobalContext> context; + + const auto stream = FetchStream(*context); + ASSERT_TRUE(stream.EncryptionKeyFingerprint().has_value()); + EXPECT_EQ(stream.EncryptionKeyFingerprint()->size(), kFingerprintHex); +} + +THEN("^the stream should report at least one KEK fingerprint$") { + cucumber::ScenarioScope<bdd::GlobalContext> context; + + EXPECT_FALSE(FetchStream(*context).EncryptionKekFingerprints().empty()); +} + +THEN("^the stream should expose no encryption key material$") { + cucumber::ScenarioScope<bdd::GlobalContext> context; + + const auto stream = FetchStream(*context); + for (const auto *secret : {"encryption_key", "encryption_key_wrapped"}) { + EXPECT_EQ(stream.Options().Explicit().count(secret), 0u) << secret << " must never be returned"; + EXPECT_EQ(stream.Options().Derived().count(secret), 0u) << secret << " must never be returned"; + } +} + +WHEN("^I create a topic named \"([^\"]+)\" with ([0-9]+) partitions? in the stream$") { + REGEX_PARAM(std::string, name); + REGEX_PARAM(int, partitions_count); + cucumber::ScenarioScope<bdd::GlobalContext> context; + + CreateTopic(*context, name, + iggy::TopicCreateOptions() + .SetPartitionsCount(static_cast<std::uint32_t>(partitions_count)) + .SetMessageExpiry(iggy::Expiry::NeverExpire())); +} + +WHEN("^I send a message with payload \"([^\"]+)\" to the topic$") { + REGEX_PARAM(std::string, payload); + cucumber::ScenarioScope<bdd::GlobalContext> context; + + SendMessage(*context, payload, rust::Vec<iggy::ffi::HeaderEntry>()); +} + +WHEN("^I send a message with payload \"([^\"]+)\" and header \"([^\"]+)\" set to \"([^\"]+)\" to the topic$") { + REGEX_PARAM(std::string, payload); + REGEX_PARAM(std::string, header); + REGEX_PARAM(std::string, value); + cucumber::ScenarioScope<bdd::GlobalContext> context; + + rust::Vec<iggy::ffi::HeaderEntry> headers; + iggy::ffi::HeaderEntry entry; + entry.key = StringField(header); + entry.value = StringField(value); + headers.push_back(std::move(entry)); + SendMessage(*context, payload, std::move(headers)); +} + +THEN("^polling the topic should return the payload \"([^\"]+)\"$") { + REGEX_PARAM(std::string, payload); + cucumber::ScenarioScope<bdd::GlobalContext> context; + + EXPECT_EQ(bdd::payload_to_string(PollFirst(*context).payload), payload); +} + +THEN("^polling the topic should return the payload \"([^\"]+)\" and header \"([^\"]+)\" set to \"([^\"]+)\"$") { + REGEX_PARAM(std::string, payload); + REGEX_PARAM(std::string, header); + REGEX_PARAM(std::string, value); + cucumber::ScenarioScope<bdd::GlobalContext> context; + + const auto message = PollFirst(*context); + EXPECT_EQ(bdd::payload_to_string(message.payload), payload); + bool found = false; + for (const auto &entry : message.user_headers) { + if (entry.key.kind == static_cast<std::uint8_t>(iggy::ffi::HeaderKind::String) && + bdd::payload_to_string(entry.key.value) == header) { + found = true; + EXPECT_EQ(entry.value.kind, static_cast<std::uint8_t>(iggy::ffi::HeaderKind::String)); + EXPECT_EQ(bdd::payload_to_string(entry.value.value), value); + } + } + EXPECT_TRUE(found) << "header " << header << " is missing"; +} + +WHEN("^I rename the stream to \"([^\"]+)\"$") { + REGEX_PARAM(std::string, name); + cucumber::ScenarioScope<bdd::GlobalContext> context; + + Sdk(*context).UpdateStream(StreamIdentifier(*context), iggy::StreamUpdateOptions().SetName(name)); +} + +THEN("^the stream name should be \"([^\"]+)\"$") { + REGEX_PARAM(std::string, name); + cucumber::ScenarioScope<bdd::GlobalContext> context; + + EXPECT_EQ(FetchStream(*context).Name(), name); +} + +WHEN("^I request a reset of the stream option \"([^\"]+)\"$") { + REGEX_PARAM(std::string, key); + cucumber::ScenarioScope<bdd::GlobalContext> context; + + const auto options = iggy::StreamUpdateOptions().Reset(KeyFromName(kStreamOptionKeys, key)); + try { + Sdk(*context).UpdateStream(StreamIdentifier(*context), options); + context->last_update_rejected = false; + } catch (const iggy::IggyException &) { + context->last_update_rejected = true; + } +} + +THEN("^the stream update should be rejected$") { + cucumber::ScenarioScope<bdd::GlobalContext> context; + + EXPECT_TRUE(context->last_update_rejected) << "the update must fail"; +} + +WHEN("^I describe the stream options$") { + cucumber::ScenarioScope<bdd::GlobalContext> context; + + context->last_option_specs = Sdk(*context).DescribeOptions(iggy::OptionsScope::Stream); +} + +THEN("^the stream option \"([^\"]+)\" should be updatable and not secret$") { + REGEX_PARAM(std::string, key); + cucumber::ScenarioScope<bdd::GlobalContext> context; + + const auto &spec = StreamOptionSpec(*context, key); + EXPECT_TRUE(spec.Updatable() && !spec.Secret() && !spec.ReadOnly()); +} + +THEN("^the stream option \"([^\"]+)\" should be create only and secret$") { + REGEX_PARAM(std::string, key); + cucumber::ScenarioScope<bdd::GlobalContext> context; + + const auto &spec = StreamOptionSpec(*context, key); + EXPECT_TRUE(!spec.Updatable() && !spec.Resettable() && spec.Secret() && !spec.ReadOnly()); +} + +THEN("^the stream option \"([^\"]+)\" should be read only$") { + REGEX_PARAM(std::string, key); + cucumber::ScenarioScope<bdd::GlobalContext> context; + + const auto &spec = StreamOptionSpec(*context, key); + EXPECT_TRUE(spec.ReadOnly() && !spec.Updatable() && !spec.Secret()); +} + +GIVEN( + "^a topic named \"([^\"]+)\" exists in the stream with message expiry ([0-9]+) microseconds and max topic " + "size ([0-9]+) bytes$") { + REGEX_PARAM(std::string, name); + REGEX_PARAM(std::uint64_t, expiry); + REGEX_PARAM(std::uint64_t, max_size); + cucumber::ScenarioScope<bdd::GlobalContext> context; + + CreateTopic(*context, name, + iggy::TopicCreateOptions() + .SetPartitionsCount(1) + .SetMessageExpiry(iggy::Expiry::Duration(expiry)) + .SetMaxTopicSize(iggy::MaxTopicSize::FromBytes(max_size))); +} + +WHEN("^I update the topic message expiry to ([0-9]+) microseconds without a name$") { + REGEX_PARAM(std::uint64_t, expiry); + cucumber::ScenarioScope<bdd::GlobalContext> context; + + UpdateTopic(*context, iggy::TopicUpdateOptions().SetMessageExpiry(iggy::Expiry::Duration(expiry))); +} + +WHEN("^I reset the topic option \"([^\"]+)\"$") { + REGEX_PARAM(std::string, key); + cucumber::ScenarioScope<bdd::GlobalContext> context; + + UpdateTopic(*context, iggy::TopicUpdateOptions().Reset(KeyFromName(kTopicOptionKeys, key))); +} + +WHEN("^I rename the topic to \"([^\"]+)\"$") { + REGEX_PARAM(std::string, name); + cucumber::ScenarioScope<bdd::GlobalContext> context; + + UpdateTopic(*context, iggy::TopicUpdateOptions().SetName(name)); +} + +THEN("^the topic name should be \"([^\"]+)\"$") { + REGEX_PARAM(std::string, name); + cucumber::ScenarioScope<bdd::GlobalContext> context; + + EXPECT_EQ(FetchTopic(*context).Name(), name); +} + +THEN("^the topic message expiry should be ([0-9]+) microseconds set explicitly$") { + REGEX_PARAM(std::uint64_t, expiry); + cucumber::ScenarioScope<bdd::GlobalContext> context; + + const auto topic = FetchTopic(*context); + EXPECT_EQ(topic.MessageExpiry(), expiry); + EXPECT_EQ(topic.Options().Explicit().count(std::string(iggy::to_string(iggy::TopicOptionKey::MessageExpiry))), 1u); +} + +THEN("^the topic max topic size should be ([0-9]+) bytes set explicitly$") { + REGEX_PARAM(std::uint64_t, max_size); + cucumber::ScenarioScope<bdd::GlobalContext> context; + + const auto topic = FetchTopic(*context); + EXPECT_EQ(topic.MaxTopicSize(), max_size); + EXPECT_EQ(topic.Options().Explicit().count(std::string(iggy::to_string(iggy::TopicOptionKey::MaxTopicSize))), 1u); +} + +THEN("^the topic max topic size should be derived from the server default$") { + cucumber::ScenarioScope<bdd::GlobalContext> context; + + const auto topic = FetchTopic(*context); + const auto key = std::string(iggy::to_string(iggy::TopicOptionKey::MaxTopicSize)); + EXPECT_EQ(topic.Options().Explicit().count(key), 0u); + EXPECT_EQ(topic.Options().Derived().count(key), 1u); + EXPECT_EQ(topic.MaxTopicSize(), std::numeric_limits<std::uint64_t>::max()); +} + +GIVEN("^a user named \"([^\"]+)\" with password \"([^\"]+)\" exists$") { + REGEX_PARAM(std::string, username); + REGEX_PARAM(std::string, password); + cucumber::ScenarioScope<bdd::GlobalContext> context; + + const auto user = + Sdk(*context).CreateUser(username, password, iggy::UserStatus::Active, std::nullopt, iggy::UserCreateOptions()); + context->last_user_id = user.Id(); +} + +WHEN("^I update the user username to \"([^\"]+)\" and status to \"([^\"]+)\"$") { + REGEX_PARAM(std::string, username); + REGEX_PARAM(std::string, status); + cucumber::ScenarioScope<bdd::GlobalContext> context; + + Sdk(*context).UpdateUser(iggy::Identifier::Numeric(context->last_user_id.value()), + iggy::UserUpdateOptions().SetUsername(username).SetStatus(UserStatusFromName(status))); +} + +THEN("^the user should have username \"([^\"]+)\"$") { + REGEX_PARAM(std::string, username); + cucumber::ScenarioScope<bdd::GlobalContext> context; + + EXPECT_EQ(FetchUser(*context).Username(), username); +} + +THEN("^the user should have status \"([^\"]+)\"$") { + REGEX_PARAM(std::string, status); + cucumber::ScenarioScope<bdd::GlobalContext> context; + + EXPECT_EQ(FetchUser(*context).Status(), UserStatusFromName(status)); +}
diff --git a/bdd/cpp/features/step_definitions/world.hpp b/bdd/cpp/features/step_definitions/world.hpp index 886be4f..a102147 100644 --- a/bdd/cpp/features/step_definitions/world.hpp +++ b/bdd/cpp/features/step_definitions/world.hpp
@@ -21,10 +21,12 @@ #include <cstdint> #include <cstdlib> +#include <optional> #include <stdexcept> #include <string> #include <vector> +#include "iggy.hpp" #include "lib.rs.h" namespace bdd { @@ -49,6 +51,14 @@ PolledData polled; std::vector<std::uint8_t> raw_response; std::string raw_error; + // Public client for the option steps, connected on first use. + std::optional<iggy::IggyBlockingClient> sdk; + std::optional<std::uint32_t> last_stream_id; + std::optional<std::uint32_t> last_topic_id; + std::optional<std::uint32_t> last_user_id; + std::optional<std::vector<std::uint8_t>> last_stream_key; + bool last_update_rejected = false; + std::vector<iggy::OptionSpec> last_option_specs; std::string call_error; GlobalContext() = default; @@ -69,7 +79,7 @@ inline std::string required_env(const char *name) { const char *value = std::getenv(name); if (value == nullptr || *value == '\0') { - throw std::runtime_error(std::string(name) + " must be set; run the suite via scripts/run-bdd-tests.sh"); + throw std::runtime_error(std::string(name) + " must be set. Run the suite via scripts/run-bdd-tests.sh"); } return std::string(value); }
diff --git a/bdd/cpp/scripts/entrypoint.sh b/bdd/cpp/scripts/entrypoint.sh index 94c4eed..f04fcfd 100755 --- a/bdd/cpp/scripts/entrypoint.sh +++ b/bdd/cpp/scripts/entrypoint.sh
@@ -58,6 +58,8 @@ case "${BDD_FEATURE:-all}" in basic_messaging) tags=(--tags @basic-messaging) ;; raw_command) tags=(--tags @raw-command) ;; + stream_options) tags=(--tags @stream-options) ;; + resource_options_patch) tags=(--tags @resource-options-patch) ;; session_contract) tags=(--tags @session-contract) ;; esac
diff --git a/bdd/docker-compose.coverage.yml b/bdd/docker-compose.coverage.yml index 7caa215..1bdf761 100644 --- a/bdd/docker-compose.coverage.yml +++ b/bdd/docker-compose.coverage.yml
@@ -39,7 +39,7 @@ trap 'cargo llvm-cov report --lcov --output-path /reports/rust-bdd-coverage.lcov' EXIT TEST_ARGS=() case "$$BDD_FEATURE" in - basic_messaging|leader_redirection|raw_command|session_contract|stream_crud) TEST_ARGS=(--test "$$BDD_FEATURE") ;; + basic_messaging|leader_redirection|raw_command|session_contract|stream_crud|stream_options|resource_options_patch) TEST_ARGS=(--test "$$BDD_FEATURE") ;; esac cargo test --locked -p bdd --features "$$BDD_RUST_FEATURES" "$${TEST_ARGS[@]}" @@ -103,6 +103,8 @@ case "$$BDD_FEATURE" in basic_messaging) FILTER='--filter-trait Category=basic-messaging' ;; leader_redirection) FILTER='--filter-trait Category=requires-leader-awareness' ;; + stream_options) FILTER='--filter-trait Category=stream-options' ;; + resource_options_patch) FILTER='--filter-trait Category=resource-options-patch' ;; session_contract) FILTER='--filter-trait Category=session-contract' ;; esac dotnet test $$FILTER --coverage --coverage-output-format cobertura --coverage-output csharp-bdd-coverage.cobertura.xml --results-directory /reports @@ -120,5 +122,7 @@ raw_command) export CUCUMBER_FILTER_TAGS='@raw-command' ;; session_contract) export CUCUMBER_FILTER_TAGS='@session-contract' ;; stream_crud) export CUCUMBER_FILTER_TAGS='@stream-crud' ;; + stream_options) export CUCUMBER_FILTER_TAGS='@stream-options' ;; + resource_options_patch) export CUCUMBER_FILTER_TAGS='@resource-options-patch' ;; esac gradle --no-daemon test jacocoTestReport && cp build/reports/jacoco/test/jacocoTestReport.xml /reports/java-bdd-coverage.xml
diff --git a/bdd/docker-compose.server.yml b/bdd/docker-compose.server.yml index dcda953..0f4c300 100644 --- a/bdd/docker-compose.server.yml +++ b/bdd/docker-compose.server.yml
@@ -70,6 +70,9 @@ - IGGY_HTTP_ADDRESS=0.0.0.0:3000 - IGGY_QUIC_ADDRESS=0.0.0.0:8080 - IGGY_WEBSOCKET_ADDRESS=0.0.0.0:8070 + # Every feature shares this server, and stream_options asserts server-generated stream keys. + - IGGY_ENCRYPTION_ENABLED=true + - IGGY_ENCRYPTION_KEY=/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE= volumes: - iggy_data:/app/local_data - ../target:/profiles
diff --git a/bdd/docker-compose.yml b/bdd/docker-compose.yml index 3dd3e10..681f689 100644 --- a/bdd/docker-compose.yml +++ b/bdd/docker-compose.yml
@@ -36,6 +36,8 @@ - ./scenarios/raw_command.feature:/app/features/raw_command.feature - ./scenarios/session_contract.feature:/app/features/session_contract.feature - ./scenarios/stream_crud.feature:/app/features/stream_crud.feature + - ./scenarios/stream_options.feature:/app/features/stream_options.feature + - ./scenarios/resource_options_patch.feature:/app/features/resource_options_patch.feature command: - sh - -c @@ -46,6 +48,8 @@ raw_command) cargo test -p bdd --features "$$BDD_RUST_FEATURES" --test raw_command ;; session_contract) cargo test -p bdd --features "$$BDD_RUST_FEATURES" --test session_contract ;; stream_crud) cargo test -p bdd --features "$$BDD_RUST_FEATURES" --test stream_crud ;; + stream_options) cargo test -p bdd --features "$$BDD_RUST_FEATURES" --test stream_options ;; + resource_options_patch) cargo test -p bdd --features "$$BDD_RUST_FEATURES" --test resource_options_patch ;; *) cargo test -p bdd --features "$$BDD_RUST_FEATURES" ;; esac networks: @@ -63,6 +67,8 @@ volumes: - ./scenarios/basic_messaging.feature:/app/features/basic_messaging.feature - ./scenarios/raw_command.feature:/app/features/raw_command.feature + - ./scenarios/stream_options.feature:/app/features/stream_options.feature + - ./scenarios/resource_options_patch.feature:/app/features/resource_options_patch.feature - ./scenarios/session_contract.feature:/app/features/session_contract.feature working_dir: /app command: @@ -70,10 +76,12 @@ - -c - | case "$$BDD_FEATURE" in - basic_messaging) TEST_PATH='tests/test_basic_messaging.py' ;; - raw_command) TEST_PATH='tests/test_raw_command.py' ;; - session_contract) TEST_PATH='tests/test_session_contract.py' ;; - *) TEST_PATH='tests/' ;; + basic_messaging) TEST_PATH='tests/test_basic_messaging.py' ;; + raw_command) TEST_PATH='tests/test_raw_command.py' ;; + session_contract) TEST_PATH='tests/test_session_contract.py' ;; + stream_options) TEST_PATH='tests/test_stream_options.py' ;; + resource_options_patch) TEST_PATH='tests/test_resource_options_patch.py' ;; + *) TEST_PATH='tests/' ;; esac uv run --project /workspace/bdd/python pytest $$TEST_PATH -v networks: @@ -90,6 +98,7 @@ volumes: - ./scenarios/basic_messaging.feature:/app/features/basic_messaging.feature - ./scenarios/raw_command.feature:/app/features/raw_command.feature + - ./scenarios/stream_options.feature:/app/features/stream_options.feature - ./scenarios/session_contract.feature:/app/features/session_contract.feature command: - sh @@ -99,6 +108,7 @@ basic_messaging) GROUP='--group basic-messaging' ;; raw_command) GROUP='--group raw-command' ;; session_contract) GROUP='--group session-contract' ;; + stream_options) GROUP='--group stream-options' ;; *) GROUP='' ;; esac ./scripts/test.sh --configuration /workspace/bdd/php/phpunit.xml.dist $$GROUP @@ -133,10 +143,12 @@ - -c - | case "$$BDD_FEATURE" in - basic_messaging) TAGS='--tags @basic-messaging' ;; - raw_command) TAGS='--tags @raw-command' ;; - session_contract) TAGS='--tags @session-contract' ;; - *) TAGS='' ;; + basic_messaging) TAGS='--tags @basic-messaging' ;; + raw_command) TAGS='--tags @raw-command' ;; + session_contract) TAGS='--tags @session-contract' ;; + stream_options) TAGS='--tags @stream-options' ;; + resource_options_patch) TAGS='--tags @resource-options-patch' ;; + *) TAGS='' ;; esac npm run test:bdd -- $$TAGS networks: @@ -160,6 +172,8 @@ basic_messaging) FILTER='--filter-trait Category=basic-messaging' ;; leader_redirection) FILTER='--filter-trait Category=requires-leader-awareness' ;; raw_command) FILTER='--filter-trait Category=raw-command' ;; + stream_options) FILTER='--filter-trait Category=stream-options' ;; + resource_options_patch) FILTER='--filter-trait Category=resource-options-patch' ;; session_contract) FILTER='--filter-trait Category=session-contract' ;; esac dotnet test $$FILTER @@ -181,6 +195,8 @@ - ./scenarios/raw_command.feature:/app/features/raw_command.feature - ./scenarios/session_contract.feature:/app/features/session_contract.feature - ./scenarios/stream_crud.feature:/app/features/stream_crud.feature + - ./scenarios/stream_options.feature:/app/features/stream_options.feature + - ./scenarios/resource_options_patch.feature:/app/features/resource_options_patch.feature command: - sh - -c @@ -191,6 +207,8 @@ raw_command) export CUCUMBER_FILTER_TAGS='@raw-command' ;; session_contract) export CUCUMBER_FILTER_TAGS='@session-contract' ;; stream_crud) export CUCUMBER_FILTER_TAGS='@stream-crud' ;; + stream_options) export CUCUMBER_FILTER_TAGS='@stream-options' ;; + resource_options_patch) export CUCUMBER_FILTER_TAGS='@resource-options-patch' ;; esac gradle --no-daemon test networks: @@ -208,6 +226,8 @@ volumes: - ./scenarios/basic_messaging.feature:/app/features/basic_messaging.feature - ./scenarios/raw_command.feature:/app/features/raw_command.feature + - ./scenarios/stream_options.feature:/app/features/stream_options.feature + - ./scenarios/resource_options_patch.feature:/app/features/resource_options_patch.feature - ./scenarios/session_contract.feature:/app/features/session_contract.feature networks: - iggy-bdd-network
diff --git a/bdd/go/tests/basic_messaging.go b/bdd/go/tests/basic_messaging.go index 21ad3e1..ef91b20 100644 --- a/bdd/go/tests/basic_messaging.go +++ b/bdd/go/tests/basic_messaging.go
@@ -228,7 +228,7 @@ func (s basicMessagingSteps) whenCreateStream(ctx context.Context, streamName string) error { c := getBasicMessagingCtx(ctx) - stream, err := c.client.CreateStream(ctx, streamName) + stream, err := c.client.CreateStream(ctx, streamName, nil) if err != nil { return fmt.Errorf("failed to create stream: %w", err) } @@ -259,14 +259,12 @@ partitionsCount uint32) error { c := getBasicMessagingCtx(ctx) streamIdentifier, _ := iggcon.NewIdentifier(streamID) + expiry := iggcon.IggyExpiryNeverExpire topic, err := c.client.CreateTopic( ctx, streamIdentifier, topicName, - partitionsCount, - iggcon.CompressionAlgorithmNone, - iggcon.IggyExpiryNeverExpire, - 0, + &iggcon.TopicOptions{PartitionsCount: &partitionsCount, MessageExpiry: &expiry}, ) if err != nil { return fmt.Errorf("failed to create topic: %w", err)
diff --git a/bdd/go/tests/leader_redirection.go b/bdd/go/tests/leader_redirection.go index 8593f55..6e8efe1 100644 --- a/bdd/go/tests/leader_redirection.go +++ b/bdd/go/tests/leader_redirection.go
@@ -334,7 +334,7 @@ return errors.New("client should be available") } - stream, err := cli.CreateStream(ctx, streamName) + stream, err := cli.CreateStream(ctx, streamName, nil) if err != nil { return err }
diff --git a/bdd/go/tests/resource_options.go b/bdd/go/tests/resource_options.go new file mode 100644 index 0000000..9a250ba --- /dev/null +++ b/bdd/go/tests/resource_options.go
@@ -0,0 +1,607 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package tests + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/hex" + "errors" + "fmt" + "slices" + + "github.com/apache/iggy/bdd/go/tests/env" + "github.com/apache/iggy/foreign/go/client" + "github.com/apache/iggy/foreign/go/client/tcp" + iggcon "github.com/apache/iggy/foreign/go/contracts" + "github.com/cucumber/godog" +) + +// resourceOptionsPartition is the partition every message of these scenarios +// goes to and is polled from. +const resourceOptionsPartition = 0 + +type resourceOptionsCtxKey struct{} + +type resourceOptionsCtx struct { + client iggcon.Client + streamId *uint32 + streamKey []byte + topicId *uint32 + userId *uint32 + createdStreams []uint32 + createdUsers []uint32 + updateError error + optionSpecs []iggcon.OptionSpec +} + +func getResourceOptionsCtx(ctx context.Context) *resourceOptionsCtx { + return ctx.Value(resourceOptionsCtxKey{}).(*resourceOptionsCtx) +} + +type resourceOptionsSteps struct{} + +func (resourceOptionsSteps) givenRunningServer(context.Context) error { + return nil +} + +func (resourceOptionsSteps) givenAuthenticationAsRoot(ctx context.Context) error { + state := getResourceOptionsCtx(ctx) + iggyClient, err := client.NewIggyClient(client.WithTcp(tcp.WithServerAddress(env.ServerAddress()))) + if err != nil { + return fmt.Errorf("create client: %w", err) + } + if err = iggyClient.Connect(ctx); err != nil { + return fmt.Errorf("connect client: %w", err) + } + username, password := env.RootCredentials() + if _, err = iggyClient.LoginUser(ctx, username, password); err != nil { + return fmt.Errorf("authenticate client: %w", err) + } + state.client = iggyClient + return nil +} + +func (s resourceOptionsSteps) whenCreateKeyedStream(ctx context.Context, name string, value int) error { + state := getResourceOptionsCtx(ctx) + key := bytes.Repeat([]byte{byte(value)}, iggcon.StreamEncryptionKeyLength) + if err := s.createStream(ctx, name, &iggcon.StreamOptions{EncryptionKey: key}); err != nil { + return err + } + state.streamKey = key + return nil +} + +func (s resourceOptionsSteps) whenCreateKeylessStream(ctx context.Context, name string) error { + getResourceOptionsCtx(ctx).streamKey = nil + return s.createStream(ctx, name, nil) +} + +func (resourceOptionsSteps) createStream(ctx context.Context, name string, options *iggcon.StreamOptions) error { + state := getResourceOptionsCtx(ctx) + stream, err := state.client.CreateStream(ctx, name, options) + if err != nil { + return fmt.Errorf("create stream %s: %w", name, err) + } + state.streamId = &stream.Id + state.createdStreams = append(state.createdStreams, stream.Id) + return nil +} + +func (s resourceOptionsSteps) thenStreamSource(ctx context.Context, want string) error { + stream, err := s.fetchStream(ctx) + if err != nil { + return err + } + source, ok := stream.EncryptionKeySource() + if !ok || source != iggcon.StreamEncryptionSource(want) { + return fmt.Errorf("encryption key source is %q (reported %t), want %q", source, ok, want) + } + return nil +} + +func (s resourceOptionsSteps) thenFingerprintMatchesKey(ctx context.Context) error { + stream, err := s.fetchStream(ctx) + if err != nil { + return err + } + want, err := iggcon.EncryptionKeyFingerprint(getResourceOptionsCtx(ctx).streamKey) + if err != nil { + return fmt.Errorf("fingerprint the stream key: %w", err) + } + if got, ok := stream.EncryptionKeyFingerprint(); !ok || got != want { + return fmt.Errorf("encryption key fingerprint is %q (reported %t), want %q", got, ok, want) + } + return nil +} + +func (s resourceOptionsSteps) thenStreamReportsFingerprint(ctx context.Context) error { + stream, err := s.fetchStream(ctx) + if err != nil { + return err + } + if fingerprint, ok := stream.EncryptionKeyFingerprint(); !ok || len(fingerprint) != 16 { + return fmt.Errorf("encryption key fingerprint is %q (reported %t), want 16 hex characters", fingerprint, ok) + } + return nil +} + +func (s resourceOptionsSteps) thenStreamReportsKek(ctx context.Context) error { + stream, err := s.fetchStream(ctx) + if err != nil { + return err + } + if len(stream.EncryptionKekFingerprints()) == 0 { + return errors.New("the stream reports no KEK fingerprint") + } + return nil +} + +func (s resourceOptionsSteps) thenNoKeyMaterial(ctx context.Context) error { + stream, err := s.fetchStream(ctx) + if err != nil { + return err + } + key := getResourceOptionsCtx(ctx).streamKey + for _, options := range []map[string]iggcon.HeaderValue{stream.Options, stream.DerivedOptions} { + for _, secret := range []string{string(iggcon.StreamOptionEncryptionKey), "encryption_key_wrapped", "critical_keys"} { + if _, found := options[secret]; found { + return fmt.Errorf("%s must never be returned", secret) + } + } + if key == nil { + continue + } + for name, value := range options { + if bytes.Contains(value.Value, key) || + bytes.Contains(value.Value, []byte(base64.StdEncoding.EncodeToString(key))) || + bytes.Contains(value.Value, []byte(hex.EncodeToString(key))) { + return fmt.Errorf("option %s carries the stream key", name) + } + } + } + return nil +} + +func (resourceOptionsSteps) whenCreateTopic(ctx context.Context, name string, partitions uint32) error { + state := getResourceOptionsCtx(ctx) + expiry := iggcon.IggyExpiryNeverExpire + topic, err := state.client.CreateTopic(ctx, streamIdentifier(state), name, + &iggcon.TopicOptions{PartitionsCount: &partitions, MessageExpiry: &expiry}) + if err != nil { + return fmt.Errorf("create topic %s: %w", name, err) + } + state.topicId = &topic.Id + return nil +} + +func (s resourceOptionsSteps) whenSendPayload(ctx context.Context, payload string) error { + return s.sendMessage(ctx, payload, nil) +} + +func (s resourceOptionsSteps) whenSendPayloadWithHeader(ctx context.Context, payload, header, value string) error { + return s.sendMessage(ctx, payload, []iggcon.HeaderEntry{{ + Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte(header)}, + Value: iggcon.HeaderValue{Kind: iggcon.String, Value: []byte(value)}, + }}) +} + +func (resourceOptionsSteps) sendMessage(ctx context.Context, payload string, headers []iggcon.HeaderEntry) error { + state := getResourceOptionsCtx(ctx) + var options []iggcon.IggyMessageOpt + if len(headers) > 0 { + options = append(options, iggcon.WithUserHeaders(headers)) + } + message, err := iggcon.NewIggyMessage([]byte(payload), options...) + if err != nil { + return fmt.Errorf("build message: %w", err) + } + _, err = state.client.SendMessages(ctx, streamIdentifier(state), topicIdentifier(state), + iggcon.PartitionId(resourceOptionsPartition), []iggcon.IggyMessage{message}) + if err != nil { + return fmt.Errorf("send message: %w", err) + } + return nil +} + +func (s resourceOptionsSteps) thenPollReturnsPayload(ctx context.Context, payload string) error { + message, err := s.pollFirst(ctx) + if err != nil { + return err + } + if string(message.Payload) != payload { + return fmt.Errorf("polled payload %q, want %q", message.Payload, payload) + } + return nil +} + +func (s resourceOptionsSteps) thenPollReturnsPayloadAndHeader(ctx context.Context, payload, header, value string) error { + message, err := s.pollFirst(ctx) + if err != nil { + return err + } + if string(message.Payload) != payload { + return fmt.Errorf("polled payload %q, want %q", message.Payload, payload) + } + headers, err := iggcon.DeserializeHeaders(message.UserHeaders) + if err != nil { + return fmt.Errorf("read user headers: %w", err) + } + want := iggcon.HeaderEntry{ + Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte(header)}, + Value: iggcon.HeaderValue{Kind: iggcon.String, Value: []byte(value)}, + } + if !slices.ContainsFunc(headers, func(entry iggcon.HeaderEntry) bool { + return entry.Key.Kind == want.Key.Kind && bytes.Equal(entry.Key.Value, want.Key.Value) && + entry.Value.Kind == want.Value.Kind && bytes.Equal(entry.Value.Value, want.Value.Value) + }) { + return fmt.Errorf("polled headers %v do not carry %s=%s", headers, header, value) + } + return nil +} + +func (resourceOptionsSteps) pollFirst(ctx context.Context) (iggcon.IggyMessage, error) { + state := getResourceOptionsCtx(ctx) + partition := uint32(resourceOptionsPartition) + polled, err := state.client.PollMessages(ctx, streamIdentifier(state), topicIdentifier(state), + iggcon.DefaultConsumer(), iggcon.OffsetPollingStrategy(0), 1, false, &partition) + if err != nil { + return iggcon.IggyMessage{}, fmt.Errorf("poll messages: %w", err) + } + if len(polled.Messages) == 0 { + return iggcon.IggyMessage{}, errors.New("the poll returned no message") + } + return polled.Messages[0], nil +} + +func (resourceOptionsSteps) whenRenameStream(ctx context.Context, name string) error { + state := getResourceOptionsCtx(ctx) + if err := state.client.UpdateStream(ctx, streamIdentifier(state), iggcon.StreamUpdate{Name: &name}); err != nil { + return fmt.Errorf("rename stream: %w", err) + } + return nil +} + +func (s resourceOptionsSteps) thenStreamName(ctx context.Context, name string) error { + stream, err := s.fetchStream(ctx) + if err != nil { + return err + } + if stream.Name != name { + return fmt.Errorf("stream name is %q, want %q", stream.Name, name) + } + return nil +} + +func (resourceOptionsSteps) whenResetStreamOption(ctx context.Context, key string) error { + state := getResourceOptionsCtx(ctx) + state.updateError = state.client.UpdateStream(ctx, streamIdentifier(state), + iggcon.StreamUpdate{Reset: []iggcon.StreamOptionKey{iggcon.StreamOptionKey(key)}}) + return nil +} + +func (resourceOptionsSteps) thenStreamUpdateRejected(ctx context.Context) error { + if getResourceOptionsCtx(ctx).updateError == nil { + return errors.New("the update must fail") + } + return nil +} + +func (resourceOptionsSteps) whenDescribeStreamOptions(ctx context.Context) error { + state := getResourceOptionsCtx(ctx) + specs, err := state.client.DescribeOptions(ctx, iggcon.OptionsScopeStream) + if err != nil { + return fmt.Errorf("describe options: %w", err) + } + state.optionSpecs = specs + return nil +} + +func (s resourceOptionsSteps) thenOptionUpdatable(ctx context.Context, key string) error { + spec, err := s.optionSpec(ctx, key) + if err != nil { + return err + } + if !spec.Updatable || spec.Secret || spec.ReadOnly { + return fmt.Errorf("%s is %+v, want updatable and not secret", key, spec) + } + return nil +} + +func (s resourceOptionsSteps) thenOptionCreateOnlySecret(ctx context.Context, key string) error { + spec, err := s.optionSpec(ctx, key) + if err != nil { + return err + } + if spec.Updatable || spec.Resettable || !spec.Secret || spec.ReadOnly { + return fmt.Errorf("%s is %+v, want create only and secret", key, spec) + } + return nil +} + +func (s resourceOptionsSteps) thenOptionReadOnly(ctx context.Context, key string) error { + spec, err := s.optionSpec(ctx, key) + if err != nil { + return err + } + if !spec.ReadOnly || spec.Updatable || spec.Secret { + return fmt.Errorf("%s is %+v, want read only", key, spec) + } + return nil +} + +func (resourceOptionsSteps) optionSpec(ctx context.Context, key string) (iggcon.OptionSpec, error) { + for _, spec := range getResourceOptionsCtx(ctx).optionSpecs { + if spec.Key == key { + return spec, nil + } + } + return iggcon.OptionSpec{}, fmt.Errorf("the catalog does not describe %s", key) +} + +func (resourceOptionsSteps) givenTopicWithSettings(ctx context.Context, name string, expiry, maxTopicSize uint64) error { + state := getResourceOptionsCtx(ctx) + partitions := uint32(1) + messageExpiry := iggcon.Duration(expiry) + topic, err := state.client.CreateTopic(ctx, streamIdentifier(state), name, &iggcon.TopicOptions{ + PartitionsCount: &partitions, + MessageExpiry: &messageExpiry, + MaxTopicSize: &maxTopicSize, + }) + if err != nil { + return fmt.Errorf("create topic %s: %w", name, err) + } + state.topicId = &topic.Id + return nil +} + +func (s resourceOptionsSteps) whenUpdateTopicExpiry(ctx context.Context, expiry uint64) error { + messageExpiry := iggcon.Duration(expiry) + return s.updateTopic(ctx, iggcon.TopicUpdate{MessageExpiry: &messageExpiry}) +} + +func (s resourceOptionsSteps) whenResetTopicOption(ctx context.Context, key string) error { + return s.updateTopic(ctx, iggcon.TopicUpdate{Reset: []iggcon.TopicOptionKey{iggcon.TopicOptionKey(key)}}) +} + +func (s resourceOptionsSteps) whenRenameTopic(ctx context.Context, name string) error { + return s.updateTopic(ctx, iggcon.TopicUpdate{Name: &name}) +} + +func (resourceOptionsSteps) updateTopic(ctx context.Context, update iggcon.TopicUpdate) error { + state := getResourceOptionsCtx(ctx) + if err := state.client.UpdateTopic(ctx, streamIdentifier(state), topicIdentifier(state), update); err != nil { + return fmt.Errorf("update topic: %w", err) + } + return nil +} + +func (s resourceOptionsSteps) thenTopicName(ctx context.Context, name string) error { + topic, err := s.fetchTopic(ctx) + if err != nil { + return err + } + if topic.Name != name { + return fmt.Errorf("topic name is %q, want %q", topic.Name, name) + } + return nil +} + +func (s resourceOptionsSteps) thenTopicExpiryExplicit(ctx context.Context, expiry uint64) error { + topic, err := s.fetchTopic(ctx) + if err != nil { + return err + } + if uint64(topic.MessageExpiry) != expiry { + return fmt.Errorf("message expiry is %d, want %d", topic.MessageExpiry, expiry) + } + if _, explicit := topic.Options[string(iggcon.TopicOptionMessageExpiry)]; !explicit { + return errors.New("message expiry must be set explicitly") + } + return nil +} + +func (s resourceOptionsSteps) thenTopicMaxSizeExplicit(ctx context.Context, maxTopicSize uint64) error { + topic, err := s.fetchTopic(ctx) + if err != nil { + return err + } + if topic.MaxTopicSize != maxTopicSize { + return fmt.Errorf("max topic size is %d, want %d", topic.MaxTopicSize, maxTopicSize) + } + if _, explicit := topic.Options[string(iggcon.TopicOptionMaxTopicSize)]; !explicit { + return errors.New("max topic size must be set explicitly") + } + return nil +} + +func (s resourceOptionsSteps) thenTopicMaxSizeDerived(ctx context.Context) error { + topic, err := s.fetchTopic(ctx) + if err != nil { + return err + } + if _, explicit := topic.Options[string(iggcon.TopicOptionMaxTopicSize)]; explicit { + return errors.New("max topic size must no longer be explicit") + } + if _, derived := topic.DerivedOptions[string(iggcon.TopicOptionMaxTopicSize)]; !derived { + return errors.New("max topic size must be derived from the server default") + } + return nil +} + +func (resourceOptionsSteps) givenUser(ctx context.Context, username, password string) error { + state := getResourceOptionsCtx(ctx) + user, err := state.client.CreateUser(ctx, username, password, iggcon.Active, nil, &iggcon.UserOptions{}) + if err != nil { + return fmt.Errorf("create user %s: %w", username, err) + } + state.userId = &user.Id + state.createdUsers = append(state.createdUsers, user.Id) + return nil +} + +func (resourceOptionsSteps) whenUpdateUser(ctx context.Context, username, status string) error { + state := getResourceOptionsCtx(ctx) + userStatus, err := parseUserStatus(status) + if err != nil { + return err + } + update := iggcon.UserUpdate{Username: &username, Status: &userStatus} + if err = state.client.UpdateUser(ctx, userIdentifier(state), update); err != nil { + return fmt.Errorf("update user: %w", err) + } + return nil +} + +func (s resourceOptionsSteps) thenUserUsername(ctx context.Context, username string) error { + user, err := s.fetchUser(ctx) + if err != nil { + return err + } + if user.Username != username { + return fmt.Errorf("username is %q, want %q", user.Username, username) + } + return nil +} + +func (s resourceOptionsSteps) thenUserStatus(ctx context.Context, status string) error { + user, err := s.fetchUser(ctx) + if err != nil { + return err + } + want, err := parseUserStatus(status) + if err != nil { + return err + } + if user.Status != want { + return fmt.Errorf("user status is %d, want %s", user.Status, status) + } + return nil +} + +func (resourceOptionsSteps) fetchStream(ctx context.Context) (*iggcon.StreamDetails, error) { + state := getResourceOptionsCtx(ctx) + stream, err := state.client.GetStream(ctx, streamIdentifier(state)) + if err != nil { + return nil, fmt.Errorf("get stream: %w", err) + } + return stream, nil +} + +func (resourceOptionsSteps) fetchTopic(ctx context.Context) (*iggcon.TopicDetails, error) { + state := getResourceOptionsCtx(ctx) + topic, err := state.client.GetTopic(ctx, streamIdentifier(state), topicIdentifier(state)) + if err != nil { + return nil, fmt.Errorf("get topic: %w", err) + } + return topic, nil +} + +func (resourceOptionsSteps) fetchUser(ctx context.Context) (*iggcon.UserInfoDetails, error) { + state := getResourceOptionsCtx(ctx) + user, err := state.client.GetUser(ctx, userIdentifier(state)) + if err != nil { + return nil, fmt.Errorf("get user: %w", err) + } + return user, nil +} + +func initResourceOptionsScenario(sc *godog.ScenarioContext) { + sc.Before(func(context.Context, *godog.Scenario) (context.Context, error) { + return context.WithValue(context.Background(), resourceOptionsCtxKey{}, &resourceOptionsCtx{}), nil + }) + s := resourceOptionsSteps{} + sc.Step(`^I have a running Iggy server$`, s.givenRunningServer) + sc.Step(`^I am authenticated as the root user$`, s.givenAuthenticationAsRoot) + sc.Step(`^I create a stream named "([^"]+)" with an encryption key of 32 bytes valued (\d+)$`, s.whenCreateKeyedStream) + sc.Step(`^a stream named "([^"]+)" with an encryption key of 32 bytes valued (\d+) exists$`, s.whenCreateKeyedStream) + sc.Step(`^I create a stream named "([^"]+)" without an encryption key$`, s.whenCreateKeylessStream) + sc.Step(`^a stream named "([^"]+)" exists$`, s.whenCreateKeylessStream) + sc.Step(`^the stream should report encryption key source "([^"]+)"$`, s.thenStreamSource) + sc.Step(`^the stream encryption key fingerprint should match the key$`, s.thenFingerprintMatchesKey) + sc.Step(`^the stream should report an encryption key fingerprint$`, s.thenStreamReportsFingerprint) + sc.Step(`^the stream should report at least one KEK fingerprint$`, s.thenStreamReportsKek) + sc.Step(`^the stream should expose no encryption key material$`, s.thenNoKeyMaterial) + sc.Step(`^I create a topic named "([^"]+)" with (\d+) partitions? in the stream$`, s.whenCreateTopic) + sc.Step(`^I send a message with payload "([^"]+)" to the topic$`, s.whenSendPayload) + sc.Step(`^I send a message with payload "([^"]+)" and header "([^"]+)" set to "([^"]+)" to the topic$`, s.whenSendPayloadWithHeader) + sc.Step(`^polling the topic should return the payload "([^"]+)"$`, s.thenPollReturnsPayload) + sc.Step(`^polling the topic should return the payload "([^"]+)" and header "([^"]+)" set to "([^"]+)"$`, s.thenPollReturnsPayloadAndHeader) + sc.Step(`^I rename the stream to "([^"]+)"$`, s.whenRenameStream) + sc.Step(`^the stream name should be "([^"]+)"$`, s.thenStreamName) + sc.Step(`^I request a reset of the stream option "([^"]+)"$`, s.whenResetStreamOption) + sc.Step(`^the stream update should be rejected$`, s.thenStreamUpdateRejected) + sc.Step(`^I describe the stream options$`, s.whenDescribeStreamOptions) + sc.Step(`^the stream option "([^"]+)" should be updatable and not secret$`, s.thenOptionUpdatable) + sc.Step(`^the stream option "([^"]+)" should be create only and secret$`, s.thenOptionCreateOnlySecret) + sc.Step(`^the stream option "([^"]+)" should be read only$`, s.thenOptionReadOnly) + sc.Step(`^a topic named "([^"]+)" exists in the stream with message expiry (\d+) microseconds and max topic size (\d+) bytes$`, s.givenTopicWithSettings) + sc.Step(`^I update the topic message expiry to (\d+) microseconds without a name$`, s.whenUpdateTopicExpiry) + sc.Step(`^I reset the topic option "([^"]+)"$`, s.whenResetTopicOption) + sc.Step(`^I rename the topic to "([^"]+)"$`, s.whenRenameTopic) + sc.Step(`^the topic name should be "([^"]+)"$`, s.thenTopicName) + sc.Step(`^the topic message expiry should be (\d+) microseconds set explicitly$`, s.thenTopicExpiryExplicit) + sc.Step(`^the topic max topic size should be (\d+) bytes set explicitly$`, s.thenTopicMaxSizeExplicit) + sc.Step(`^the topic max topic size should be derived from the server default$`, s.thenTopicMaxSizeDerived) + sc.Step(`^a user named "([^"]+)" with password "([^"]+)" exists$`, s.givenUser) + sc.Step(`^I update the user username to "([^"]+)" and status to "([^"]+)"$`, s.whenUpdateUser) + sc.Step(`^the user should have username "([^"]+)"$`, s.thenUserUsername) + sc.Step(`^the user should have status "([^"]+)"$`, s.thenUserStatus) + sc.After(func(ctx context.Context, _ *godog.Scenario, scenarioError error) (context.Context, error) { + state := getResourceOptionsCtx(ctx) + if state.client == nil { + return ctx, scenarioError + } + for _, id := range state.createdStreams { + streamId, err := iggcon.NewIdentifier(id) + scenarioError = errors.Join(scenarioError, err, state.client.DeleteStream(ctx, streamId)) + } + for _, id := range state.createdUsers { + userId, err := iggcon.NewIdentifier(id) + scenarioError = errors.Join(scenarioError, err, state.client.DeleteUser(ctx, userId)) + } + return ctx, errors.Join(scenarioError, state.client.Close()) + }) +} + +func streamIdentifier(state *resourceOptionsCtx) iggcon.Identifier { + identifier, _ := iggcon.NewIdentifier(*state.streamId) + return identifier +} + +func topicIdentifier(state *resourceOptionsCtx) iggcon.Identifier { + identifier, _ := iggcon.NewIdentifier(*state.topicId) + return identifier +} + +func userIdentifier(state *resourceOptionsCtx) iggcon.Identifier { + identifier, _ := iggcon.NewIdentifier(*state.userId) + return identifier +} + +func parseUserStatus(status string) (iggcon.UserStatus, error) { + switch status { + case "active": + return iggcon.Active, nil + case "inactive": + return iggcon.Inactive, nil + default: + return 0, fmt.Errorf("unknown user status %q", status) + } +}
diff --git a/bdd/go/tests/suite_test.go b/bdd/go/tests/suite_test.go index 7841899..3adf9c1 100644 --- a/bdd/go/tests/suite_test.go +++ b/bdd/go/tests/suite_test.go
@@ -76,6 +76,29 @@ }) } + if feature == "all" || feature == "stream_options" { + suites = append(suites, godog.TestSuite{ + ScenarioInitializer: initResourceOptionsScenario, + Options: &godog.Options{ + Format: "pretty", + Paths: []string{"../../scenarios/stream_options.feature"}, + Strict: true, + TestingT: t, + }, + }) + } + if feature == "all" || feature == "resource_options_patch" { + suites = append(suites, godog.TestSuite{ + ScenarioInitializer: initResourceOptionsScenario, + Options: &godog.Options{ + Format: "pretty", + Paths: []string{"../../scenarios/resource_options_patch.feature"}, + Strict: true, + TestingT: t, + }, + }) + } + if len(suites) == 0 { t.Fatalf("unknown BDD_FEATURE=%q", feature) }
diff --git a/bdd/go/tests/tcp_test/stream_feature_create.go b/bdd/go/tests/tcp_test/stream_feature_create.go index 5502c79..4271f95 100644 --- a/bdd/go/tests/tcp_test/stream_feature_create.go +++ b/bdd/go/tests/tcp_test/stream_feature_create.go
@@ -30,7 +30,7 @@ client := createAuthorizedConnection() name := createRandomString(32) - stream, err := client.CreateStream(context.Background(), name) + stream, err := client.CreateStream(context.Background(), name, nil) defer deleteStreamAfterTests(stream.Id, client) itShouldNotReturnError(err) @@ -41,13 +41,13 @@ client := createAuthorizedConnection() name := createRandomString(32) - stream, err := client.CreateStream(context.Background(), name) + stream, err := client.CreateStream(context.Background(), name, nil) defer deleteStreamAfterTests(stream.Id, client) itShouldNotReturnError(err) itShouldSuccessfullyCreateStream(stream.Id, name, client) - _, err = client.CreateStream(context.Background(), name) + _, err = client.CreateStream(context.Background(), name, nil) itShouldReturnSpecificError(err, ierror.ErrStreamNameAlreadyExists) }) @@ -56,7 +56,7 @@ client := createAuthorizedConnection() name := createRandomString(256) - _, err := client.CreateStream(context.Background(), name) + _, err := client.CreateStream(context.Background(), name, nil) itShouldReturnSpecificError(err, ierror.ErrInvalidStreamName) }) @@ -65,7 +65,7 @@ ginkgo.When("User is not logged in", func() { ginkgo.Context("and tries to create stream", func() { client := createClient() - _, err := client.CreateStream(context.Background(), createRandomString(32)) + _, err := client.CreateStream(context.Background(), createRandomString(32), nil) itShouldReturnUnauthenticatedError(err) })
diff --git a/bdd/go/tests/tcp_test/stream_feature_get_by_id.go b/bdd/go/tests/tcp_test/stream_feature_get_by_id.go index 71d4cc4..f4a001b 100644 --- a/bdd/go/tests/tcp_test/stream_feature_get_by_id.go +++ b/bdd/go/tests/tcp_test/stream_feature_get_by_id.go
@@ -58,21 +58,13 @@ // create two topics t1Name := createRandomString(32) t2Name := createRandomString(32) - t1, err := client.CreateTopic(context.Background(), streamIdentifier, - t1Name, - 2, - iggcon.CompressionAlgorithmNone, - iggcon.Millisecond, - math.MaxUint64) + partitions := uint32(2) + expiry := iggcon.Millisecond + maxTopicSize := uint64(math.MaxUint64) + options := &iggcon.TopicOptions{PartitionsCount: &partitions, MessageExpiry: &expiry, MaxTopicSize: &maxTopicSize} + t1, err := client.CreateTopic(context.Background(), streamIdentifier, t1Name, options) itShouldNotReturnError(err) - t2, err := client.CreateTopic( - context.Background(), - streamIdentifier, - t2Name, - 2, - iggcon.CompressionAlgorithmNone, - iggcon.Millisecond, - math.MaxUint64) + t2, err := client.CreateTopic(context.Background(), streamIdentifier, t2Name, options) itShouldNotReturnError(err) itShouldSuccessfullyCreateTopic(streamId, t1.Id, t1Name, client) itShouldSuccessfullyCreateTopic(streamId, t2.Id, t2Name, client)
diff --git a/bdd/go/tests/tcp_test/stream_feature_update.go b/bdd/go/tests/tcp_test/stream_feature_update.go index d8ea7a5..c20b379 100644 --- a/bdd/go/tests/tcp_test/stream_feature_update.go +++ b/bdd/go/tests/tcp_test/stream_feature_update.go
@@ -34,7 +34,7 @@ defer deleteStreamAfterTests(streamId, client) newName := createRandomString(128) streamIdentifier, _ := iggcon.NewIdentifier(streamId) - err := client.UpdateStream(context.Background(), streamIdentifier, newName) + err := client.UpdateStream(context.Background(), streamIdentifier, iggcon.StreamUpdate{Name: &newName}) itShouldNotReturnError(err) itShouldSuccessfullyUpdateStream(streamId, newName, client) }) @@ -47,14 +47,15 @@ defer deleteStreamAfterTests(stream2Id, client) stream2Identifier, _ := iggcon.NewIdentifier(stream2Id) - err := client.UpdateStream(context.Background(), stream2Identifier, stream1Name) + err := client.UpdateStream(context.Background(), stream2Identifier, iggcon.StreamUpdate{Name: &stream1Name}) itShouldReturnSpecificError(err, ierror.ErrStreamNameAlreadyExists) }) ginkgo.Context("and tries to update non-existing stream", func() { client := createAuthorizedConnection() - err := client.UpdateStream(context.Background(), randomU32Identifier(), createRandomString(128)) + name := createRandomString(128) + err := client.UpdateStream(context.Background(), randomU32Identifier(), iggcon.StreamUpdate{Name: &name}) itShouldReturnSpecificError(err, ierror.ErrStreamIdNotFound) }) @@ -64,7 +65,8 @@ streamId, _ := successfullyCreateStream(prefix, client) defer deleteStreamAfterTests(streamId, createAuthorizedConnection()) streamIdentifier, _ := iggcon.NewIdentifier(streamId) - err := client.UpdateStream(context.Background(), streamIdentifier, createRandomString(256)) + name := createRandomString(256) + err := client.UpdateStream(context.Background(), streamIdentifier, iggcon.StreamUpdate{Name: &name}) itShouldReturnSpecificError(err, ierror.ErrInvalidStreamName) }) @@ -73,7 +75,8 @@ ginkgo.When("User is not logged in", func() { ginkgo.Context("and tries to update stream", func() { client := createClient() - err := client.UpdateStream(context.Background(), randomU32Identifier(), createRandomString(128)) + name := createRandomString(128) + err := client.UpdateStream(context.Background(), randomU32Identifier(), iggcon.StreamUpdate{Name: &name}) itShouldReturnUnauthenticatedError(err) })
diff --git a/bdd/go/tests/tcp_test/stream_steps.go b/bdd/go/tests/tcp_test/stream_steps.go index 2789bf4..47d1ea2 100644 --- a/bdd/go/tests/tcp_test/stream_steps.go +++ b/bdd/go/tests/tcp_test/stream_steps.go
@@ -32,7 +32,7 @@ func successfullyCreateStream(prefix string, client iggcon.Client) (uint32, string) { name := createRandomStringWithPrefix(prefix, 128) - stream, err := client.CreateStream(context.Background(), name) + stream, err := client.CreateStream(context.Background(), name, nil) itShouldNotReturnError(err) itShouldSuccessfullyCreateStream(stream.Id, name, client)
diff --git a/bdd/go/tests/tcp_test/topic_feature_create.go b/bdd/go/tests/tcp_test/topic_feature_create.go index 902abcd..07b4beb 100644 --- a/bdd/go/tests/tcp_test/topic_feature_create.go +++ b/bdd/go/tests/tcp_test/topic_feature_create.go
@@ -35,14 +35,14 @@ name := createRandomString(32) defer deleteStreamAfterTests(streamId, client) streamIdentifier, _ := iggcon.NewIdentifier(streamId) + partitions := uint32(2) + expiry := iggcon.Millisecond + maxTopicSize := uint64(math.MaxUint64) _, err := client.CreateTopic( context.Background(), streamIdentifier, name, - 2, - iggcon.CompressionAlgorithmNone, - iggcon.Millisecond, - math.MaxUint64) + &iggcon.TopicOptions{PartitionsCount: &partitions, MessageExpiry: &expiry, MaxTopicSize: &maxTopicSize}) itShouldNotReturnError(err) }) @@ -52,14 +52,14 @@ streamId := createRandomUInt32() name := createRandomString(32) streamIdentifier, _ := iggcon.NewIdentifier(streamId) + partitions := uint32(2) + expiry := iggcon.Millisecond + maxTopicSize := uint64(math.MaxUint64) _, err := client.CreateTopic( context.Background(), streamIdentifier, name, - 2, - iggcon.CompressionAlgorithmNone, - iggcon.Millisecond, - math.MaxUint64) + &iggcon.TopicOptions{PartitionsCount: &partitions, MessageExpiry: &expiry, MaxTopicSize: &maxTopicSize}) itShouldReturnSpecificError(err, ierror.ErrStreamIdNotFound) }) @@ -71,14 +71,13 @@ _, name := successfullyCreateTopic(streamId, client) streamIdentifier, _ := iggcon.NewIdentifier(streamId) + partitions := uint32(2) + maxTopicSize := uint64(math.MaxUint64) _, err := client.CreateTopic( context.Background(), streamIdentifier, name, - 2, - iggcon.CompressionAlgorithmNone, - iggcon.IggyExpiryServerDefault, - math.MaxUint64) + &iggcon.TopicOptions{PartitionsCount: &partitions, MaxTopicSize: &maxTopicSize}) itShouldReturnSpecificError(err, ierror.ErrTopicNameAlreadyExists) }) @@ -88,14 +87,13 @@ defer deleteStreamAfterTests(streamId, createAuthorizedConnection()) streamIdentifier, _ := iggcon.NewIdentifier(streamId) + partitions := uint32(2) + maxTopicSize := uint64(math.MaxUint64) _, err := client.CreateTopic( context.Background(), streamIdentifier, createRandomString(256), - 2, - iggcon.CompressionAlgorithmNone, - iggcon.IggyExpiryServerDefault, - math.MaxUint64) + &iggcon.TopicOptions{PartitionsCount: &partitions, MaxTopicSize: &maxTopicSize}) itShouldReturnSpecificError(err, ierror.ErrInvalidTopicName) }) @@ -105,14 +103,13 @@ ginkgo.Context("and tries to create topic", func() { client := createClient() streamIdentifier, _ := iggcon.NewIdentifier[uint32](10) + partitions := uint32(2) + maxTopicSize := uint64(math.MaxUint64) _, err := client.CreateTopic( context.Background(), streamIdentifier, "name", - 2, - iggcon.CompressionAlgorithmNone, - iggcon.IggyExpiryServerDefault, - math.MaxUint64) + &iggcon.TopicOptions{PartitionsCount: &partitions, MaxTopicSize: &maxTopicSize}) itShouldReturnUnauthenticatedError(err) })
diff --git a/bdd/go/tests/tcp_test/topic_feature_update.go b/bdd/go/tests/tcp_test/topic_feature_update.go index 2928830..d375d11 100644 --- a/bdd/go/tests/tcp_test/topic_feature_update.go +++ b/bdd/go/tests/tcp_test/topic_feature_update.go
@@ -37,14 +37,13 @@ newName := createRandomString(128) streamIdentifier, _ := iggcon.NewIdentifier(streamId) topicIdentifier, _ := iggcon.NewIdentifier(topicId) + expiry := iggcon.Microsecond + maxTopicSize := uint64(math.MaxUint64) err := client.UpdateTopic( context.Background(), streamIdentifier, topicIdentifier, - newName, - iggcon.CompressionAlgorithmNone, - iggcon.Microsecond, - math.MaxUint64) + iggcon.TopicUpdate{Name: &newName, MessageExpiry: &expiry, MaxTopicSize: &maxTopicSize}) itShouldNotReturnError(err) itShouldSuccessfullyUpdateTopic(streamId, topicId, newName, client) }) @@ -57,28 +56,25 @@ topic2Id, _ := successfullyCreateTopic(streamId, client) streamIdentifier, _ := iggcon.NewIdentifier(streamId) topic2Identifier, _ := iggcon.NewIdentifier(topic2Id) + maxTopicSize := uint64(math.MaxUint64) err := client.UpdateTopic( context.Background(), streamIdentifier, topic2Identifier, - topic1Name, - iggcon.CompressionAlgorithmNone, - iggcon.IggyExpiryServerDefault, - math.MaxUint64) + iggcon.TopicUpdate{Name: &topic1Name, MaxTopicSize: &maxTopicSize}) itShouldReturnSpecificError(err, ierror.ErrTopicNameAlreadyExists) }) ginkgo.Context("and tries to update non-existing topic", func() { client := createAuthorizedConnection() + name := createRandomString(128) + maxTopicSize := uint64(math.MaxUint64) err := client.UpdateTopic( context.Background(), randomU32Identifier(), randomU32Identifier(), - createRandomString(128), - 1, - 0, - math.MaxUint64) + iggcon.TopicUpdate{Name: &name, MaxTopicSize: &maxTopicSize}) itShouldReturnSpecificError(err, ierror.ErrStreamIdNotFound) }) @@ -88,14 +84,13 @@ streamId, _ := successfullyCreateStream(prefix, client) defer deleteStreamAfterTests(streamId, createAuthorizedConnection()) streamIdentifier, _ := iggcon.NewIdentifier(streamId) + name := createRandomString(128) + maxTopicSize := uint64(math.MaxUint64) err := client.UpdateTopic( context.Background(), streamIdentifier, randomU32Identifier(), - createRandomString(128), - 1, - 0, - math.MaxUint64) + iggcon.TopicUpdate{Name: &name, MaxTopicSize: &maxTopicSize}) itShouldReturnSpecificError(err, ierror.ErrTopicIdNotFound) }) @@ -107,14 +102,13 @@ topicId, _ := successfullyCreateTopic(streamId, client) streamIdentifier, _ := iggcon.NewIdentifier(streamId) topicIdentifier, _ := iggcon.NewIdentifier(topicId) + name := createRandomString(256) + maxTopicSize := uint64(math.MaxUint64) err := client.UpdateTopic( context.Background(), streamIdentifier, topicIdentifier, - createRandomString(256), - 1, - 0, - math.MaxUint64) + iggcon.TopicUpdate{Name: &name, MaxTopicSize: &maxTopicSize}) itShouldReturnSpecificError(err, ierror.ErrInvalidTopicName) }) @@ -123,7 +117,8 @@ ginkgo.When("User is not logged in", func() { ginkgo.Context("and tries to update stream", func() { client := createClient() - err := client.UpdateStream(context.Background(), randomU32Identifier(), createRandomString(128)) + name := createRandomString(128) + err := client.UpdateStream(context.Background(), randomU32Identifier(), iggcon.StreamUpdate{Name: &name}) itShouldReturnUnauthenticatedError(err) })
diff --git a/bdd/go/tests/tcp_test/topic_steps.go b/bdd/go/tests/tcp_test/topic_steps.go index a359f68..d1715c4 100644 --- a/bdd/go/tests/tcp_test/topic_steps.go +++ b/bdd/go/tests/tcp_test/topic_steps.go
@@ -33,14 +33,13 @@ func successfullyCreateTopic(streamId uint32, client iggcon.Client) (uint32, string) { name := createRandomString(128) streamIdentifier, _ := iggcon.NewIdentifier(streamId) + partitions := uint32(2) + maxTopicSize := uint64(math.MaxUint64) topic, err := client.CreateTopic( context.Background(), streamIdentifier, name, - 2, - 1, - 0, - math.MaxUint64) + &iggcon.TopicOptions{PartitionsCount: &partitions, MaxTopicSize: &maxTopicSize}) itShouldSuccessfullyCreateTopic(streamId, topic.Id, name, client) itShouldNotReturnError(err)
diff --git a/bdd/go/tests/tcp_test/users_feature_create.go b/bdd/go/tests/tcp_test/users_feature_create.go index 033267c..523d4f8 100644 --- a/bdd/go/tests/tcp_test/users_feature_create.go +++ b/bdd/go/tests/tcp_test/users_feature_create.go
@@ -48,7 +48,7 @@ PollMessages: true, SendMessages: true, }, - }) + }, nil) identifier, _ := iggcon.NewIdentifier(username) defer deleteUserAfterTests(identifier, client) @@ -104,7 +104,7 @@ SendMessages: true, }, Streams: userStreamPermissions, - }) + }, nil) identifier, _ := iggcon.NewIdentifier(username) defer deleteUserAfterTests(identifier, client) defer deleteStreamAfterTests(streamId, client) @@ -137,7 +137,7 @@ PollMessages: true, SendMessages: true, }, - }) + }, nil) itShouldReturnUnauthenticatedError(err) })
diff --git a/bdd/go/tests/tcp_test/users_feature_password.go b/bdd/go/tests/tcp_test/users_feature_password.go index 78d7562..7280395 100644 --- a/bdd/go/tests/tcp_test/users_feature_password.go +++ b/bdd/go/tests/tcp_test/users_feature_password.go
@@ -49,7 +49,7 @@ PollMessages: true, SendMessages: true, }, - }) + }, nil) itShouldNotReturnError(err) identifier, _ := iggcon.NewIdentifier(username) defer deleteUserAfterTests(identifier, client)
diff --git a/bdd/go/tests/tcp_test/users_feature_permissions.go b/bdd/go/tests/tcp_test/users_feature_permissions.go index 55b734c..e70f762 100644 --- a/bdd/go/tests/tcp_test/users_feature_permissions.go +++ b/bdd/go/tests/tcp_test/users_feature_permissions.go
@@ -62,8 +62,7 @@ err := client.UpdateUser( context.Background(), randomU32Identifier(), - &username, - nil, + iggcon.UserUpdate{Username: &username}, ) itShouldReturnUnauthenticatedError(err) })
diff --git a/bdd/go/tests/tcp_test/users_feature_update.go b/bdd/go/tests/tcp_test/users_feature_update.go index 2a6df4c..3b50bb1 100644 --- a/bdd/go/tests/tcp_test/users_feature_update.go +++ b/bdd/go/tests/tcp_test/users_feature_update.go
@@ -36,8 +36,7 @@ err := client.UpdateUser( context.Background(), identifier, - &username, - nil, + iggcon.UserUpdate{Username: &username}, ) itShouldNotReturnError(err) @@ -53,8 +52,7 @@ err := client.UpdateUser( context.Background(), randomU32Identifier(), - &username, - nil, + iggcon.UserUpdate{Username: &username}, ) itShouldReturnUnauthenticatedError(err) })
diff --git a/bdd/go/tests/tcp_test/users_steps.go b/bdd/go/tests/tcp_test/users_steps.go index 22a71dc..908806c 100644 --- a/bdd/go/tests/tcp_test/users_steps.go +++ b/bdd/go/tests/tcp_test/users_steps.go
@@ -47,7 +47,7 @@ PollMessages: true, SendMessages: true, }, - }) + }, nil) itShouldNotReturnError(err) nameIdentifier, _ := iggcon.NewIdentifier(name) user, err := client.GetUser(context.Background(), nameIdentifier)
diff --git a/bdd/java/src/test/java/org/apache/iggy/bdd/BasicMessagingSteps.java b/bdd/java/src/test/java/org/apache/iggy/bdd/BasicMessagingSteps.java index 34b8008..823ec2d 100644 --- a/bdd/java/src/test/java/org/apache/iggy/bdd/BasicMessagingSteps.java +++ b/bdd/java/src/test/java/org/apache/iggy/bdd/BasicMessagingSteps.java
@@ -31,7 +31,7 @@ import org.apache.iggy.message.PollingStrategy; import org.apache.iggy.stream.StreamBase; import org.apache.iggy.stream.StreamDetails; -import org.apache.iggy.topic.CompressionAlgorithm; +import org.apache.iggy.stream.StreamUpdate; import org.apache.iggy.topic.TopicDetails; import java.math.BigInteger; @@ -126,7 +126,11 @@ @When("I update the stream name to {string}") public void updateStreamName(String streamName) { - getClient().streams().updateStream(context.lastStreamId, streamName); + getClient() + .streams() + .updateStream( + context.lastStreamId, + StreamUpdate.builder().name(streamName).build()); } @Then("getting the stream by its numeric ID should return name {string}") @@ -156,15 +160,7 @@ @When("I create a topic with name {string} in stream {int} with {int} partitions") public void createTopic(String topicName, int streamId, int partitions) { - TopicDetails topic = getClient() - .topics() - .createTopic( - (long) streamId, - (long) partitions, - CompressionAlgorithm.None, - BigInteger.ZERO, - BigInteger.ZERO, - topicName); + TopicDetails topic = getClient().topics().createTopic((long) streamId, topicName, (long) partitions); context.lastTopicId = topic.id(); context.lastTopicName = topic.name();
diff --git a/bdd/java/src/test/java/org/apache/iggy/bdd/ResourceOptionsSteps.java b/bdd/java/src/test/java/org/apache/iggy/bdd/ResourceOptionsSteps.java new file mode 100644 index 0000000..c7501d9 --- /dev/null +++ b/bdd/java/src/test/java/org/apache/iggy/bdd/ResourceOptionsSteps.java
@@ -0,0 +1,415 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.bdd; + +import io.cucumber.java.After; +import io.cucumber.java.en.Given; +import io.cucumber.java.en.Then; +import io.cucumber.java.en.When; +import org.apache.iggy.client.blocking.tcp.IggyTcpClient; +import org.apache.iggy.consumergroup.Consumer; +import org.apache.iggy.identifier.StreamId; +import org.apache.iggy.identifier.TopicId; +import org.apache.iggy.identifier.UserId; +import org.apache.iggy.message.HeaderKey; +import org.apache.iggy.message.HeaderValue; +import org.apache.iggy.message.Message; +import org.apache.iggy.message.Partitioning; +import org.apache.iggy.message.PollingStrategy; +import org.apache.iggy.stream.EncryptionKeySource; +import org.apache.iggy.stream.EncryptionKeys; +import org.apache.iggy.stream.StreamDetails; +import org.apache.iggy.stream.StreamOptionKey; +import org.apache.iggy.stream.StreamOptions; +import org.apache.iggy.stream.StreamUpdate; +import org.apache.iggy.system.OptionSpec; +import org.apache.iggy.system.OptionsScope; +import org.apache.iggy.topic.TopicDetails; +import org.apache.iggy.topic.TopicOptionKey; +import org.apache.iggy.topic.TopicOptions; +import org.apache.iggy.topic.TopicUpdate; +import org.apache.iggy.user.UserOptions; +import org.apache.iggy.user.UserStatus; +import org.apache.iggy.user.UserUpdate; + +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public final class ResourceOptionsSteps { + + private IggyTcpClient client; + private Long streamId; + private Long topicId; + private Long userId; + private String streamName; + private byte[] streamKey; + private boolean updateRejected; + + @After + public void close() { + if (client == null) { + return; + } + if (topicId != null && streamId != null) { + client.topics().deleteTopic(streamId, topicId); + } + if (streamId != null) { + client.streams().deleteStream(streamId); + } + if (userId != null) { + client.users().deleteUser(userId); + } + client.close(); + } + + @When("I create a stream named {string} with an encryption key of 32 bytes valued {int}") + public void createKeyedStream(String name, int value) { + streamKey = new byte[32]; + Arrays.fill(streamKey, (byte) value); + var created = client().streams() + .createStream( + name, StreamOptions.builder().encryptionKey(streamKey).build()); + streamId = created.id(); + streamName = name; + } + + @Given("a stream named {string} with an encryption key of 32 bytes valued {int} exists") + public void keyedStreamExists(String name, int value) { + createKeyedStream(name, value); + } + + @When("I create a stream named {string} without an encryption key") + public void createKeylessStream(String name) { + var created = client().streams().createStream(name); + streamId = created.id(); + streamName = name; + } + + @Given("a stream named {string} exists") + public void streamExists(String name) { + createKeylessStream(name); + } + + @Then("the stream should report encryption key source {string}") + public void streamSource(String source) { + assertEquals( + EncryptionKeySource.valueOf(source.toUpperCase()), + stream().encryptionKeySource().orElseThrow()); + } + + @Then("the stream encryption key fingerprint should match the key") + public void streamFingerprintMatches() { + assertNotNull(streamKey); + assertEquals( + EncryptionKeys.fingerprint(streamKey), + stream().encryptionKeyFingerprint().orElseThrow()); + } + + @Then("the stream should report an encryption key fingerprint") + public void streamFingerprintExists() { + assertTrue(stream().encryptionKeyFingerprint().orElseThrow().matches("[0-9a-f]{16}")); + } + + @Then("the stream should report at least one KEK fingerprint") + public void kekFingerprintExists() { + List<String> fingerprints = stream().encryptionKekFingerprints(); + assertFalse(fingerprints.isEmpty()); + assertTrue(fingerprints.stream().allMatch(value -> value.matches("[0-9a-f]{16}"))); + } + + @Then("the stream should expose no encryption key material") + public void streamDoesNotExposeKey() { + var details = stream(); + for (Map<String, HeaderValue> options : List.of(details.options(), details.derivedOptions())) { + for (String key : List.of("encryption_key", "encryption_key_wrapped", "critical_keys")) { + assertFalse(options.containsKey(key)); + } + } + } + + @When("I create a topic named {string} with {int} partition in the stream") + public void createTopic(String name, int partitions) { + var created = client().topics().createTopic(StreamId.of(requireStreamId()), name, partitions); + topicId = created.id(); + } + + @When("I send a message with payload {string} to the topic") + public void sendPayload(String payload) { + client().messages() + .sendMessages( + StreamId.of(requireStreamId()), + TopicId.of(requireTopicId()), + Partitioning.partitionId(0L), + List.of(Message.of(payload))); + } + + @When("I send a message with payload {string} and header {string} set to {string} to the topic") + public void sendPayloadWithHeader(String payload, String key, String value) { + client().messages() + .sendMessages( + StreamId.of(requireStreamId()), + TopicId.of(requireTopicId()), + Partitioning.partitionId(0L), + List.of(Message.of(payload, Map.of(HeaderKey.fromString(key), HeaderValue.fromString(value))))); + } + + @Then("polling the topic should return the payload {string}") + public void pollPayload(String expected) { + var messages = poll(); + assertEquals(1, messages.size()); + assertEquals(expected, new String(messages.get(0).payload(), StandardCharsets.UTF_8)); + } + + @Then("polling the topic should return the payload {string} and header {string} set to {string}") + public void pollPayloadWithHeader(String expected, String key, String value) { + var messages = poll(); + assertEquals(1, messages.size()); + assertEquals(expected, new String(messages.get(0).payload(), StandardCharsets.UTF_8)); + assertEquals( + HeaderValue.fromString(value), messages.get(0).userHeaders().get(HeaderKey.fromString(key))); + } + + @When("I rename the stream to {string}") + public void renameStream(String name) { + client().streams() + .updateStream( + StreamId.of(requireStreamId()), + StreamUpdate.builder().name(name).build()); + streamName = name; + } + + @Then("the stream name should be {string}") + public void streamName(String name) { + assertEquals(name, stream().name()); + } + + @When("I request a reset of the stream option {string}") + public void resetStreamOption(String key) { + try { + var update = StreamUpdate.builder() + .reset(StreamOptionKey.valueOf(key.toUpperCase())) + .build(); + client().streams().updateStream(StreamId.of(requireStreamId()), update); + } catch (RuntimeException rejected) { + updateRejected = true; + } + } + + @Then("the stream update should be rejected") + public void streamUpdateRejected() { + assertTrue(updateRejected); + } + + @When("I describe the stream options") + public void describeStreamOptions() { + client().system().describeOptions(OptionsScope.Stream); + } + + @Then("the stream option {string} should be updatable and not secret") + public void updatableStreamOption(String key) { + OptionSpec option = streamOption(key); + assertTrue(option.updatable()); + assertFalse(option.secret()); + } + + @Then("the stream option {string} should be create only and secret") + public void secretStreamOption(String key) { + OptionSpec option = streamOption(key); + assertFalse(option.updatable()); + assertTrue(option.secret()); + } + + @Then("the stream option {string} should be read only") + public void readOnlyStreamOption(String key) { + assertTrue(streamOption(key).readOnly()); + } + + @Given( + "a topic named {string} exists in the stream with message expiry {long} microseconds and max topic size {long} bytes") + public void topicWithOptions(String name, long expiry, long maxSize) { + var options = TopicOptions.builder() + .messageExpiry(BigInteger.valueOf(expiry)) + .maxTopicSize(BigInteger.valueOf(maxSize)) + .build(); + var created = client().topics().createTopic(StreamId.of(requireStreamId()), name, 1L, options); + topicId = created.id(); + } + + @When("I update the topic message expiry to {long} microseconds without a name") + public void updateTopicExpiry(long expiry) { + client().topics() + .updateTopic( + StreamId.of(requireStreamId()), + TopicId.of(requireTopicId()), + TopicUpdate.builder() + .messageExpiry(BigInteger.valueOf(expiry)) + .build()); + } + + @When("I reset the topic option {string}") + public void resetTopicOption(String key) { + client().topics() + .updateTopic( + StreamId.of(requireStreamId()), + TopicId.of(requireTopicId()), + TopicUpdate.builder() + .reset(TopicOptionKey.valueOf(key.toUpperCase())) + .build()); + } + + @When("I rename the topic to {string}") + public void renameTopic(String name) { + client().topics() + .updateTopic( + StreamId.of(requireStreamId()), + TopicId.of(requireTopicId()), + TopicUpdate.builder().name(name).build()); + } + + @Then("the topic name should be {string}") + public void topicName(String name) { + assertEquals(name, topic().name()); + } + + @Then("the topic message expiry should be {long} microseconds set explicitly") + public void topicExpiry(long expiry) { + assertEquals( + HeaderValue.fromUint64(BigInteger.valueOf(expiry)), + topic().options().get("message_expiry")); + } + + @Then("the topic max topic size should be {long} bytes set explicitly") + public void topicMaxSize(long bytes) { + assertEquals( + HeaderValue.fromUint64(BigInteger.valueOf(bytes)), + topic().options().get("max_topic_size")); + } + + @Then("the topic max topic size should be derived from the server default") + public void defaultTopicMaxSize() { + assertEquals( + BigInteger.ONE.shiftLeft(64).subtract(BigInteger.ONE), + topic().derivedOptions().get("max_topic_size").asUint64()); + } + + @Given("a user named {string} with password {string} exists") + public void createUser(String name, String password) { + var created = + client().users().createUser(name, password, UserStatus.Active, Optional.empty(), UserOptions.empty()); + userId = created.id(); + } + + @When("I update the user username to {string} and status to {string}") + public void updateUser(String name, String status) { + client().users() + .updateUser( + UserId.of(requireUserId()), + UserUpdate.builder() + .username(name) + .status(UserStatus.valueOf( + status.substring(0, 1).toUpperCase() + status.substring(1))) + .build()); + } + + @Then("the user should have username {string}") + public void userName(String name) { + assertEquals( + name, client().users().getUser(requireUserId()).orElseThrow().username()); + } + + @Then("the user should have status {string}") + public void userStatus(String status) { + assertEquals( + status, + client().users() + .getUser(requireUserId()) + .orElseThrow() + .status() + .name() + .toLowerCase()); + } + + private IggyTcpClient client() { + if (client == null) { + String[] address = TestEnvironment.serverAddress().split(":", 2); + client = IggyTcpClient.builder() + .host(address[0]) + .port(Integer.parseInt(address[1])) + .build(); + client.connect(); + client.users().login(TestEnvironment.rootUsername(), TestEnvironment.rootPassword()); + } + return client; + } + + private StreamDetails stream() { + return client().streams().getStream(requireStreamId()).orElseThrow(); + } + + private TopicDetails topic() { + return client().topics() + .getTopic(StreamId.of(requireStreamId()), TopicId.of(requireTopicId())) + .orElseThrow(); + } + + private OptionSpec streamOption(String key) { + return client().system().describeOptions(OptionsScope.Stream).stream() + .filter(option -> option.key().equals(key)) + .findFirst() + .orElseThrow(); + } + + private List<Message> poll() { + return client().messages() + .pollMessages( + StreamId.of(requireStreamId()), + TopicId.of(requireTopicId()), + Optional.of(0L), + Consumer.of(0L), + PollingStrategy.offset(BigInteger.ZERO), + 10L, + false) + .messages(); + } + + private Long requireStreamId() { + assertNotNull(streamId); + return streamId; + } + + private Long requireTopicId() { + assertNotNull(topicId); + return topicId; + } + + private Long requireUserId() { + assertNotNull(userId); + return userId; + } +}
diff --git a/bdd/node/Dockerfile b/bdd/node/Dockerfile index e11d197..a76c12b 100644 --- a/bdd/node/Dockerfile +++ b/bdd/node/Dockerfile
@@ -20,6 +20,8 @@ COPY ./foreign/node . COPY ./bdd/scenarios/basic_messaging.feature ./bdd/ COPY ./bdd/scenarios/raw_command.feature ./bdd/ +COPY ./bdd/scenarios/stream_options.feature ./bdd/ +COPY ./bdd/scenarios/resource_options_patch.feature ./bdd/ COPY ./bdd/scenarios/session_contract.feature ./bdd/ RUN npm ci
diff --git a/bdd/php/tests/SharedFeatureParser.php b/bdd/php/tests/SharedFeatureParser.php index 9a0d070..a258556 100644 --- a/bdd/php/tests/SharedFeatureParser.php +++ b/bdd/php/tests/SharedFeatureParser.php
@@ -20,7 +20,7 @@ final class SharedFeatureParser { - public static function load(string $featureFile): array + public static function load(string $featureFile, array $excludedTags = []): array { if (!is_file($featureFile)) { throw new RuntimeException("feature file not found at {$featureFile}"); @@ -35,10 +35,22 @@ $scenarios = []; $currentScenario = null; $currentSteps = []; + $currentTags = []; + $pendingTags = []; $section = null; foreach ($lines as $line) { $line = trim($line); - if ($line === '' || str_starts_with($line, '#') || str_starts_with($line, '@')) { + if ($line === '' || str_starts_with($line, '#')) { + continue; + } + + if (str_starts_with($line, '@')) { + array_push($pendingTags, ...preg_split('/\s+/', $line)); + continue; + } + + if (str_starts_with($line, 'Feature:')) { + $pendingTags = []; continue; } @@ -48,11 +60,13 @@ } if (str_starts_with($line, 'Scenario:')) { - if ($currentScenario !== null) { + if ($currentScenario !== null && array_intersect($currentTags, $excludedTags) === []) { $scenarios[$currentScenario] = $currentSteps; } $currentScenario = trim(substr($line, strlen('Scenario:'))); $currentSteps = []; + $currentTags = $pendingTags; + $pendingTags = []; $section = 'scenario'; continue; } @@ -74,7 +88,7 @@ } } - if ($currentScenario !== null) { + if ($currentScenario !== null && array_intersect($currentTags, $excludedTags) === []) { $scenarios[$currentScenario] = $currentSteps; } if ($scenarios === []) {
diff --git a/bdd/php/tests/SharedFeatureParserTest.php b/bdd/php/tests/SharedFeatureParserTest.php new file mode 100644 index 0000000..be7004f --- /dev/null +++ b/bdd/php/tests/SharedFeatureParserTest.php
@@ -0,0 +1,46 @@ +<?php +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +declare(strict_types=1); + +require_once __DIR__ . '/SharedFeatureParser.php'; + +use PHPUnit\Framework\Attributes\Group; +use PHPUnit\Framework\TestCase; + +final class SharedFeatureParserTest extends TestCase +{ + #[Group('stream-options')] + public function testGivenTaggedScenariosWhenLoadingWithExclusionsShouldKeepSupportedCases(): void + { + $cases = SharedFeatureParser::load( + __DIR__ . '/../../scenarios/stream_options.feature', + ['@describe-options', '@message-headers'], + ); + + self::assertCount(4, $cases); + self::assertArrayNotHasKey( + 'Describe stream options', + $cases, + ); + self::assertArrayNotHasKey( + 'Send and poll a message with headers on a keyed stream', + $cases, + ); + } +}
diff --git a/bdd/php/tests/StreamOptionsFeatureTest.php b/bdd/php/tests/StreamOptionsFeatureTest.php new file mode 100644 index 0000000..52d1e2d --- /dev/null +++ b/bdd/php/tests/StreamOptionsFeatureTest.php
@@ -0,0 +1,249 @@ +<?php +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +declare(strict_types=1); + +require_once __DIR__ . '/SharedFeatureParser.php'; +require_once __DIR__ . '/TestEnvironment.php'; + +use Iggy\Client as IggyClient; +use Iggy\OptionSpec; +use Iggy\OptionsScope; +use Iggy\PollingStrategy; +use Iggy\SendMessage; +use Iggy\StreamDetails; +use Iggy\StreamOptionKey; +use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\Attributes\Group; +use PHPUnit\Framework\TestCase; + +final class StreamOptionsFeatureTest extends TestCase +{ + private ?IggyClient $client = null; + private ?int $streamId = null; + private ?string $streamName = null; + private ?string $topicName = null; + private ?string $keyBase64 = null; + private bool $updateRejected = false; + /** @var array<string, OptionSpec> */ + private array $streamOptionSpecs = []; + + public static function scenarioCases(): array + { + return SharedFeatureParser::load( + __DIR__ . '/../../scenarios/stream_options.feature', + ['@message-headers'], + ); + } + + #[DataProvider('scenarioCases')] + #[Group('stream-options')] + public function testStreamOptionsScenario(string $scenarioName, array $steps): void + { + assert_true($scenarioName !== '', 'scenario name must not be empty'); + foreach ($steps as $step) { + $this->runStep($step); + } + } + + protected function tearDown(): void + { + if ($this->client !== null && $this->streamName !== null) { + cleanup_stream_with_topics( + $this->client, + $this->streamName, + $this->topicName === null ? [] : [$this->topicName], + ); + } + + parent::tearDown(); + } + + private function runStep(string $step): void + { + if ($step === 'I have a running Iggy server') { + $this->client = new IggyClient(TestEnvironment::serverAddress()); + $this->client->connect(); + $this->client->ping(); + return; + } + + if ($step === 'I am authenticated as the root user') { + $this->requireClient()->loginUser(TestEnvironment::rootUsername(), TestEnvironment::rootPassword()); + return; + } + + if (preg_match('/^I create a stream named "([^"]+)" with an encryption key of 32 bytes valued (\d+)$/', $step, $matches) === 1 + || preg_match('/^a stream named "([^"]+)" with an encryption key of 32 bytes valued (\d+) exists$/', $step, $matches) === 1) { + $this->keyBase64 = base64_encode(str_repeat(chr((int) $matches[2]), 32)); + $this->createStream($matches[1], $this->keyBase64); + return; + } + + if (preg_match('/^I create a stream named "([^"]+)" without an encryption key$/', $step, $matches) === 1) { + $this->createStream($matches[1], null); + return; + } + + if (preg_match('/^the stream should report encryption key source "([^"]+)"$/', $step, $matches) === 1) { + assert_same($matches[1], $this->requireStream()->encryption_key_source); + return; + } + + if ($step === 'the stream encryption key fingerprint should match the key') { + assert_not_null($this->keyBase64); + assert_same( + IggyClient::encryptionKeyFingerprint($this->keyBase64), + $this->requireStream()->encryption_key_fingerprint, + ); + return; + } + + if ($step === 'the stream should report an encryption key fingerprint') { + self::assertMatchesRegularExpression('/^[0-9a-f]{16}$/', (string) $this->requireStream()->encryption_key_fingerprint); + return; + } + + if ($step === 'the stream should report at least one KEK fingerprint') { + $fingerprints = $this->requireStream()->encryption_kek_fingerprint; + assert_not_null($fingerprints); + foreach (explode(',', $fingerprints) as $fingerprint) { + self::assertMatchesRegularExpression('/^[0-9a-f]{16}$/', $fingerprint); + } + return; + } + + if ($step === 'the stream should expose no encryption key material') { + $stream = $this->requireStream(); + foreach (['encryption_key', 'encryption_key_wrapped', 'critical_keys'] as $secret) { + self::assertArrayNotHasKey($secret, $stream->options); + self::assertArrayNotHasKey($secret, $stream->derived_options); + } + return; + } + + if (preg_match('/^I create a topic named "([^"]+)" with (\d+) partition in the stream$/', $step, $matches) === 1) { + $this->topicName = $matches[1]; + $this->requireClient()->createTopic($this->requireStreamId(), $this->topicName, (int) $matches[2], null, null, null, null); + return; + } + + if (preg_match('/^I send a message with payload "([^"]+)" to the topic$/', $step, $matches) === 1) { + assert_not_null($this->topicName); + $this->requireClient()->sendMessages($this->requireStreamId(), $this->topicName, 0, [new SendMessage($matches[1])]); + return; + } + + if (preg_match('/^polling the topic should return the payload "([^"]+)"$/', $step, $matches) === 1) { + assert_not_null($this->topicName); + $messages = $this->requireClient()->pollMessages( + $this->requireStreamId(), $this->topicName, 0, PollingStrategy::offset(0), 10, false, + ); + assert_same([$matches[1]], collect_payloads($messages)); + return; + } + + if (preg_match('/^I rename the stream to "([^"]+)"$/', $step, $matches) === 1) { + $this->requireClient()->updateStream($this->requireStreamId(), name: $matches[1]); + $this->streamName = $matches[1]; + return; + } + + if (preg_match('/^the stream name should be "([^"]+)"$/', $step, $matches) === 1) { + assert_same($matches[1], $this->requireStream()->name); + return; + } + + if (preg_match('/^I request a reset of the stream option "([^"]+)"$/', $step, $matches) === 1) { + assert_throws( + fn () => $this->requireClient()->updateStream( + $this->requireStreamId(), reset: [StreamOptionKey::from($matches[1])], + ), + 'Option cannot be updated after creation', + ); + $this->updateRejected = true; + return; + } + + if ($step === 'I describe the stream options') { + foreach ($this->requireClient()->describeOptions(OptionsScope::Stream) as $spec) { + $this->streamOptionSpecs[$spec->key] = $spec; + } + return; + } + + if (preg_match('/^the stream option "([^"]+)" should be updatable and not secret$/', $step, $matches) === 1) { + $spec = $this->describedOption($matches[1]); + assert_true($spec->updatable && !$spec->secret, "{$matches[1]} must be updatable and not secret"); + return; + } + + if (preg_match('/^the stream option "([^"]+)" should be create only and secret$/', $step, $matches) === 1) { + $spec = $this->describedOption($matches[1]); + assert_true(!$spec->updatable && $spec->secret, "{$matches[1]} must be create only and secret"); + return; + } + + if (preg_match('/^the stream option "([^"]+)" should be read only$/', $step, $matches) === 1) { + assert_true($this->describedOption($matches[1])->read_only, "{$matches[1]} must be read only"); + return; + } + + if ($step === 'the stream update should be rejected') { + assert_true($this->updateRejected, 'the reset must be rejected'); + return; + } + + self::fail("Unsupported BDD step: {$step}"); + } + + private function describedOption(string $key): OptionSpec + { + assert_true(array_key_exists($key, $this->streamOptionSpecs), "{$key} must be described"); + + return $this->streamOptionSpecs[$key]; + } + + private function createStream(string $name, ?string $key): void + { + $this->requireClient()->createStream($name, $key); + $stream = $this->requireClient()->getStream($name); + assert_not_null($stream); + $this->streamId = $stream->id; + $this->streamName = $name; + } + + private function requireClient(): IggyClient + { + assert_not_null($this->client); + return $this->client; + } + + private function requireStreamId(): int + { + assert_not_null($this->streamId); + return $this->streamId; + } + + private function requireStream(): StreamDetails + { + $stream = $this->requireClient()->getStream($this->requireStreamId()); + assert_not_null($stream); + return $stream; + } +}
diff --git a/bdd/python/tests/conftest.py b/bdd/python/tests/conftest.py index 0b1ddce..9930e66 100644 --- a/bdd/python/tests/conftest.py +++ b/bdd/python/tests/conftest.py
@@ -24,7 +24,7 @@ from dataclasses import dataclass import pytest -from apache_iggy import IggyClient, ReceiveMessage +from apache_iggy import IggyClient, OptionSpec, ReceiveMessage @dataclass @@ -42,9 +42,31 @@ last_sent_message: str | None = None # Store message payload as string last_raw_response: bytes | None = None last_raw_error: RuntimeError | None = None + last_stream_key: bytes | None = None + last_update_error: RuntimeError | None = None + last_option_specs: list[OptionSpec] | None = None + last_user_id: int | None = None last_call_error: RuntimeError | None = None +def pytest_configure(config: pytest.Config) -> None: + """Register the shared feature tags as markers. + + The Docker image runs pytest from /app, where bdd/python/pyproject.toml is + not the rootdir config, so markers declared there would never load. + """ + for marker in ( + "basic-messaging: shared basic messaging scenarios", + "raw-command: shared raw command scenarios", + "session-contract: shared session contract scenarios", + "stream-options: shared stream option scenarios", + "resource-options-patch: shared resource patch scenarios", + "message-headers: scenarios that assert message headers", + "describe-options: scenarios that assert the option catalog", + ): + config.addinivalue_line("markers", marker) + + def required_env(name: str) -> str: """Read a variable the suite cannot run without.
diff --git a/bdd/python/tests/test_basic_messaging.py b/bdd/python/tests/test_basic_messaging.py index b34d0cd..6684db0 100644 --- a/bdd/python/tests/test_basic_messaging.py +++ b/bdd/python/tests/test_basic_messaging.py
@@ -22,7 +22,7 @@ import asyncio import socket -from apache_iggy import Consumer, IggyClient, PollingStrategy, SendMessage +from apache_iggy import Consumer, IggyClient, PollingStrategy, SendMessage, TopicOptions from pytest_bdd import given, parsers, scenarios, then, when # Load scenarios from the shared feature file @@ -119,7 +119,9 @@ async def _create(): await context.client.create_topic( - stream=stream_id, name=topic_name, partitions_count=partitions + stream=stream_id, + name=topic_name, + options=TopicOptions(partitions_count=partitions), ) topic = await context.client.get_topic(stream_id, topic_name)
diff --git a/bdd/python/tests/test_resource_options_patch.py b/bdd/python/tests/test_resource_options_patch.py new file mode 100644 index 0000000..cbe3dcd --- /dev/null +++ b/bdd/python/tests/test_resource_options_patch.py
@@ -0,0 +1,248 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Resource option update BDD steps for the Python SDK. +""" + +import asyncio +import socket +from datetime import timedelta +from pathlib import Path + +from apache_iggy import ( + IggyClient, + IggyExpiry, + MaxTopicSize, + OptionKey, + TopicDetails, + TopicOptions, + UserInfoDetails, + UserOptions, + UserStatus, +) +from pytest_bdd import given, parsers, scenarios, then, when + +FEATURE_FILE = Path("/app/features/resource_options_patch.feature") +if not FEATURE_FILE.exists(): + FEATURE_FILE = ( + Path(__file__).resolve().parents[2] / "scenarios/resource_options_patch.feature" + ) +scenarios(str(FEATURE_FILE)) + + +@given("I have a running Iggy server") +def running_server(context): + async def connect(): + host, port = context.server_addr.split(":") + try: + address = f"{socket.gethostbyname(host)}:{port}" + except socket.gaierror: + address = context.server_addr + + context.client = IggyClient(address) + await context.client.connect() + await context.client.ping() + + asyncio.run(connect()) + + +@given("I am authenticated as the root user") +def authenticated_root_user(context, root_credentials): + async def login(): + await context.client.login_user(*root_credentials) + + asyncio.run(login()) + + +@given(parsers.re(r'a stream named "(?P<name>[^"]+)" exists')) +def stream_exists(context, name): + async def create(): + stream = await context.client.create_stream(name) + context.last_stream_id = stream.id + context.last_stream_name = stream.name + + asyncio.run(create()) + + +@given( + parsers.re( + r'a topic named "(?P<name>[^"]+)" exists in the stream with message expiry ' + r"(?P<expiry>\d+) microseconds and max topic size (?P<max_size>\d+) bytes" + ), + converters={"expiry": int, "max_size": int}, +) +def topic_exists_with_settings(context, name, expiry, max_size): + async def create(): + await context.client.create_topic( + context.last_stream_id, + name, + TopicOptions( + message_expiry=IggyExpiry.ExpireDuration( + duration=timedelta(microseconds=expiry) + ), + max_topic_size=MaxTopicSize.Custom(bytes=max_size), + ), + ) + topic = await context.client.get_topic(context.last_stream_id, name) + assert topic is not None + context.last_topic_id = topic.id + context.last_topic_name = topic.name + + asyncio.run(create()) + + +@when( + parsers.re( + r"I update the topic message expiry to (?P<expiry>\d+) microseconds " + r"without a name" + ), + converters={"expiry": int}, +) +def update_topic_expiry(context, expiry): + update_topic( + context, + message_expiry=IggyExpiry.ExpireDuration( + duration=timedelta(microseconds=expiry) + ), + ) + + +@when(parsers.re(r'I reset the topic option "(?P<key>[^"]+)"')) +def reset_topic_option(context, key): + update_topic(context, reset=[option_key(key)]) + + +@when(parsers.re(r'I rename the topic to "(?P<name>[^"]+)"')) +def rename_topic(context, name): + update_topic(context, name=name) + + +@then(parsers.re(r'the topic name should be "(?P<name>[^"]+)"')) +def topic_name_is(context, name): + assert fetch_topic(context).name == name + + +@then( + parsers.re( + r"the topic message expiry should be (?P<expiry>\d+) microseconds " + r"set explicitly" + ), + converters={"expiry": int}, +) +def topic_expiry_explicit(context, expiry): + topic = fetch_topic(context) + assert isinstance(topic.message_expiry, IggyExpiry.ExpireDuration) + assert topic.message_expiry.duration == timedelta(microseconds=expiry) + assert topic.options.to_scalar_dict()["message_expiry"] == expiry + + +@then( + parsers.re( + r"the topic max topic size should be (?P<max_size>\d+) bytes set explicitly" + ), + converters={"max_size": int}, +) +def topic_max_size_explicit(context, max_size): + topic = fetch_topic(context) + assert isinstance(topic.max_topic_size, MaxTopicSize.Custom) + assert topic.max_topic_size.bytes == max_size + assert topic.options.to_scalar_dict()["max_topic_size"] == max_size + + +@then("the topic max topic size should be derived from the server default") +def topic_max_size_derived(context): + topic = fetch_topic(context) + assert "max_topic_size" not in topic.options.to_scalar_dict() + assert "max_topic_size" in topic.derived_options.to_scalar_dict() + + +@given( + parsers.re( + r'a user named "(?P<username>[^"]+)" with password "(?P<password>[^"]+)" exists' + ) +) +def user_exists(context, username, password): + async def create(): + user = await context.client.create_user( + username, password, options=UserOptions() + ) + context.last_user_id = user.id + + asyncio.run(create()) + + +@when( + parsers.re( + r'I update the user username to "(?P<username>[^"]+)" and status to ' + r'"(?P<status>[^"]+)"' + ) +) +def update_user(context, username, status): + async def update(): + await context.client.update_user( + context.last_user_id, username=username, status=user_status(status) + ) + + asyncio.run(update()) + + +@then(parsers.re(r'the user should have username "(?P<username>[^"]+)"')) +def user_has_username(context, username): + assert fetch_user(context).username == username + + +@then(parsers.re(r'the user should have status "(?P<status>[^"]+)"')) +def user_has_status(context, status): + assert fetch_user(context).status == user_status(status) + + +def update_topic(context, **changes) -> None: + async def update(): + await context.client.update_topic( + context.last_stream_id, context.last_topic_id, **changes + ) + + asyncio.run(update()) + + +def fetch_topic(context) -> TopicDetails: + async def fetch(): + return await context.client.get_topic( + context.last_stream_id, context.last_topic_id + ) + + topic = asyncio.run(fetch()) + assert topic is not None + return topic + + +def fetch_user(context) -> UserInfoDetails: + async def fetch(): + return await context.client.get_user(context.last_user_id) + + user = asyncio.run(fetch()) + assert user is not None + return user + + +def option_key(key: str) -> OptionKey: + return getattr(OptionKey, "".join(part.title() for part in key.split("_"))) + + +def user_status(status: str) -> UserStatus: + return getattr(UserStatus, status.title())
diff --git a/bdd/python/tests/test_stream_options.py b/bdd/python/tests/test_stream_options.py new file mode 100644 index 0000000..2a4b9e0 --- /dev/null +++ b/bdd/python/tests/test_stream_options.py
@@ -0,0 +1,331 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Stream options BDD steps for the Python SDK. +""" + +import asyncio +import base64 +import socket +from pathlib import Path + +from apache_iggy import ( + Consumer, + IggyClient, + IggyExpiry, + OptionKey, + PollingStrategy, + SendMessage, + StreamDetails, + StreamOptions, + TopicOptions, + encryption_key_fingerprint, +) +from pytest_bdd import given, parsers, scenarios, then, when + +FEATURE_FILE = Path("/app/features/stream_options.feature") +if not FEATURE_FILE.exists(): + FEATURE_FILE = ( + Path(__file__).resolve().parents[2] / "scenarios/stream_options.feature" + ) +scenarios(str(FEATURE_FILE)) + +KEY_LENGTH = 32 +FINGERPRINT_LENGTH = 16 +PARTITION_ID = 0 +NEVER_RETURNED_KEYS = {"encryption_key", "encryption_key_wrapped", "critical_keys"} + + +@given("I have a running Iggy server") +def running_server(context): + async def connect(): + host, port = context.server_addr.split(":") + try: + address = f"{socket.gethostbyname(host)}:{port}" + except socket.gaierror: + address = context.server_addr + + context.client = IggyClient(address) + await context.client.connect() + await context.client.ping() + + asyncio.run(connect()) + + +@given("I am authenticated as the root user") +def authenticated_root_user(context, root_credentials): + async def login(): + await context.client.login_user(*root_credentials) + + asyncio.run(login()) + + +@when( + parsers.re( + r'I create a stream named "(?P<name>[^"]+)" with an encryption key ' + r"of 32 bytes valued (?P<byte>\d+)" + ), + converters={"byte": int}, +) +def create_stream_with_key(context, name, byte): + create_keyed_stream(context, name, byte) + + +@given( + parsers.re( + r'a stream named "(?P<name>[^"]+)" with an encryption key ' + r"of 32 bytes valued (?P<byte>\d+) exists" + ), + converters={"byte": int}, +) +def keyed_stream_exists(context, name, byte): + create_keyed_stream(context, name, byte) + + +@when( + parsers.re(r'I create a stream named "(?P<name>[^"]+)" without an encryption key') +) +def create_stream_without_key(context, name): + async def create(): + stream = await context.client.create_stream(name) + context.last_stream_id = stream.id + context.last_stream_name = stream.name + context.last_stream_key = None + + asyncio.run(create()) + + +@then(parsers.re(r'the stream should report encryption key source "(?P<source>[^"]+)"')) +def stream_reports_source(context, source): + assert fetch_stream(context).encryption_key_source == source + + +@then("the stream encryption key fingerprint should match the key") +def stream_fingerprint_matches_key(context): + assert context.last_stream_key is not None + expected = encryption_key_fingerprint(context.last_stream_key) + assert fetch_stream(context).encryption_key_fingerprint == expected + + +@then("the stream should report an encryption key fingerprint") +def stream_reports_fingerprint(context): + fingerprint = fetch_stream(context).encryption_key_fingerprint + assert fingerprint is not None + assert len(fingerprint) == FINGERPRINT_LENGTH + + +@then("the stream should report at least one KEK fingerprint") +def stream_reports_kek_fingerprint(context): + assert fetch_stream(context).encryption_kek_fingerprints + + +@then("the stream should expose no encryption key material") +def stream_exposes_no_key_material(context): + stream = fetch_stream(context) + explicit = stream.options.to_scalar_dict() + derived = stream.derived_options.to_scalar_dict() + assert not NEVER_RETURNED_KEYS & (set(explicit) | set(derived)) + if context.last_stream_key is not None: + values = [*explicit.values(), *derived.values()] + assert context.last_stream_key not in values + assert base64.b64encode(context.last_stream_key).decode() not in values + + +@when( + parsers.re( + r'I create a topic named "(?P<name>[^"]+)" with (?P<partitions>\d+) ' + r"partitions? in the stream" + ), + converters={"partitions": int}, +) +def create_topic_in_stream(context, name, partitions): + async def create(): + await context.client.create_topic( + context.last_stream_id, + name, + TopicOptions( + partitions_count=partitions, + message_expiry=IggyExpiry.NeverExpire(), + ), + ) + topic = await context.client.get_topic(context.last_stream_id, name) + assert topic is not None + context.last_topic_id = topic.id + context.last_topic_name = topic.name + context.last_topic_partitions = partitions + + asyncio.run(create()) + + +@when(parsers.re(r'I send a message with payload "(?P<payload>[^"]+)" to the topic')) +def send_payload(context, payload): + send_message(context, SendMessage(payload)) + + +@when( + parsers.re( + r'I send a message with payload "(?P<payload>[^"]+)" and header ' + r'"(?P<header>[^"]+)" set to "(?P<value>[^"]+)" to the topic' + ) +) +def send_payload_with_header(context, payload, header, value): + send_message(context, SendMessage(payload, user_headers={header: value})) + + +@then(parsers.re(r'polling the topic should return the payload "(?P<payload>[^"]+)"')) +def poll_returns_payload(context, payload): + message = poll_first(context) + assert message.payload() == payload.encode() + + +@then( + parsers.re( + r'polling the topic should return the payload "(?P<payload>[^"]+)" ' + r'and header "(?P<header>[^"]+)" set to "(?P<value>[^"]+)"' + ) +) +def poll_returns_payload_and_header(context, payload, header, value): + message = poll_first(context) + assert message.payload() == payload.encode() + headers = message.user_headers() + assert headers is not None + assert headers.to_scalar_dict()[header] == value + + +@when(parsers.re(r'I rename the stream to "(?P<name>[^"]+)"')) +def rename_stream(context, name): + async def rename(): + await context.client.update_stream(context.last_stream_id, name=name) + + asyncio.run(rename()) + + +@then(parsers.re(r'the stream name should be "(?P<name>[^"]+)"')) +def stream_name_is(context, name): + assert fetch_stream(context).name == name + + +@when(parsers.re(r'I request a reset of the stream option "(?P<key>[^"]+)"')) +def request_stream_reset(context, key): + async def reset(): + try: + await context.client.update_stream( + context.last_stream_id, reset=[option_key(key)] + ) + context.last_update_error = None + except RuntimeError as error: + context.last_update_error = error + + asyncio.run(reset()) + + +@then("the stream update should be rejected") +def stream_update_rejected(context): + assert context.last_update_error is not None + + +@when("I describe the stream options") +def describe_stream_options(context): + async def describe(): + context.last_option_specs = await context.client.describe_options("stream") + + asyncio.run(describe()) + + +@then( + parsers.re(r'the stream option "(?P<key>[^"]+)" should be updatable and not secret') +) +def option_updatable(context, key): + spec = option_spec(context, key) + assert spec.updatable and not spec.secret and not spec.read_only + + +@then( + parsers.re(r'the stream option "(?P<key>[^"]+)" should be create only and secret') +) +def option_create_only_secret(context, key): + spec = option_spec(context, key) + assert not spec.updatable and not spec.resettable + assert spec.secret and not spec.read_only + + +@then(parsers.re(r'the stream option "(?P<key>[^"]+)" should be read only')) +def option_read_only(context, key): + spec = option_spec(context, key) + assert spec.read_only and not spec.updatable and not spec.secret + + +def create_keyed_stream(context, name: str, byte: int) -> None: + key = bytes([byte]) * KEY_LENGTH + + async def create(): + stream = await context.client.create_stream( + name, StreamOptions(encryption_key=key) + ) + context.last_stream_id = stream.id + context.last_stream_name = stream.name + context.last_stream_key = key + + asyncio.run(create()) + + +def fetch_stream(context) -> StreamDetails: + async def fetch(): + return await context.client.get_stream(context.last_stream_id) + + stream = asyncio.run(fetch()) + assert stream is not None + return stream + + +def send_message(context, message: SendMessage) -> None: + async def send(): + await context.client.send_messages( + stream=context.last_stream_id, + topic=context.last_topic_id, + partitioning=PARTITION_ID, + messages=[message], + ) + + asyncio.run(send()) + + +def poll_first(context): + async def poll(): + return await context.client.poll_messages( + stream=context.last_stream_id, + topic=context.last_topic_id, + consumer=Consumer.Single(1), + partition_id=PARTITION_ID, + polling_strategy=PollingStrategy.Offset(value=0), + count=1, + auto_commit=False, + ) + + messages = asyncio.run(poll()) + assert messages + return messages[0] + + +def option_key(key: str) -> OptionKey: + return getattr(OptionKey, "".join(part.title() for part in key.split("_"))) + + +def option_spec(context, key: str): + assert context.last_option_specs is not None + return next(spec for spec in context.last_option_specs if spec.key == key)
diff --git a/bdd/python/uv.lock b/bdd/python/uv.lock index cb82f87..4a602d5 100644 --- a/bdd/python/uv.lock +++ b/bdd/python/uv.lock
@@ -8,7 +8,7 @@ [[package]] name = "apache-iggy" -version = "0.9.1.dev1" +version = "0.9.1.dev2" source = { directory = "../../foreign/python" } [package.metadata]
diff --git a/bdd/rust/Cargo.toml b/bdd/rust/Cargo.toml index 03207ef..cb7249d 100644 --- a/bdd/rust/Cargo.toml +++ b/bdd/rust/Cargo.toml
@@ -60,5 +60,15 @@ harness = false required-features = ["bdd"] +[[test]] +name = "stream_options" +harness = false +required-features = ["bdd"] + +[[test]] +name = "resource_options_patch" +harness = false +required-features = ["bdd"] + [lints] workspace = true
diff --git a/bdd/rust/tests/common/global_context.rs b/bdd/rust/tests/common/global_context.rs index 3313c00..b7d36b8 100644 --- a/bdd/rust/tests/common/global_context.rs +++ b/bdd/rust/tests/common/global_context.rs
@@ -18,7 +18,7 @@ use bytes::Bytes; use cucumber::World; use iggy::clients::client::IggyClient; -use iggy::prelude::{IggyError, IggyMessage, PolledMessages}; +use iggy::prelude::{IggyError, IggyMessage, OptionSpec, PolledMessages, StreamEncryptionKey}; #[derive(Debug, World, Default)] pub struct GlobalContext { @@ -33,5 +33,9 @@ pub last_polled_messages: Option<PolledMessages>, pub last_sent_message: Option<IggyMessage>, pub last_raw_result: Option<Result<Bytes, IggyError>>, + pub last_stream_key: Option<StreamEncryptionKey>, + pub last_update_error: Option<IggyError>, + pub last_option_specs: Vec<OptionSpec>, + pub last_user_id: Option<u32>, pub last_call_result: Option<Result<(), IggyError>>, }
diff --git a/core/binary_protocol/src/responses/users/update_user.rs b/bdd/rust/tests/resource_options_patch.rs similarity index 70% copy from core/binary_protocol/src/responses/users/update_user.rs copy to bdd/rust/tests/resource_options_patch.rs index 2b4a4e4..43739ea 100644 --- a/core/binary_protocol/src/responses/users/update_user.rs +++ b/bdd/rust/tests/resource_options_patch.rs
@@ -15,4 +15,17 @@ // specific language governing permissions and limitations // under the License. -pub type UpdateUserResponse = super::EmptyResponse; +pub(crate) mod common; +pub(crate) mod helpers; +pub(crate) mod steps; + +use crate::common::global_context::GlobalContext; +use cucumber::World; + +#[tokio::main] +async fn main() { + GlobalContext::cucumber() + .fail_on_skipped() + .run_and_exit("../../bdd/scenarios/resource_options_patch.feature") + .await; +}
diff --git a/bdd/rust/tests/steps/leader_redirection.rs b/bdd/rust/tests/steps/leader_redirection.rs index dde838f..bcbf891 100644 --- a/bdd/rust/tests/steps/leader_redirection.rs +++ b/bdd/rust/tests/steps/leader_redirection.rs
@@ -180,7 +180,7 @@ .expect("Client should be available"); let stream = client - .create_stream(&stream_name) + .create_stream(&stream_name, &StreamCreateOptions::default()) .await .expect("Should be able to create stream");
diff --git a/bdd/rust/tests/steps/mod.rs b/bdd/rust/tests/steps/mod.rs index e6c8814..a091ca2 100644 --- a/bdd/rust/tests/steps/mod.rs +++ b/bdd/rust/tests/steps/mod.rs
@@ -19,6 +19,7 @@ pub mod leader_redirection; pub mod messages; pub mod raw_command; +pub mod resource_options; pub mod server; pub mod session; pub mod streams;
diff --git a/bdd/rust/tests/steps/resource_options.rs b/bdd/rust/tests/steps/resource_options.rs new file mode 100644 index 0000000..de6d6dd --- /dev/null +++ b/bdd/rust/tests/steps/resource_options.rs
@@ -0,0 +1,509 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::common::global_context::GlobalContext; +use cucumber::{given, then, when}; +use iggy::prelude::{ + Consumer, HeaderKey, HeaderValue, Identifier, IggyExpiry, IggyMessage, MaxTopicSize, + MessageClient, OptionKey, OptionSpec, OptionsScope, Partitioning, PollingStrategy, + StreamClient, StreamCreateOptions, StreamDetails, StreamEncryptionKey, StreamEncryptionSource, + StreamUpdateOptions, SystemClient, TopicClient, TopicCreateOptions, TopicDetails, + TopicUpdateOptions, UserClient, UserCreateOptions, UserStatus, UserUpdateOptions, +}; +use std::collections::{BTreeMap, BTreeSet}; +use std::str::FromStr; + +const KEY_LENGTH: usize = 32; +const PARTITION_ID: u32 = 0; + +#[when( + regex = r#"^I create a stream named "([^"]+)" with an encryption key of 32 bytes valued (\d+)$"# +)] +pub async fn when_create_stream_with_key(world: &mut GlobalContext, name: String, byte: u8) { + create_keyed_stream(world, &name, byte).await; +} + +#[given( + regex = r#"^a stream named "([^"]+)" with an encryption key of 32 bytes valued (\d+) exists$"# +)] +pub async fn given_keyed_stream(world: &mut GlobalContext, name: String, byte: u8) { + create_keyed_stream(world, &name, byte).await; +} + +#[when(regex = r#"^I create a stream named "([^"]+)" without an encryption key$"#)] +pub async fn when_create_stream_without_key(world: &mut GlobalContext, name: String) { + let client = world.client.as_ref().expect("client"); + let stream = client + .create_stream(&name, &StreamCreateOptions::default()) + .await + .expect("create stream"); + world.last_stream_id = Some(stream.id); + world.last_stream_name = Some(stream.name); + world.last_stream_key = None; +} + +#[given(regex = r#"^a stream named "([^"]+)" exists$"#)] +pub async fn given_plain_stream(world: &mut GlobalContext, name: String) { + when_create_stream_without_key(world, name).await; +} + +#[then(regex = r#"^the stream should report encryption key source "([^"]+)"$"#)] +pub async fn then_stream_source(world: &mut GlobalContext, source: String) { + let stream = fetch_stream(world).await; + let expected = StreamEncryptionSource::from_str(&source).expect("known source"); + assert_eq!(stream.encryption_key_source(), Some(expected)); +} + +#[then("the stream encryption key fingerprint should match the key")] +pub async fn then_fingerprint_matches_key(world: &mut GlobalContext) { + let stream = fetch_stream(world).await; + let key = world.last_stream_key.as_ref().expect("stream key"); + assert_eq!(stream.encryption_key_fingerprint(), Some(key.fingerprint())); +} + +#[then("the stream should report an encryption key fingerprint")] +pub async fn then_stream_reports_fingerprint(world: &mut GlobalContext) { + let stream = fetch_stream(world).await; + let fingerprint = stream.encryption_key_fingerprint().expect("fingerprint"); + assert_eq!(fingerprint.len(), 16); +} + +#[then("the stream should report at least one KEK fingerprint")] +pub async fn then_stream_reports_kek(world: &mut GlobalContext) { + let stream = fetch_stream(world).await; + assert!(!stream.encryption_kek_fingerprints().is_empty()); +} + +#[then("the stream should expose no encryption key material")] +pub async fn then_no_key_material(world: &mut GlobalContext) { + let stream = fetch_stream(world).await; + for secret in [OptionKey::EncryptionKey, OptionKey::EncryptionKeyWrapped] { + assert!( + !stream.options.contains_key(&HeaderKey::from(secret)), + "{secret} must never be returned" + ); + } +} + +#[when(regex = r#"^I create a topic named "([^"]+)" with (\d+) partitions? in the stream$"#)] +pub async fn when_create_topic_in_stream( + world: &mut GlobalContext, + name: String, + partitions_count: u32, +) { + let client = world.client.as_ref().expect("client"); + let topic = client + .create_topic( + &stream_identifier(world), + &name, + &TopicCreateOptions { + partitions_count: Some(partitions_count), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, + ) + .await + .expect("create topic"); + world.last_topic_id = Some(topic.id); + world.last_topic_name = Some(topic.name); +} + +#[when(regex = r#"^I send a message with payload "([^"]+)" to the topic$"#)] +pub async fn when_send_payload(world: &mut GlobalContext, payload: String) { + send_message(world, &payload, BTreeMap::new()).await; +} + +#[when( + regex = r#"^I send a message with payload "([^"]+)" and header "([^"]+)" set to "([^"]+)" to the topic$"# +)] +pub async fn when_send_payload_with_header( + world: &mut GlobalContext, + payload: String, + header: String, + value: String, +) { + let headers = BTreeMap::from([( + HeaderKey::from_str(&header).expect("header key"), + HeaderValue::from_str(&value).expect("header value"), + )]); + send_message(world, &payload, headers).await; +} + +#[then(regex = r#"^polling the topic should return the payload "([^"]+)"$"#)] +pub async fn then_poll_returns_payload(world: &mut GlobalContext, payload: String) { + let message = poll_first(world).await; + assert_eq!(message.payload.as_ref(), payload.as_bytes()); +} + +#[then( + regex = r#"^polling the topic should return the payload "([^"]+)" and header "([^"]+)" set to "([^"]+)"$"# +)] +pub async fn then_poll_returns_payload_and_header( + world: &mut GlobalContext, + payload: String, + header: String, + value: String, +) { + let message = poll_first(world).await; + assert_eq!(message.payload.as_ref(), payload.as_bytes()); + let headers = message + .user_headers_map() + .expect("readable headers") + .expect("headers present"); + assert_eq!( + headers.get(&HeaderKey::from_str(&header).expect("header key")), + Some(&HeaderValue::from_str(&value).expect("header value")) + ); +} + +#[when(regex = r#"^I rename the stream to "([^"]+)"$"#)] +pub async fn when_rename_stream(world: &mut GlobalContext, name: String) { + let client = world.client.as_ref().expect("client"); + client + .update_stream( + &stream_identifier(world), + &StreamUpdateOptions { + name: Some(name), + ..StreamUpdateOptions::default() + }, + ) + .await + .expect("rename stream"); +} + +#[then(regex = r#"^the stream name should be "([^"]+)"$"#)] +pub async fn then_stream_name(world: &mut GlobalContext, name: String) { + assert_eq!(fetch_stream(world).await.name, name); +} + +#[when(regex = r#"^I request a reset of the stream option "([^"]+)"$"#)] +pub async fn when_reset_stream_option(world: &mut GlobalContext, key: String) { + let client = world.client.as_ref().expect("client"); + let result = client + .update_stream( + &stream_identifier(world), + &StreamUpdateOptions { + reset: BTreeSet::from([OptionKey::from_str(&key).expect("known key")]), + ..StreamUpdateOptions::default() + }, + ) + .await; + world.last_update_error = result.err(); +} + +#[then("the stream update should be rejected")] +pub async fn then_stream_update_rejected(world: &mut GlobalContext) { + assert!(world.last_update_error.is_some(), "the update must fail"); +} + +#[when("I describe the stream options")] +pub async fn when_describe_stream_options(world: &mut GlobalContext) { + let client = world.client.as_ref().expect("client"); + world.last_option_specs = client + .describe_options(OptionsScope::Stream) + .await + .expect("describe options"); +} + +#[then(regex = r#"^the stream option "([^"]+)" should be updatable and not secret$"#)] +pub async fn then_option_updatable(world: &mut GlobalContext, key: String) { + let spec = option_spec(world, &key); + assert!(spec.updatable && !spec.secret && !spec.read_only); +} + +#[then(regex = r#"^the stream option "([^"]+)" should be create only and secret$"#)] +pub async fn then_option_create_only_secret(world: &mut GlobalContext, key: String) { + let spec = option_spec(world, &key); + assert!(!spec.updatable && !spec.resettable && spec.secret && !spec.read_only); +} + +#[then(regex = r#"^the stream option "([^"]+)" should be read only$"#)] +pub async fn then_option_read_only(world: &mut GlobalContext, key: String) { + let spec = option_spec(world, &key); + assert!(spec.read_only && !spec.updatable && !spec.secret); +} + +#[given( + regex = r#"^a topic named "([^"]+)" exists in the stream with message expiry (\d+) microseconds and max topic size (\d+) bytes$"# +)] +pub async fn given_topic_with_settings( + world: &mut GlobalContext, + name: String, + expiry: u64, + max_size: u64, +) { + let client = world.client.as_ref().expect("client"); + let topic = client + .create_topic( + &stream_identifier(world), + &name, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::from(expiry)), + max_topic_size: Some(MaxTopicSize::from(max_size)), + ..TopicCreateOptions::default() + }, + ) + .await + .expect("create topic"); + world.last_topic_id = Some(topic.id); + world.last_topic_name = Some(topic.name); +} + +#[when(regex = r"^I update the topic message expiry to (\d+) microseconds without a name$")] +pub async fn when_update_topic_expiry(world: &mut GlobalContext, expiry: u64) { + update_topic( + world, + TopicUpdateOptions { + message_expiry: Some(IggyExpiry::from(expiry)), + ..TopicUpdateOptions::default() + }, + ) + .await; +} + +#[when(regex = r#"^I reset the topic option "([^"]+)"$"#)] +pub async fn when_reset_topic_option(world: &mut GlobalContext, key: String) { + update_topic( + world, + TopicUpdateOptions { + reset: BTreeSet::from([OptionKey::from_str(&key).expect("known key")]), + ..TopicUpdateOptions::default() + }, + ) + .await; +} + +#[when(regex = r#"^I rename the topic to "([^"]+)"$"#)] +pub async fn when_rename_topic(world: &mut GlobalContext, name: String) { + update_topic( + world, + TopicUpdateOptions { + name: Some(name), + ..TopicUpdateOptions::default() + }, + ) + .await; +} + +#[then(regex = r#"^the topic name should be "([^"]+)"$"#)] +pub async fn then_topic_name(world: &mut GlobalContext, name: String) { + assert_eq!(fetch_topic(world).await.name, name); +} + +#[then(regex = r"^the topic message expiry should be (\d+) microseconds set explicitly$")] +pub async fn then_topic_expiry_explicit(world: &mut GlobalContext, expiry: u64) { + let topic = fetch_topic(world).await; + assert_eq!( + topic.resolved_options().message_expiry, + IggyExpiry::from(expiry) + ); + assert!(topic_option_explicit(&topic, OptionKey::MessageExpiry)); +} + +#[then(regex = r"^the topic max topic size should be (\d+) bytes set explicitly$")] +pub async fn then_topic_max_size_explicit(world: &mut GlobalContext, max_size: u64) { + let topic = fetch_topic(world).await; + assert_eq!( + topic.resolved_options().max_topic_size, + MaxTopicSize::from(max_size) + ); + assert!(topic_option_explicit(&topic, OptionKey::MaxTopicSize)); +} + +#[then("the topic max topic size should be derived from the server default")] +pub async fn then_topic_max_size_derived(world: &mut GlobalContext) { + let topic = fetch_topic(world).await; + assert!(!topic_option_explicit(&topic, OptionKey::MaxTopicSize)); +} + +#[given(regex = r#"^a user named "([^"]+)" with password "([^"]+)" exists$"#)] +pub async fn given_user(world: &mut GlobalContext, username: String, password: String) { + let client = world.client.as_ref().expect("client"); + let user = client + .create_user( + &username, + &password, + UserStatus::Active, + None, + &UserCreateOptions::default(), + ) + .await + .expect("create user"); + world.last_user_id = Some(user.id); +} + +#[when(regex = r#"^I update the user username to "([^"]+)" and status to "([^"]+)"$"#)] +pub async fn when_update_user(world: &mut GlobalContext, username: String, status: String) { + let client = world.client.as_ref().expect("client"); + client + .update_user( + &user_identifier(world), + &UserUpdateOptions { + username: Some(username), + status: Some(UserStatus::from_str(&status).expect("known status")), + ..UserUpdateOptions::default() + }, + ) + .await + .expect("update user"); +} + +#[then(regex = r#"^the user should have username "([^"]+)"$"#)] +pub async fn then_user_username(world: &mut GlobalContext, username: String) { + let client = world.client.as_ref().expect("client"); + let user = client + .get_user(&user_identifier(world)) + .await + .expect("get user") + .expect("user exists"); + assert_eq!(user.username, username); +} + +#[then(regex = r#"^the user should have status "([^"]+)"$"#)] +pub async fn then_user_status(world: &mut GlobalContext, status: String) { + let client = world.client.as_ref().expect("client"); + let user = client + .get_user(&user_identifier(world)) + .await + .expect("get user") + .expect("user exists"); + assert_eq!( + user.status, + UserStatus::from_str(&status).expect("known status") + ); +} + +async fn create_keyed_stream(world: &mut GlobalContext, name: &str, byte: u8) { + let key = StreamEncryptionKey::try_from([byte; KEY_LENGTH].as_slice()).expect("32 bytes"); + let client = world.client.as_ref().expect("client"); + let stream = client + .create_stream( + name, + &StreamCreateOptions { + encryption_key: Some(key.clone()), + ..StreamCreateOptions::default() + }, + ) + .await + .expect("create keyed stream"); + world.last_stream_id = Some(stream.id); + world.last_stream_name = Some(stream.name); + world.last_stream_key = Some(key); +} + +async fn fetch_stream(world: &GlobalContext) -> StreamDetails { + let client = world.client.as_ref().expect("client"); + client + .get_stream(&stream_identifier(world)) + .await + .expect("get stream") + .expect("stream exists") +} + +async fn fetch_topic(world: &GlobalContext) -> TopicDetails { + let client = world.client.as_ref().expect("client"); + client + .get_topic(&stream_identifier(world), &topic_identifier(world)) + .await + .expect("get topic") + .expect("topic exists") +} + +async fn update_topic(world: &GlobalContext, options: TopicUpdateOptions) { + let client = world.client.as_ref().expect("client"); + client + .update_topic( + &stream_identifier(world), + &topic_identifier(world), + &options, + ) + .await + .expect("update topic"); +} + +async fn send_message( + world: &GlobalContext, + payload: &str, + headers: BTreeMap<HeaderKey, HeaderValue>, +) { + let client = world.client.as_ref().expect("client"); + let builder = IggyMessage::builder().payload(payload.to_owned().into()); + let message = if headers.is_empty() { + builder.build() + } else { + builder.user_headers(headers).build() + } + .expect("message"); + client + .send_messages( + &stream_identifier(world), + &topic_identifier(world), + &Partitioning::partition_id(PARTITION_ID), + &mut [message], + ) + .await + .expect("send message"); +} + +async fn poll_first(world: &GlobalContext) -> IggyMessage { + let client = world.client.as_ref().expect("client"); + let polled = client + .poll_messages( + &stream_identifier(world), + &topic_identifier(world), + Some(PARTITION_ID), + &Consumer::default(), + &PollingStrategy::offset(0), + 1, + false, + ) + .await + .expect("poll messages"); + polled + .messages + .into_iter() + .next() + .expect("one message polled") +} + +fn option_spec<'a>(world: &'a GlobalContext, key: &str) -> &'a OptionSpec { + world + .last_option_specs + .iter() + .find(|spec| spec.key == key) + .expect("described key") +} + +fn topic_option_explicit(topic: &TopicDetails, key: OptionKey) -> bool { + topic + .options + .get(&HeaderKey::from(key)) + .is_some_and(|option| option.explicit) +} + +fn stream_identifier(world: &GlobalContext) -> Identifier { + Identifier::numeric(world.last_stream_id.expect("stream created")).expect("stream id") +} + +fn topic_identifier(world: &GlobalContext) -> Identifier { + Identifier::numeric(world.last_topic_id.expect("topic created")).expect("topic id") +} + +fn user_identifier(world: &GlobalContext) -> Identifier { + Identifier::numeric(world.last_user_id.expect("user created")).expect("user id") +}
diff --git a/bdd/rust/tests/steps/streams.rs b/bdd/rust/tests/steps/streams.rs index f94ef40..9d9ce3e 100644 --- a/bdd/rust/tests/steps/streams.rs +++ b/bdd/rust/tests/steps/streams.rs
@@ -17,6 +17,7 @@ use crate::common::global_context::GlobalContext; use cucumber::{given, then, when}; +use iggy::prelude::StreamCreateOptions; use iggy::prelude::{Identifier, StreamClient, StreamUpdateOptions}; #[given("I have no streams in the system")] @@ -96,8 +97,10 @@ client .update_stream( &Identifier::numeric(stream_id).expect("Stream ID should be valid"), - &stream_name, - &StreamUpdateOptions::default(), + &StreamUpdateOptions { + name: Some(stream_name.to_owned()), + ..StreamUpdateOptions::default() + }, ) .await .expect("Should be able to update stream"); @@ -142,7 +145,7 @@ async fn create_stream(world: &mut GlobalContext, stream_name: &str) { let client = world.client.as_ref().expect("Client should be available"); let stream = client - .create_stream(stream_name) + .create_stream(stream_name, &StreamCreateOptions::default()) .await .expect("Should be able to create stream");
diff --git a/foreign/csharp/Iggy_SDK/Contracts/Http/Auth/UpdateUserRequest.cs b/bdd/rust/tests/stream_options.rs similarity index 71% rename from foreign/csharp/Iggy_SDK/Contracts/Http/Auth/UpdateUserRequest.cs rename to bdd/rust/tests/stream_options.rs index 7ef9c59..ce5c563 100644 --- a/foreign/csharp/Iggy_SDK/Contracts/Http/Auth/UpdateUserRequest.cs +++ b/bdd/rust/tests/stream_options.rs
@@ -15,8 +15,17 @@ // specific language governing permissions and limitations // under the License. -using Apache.Iggy.Enums; +pub(crate) mod common; +pub(crate) mod helpers; +pub(crate) mod steps; -namespace Apache.Iggy.Contracts.Http.Auth; +use crate::common::global_context::GlobalContext; +use cucumber::World; -internal sealed record UpdateUserRequest(string? Username, UserStatus? UserStatus); +#[tokio::main] +async fn main() { + GlobalContext::cucumber() + .fail_on_skipped() + .run_and_exit("../../bdd/scenarios/stream_options.feature") + .await; +}
diff --git a/bdd/scenarios/resource_options_patch.feature b/bdd/scenarios/resource_options_patch.feature new file mode 100644 index 0000000..348156f --- /dev/null +++ b/bdd/scenarios/resource_options_patch.feature
@@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +@resource-options-patch +Feature: Resource option updates + As a developer using Apache Iggy + I want to change only the settings I name on a topic or a user + So that an update never resends or silently resets the others + + Background: + Given I have a running Iggy server + And I am authenticated as the root user + + Scenario: Update one topic setting + Given a stream named "patch-topic-stream" exists + And a topic named "patch-topic" exists in the stream with message expiry 3600000000 microseconds and max topic size 1073741824 bytes + When I update the topic message expiry to 7200000000 microseconds without a name + Then the topic name should be "patch-topic" + And the topic message expiry should be 7200000000 microseconds set explicitly + And the topic max topic size should be 1073741824 bytes set explicitly + + Scenario: Reset a topic setting to the server default + Given a stream named "patch-reset-stream" exists + And a topic named "patch-reset" exists in the stream with message expiry 3600000000 microseconds and max topic size 1073741824 bytes + When I reset the topic option "max_topic_size" + Then the topic max topic size should be derived from the server default + And the topic message expiry should be 3600000000 microseconds set explicitly + + Scenario: Rename a topic + Given a stream named "patch-rename-stream" exists + And a topic named "patch-rename" exists in the stream with message expiry 3600000000 microseconds and max topic size 1073741824 bytes + When I rename the topic to "patch-renamed" + Then the topic name should be "patch-renamed" + And the topic message expiry should be 3600000000 microseconds set explicitly + And the topic max topic size should be 1073741824 bytes set explicitly + + Scenario: Update a user username and status + Given a user named "patch-user" with password "patch-password" exists + When I update the user username to "patch-user-renamed" and status to "inactive" + Then the user should have username "patch-user-renamed" + And the user should have status "inactive"
diff --git a/bdd/scenarios/stream_options.feature b/bdd/scenarios/stream_options.feature new file mode 100644 index 0000000..c1a56e7 --- /dev/null +++ b/bdd/scenarios/stream_options.feature
@@ -0,0 +1,74 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +@stream-options +Feature: Stream options + As a developer using Apache Iggy + I want to give a stream its own encryption key through typed options + So that each stream keeps its data at rest under its own key + + Background: + Given I have a running Iggy server + And I am authenticated as the root user + + Scenario: Create a stream with its own key + When I create a stream named "stream-options-own-key" with an encryption key of 32 bytes valued 7 + Then the stream should report encryption key source "stream" + And the stream encryption key fingerprint should match the key + And the stream should report at least one KEK fingerprint + And the stream should expose no encryption key material + + Scenario: Create a stream with a server generated key + When I create a stream named "stream-options-generated" without an encryption key + Then the stream should report encryption key source "server" + And the stream should report an encryption key fingerprint + And the stream should report at least one KEK fingerprint + And the stream should expose no encryption key material + When I create a topic named "events" with 1 partition in the stream + And I send a message with payload "generated-key-payload" to the topic + Then polling the topic should return the payload "generated-key-payload" + + @message-headers + Scenario: Send and poll a message with headers on a keyed stream + Given a stream named "stream-options-custom" with an encryption key of 32 bytes valued 9 exists + When I create a topic named "events" with 1 partition in the stream + And I send a message with payload "custom-key-payload" and header "trace" set to "abc" to the topic + Then polling the topic should return the payload "custom-key-payload" and header "trace" set to "abc" + + Scenario: Rename a keyed stream + Given a stream named "stream-options-rename" with an encryption key of 32 bytes valued 11 exists + When I create a topic named "events" with 1 partition in the stream + And I send a message with payload "before-rename" to the topic + And I rename the stream to "stream-options-renamed" + Then the stream name should be "stream-options-renamed" + And the stream should report encryption key source "stream" + And the stream encryption key fingerprint should match the key + And polling the topic should return the payload "before-rename" + + Scenario: Stream key reset is rejected + Given a stream named "stream-options-reset" with an encryption key of 32 bytes valued 13 exists + When I request a reset of the stream option "encryption_key" + Then the stream update should be rejected + And the stream name should be "stream-options-reset" + And the stream encryption key fingerprint should match the key + + @describe-options + Scenario: Describe stream options + When I describe the stream options + Then the stream option "name" should be updatable and not secret + And the stream option "encryption_key" should be create only and secret + And the stream option "encryption_key_fingerprint" should be read only
diff --git a/core/ai/mcp/Cargo.toml b/core/ai/mcp/Cargo.toml index a912c02..1fc411c 100644 --- a/core/ai/mcp/Cargo.toml +++ b/core/ai/mcp/Cargo.toml
@@ -17,7 +17,7 @@ [package] name = "iggy-mcp" -version = "0.5.1-edge.1" +version = "0.5.1-edge.2" description = "MCP Server for Iggy message streaming platform" edition = "2024" license = "Apache-2.0"
diff --git a/core/ai/mcp/src/service/mod.rs b/core/ai/mcp/src/service/mod.rs index f129522..5fbe234 100644 --- a/core/ai/mcp/src/service/mod.rs +++ b/core/ai/mcp/src/service/mod.rs
@@ -18,10 +18,11 @@ use iggy::prelude::{ Client, ClusterClient, CompressionAlgorithm, Consumer, ConsumerGroupClient, ConsumerOffsetClient, Identifier, IggyClient, IggyError, IggyExpiry, IggyMessage, - IggyTimestamp, MaxTopicSize, MessageClient, PartitionClient, Partitioning, + IggyTimestamp, MaxTopicSize, MessageClient, OptionKey, PartitionClient, Partitioning, PersonalAccessTokenClient, PollingKind, PollingStrategy, SegmentClient, StreamClient, - StreamUpdateOptions, SystemClient, SystemSnapshotType, TopicClient, TopicCreateOptions, - TopicUpdateOptions, UserClient, UserStatus, UserUpdateOptions, + StreamCreateOptions, StreamUpdateOptions, SystemClient, SystemSnapshotType, TopicClient, + TopicCreateOptions, TopicUpdateOptions, UserClient, UserCreateOptions, UserStatus, + UserUpdateOptions, }; use requests::*; use rmcp::{ @@ -31,6 +32,9 @@ tool, tool_handler, tool_router, }; use serde::Serialize; +use std::collections::BTreeSet; +use std::fmt::Display; +use std::str::FromStr; use std::sync::Arc; use std::time::{Duration, Instant}; use tracing::error; @@ -90,18 +94,23 @@ Parameters(CreateStream { name }): Parameters<CreateStream>, ) -> Result<CallToolResult, ErrorData> { self.permissions.ensure_create()?; - request(self.client.create_stream(&name).await) + request( + self.client + .create_stream(&name, &StreamCreateOptions::default()) + .await, + ) } #[tool(description = "Update stream")] pub async fn update_stream( &self, - Parameters(UpdateStream { stream_id, name }): Parameters<UpdateStream>, + Parameters(update): Parameters<UpdateStream>, ) -> Result<CallToolResult, ErrorData> { self.permissions.ensure_update()?; + let stream_id = id(&update.stream_id)?; request( self.client - .update_stream(&id(&stream_id)?, &name, &StreamUpdateOptions::default()) + .update_stream(&stream_id, &StreamUpdateOptions::from(update)) .await, ) } @@ -189,28 +198,15 @@ #[tool(description = "Update topic")] pub async fn update_topic( &self, - Parameters(UpdateTopic { - stream_id, - topic_id, - name, - compression_algorithm, - message_expiry, - max_size, - options, - }): Parameters<UpdateTopic>, + Parameters(update): Parameters<UpdateTopic>, ) -> Result<CallToolResult, ErrorData> { self.permissions.ensure_update()?; - // Absent means "leave alone" now, so an unparsable value must not - // silently become a reset to the server default. - let update_options = TopicUpdateOptions { - compression_algorithm: compression_algorithm.and_then(|ca| ca.parse().ok()), - message_expiry: message_expiry.and_then(|me| me.parse().ok()), - max_topic_size: max_size.and_then(|ms| ms.parse().ok()), - raw: options, - }; + let stream_id = id(&update.stream_id)?; + let topic_id = id(&update.topic_id)?; + let update_options = TopicUpdateOptions::try_from(update)?; request( self.client - .update_topic(&id(&stream_id)?, &id(&topic_id)?, &name, &update_options) + .update_topic(&stream_id, &topic_id, &update_options) .await, ) } @@ -637,7 +633,8 @@ ) -> Result<CallToolResult, ErrorData> { self.permissions.ensure_create()?; let expiry = expiry - .and_then(|expiry| expiry.parse().ok()) + .map(|expiry| parsed(&expiry, "personal access token expiry")) + .transpose()? .unwrap_or_default(); request( self.client @@ -689,7 +686,13 @@ let permissions = permissions.map(|p| p.into()); request( self.client - .create_user(&username, &password, status, permissions) + .create_user( + &username, + &password, + status, + permissions, + &UserCreateOptions::default(), + ) .await, ) } @@ -697,28 +700,13 @@ #[tool(description = "Update user")] pub async fn update_user( &self, - Parameters(UpdateUser { - user_id, - username, - active, - }): Parameters<UpdateUser>, + Parameters(update): Parameters<UpdateUser>, ) -> Result<CallToolResult, ErrorData> { self.permissions.ensure_update()?; - let status = active.map(|active| { - if active { - UserStatus::Active - } else { - UserStatus::Inactive - } - }); + let user_id = id(&update.user_id)?; request( self.client - .update_user( - &id(&user_id)?, - username.as_deref(), - status, - &UserUpdateOptions::default(), - ) + .update_user(&user_id, &UserUpdateOptions::from(update)) .await, ) } @@ -775,6 +763,88 @@ } } +impl From<UpdateStream> for StreamUpdateOptions { + fn from(update: UpdateStream) -> Self { + let (reset, raw_reset) = split_reset(update.reset); + StreamUpdateOptions { + name: update.name, + reset, + raw_reset, + ..StreamUpdateOptions::default() + } + } +} + +impl TryFrom<UpdateTopic> for TopicUpdateOptions { + type Error = ErrorData; + + fn try_from(update: UpdateTopic) -> Result<Self, Self::Error> { + let (reset, raw_reset) = split_reset(update.reset); + Ok(TopicUpdateOptions { + name: update.name, + message_expiry: update + .message_expiry + .map(|value| parsed(&value, "message expiry")) + .transpose()?, + max_topic_size: update + .max_size + .map(|value| parsed(&value, "maximum size")) + .transpose()?, + reset, + raw_reset, + raw: update.options, + }) + } +} + +impl From<UpdateUser> for UserUpdateOptions { + fn from(update: UpdateUser) -> Self { + let (reset, raw_reset) = split_reset(update.reset); + UserUpdateOptions { + username: update.username, + status: update.active.map(|active| { + if active { + UserStatus::Active + } else { + UserStatus::Inactive + } + }), + reset, + raw_reset, + raw: update.options, + } + } +} + +/// Split reset names into keys this build checks before sending and newer keys +/// the server decides on. +fn split_reset(names: Vec<String>) -> (BTreeSet<OptionKey>, BTreeSet<String>) { + let mut known = BTreeSet::new(); + let mut newer = BTreeSet::new(); + for name in names { + match name.parse::<OptionKey>() { + Ok(key) => { + known.insert(key); + } + Err(_) => { + newer.insert(name); + } + } + } + (known, newer) +} + +fn parsed<T: FromStr>(value: &str, what: &str) -> Result<T, ErrorData> +where + T::Err: Display, +{ + value.parse().map_err(|error: T::Err| { + let message = format!("Failed to parse {what}. {error}"); + error!(message); + ErrorData::invalid_request(message, None) + }) +} + fn id(id: &str) -> Result<Identifier, ErrorData> { Identifier::from_str_value(id).map_err(|e| { let message = format!("Failed to parse identifier. {e}"); @@ -814,6 +884,7 @@ #[cfg(test)] mod tests { use super::*; + use std::collections::BTreeMap; #[test] fn given_uncertain_or_blocked_writes_when_reporting_errors_should_preserve_exact_operation_ids() @@ -828,4 +899,82 @@ assert_eq!(operation_id_value(operation).unwrap(), u64::MAX); } } + + #[test] + fn given_update_topic_values_when_converting_should_parse_them_into_typed_options() { + let update = update_topic( + r#"{"name":"events-v2","message_expiry":"7days","max_size":"15GB","options":{"future_key":"1"}}"#, + ); + + let options = TopicUpdateOptions::try_from(update).expect("valid values convert"); + + assert_eq!( + options, + TopicUpdateOptions { + name: Some("events-v2".to_owned()), + message_expiry: Some("7days".parse().expect("valid expiry")), + max_topic_size: Some("15GB".parse().expect("valid size")), + raw: BTreeMap::from([("future_key".to_owned(), "1".to_owned())]), + ..TopicUpdateOptions::default() + } + ); + } + + #[test] + fn given_unparsable_update_topic_value_when_converting_should_refuse() { + for body in [r#"{"message_expiry":"soon"}"#, r#"{"max_size":"big"}"#] { + assert!( + TopicUpdateOptions::try_from(update_topic(body)).is_err(), + "{body}" + ); + } + } + + #[test] + fn given_reset_names_when_converting_should_send_known_typed_and_newer_raw() { + let update = update_topic(r#"{"reset":["message_expiry","future_key"]}"#); + + let options = TopicUpdateOptions::try_from(update).expect("reset names convert"); + + assert_eq!( + options, + TopicUpdateOptions { + reset: BTreeSet::from([OptionKey::MessageExpiry]), + raw_reset: BTreeSet::from(["future_key".to_owned()]), + ..TopicUpdateOptions::default() + } + ); + } + + #[test] + fn given_stream_and_user_resets_when_converting_should_keep_the_name_optional() { + let stream: UpdateStream = + serde_json::from_str(r#"{"stream_id":"orders","reset":["future_key"]}"#) + .expect("a reset without a name parses"); + let user: UpdateUser = serde_json::from_str( + r#"{"user_id":"alice","reset":["future_key"],"options":{"newer_key":"x"}}"#, + ) + .expect("a user patch parses"); + + assert_eq!( + StreamUpdateOptions::from(stream), + StreamUpdateOptions { + raw_reset: BTreeSet::from(["future_key".to_owned()]), + ..StreamUpdateOptions::default() + } + ); + assert_eq!( + UserUpdateOptions::from(user), + UserUpdateOptions { + raw_reset: BTreeSet::from(["future_key".to_owned()]), + raw: BTreeMap::from([("newer_key".to_owned(), "x".to_owned())]), + ..UserUpdateOptions::default() + } + ); + } + + fn update_topic(fields: &str) -> UpdateTopic { + let ids = r#"{"stream_id":"orders","topic_id":"events","#; + serde_json::from_str(&format!("{ids}{}", &fields[1..])).expect("valid update_topic body") + } }
diff --git a/core/ai/mcp/src/service/requests.rs b/core/ai/mcp/src/service/requests.rs index dd3dbcc..05eb380 100644 --- a/core/ai/mcp/src/service/requests.rs +++ b/core/ai/mcp/src/service/requests.rs
@@ -36,17 +36,24 @@ } #[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] pub struct CreateStream { #[schemars(description = "stream name (required, must be unique)")] pub name: String, } #[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] pub struct UpdateStream { #[schemars(description = "stream identifier (name or number)")] pub stream_id: String, - #[schemars(description = "stream name (required, must be unique)")] - pub name: String, + #[schemars(description = "new name, must be unique (optional)")] + pub name: Option<String>, + #[schemars( + description = "setting names to return to their defaults, checked by the Iggy server (optional)" + )] + #[serde(default)] + pub reset: Vec<String>, } #[derive(Debug, Deserialize, JsonSchema)] @@ -99,6 +106,7 @@ } #[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] pub struct UpdateTopic { #[schemars(description = "stream identifier (name or number)")] pub stream_id: String, @@ -106,22 +114,25 @@ #[schemars(description = "topic identifier (name or number)")] pub topic_id: String, - #[schemars(description = "name (required, must be unique)")] - pub name: String, + #[schemars(description = "new name, must be unique in the stream (optional)")] + pub name: Option<String>, - #[schemars(description = "compression algorithm (optional, can be one of 'none', 'gzip')")] - pub compression_algorithm: Option<String>, - - #[schemars(description = "message expiry (optional)")] + #[schemars(description = "message expiry like '7 days' or 'unlimited' (optional)")] pub message_expiry: Option<String>, - #[schemars(description = "maximum size (optional)")] + #[schemars(description = "maximum size like '15 GB' or 'unlimited' (optional)")] pub max_size: Option<String>, #[schemars( - description = "options to change, as string key-values; only compression_algorithm, \ - message_expiry and max_topic_size may be updated, and a key left out \ - keeps its current value (optional)" + description = "setting names to return to their defaults, e.g. message_expiry. Keys are \ + validated by the Iggy server (optional)" + )] + #[serde(default)] + pub reset: Vec<String>, + + #[schemars( + description = "other updatable settings as string key-values, a key left out keeps its \ + current value (optional)" )] #[serde(default)] pub options: BTreeMap<String, String>, @@ -377,6 +388,19 @@ #[schemars(description = "active (optional)")] pub active: Option<bool>, + + #[schemars( + description = "setting names to return to their defaults, checked by the Iggy server (optional)" + )] + #[serde(default)] + pub reset: Vec<String>, + + #[schemars( + description = "other updatable settings as string key-values, a key left out keeps its \ + current value (optional)" + )] + #[serde(default)] + pub options: BTreeMap<String, String>, } #[derive(Debug, Deserialize, JsonSchema)] @@ -550,3 +574,47 @@ } } } + +#[cfg(test)] +mod tests { + use super::*; + use rmcp::schemars::schema_for; + + #[test] + fn given_stream_tool_schema_when_describing_should_exclude_encryption_key() { + for schema in [schema_for!(CreateStream), schema_for!(UpdateStream)] { + let properties = schema + .get("properties") + .and_then(|properties| properties.as_object()) + .expect("object schema"); + for secret_route in ["encryption_key", "options", "raw"] { + assert!( + !properties.contains_key(secret_route), + "stream tools must not accept {secret_route}" + ); + } + } + + let smuggled = [ + r#"{"name":"orders","encryption_key":"AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8="}"#, + r#"{"name":"orders","options":{"encryption_key":"AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8="}}"#, + ]; + for body in smuggled { + assert!( + serde_json::from_str::<CreateStream>(body).is_err(), + "{body}" + ); + } + let rename: UpdateStream = + serde_json::from_str(r#"{"stream_id":"orders","name":"orders-v2"}"#) + .expect("a rename still parses"); + assert_eq!(rename.name.as_deref(), Some("orders-v2")); + } + + #[test] + fn given_update_topic_with_retired_compression_when_parsing_should_refuse() { + let body = r#"{"stream_id":"orders","topic_id":"events","name":"events-v2","compression_algorithm":"gzip"}"#; + + assert!(serde_json::from_str::<UpdateTopic>(body).is_err()); + } +}
diff --git a/core/bench/Cargo.toml b/core/bench/Cargo.toml index 0b90798..f05e136 100644 --- a/core/bench/Cargo.toml +++ b/core/bench/Cargo.toml
@@ -17,7 +17,7 @@ [package] name = "iggy-bench" -version = "0.6.1-edge.1" +version = "0.6.1-edge.2" edition = "2024" license = "Apache-2.0" repository = "https://github.com/apache/iggy"
diff --git a/core/bench/src/benchmarks/benchmark.rs b/core/bench/src/benchmarks/benchmark.rs index c5359de..a37a219 100644 --- a/core/bench/src/benchmarks/benchmark.rs +++ b/core/bench/src/benchmarks/benchmark.rs
@@ -116,7 +116,9 @@ } info!("Creating the test stream '{}'", stream_name); - client.create_stream(&stream_name).await?; + client + .create_stream(&stream_name, &StreamCreateOptions::default()) + .await?; let topic_name = "topic-1".to_string(); let max_topic_size = self .args()
diff --git a/core/binary_protocol/Cargo.toml b/core/binary_protocol/Cargo.toml index 9c5e86d..531d478 100644 --- a/core/binary_protocol/Cargo.toml +++ b/core/binary_protocol/Cargo.toml
@@ -17,7 +17,7 @@ [package] name = "iggy_binary_protocol" -version = "0.11.1-edge.1" +version = "0.11.1-edge.2" description = "Wire protocol types and codec for the Iggy binary protocol. Shared between server and SDK." edition = "2024" rust-version.workspace = true @@ -33,6 +33,7 @@ bytemuck = { workspace = true } bytes = { workspace = true } enumset = { workspace = true } +ring = { workspace = true } secrecy = { workspace = true } serde = { workspace = true } thiserror = { workspace = true }
diff --git a/core/binary_protocol/src/codes.rs b/core/binary_protocol/src/codes.rs index d90f446..ab8e8f9 100644 --- a/core/binary_protocol/src/codes.rs +++ b/core/binary_protocol/src/codes.rs
@@ -28,6 +28,7 @@ pub const DESCRIBE_OPTIONS_CODE: u32 = 13; // Code 14 is retired. pub const BIND_SESSION_CODE: u32 = 15; +pub const UPDATE_RESOURCE_CODE: u32 = 16; pub const GET_ME_CODE: u32 = 20; pub const GET_CLIENT_CODE: u32 = 21; pub const GET_CLIENTS_CODE: u32 = 22; @@ -37,7 +38,6 @@ pub const GET_USERS_CODE: u32 = 32; pub const CREATE_USER_CODE: u32 = 33; pub const DELETE_USER_CODE: u32 = 34; -pub const UPDATE_USER_CODE: u32 = 35; pub const UPDATE_PERMISSIONS_CODE: u32 = 36; pub const CHANGE_PASSWORD_CODE: u32 = 37; pub const LOGIN_USER_CODE: u32 = 38; @@ -73,7 +73,6 @@ pub const GET_STREAMS_CODE: u32 = 201; pub const CREATE_STREAM_CODE: u32 = 202; pub const DELETE_STREAM_CODE: u32 = 203; -pub const UPDATE_STREAM_CODE: u32 = 204; // 205 was PURGE_STREAM. Never reuse it. // -- Topics -- @@ -81,7 +80,6 @@ pub const GET_TOPICS_CODE: u32 = 301; pub const CREATE_TOPIC_CODE: u32 = 302; pub const DELETE_TOPIC_CODE: u32 = 303; -pub const UPDATE_TOPIC_CODE: u32 = 304; // 305 was PURGE_TOPIC. Never reuse it. // -- Partitions -- @@ -115,7 +113,7 @@ mod tests { use super::*; - const RETIRED_CODES: &[u32] = &[14, 102, 103, 104, 123, 124, 205, 305]; + const RETIRED_CODES: &[u32] = &[14, 35, 102, 103, 104, 123, 124, 204, 205, 304, 305]; const ALL_CODES: &[u32] = &[ PING_CODE, @@ -124,6 +122,7 @@ GET_CLUSTER_METADATA_CODE, DESCRIBE_OPTIONS_CODE, BIND_SESSION_CODE, + UPDATE_RESOURCE_CODE, GET_ME_CODE, GET_CLIENT_CODE, GET_CLIENTS_CODE, @@ -131,7 +130,6 @@ GET_USERS_CODE, CREATE_USER_CODE, DELETE_USER_CODE, - UPDATE_USER_CODE, UPDATE_PERMISSIONS_CODE, CHANGE_PASSWORD_CODE, LOGIN_USER_CODE, @@ -152,12 +150,10 @@ GET_STREAMS_CODE, CREATE_STREAM_CODE, DELETE_STREAM_CODE, - UPDATE_STREAM_CODE, GET_TOPIC_CODE, GET_TOPICS_CODE, CREATE_TOPIC_CODE, DELETE_TOPIC_CODE, - UPDATE_TOPIC_CODE, CREATE_PARTITIONS_CODE, DELETE_PARTITIONS_CODE, DELETE_SEGMENTS_CODE,
diff --git a/core/binary_protocol/src/consensus/header.rs b/core/binary_protocol/src/consensus/header.rs index ee09569..0d64b42 100644 --- a/core/binary_protocol/src/consensus/header.rs +++ b/core/binary_protocol/src/consensus/header.rs
@@ -298,13 +298,10 @@ pub reserved_frame: [u8; 66], pub client: u128, - /// Optional XXH3 integrity stamp over the client's request payload. Zero - /// means unstamped; the server verifies a nonzero stamp before routing or - /// rewriting the body. The Rust SDK stamps metadata/session operations - /// and `DeleteSegments`; partition and non-replicated operations leave it - /// zero. Retry identity uses the client, session, request, group and - /// operation, so matching retries replay the first receipt without - /// comparing stamps. + /// Random identifier for a logical metadata or session request. The client + /// keeps it across retries and reconnect resends. Partition and + /// non-replicated operations leave it zero. The server seals the prepared + /// body separately with `checksum_body`. pub request_checksum: u128, pub timestamp: u64, pub request: u64,
diff --git a/core/binary_protocol/src/consensus/operation.rs b/core/binary_protocol/src/consensus/operation.rs index 7961e21..44f99da 100644 --- a/core/binary_protocol/src/consensus/operation.rs +++ b/core/binary_protocol/src/consensus/operation.rs
@@ -64,14 +64,26 @@ TruncatePartition = 68, FinalizeSession = 70, CompleteLifecycle = 71, + /// Prepare-resolved `UpdateResource` on a stream: column changes extracted, + /// resets resolved to their defaults. Claimed by the streams state machine. + UpdateStreamResolved = 72, + /// Prepare-resolved `UpdateResource` on a topic. + UpdateTopicResolved = 73, + /// Prepare-resolved `UpdateResource` on a user. Claimed by the users state + /// machine. + UpdateUserResolved = 74, + /// Server-originated: replace encrypted streams' wrap sets during a + /// key-encryption key rotation. Proposed by the metadata primary, no + /// client wire code. + RewrapStreamKeys = 75, // Metadata operations (shard 0) CreateStream = 128, - UpdateStream = 129, + // 129 was UpdateStream. Never reuse it. DeleteStream = 130, // 131 was PurgeStream. Never reuse it. CreateTopic = 132, - UpdateTopic = 133, + // 133 was UpdateTopic. Never reuse it. DeleteTopic = 134, // 135 was PurgeTopic. Never reuse it. CreatePartitions = 136, @@ -84,7 +96,7 @@ CreateConsumerGroup = 139, DeleteConsumerGroup = 140, CreateUser = 141, - UpdateUser = 142, + // 142 was UpdateUser. Never reuse it. DeleteUser = 143, ChangePassword = 144, UpdatePermissions = 145, @@ -92,6 +104,9 @@ DeletePersonalAccessToken = 147, JoinConsumerGroup = 148, LeaveConsumerGroup = 149, + /// Generic option patch for a stream, topic or user. Prepare rewrites it to + /// the resolved operation of the target's scope. + UpdateResource = 150, // Partition operations (routed by namespace) SendMessages = 160, @@ -128,6 +143,9 @@ match self { Self::CreateTopicWithAssignments => Self::CreateTopic, Self::CreatePartitionsWithAssignments => Self::CreatePartitions, + Self::UpdateStreamResolved | Self::UpdateTopicResolved | Self::UpdateUserResolved => { + Self::UpdateResource + } Self::TruncatePartition => Self::DeleteSegments, operation => *operation, } @@ -172,17 +190,14 @@ matches!( self, Self::CreateStream - | Self::UpdateStream | Self::DeleteStream | Self::CreateTopic - | Self::UpdateTopic | Self::DeleteTopic | Self::CreatePartitions | Self::DeletePartitions | Self::CreateConsumerGroup | Self::DeleteConsumerGroup | Self::CreateUser - | Self::UpdateUser | Self::DeleteUser | Self::ChangePassword | Self::UpdatePermissions @@ -190,6 +205,7 @@ | Self::DeletePersonalAccessToken | Self::JoinConsumerGroup | Self::LeaveConsumerGroup + | Self::UpdateResource ) } @@ -269,6 +285,10 @@ | Self::NonReplicated | Self::CreateTopicWithAssignments | Self::CreatePartitionsWithAssignments + | Self::UpdateStreamResolved + | Self::UpdateTopicResolved + | Self::UpdateUserResolved + | Self::RewrapStreamKeys | Self::RemoveConsumerGroupMember | Self::CompleteConsumerGroupRevocation | Self::TruncatePartition @@ -279,10 +299,8 @@ | Self::TransitionPartitionHistory | Self::RetireConsumerGroupOwners => None, Self::CreateStream - | Self::UpdateStream | Self::DeleteStream | Self::CreateTopic - | Self::UpdateTopic | Self::DeleteTopic | Self::CreatePartitions | Self::DeletePartitions @@ -290,7 +308,6 @@ | Self::CreateConsumerGroup | Self::DeleteConsumerGroup | Self::CreateUser - | Self::UpdateUser | Self::DeleteUser | Self::ChangePassword | Self::UpdatePermissions @@ -298,6 +315,7 @@ | Self::DeletePersonalAccessToken | Self::JoinConsumerGroup | Self::LeaveConsumerGroup + | Self::UpdateResource | Self::SendMessages | Self::StoreConsumerOffset | Self::DeleteConsumerOffset => match crate::dispatch::lookup_by_operation(*self) { @@ -327,10 +345,8 @@ fn command_code_roundtrip() { let ops = [ Operation::CreateStream, - Operation::UpdateStream, Operation::DeleteStream, Operation::CreateTopic, - Operation::UpdateTopic, Operation::DeleteTopic, Operation::CreatePartitions, Operation::DeletePartitions, @@ -338,7 +354,6 @@ Operation::CreateConsumerGroup, Operation::DeleteConsumerGroup, Operation::CreateUser, - Operation::UpdateUser, Operation::DeleteUser, Operation::ChangePassword, Operation::UpdatePermissions, @@ -346,6 +361,7 @@ Operation::DeletePersonalAccessToken, Operation::JoinConsumerGroup, Operation::LeaveConsumerGroup, + Operation::UpdateResource, Operation::SendMessages, Operation::StoreConsumerOffset, Operation::DeleteConsumerOffset, @@ -461,4 +477,43 @@ ); } } + + #[test] + fn given_rewrap_operation_when_classifying_should_be_internal_metadata_and_not_client_allowed() + { + let rewrap = Operation::RewrapStreamKeys; + assert_eq!(rewrap as u8, 75); + assert!(rewrap.is_internal()); + assert!(rewrap.is_metadata()); + assert!(rewrap.is_metadata_plane()); + assert!(!rewrap.is_client_allowed()); + assert_eq!(rewrap.to_command_code(), None); + assert_eq!(rewrap.request_operation(), rewrap); + assert!(!Operation::is_known_code(69)); + } + + #[test] + fn given_resolved_update_operations_when_classifying_should_be_internal_metadata() { + for resolved in [ + Operation::UpdateStreamResolved, + Operation::UpdateTopicResolved, + Operation::UpdateUserResolved, + ] { + assert!(resolved.is_internal(), "{resolved:?}"); + assert!(resolved.is_metadata(), "{resolved:?}"); + assert!(resolved.is_result_framed(), "{resolved:?}"); + assert!(!resolved.is_client_allowed(), "{resolved:?}"); + assert_eq!(resolved.to_command_code(), None); + assert_eq!(resolved.request_operation(), Operation::UpdateResource); + } + assert!(Operation::UpdateResource.is_metadata()); + assert!(Operation::UpdateResource.is_client_allowed()); + assert_eq!(Operation::UpdateResource as u8, 150); + for retired in [129u8, 133, 142] { + assert!( + !Operation::is_known_code(retired), + "{retired} must stay retired" + ); + } + } }
diff --git a/core/binary_protocol/src/consensus/request.rs b/core/binary_protocol/src/consensus/request.rs index 48dab12..02a8446 100644 --- a/core/binary_protocol/src/consensus/request.rs +++ b/core/binary_protocol/src/consensus/request.rs
@@ -18,8 +18,8 @@ use crate::codes::{LOGIN_REGISTER_CODE, LOGIN_REGISTER_WITH_PAT_CODE, LOGOUT_USER_CODE}; use crate::consensus::{Command, HEADER_SIZE, Operation, RequestHeader}; use crate::error::WireError; +use ring::rand::{SecureRandom, SystemRandom}; use std::ops::Range; -use twox_hash::XxHash3_64; /// Where `RequestHeader::reserved` carries the legacy command code of a /// `NonReplicated` request, as little-endian `u32`. @@ -48,8 +48,8 @@ /// Every rule that depends only on the code and the payload lives here so /// that each client, and the fixtures other SDKs verify against, agree on /// the bytes: metadata and session operations stamp `request_checksum` - /// with the payload hash, partition and non-replicated operations leave - /// it zero (send batches carry their own checksum and non-replicated + /// with a random per-request identifier, partition and non-replicated + /// operations leave it zero (send batches carry their own checksum and non-replicated /// requests bypass dedup), and a non-replicated request carries its /// command code in `reserved`. /// @@ -62,7 +62,8 @@ /// # Errors /// /// Returns `WireError::PayloadTooLarge` when the frame would not fit the - /// `u32` size field. + /// `u32` size field, or `WireError::RandomSourceUnavailable` if the secure + /// random source fails for a metadata or session request. pub fn for_request( code: u32, client: u128, @@ -80,7 +81,17 @@ { 0 } else { - u128::from(XxHash3_64::oneshot(payload)) + let random = SystemRandom::new(); + loop { + let mut bytes = [0u8; 16]; + random + .fill(&mut bytes) + .map_err(|_| WireError::RandomSourceUnavailable)?; + let id = u128::from_le_bytes(bytes); + if id != 0 { + break id; + } + } }; let mut reserved = [0; 4]; if operation == Operation::NonReplicated { @@ -107,6 +118,7 @@ mod tests { use super::*; use crate::codes::{CREATE_STREAM_CODE, PING_CODE, SEND_MESSAGES_CODE}; + use twox_hash::XxHash3_64; #[test] fn login_codes_register_and_logout_logs_out() { @@ -120,13 +132,16 @@ } #[test] - fn metadata_requests_are_stamped_with_the_payload_hash() { + fn given_identical_metadata_bodies_when_building_requests_should_stamp_distinct_random_ids() { let payload = [1, 2, 3]; let header = RequestHeader::for_request(CREATE_STREAM_CODE, 7, 8, 9, &payload).unwrap(); + let another = RequestHeader::for_request(CREATE_STREAM_CODE, 7, 9, 9, &payload).unwrap(); assert_eq!(header.command, Command::Request); assert_eq!(header.size as usize, HEADER_SIZE + payload.len()); assert_eq!((header.client, header.request, header.session), (7, 8, 9)); - assert_eq!( + assert_ne!(header.request_checksum, 0); + assert_ne!(header.request_checksum, another.request_checksum); + assert_ne!( header.request_checksum, u128::from(XxHash3_64::oneshot(&payload)) );
diff --git a/core/binary_protocol/src/dispatch.rs b/core/binary_protocol/src/dispatch.rs index 93be523..0e809b4 100644 --- a/core/binary_protocol/src/dispatch.rs +++ b/core/binary_protocol/src/dispatch.rs
@@ -77,7 +77,6 @@ CommandMeta::non_replicated(GET_USERS_CODE, "user.list"), CommandMeta::replicated(CREATE_USER_CODE, "user.create", Operation::CreateUser), CommandMeta::replicated(DELETE_USER_CODE, "user.delete", Operation::DeleteUser), - CommandMeta::replicated(UPDATE_USER_CODE, "user.update", Operation::UpdateUser), CommandMeta::replicated( UPDATE_PERMISSIONS_CODE, "user.permissions", @@ -130,13 +129,11 @@ CommandMeta::non_replicated(GET_STREAMS_CODE, "stream.list"), CommandMeta::replicated(CREATE_STREAM_CODE, "stream.create", Operation::CreateStream), CommandMeta::replicated(DELETE_STREAM_CODE, "stream.delete", Operation::DeleteStream), - CommandMeta::replicated(UPDATE_STREAM_CODE, "stream.update", Operation::UpdateStream), // Topics CommandMeta::non_replicated(GET_TOPIC_CODE, "topic.get"), CommandMeta::non_replicated(GET_TOPICS_CODE, "topic.list"), CommandMeta::replicated(CREATE_TOPIC_CODE, "topic.create", Operation::CreateTopic), CommandMeta::replicated(DELETE_TOPIC_CODE, "topic.delete", Operation::DeleteTopic), - CommandMeta::replicated(UPDATE_TOPIC_CODE, "topic.update", Operation::UpdateTopic), // Partitions CommandMeta::replicated( CREATE_PARTITIONS_CODE, @@ -184,6 +181,12 @@ CommandMeta::non_replicated(DESCRIBE_OPTIONS_CODE, "options.describe"), CommandMeta::non_replicated(BIND_SESSION_CODE, "session.bind"), CommandMeta::non_replicated(GET_PARTITION_ROUTE_CODE, "partition.route"), + // Generic option patch for streams, topics and users + CommandMeta::replicated( + UPDATE_RESOURCE_CODE, + "resource.update", + Operation::UpdateResource, + ), ]; /// Lookup command metadata by command code. @@ -206,45 +209,43 @@ GET_USERS_CODE => 8, CREATE_USER_CODE => 9, DELETE_USER_CODE => 10, - UPDATE_USER_CODE => 11, - UPDATE_PERMISSIONS_CODE => 12, - CHANGE_PASSWORD_CODE => 13, - LOGIN_USER_CODE => 14, - LOGOUT_USER_CODE => 15, - LOGIN_REGISTER_CODE => 16, - GET_PERSONAL_ACCESS_TOKENS_CODE => 17, - CREATE_PERSONAL_ACCESS_TOKEN_CODE => 18, - DELETE_PERSONAL_ACCESS_TOKEN_CODE => 19, - LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE => 20, - POLL_MESSAGES_CODE => 21, - SEND_MESSAGES_CODE => 22, - GET_CONSUMER_OFFSET_CODE => 23, - STORE_CONSUMER_OFFSET_CODE => 24, - DELETE_CONSUMER_OFFSET_CODE => 25, - GET_STREAM_CODE => 26, - GET_STREAMS_CODE => 27, - CREATE_STREAM_CODE => 28, - DELETE_STREAM_CODE => 29, - UPDATE_STREAM_CODE => 30, - GET_TOPIC_CODE => 31, - GET_TOPICS_CODE => 32, - CREATE_TOPIC_CODE => 33, - DELETE_TOPIC_CODE => 34, - UPDATE_TOPIC_CODE => 35, - CREATE_PARTITIONS_CODE => 36, - DELETE_PARTITIONS_CODE => 37, - DELETE_SEGMENTS_CODE => 38, - GET_CONSUMER_GROUP_CODE => 39, - GET_CONSUMER_GROUPS_CODE => 40, - CREATE_CONSUMER_GROUP_CODE => 41, - DELETE_CONSUMER_GROUP_CODE => 42, - JOIN_CONSUMER_GROUP_CODE => 43, - LEAVE_CONSUMER_GROUP_CODE => 44, - SYNC_CONSUMER_GROUP_CODE => 45, - LOGIN_REGISTER_WITH_PAT_CODE => 46, - DESCRIBE_OPTIONS_CODE => 47, - BIND_SESSION_CODE => 48, - GET_PARTITION_ROUTE_CODE => 49, + UPDATE_PERMISSIONS_CODE => 11, + CHANGE_PASSWORD_CODE => 12, + LOGIN_USER_CODE => 13, + LOGOUT_USER_CODE => 14, + LOGIN_REGISTER_CODE => 15, + GET_PERSONAL_ACCESS_TOKENS_CODE => 16, + CREATE_PERSONAL_ACCESS_TOKEN_CODE => 17, + DELETE_PERSONAL_ACCESS_TOKEN_CODE => 18, + LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE => 19, + POLL_MESSAGES_CODE => 20, + SEND_MESSAGES_CODE => 21, + GET_CONSUMER_OFFSET_CODE => 22, + STORE_CONSUMER_OFFSET_CODE => 23, + DELETE_CONSUMER_OFFSET_CODE => 24, + GET_STREAM_CODE => 25, + GET_STREAMS_CODE => 26, + CREATE_STREAM_CODE => 27, + DELETE_STREAM_CODE => 28, + GET_TOPIC_CODE => 29, + GET_TOPICS_CODE => 30, + CREATE_TOPIC_CODE => 31, + DELETE_TOPIC_CODE => 32, + CREATE_PARTITIONS_CODE => 33, + DELETE_PARTITIONS_CODE => 34, + DELETE_SEGMENTS_CODE => 35, + GET_CONSUMER_GROUP_CODE => 36, + GET_CONSUMER_GROUPS_CODE => 37, + CREATE_CONSUMER_GROUP_CODE => 38, + DELETE_CONSUMER_GROUP_CODE => 39, + JOIN_CONSUMER_GROUP_CODE => 40, + LEAVE_CONSUMER_GROUP_CODE => 41, + SYNC_CONSUMER_GROUP_CODE => 42, + LOGIN_REGISTER_WITH_PAT_CODE => 43, + DESCRIBE_OPTIONS_CODE => 44, + BIND_SESSION_CODE => 45, + GET_PARTITION_ROUTE_CODE => 46, + UPDATE_RESOURCE_CODE => 47, _ => return None, }; Some(&COMMAND_TABLE[idx]) @@ -258,30 +259,32 @@ pub const fn lookup_by_operation(op: Operation) -> Option<&'static CommandMeta> { // Indices must match the order of entries in COMMAND_TABLE above. let idx = match op { - Operation::CreateStream => 28, - Operation::UpdateStream => 30, - Operation::DeleteStream => 29, - Operation::CreateTopic => 33, - Operation::UpdateTopic => 35, - Operation::DeleteTopic => 34, - Operation::CreatePartitions => 36, - Operation::DeletePartitions => 37, - Operation::DeleteSegments => 38, - Operation::CreateConsumerGroup => 41, - Operation::DeleteConsumerGroup => 42, - Operation::JoinConsumerGroup => 43, - Operation::LeaveConsumerGroup => 44, Operation::CreateUser => 9, - Operation::UpdateUser => 11, Operation::DeleteUser => 10, - Operation::ChangePassword => 13, - Operation::UpdatePermissions => 12, - Operation::CreatePersonalAccessToken => 18, - Operation::DeletePersonalAccessToken => 19, - Operation::SendMessages => 22, - Operation::StoreConsumerOffset => 24, - Operation::DeleteConsumerOffset => 25, + Operation::UpdatePermissions => 11, + Operation::ChangePassword => 12, + Operation::CreatePersonalAccessToken => 17, + Operation::DeletePersonalAccessToken => 18, + Operation::SendMessages => 21, + Operation::StoreConsumerOffset => 23, + Operation::DeleteConsumerOffset => 24, + Operation::CreateStream => 27, + Operation::DeleteStream => 28, + Operation::CreateTopic => 31, + Operation::DeleteTopic => 32, + Operation::CreatePartitions => 33, + Operation::DeletePartitions => 34, + Operation::DeleteSegments => 35, + Operation::CreateConsumerGroup => 38, + Operation::DeleteConsumerGroup => 39, + Operation::JoinConsumerGroup => 40, + Operation::LeaveConsumerGroup => 41, + Operation::UpdateResource => 47, Operation::CreateTopicWithAssignments + | Operation::UpdateStreamResolved + | Operation::UpdateTopicResolved + | Operation::UpdateUserResolved + | Operation::RewrapStreamKeys | Operation::CreatePartitionsWithAssignments | Operation::RemoveConsumerGroupMember | Operation::CompleteConsumerGroupRevocation
diff --git a/core/binary_protocol/src/error.rs b/core/binary_protocol/src/error.rs index 208b15c..619afb3 100644 --- a/core/binary_protocol/src/error.rs +++ b/core/binary_protocol/src/error.rs
@@ -47,6 +47,9 @@ #[error("payload too large: {size} bytes, max {max}")] PayloadTooLarge { size: usize, max: usize }, + #[error("secure random source unavailable")] + RandomSourceUnavailable, + #[error("validation failed: {0}")] Validation(Cow<'static, str>),
diff --git a/core/binary_protocol/src/primitives/options.rs b/core/binary_protocol/src/primitives/options.rs index 16e781b..bc6be40 100644 --- a/core/binary_protocol/src/primitives/options.rs +++ b/core/binary_protocol/src/primitives/options.rs
@@ -26,6 +26,7 @@ //! admission. use std::borrow::Cow; +use std::fmt::{Debug, Formatter, Result as FmtResult}; use bytes::{BufMut, Bytes, BytesMut}; @@ -176,9 +177,18 @@ /// /// Like `WireUserHeaders`, this type is opaque at the wire layer. Domain /// interpretation of value kind codes happens in `iggy_common`. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Clone, PartialEq, Eq)] pub struct WireOptions(Bytes); +impl Debug for WireOptions { + fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { + formatter + .debug_struct("WireOptions") + .field("bytes_len", &self.0.len()) + .finish() + } +} + impl WireOptions { /// Validate and wrap a borrowed byte slice (copies into new `Bytes`). /// @@ -271,6 +281,19 @@ } #[test] + fn given_stream_key_when_debugging_options_should_hide_key_bytes_and_keep_the_block() { + let key = [b'k'; 32]; + let buf = encode(&[(STRING, b"encryption_key", 1, &key)]); + let options = WireOptions::from_slice(&buf).expect("valid stream options"); + + let debugged = format!("{options:?}"); + + assert!(!debugged.contains("kkkkkkkk")); + assert!(!debugged.contains("encryption_key")); + assert_eq!(options.iter().next().expect("key entry").value, key); + } + + #[test] fn empty_buffer_is_zero_options() { assert_eq!(validate_options(&[]).unwrap(), 0); assert!(WireOptions::empty().is_empty());
diff --git a/core/binary_protocol/src/requests/streams/mod.rs b/core/binary_protocol/src/requests/streams/mod.rs index 837da7f..0eb7765 100644 --- a/core/binary_protocol/src/requests/streams/mod.rs +++ b/core/binary_protocol/src/requests/streams/mod.rs
@@ -19,10 +19,15 @@ pub mod delete_stream; pub mod get_stream; pub mod get_streams; -pub mod update_stream; +pub mod rewrap_stream_keys; +pub mod update_stream_resolved; pub use create_stream::CreateStreamRequest; pub use delete_stream::DeleteStreamRequest; pub use get_stream::GetStreamRequest; pub use get_streams::GetStreamsRequest; -pub use update_stream::UpdateStreamRequest; +pub use rewrap_stream_keys::{ + MAX_WRAP_SET_ENVELOPES, RewrapStreamKeyEntry, RewrapStreamKeysRequest, + WRAPPED_STREAM_KEY_LENGTH, +}; +pub use update_stream_resolved::UpdateStreamResolvedRequest;
diff --git a/core/binary_protocol/src/requests/streams/rewrap_stream_keys.rs b/core/binary_protocol/src/requests/streams/rewrap_stream_keys.rs new file mode 100644 index 0000000..68548c6 --- /dev/null +++ b/core/binary_protocol/src/requests/streams/rewrap_stream_keys.rs
@@ -0,0 +1,216 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::WireError; +use crate::codec::{WireDecode, WireEncode, read_bytes, read_u8, read_u32_le, read_u64_le}; +use bytes::{BufMut, Bytes, BytesMut}; +use std::borrow::Cow; +use std::fmt::{Debug, Formatter}; + +/// Bytes of one wrapped stream key envelope. +pub const WRAPPED_STREAM_KEY_LENGTH: usize = 78; +/// Most envelopes one wrap set holds. +pub const MAX_WRAP_SET_ENVELOPES: usize = 3; +const KEY_FINGERPRINT_LENGTH: usize = 8; +const ENTRY_FIXED_SIZE: usize = 4 + 8 + KEY_FINGERPRINT_LENGTH + 1; + +/// Server-originated: replace the wrap sets of encrypted streams after a +/// key-encryption key rotation. Apply checks each entry against the stored +/// stream and skips one that no longer matches. +/// +/// Wire format: +/// ```text +/// [count:u32]([stream_id:u32][created_op:u64][key_fingerprint:8][set_len:u8][set])* +/// ``` +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RewrapStreamKeysRequest { + pub entries: Vec<RewrapStreamKeyEntry>, +} + +/// One stream incarnation and the wrap set it should store. +#[derive(Clone, PartialEq, Eq)] +pub struct RewrapStreamKeyEntry { + pub stream_id: u32, + pub created_op: u64, + pub key_fingerprint: [u8; KEY_FINGERPRINT_LENGTH], + pub wrap_set: Bytes, +} + +impl Debug for RewrapStreamKeyEntry { + /// Prints the envelope count, never the wrapped key bytes. + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("RewrapStreamKeyEntry") + .field("stream_id", &self.stream_id) + .field("created_op", &self.created_op) + .field("key_fingerprint", &self.key_fingerprint) + .field( + "envelopes", + &(self.wrap_set.len() / WRAPPED_STREAM_KEY_LENGTH), + ) + .finish() + } +} + +impl WireEncode for RewrapStreamKeysRequest { + fn encoded_size(&self) -> usize { + 4 + self + .entries + .iter() + .map(|entry| ENTRY_FIXED_SIZE + entry.wrap_set.len()) + .sum::<usize>() + } + + #[allow(clippy::cast_possible_truncation)] + fn encode(&self, buf: &mut BytesMut) { + buf.put_u32_le(self.entries.len() as u32); + for entry in &self.entries { + buf.put_u32_le(entry.stream_id); + buf.put_u64_le(entry.created_op); + buf.put_slice(&entry.key_fingerprint); + buf.put_u8(entry.wrap_set.len() as u8); + buf.put_slice(&entry.wrap_set); + } + } +} + +impl WireDecode for RewrapStreamKeysRequest { + fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> { + let count = read_u32_le(buf, 0)? as usize; + let mut position = 4; + // Each entry takes at least its fixed part plus one envelope, so a + // count the body cannot hold is refused before allocating for it. + let minimum = ENTRY_FIXED_SIZE + WRAPPED_STREAM_KEY_LENGTH; + if count > buf.len().saturating_sub(position) / minimum { + return Err(validation("rewrap entry count exceeds the body")); + } + let mut entries = Vec::with_capacity(count); + for _ in 0..count { + let stream_id = read_u32_le(buf, position)?; + let created_op = read_u64_le(buf, position + 4)?; + let fingerprint = read_bytes(buf, position + 12, KEY_FINGERPRINT_LENGTH)?; + let set_length = usize::from(read_u8(buf, position + 20)?); + if set_length == 0 + || !set_length.is_multiple_of(WRAPPED_STREAM_KEY_LENGTH) + || set_length > MAX_WRAP_SET_ENVELOPES * WRAPPED_STREAM_KEY_LENGTH + { + return Err(validation( + "rewrap set length is not one to three envelopes", + )); + } + let wrap_set = read_bytes(buf, position + ENTRY_FIXED_SIZE, set_length)?; + let mut key_fingerprint = [0u8; KEY_FINGERPRINT_LENGTH]; + key_fingerprint.copy_from_slice(fingerprint); + entries.push(RewrapStreamKeyEntry { + stream_id, + created_op, + key_fingerprint, + wrap_set: Bytes::copy_from_slice(wrap_set), + }); + position += ENTRY_FIXED_SIZE + set_length; + } + if position != buf.len() { + return Err(validation("trailing bytes after the rewrap entries")); + } + Ok((Self { entries }, position)) + } +} + +const fn validation(reason: &'static str) -> WireError { + WireError::Validation(Cow::Borrowed(reason)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn entry(stream_id: u32, envelopes: usize) -> RewrapStreamKeyEntry { + RewrapStreamKeyEntry { + stream_id, + created_op: 7, + key_fingerprint: [0xAB; KEY_FINGERPRINT_LENGTH], + wrap_set: Bytes::from(vec![0x5A; envelopes * WRAPPED_STREAM_KEY_LENGTH]), + } + } + + #[test] + fn given_rewrap_entries_when_roundtripping_should_match_golden_bytes() { + let single = RewrapStreamKeysRequest { + entries: vec![entry(3, 1)], + }; + let bytes = single.to_bytes(); + let mut golden = vec![1, 0, 0, 0, 3, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0]; + golden.extend_from_slice(&[0xAB; KEY_FINGERPRINT_LENGTH]); + golden.push(78); + golden.extend_from_slice(&[0x5A; WRAPPED_STREAM_KEY_LENGTH]); + assert_eq!(bytes.as_ref(), golden.as_slice()); + assert_eq!(single.encoded_size(), golden.len()); + + let several = RewrapStreamKeysRequest { + entries: vec![entry(1, 2), entry(2, 3), entry(9, 1)], + }; + let bytes = several.to_bytes(); + let (decoded, consumed) = RewrapStreamKeysRequest::decode(&bytes).unwrap(); + assert_eq!(consumed, bytes.len()); + assert_eq!(decoded, several); + } + + #[test] + fn given_malformed_rewrap_body_when_decoding_should_reject_without_panicking() { + let bytes = RewrapStreamKeysRequest { + entries: vec![entry(1, 1), entry(2, 2)], + } + .to_bytes(); + for length in 0..bytes.len() { + assert!( + RewrapStreamKeysRequest::decode(&bytes[..length]).is_err(), + "a body cut at {length} must be refused" + ); + } + let mut trailing = bytes.to_vec(); + trailing.push(0); + assert!(RewrapStreamKeysRequest::decode(&trailing).is_err()); + + let set_length_at = 4 + 4 + 8 + KEY_FINGERPRINT_LENGTH; + for invalid in [0u8, 77, 255] { + let mut body = RewrapStreamKeysRequest { + entries: vec![entry(1, 1)], + } + .to_bytes() + .to_vec(); + body[set_length_at] = invalid; + assert!(RewrapStreamKeysRequest::decode(&body).is_err(), "{invalid}"); + } + let oversized_count = [0xFF, 0xFF, 0xFF, 0xFF]; + assert!(RewrapStreamKeysRequest::decode(&oversized_count).is_err()); + } + + #[test] + fn given_rewrap_request_when_debugging_should_not_print_wrapped_keys() { + let rendered = format!( + "{:?}", + RewrapStreamKeysRequest { + entries: vec![entry(1, 2)], + } + ); + assert!( + !rendered.contains("ZZ") && !rendered.contains("90"), + "{rendered}" + ); + assert!(rendered.contains("envelopes: 2"), "{rendered}"); + } +}
diff --git a/core/binary_protocol/src/requests/streams/update_stream.rs b/core/binary_protocol/src/requests/streams/update_stream.rs deleted file mode 100644 index d36fadd..0000000 --- a/core/binary_protocol/src/requests/streams/update_stream.rs +++ /dev/null
@@ -1,127 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::WireError; -use crate::WireIdentifier; -use crate::codec::{WireDecode, WireEncode}; -use crate::primitives::identifier::WireName; -use crate::primitives::options::WireOptions; -use bytes::BytesMut; - -/// `UpdateStream` request. -/// -/// Wire format: `[identifier][name_len:1][name:N][options TLV to end]` -/// -/// The options block mirrors `CreateStream`'s, so a stream setting added to -/// the catalog is updatable without another layout change. Keys absent from -/// the block are left alone rather than reset: an update patches the stored -/// map, so a client built before a key existed cannot erase it. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct UpdateStreamRequest { - pub stream_id: WireIdentifier, - pub name: WireName, - pub options: WireOptions, -} - -impl WireEncode for UpdateStreamRequest { - fn encoded_size(&self) -> usize { - self.stream_id.encoded_size() + self.name.encoded_size() + self.options.encoded_size() - } - - fn encode(&self, buf: &mut BytesMut) { - self.stream_id.encode(buf); - self.name.encode(buf); - self.options.encode(buf); - } -} - -impl WireDecode for UpdateStreamRequest { - fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> { - let (stream_id, mut pos) = WireIdentifier::decode(buf)?; - let (name, consumed) = WireName::decode(&buf[pos..])?; - pos += consumed; - let options = WireOptions::from_slice(&buf[pos..])?; - Ok(( - Self { - stream_id, - name, - options, - }, - buf.len(), - )) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::primitives::user_headers::encode_user_headers; - - fn sample_options() -> WireOptions { - let mut buf = BytesMut::new(); - encode_user_headers(&[(2, b"future_key", 2, b"value")], &mut buf); - WireOptions::from_bytes(buf.freeze()).unwrap() - } - - #[test] - fn roundtrip() { - let req = UpdateStreamRequest { - stream_id: WireIdentifier::named("old-name").unwrap(), - name: WireName::new("new-name").unwrap(), - options: sample_options(), - }; - let bytes = req.to_bytes(); - let (decoded, consumed) = UpdateStreamRequest::decode(&bytes).unwrap(); - assert_eq!(consumed, bytes.len()); - assert_eq!(decoded, req); - } - - #[test] - fn truncated_fixed_fields_return_error() { - let req = UpdateStreamRequest { - stream_id: WireIdentifier::numeric(1), - name: WireName::new("test").unwrap(), - options: WireOptions::empty(), - }; - let bytes = req.to_bytes(); - for i in 0..bytes.len() { - assert!( - UpdateStreamRequest::decode(&bytes[..i]).is_err(), - "expected error for truncation at byte {i}" - ); - } - } - - #[test] - fn truncated_options_return_error() { - // One pair, so any strict-interior truncation is invalid. Cutting at - // the block boundary is a legitimate update carrying no options. - let req = UpdateStreamRequest { - stream_id: WireIdentifier::numeric(1), - name: WireName::new("test").unwrap(), - options: sample_options(), - }; - let bytes = req.to_bytes(); - let fixed_end = bytes.len() - req.options.encoded_size(); - for i in fixed_end + 1..bytes.len() { - assert!( - UpdateStreamRequest::decode(&bytes[..i]).is_err(), - "expected error for truncation at byte {i}" - ); - } - } -}
diff --git a/core/binary_protocol/src/requests/streams/update_stream_resolved.rs b/core/binary_protocol/src/requests/streams/update_stream_resolved.rs new file mode 100644 index 0000000..7db1263 --- /dev/null +++ b/core/binary_protocol/src/requests/streams/update_stream_resolved.rs
@@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::WireError; +use crate::WireIdentifier; +use crate::codec::{WireDecode, WireEncode}; +use crate::primitives::identifier::WireName; +use crate::primitives::options::WireOptions; +use crate::requests::system::update_resource::{ + decode_optional_name, decode_resolved_blocks, encode_optional_name, encode_resolved_blocks, + optional_name_size, resolved_blocks_size, +}; +use bytes::BytesMut; + +/// An `UpdateResource` on a stream after prepare: the rename extracted, the +/// set keys canonical in `explicit_options`, the reset keys resolved to their +/// defaults in `derived_options`. +/// +/// Wire format: +/// ```text +/// [stream_id][has_name:u8][name?][explicit_len:u32][explicit][derived_len:u32][derived] +/// ``` +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UpdateStreamResolvedRequest { + pub stream_id: WireIdentifier, + pub name: Option<WireName>, + pub explicit_options: WireOptions, + pub derived_options: WireOptions, +} + +impl WireEncode for UpdateStreamResolvedRequest { + fn encoded_size(&self) -> usize { + self.stream_id.encoded_size() + + optional_name_size(self.name.as_ref()) + + resolved_blocks_size(&self.explicit_options, &self.derived_options) + } + + fn encode(&self, buf: &mut BytesMut) { + self.stream_id.encode(buf); + encode_optional_name(self.name.as_ref(), buf); + encode_resolved_blocks(&self.explicit_options, &self.derived_options, buf); + } +} + +impl WireDecode for UpdateStreamResolvedRequest { + fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> { + let (stream_id, mut position) = WireIdentifier::decode(buf)?; + let (name, consumed) = decode_optional_name(buf, position)?; + position += consumed; + let (explicit_options, derived_options) = decode_resolved_blocks(buf, position)?; + Ok(( + Self { + stream_id, + name, + explicit_options, + derived_options, + }, + buf.len(), + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::primitives::user_headers::encode_user_headers; + + #[test] + fn given_resolved_patch_when_roundtripping_should_preserve_columns_and_provenance() { + for name in [None, Some(WireName::new("renamed").unwrap())] { + let request = UpdateStreamResolvedRequest { + stream_id: WireIdentifier::named("events").unwrap(), + name, + explicit_options: options(b"future_key", b"value"), + derived_options: WireOptions::empty(), + }; + let bytes = request.to_bytes(); + assert_eq!(bytes.len(), request.encoded_size()); + assert_eq!( + UpdateStreamResolvedRequest::decode_from(&bytes).unwrap(), + request + ); + for length in 0..bytes.len() { + assert!(UpdateStreamResolvedRequest::decode(&bytes[..length]).is_err()); + } + let mut trailing = bytes.to_vec(); + trailing.push(0); + assert!(UpdateStreamResolvedRequest::decode(&trailing).is_err()); + } + let mut bad_presence = WireIdentifier::numeric(1).to_bytes().to_vec(); + bad_presence.push(2); + assert!(UpdateStreamResolvedRequest::decode(&bad_presence).is_err()); + } + + fn options(key: &[u8], value: &[u8]) -> WireOptions { + let mut buf = BytesMut::new(); + encode_user_headers(&[(2, key, 2, value)], &mut buf); + WireOptions::from_bytes(buf.freeze()).unwrap() + } +}
diff --git a/core/binary_protocol/src/requests/system/mod.rs b/core/binary_protocol/src/requests/system/mod.rs index 9be7a86..9d5c568 100644 --- a/core/binary_protocol/src/requests/system/mod.rs +++ b/core/binary_protocol/src/requests/system/mod.rs
@@ -26,6 +26,7 @@ pub mod get_stats; pub mod ping; pub mod retire_sessions; +pub mod update_resource; pub use bind_session::{BindSessionRequest, SessionIdentity}; pub use describe_options::{ @@ -40,3 +41,4 @@ pub use get_stats::GetStatsRequest; pub use ping::PingRequest; pub use retire_sessions::{MAX_SESSIONS_PER_RETIREMENT, RetireSessionsRequest}; +pub use update_resource::{MAX_RESET_KEYS, ResourceTarget, UpdateResourceRequest, WireResetKeys};
diff --git a/core/binary_protocol/src/requests/system/update_resource.rs b/core/binary_protocol/src/requests/system/update_resource.rs new file mode 100644 index 0000000..799e634 --- /dev/null +++ b/core/binary_protocol/src/requests/system/update_resource.rs
@@ -0,0 +1,476 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::describe_options::{OPTIONS_SCOPE_STREAM, OPTIONS_SCOPE_TOPIC, OPTIONS_SCOPE_USER}; +use crate::WireError; +use crate::WireIdentifier; +use crate::codec::{WireDecode, WireEncode, read_u8}; +use crate::primitives::identifier::WireName; +use crate::primitives::options::{ + WireOptions, decode_options_prefixed, encode_options_prefixed, options_prefixed_size, +}; +use bytes::{BufMut, BytesMut}; +use std::borrow::Cow; +use std::collections::BTreeSet; +use std::ops::Deref; +use std::slice::Iter; + +/// Most reset names one patch carries, the reach of its count byte. +pub const MAX_RESET_KEYS: usize = u8::MAX as usize; + +/// The resource an `UpdateResource` patches, tagged by its options scope code. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ResourceTarget { + Topic { + stream_id: WireIdentifier, + topic_id: WireIdentifier, + }, + Stream { + stream_id: WireIdentifier, + }, + User { + user_id: WireIdentifier, + }, +} + +/// Patch one resource's options: set the keys in `options`, reset the keys in +/// `reset` to their defaults, leave every other key alone. +/// +/// Wire format: +/// ```text +/// [scope:u8][identifiers][reset_count:u8][reset_key:WireName]*[options to end of body] +/// ``` +/// Identifiers are `[stream_id][topic_id]` for a topic, `[stream_id]` for a +/// stream and `[user_id]` for a user. A patch with nothing to set or reset, a +/// key named twice, or a key both set and reset is refused at decode. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UpdateResourceRequest { + pub target: ResourceTarget, + pub reset: WireResetKeys, + pub options: WireOptions, +} + +/// The names a patch resets, at most [`MAX_RESET_KEYS`] so the count byte +/// always matches the names after it. +/// +/// Wire format: `[count:u8][name:WireName]*` +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct WireResetKeys(Vec<WireName>); + +impl ResourceTarget { + /// The options scope code the target is tagged with on the wire. + #[must_use] + pub const fn scope_code(&self) -> u8 { + match self { + Self::Topic { .. } => OPTIONS_SCOPE_TOPIC, + Self::Stream { .. } => OPTIONS_SCOPE_STREAM, + Self::User { .. } => OPTIONS_SCOPE_USER, + } + } +} + +impl TryFrom<Vec<WireName>> for WireResetKeys { + type Error = WireError; + + fn try_from(names: Vec<WireName>) -> Result<Self, Self::Error> { + if names.len() > MAX_RESET_KEYS { + return Err(validation("more reset keys than one patch carries")); + } + Ok(Self(names)) + } +} + +impl Deref for WireResetKeys { + type Target = [WireName]; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl<'a> IntoIterator for &'a WireResetKeys { + type Item = &'a WireName; + type IntoIter = Iter<'a, WireName>; + + fn into_iter(self) -> Self::IntoIter { + self.0.iter() + } +} + +impl WireEncode for WireResetKeys { + fn encoded_size(&self) -> usize { + 1 + self.0.iter().map(WireEncode::encoded_size).sum::<usize>() + } + + fn encode(&self, buf: &mut BytesMut) { + // The constructor bounds the length to what the count byte holds. + #[allow(clippy::cast_possible_truncation)] + buf.put_u8(self.0.len() as u8); + for name in &self.0 { + name.encode(buf); + } + } +} + +impl WireDecode for WireResetKeys { + fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> { + let count = read_u8(buf, 0)?; + let mut position = 1; + let mut names = Vec::with_capacity(usize::from(count)); + let mut seen = BTreeSet::new(); + for _ in 0..count { + let (name, consumed) = WireName::decode(&buf[position..])?; + position += consumed; + if !seen.insert(name.as_str().to_owned()) { + return Err(validation("a reset key is named twice")); + } + names.push(name); + } + Ok((Self(names), position)) + } +} + +impl WireEncode for ResourceTarget { + fn encoded_size(&self) -> usize { + 1 + match self { + Self::Topic { + stream_id, + topic_id, + } => stream_id.encoded_size() + topic_id.encoded_size(), + Self::Stream { stream_id } => stream_id.encoded_size(), + Self::User { user_id } => user_id.encoded_size(), + } + } + + fn encode(&self, buf: &mut BytesMut) { + buf.put_u8(self.scope_code()); + match self { + Self::Topic { + stream_id, + topic_id, + } => { + stream_id.encode(buf); + topic_id.encode(buf); + } + Self::Stream { stream_id } => stream_id.encode(buf), + Self::User { user_id } => user_id.encode(buf), + } + } +} + +impl WireDecode for ResourceTarget { + fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> { + let scope = read_u8(buf, 0)?; + let mut position = 1; + let target = match scope { + OPTIONS_SCOPE_TOPIC => { + let (stream_id, consumed) = WireIdentifier::decode(&buf[position..])?; + position += consumed; + let (topic_id, consumed) = WireIdentifier::decode(&buf[position..])?; + position += consumed; + Self::Topic { + stream_id, + topic_id, + } + } + OPTIONS_SCOPE_STREAM => { + let (stream_id, consumed) = WireIdentifier::decode(&buf[position..])?; + position += consumed; + Self::Stream { stream_id } + } + OPTIONS_SCOPE_USER => { + let (user_id, consumed) = WireIdentifier::decode(&buf[position..])?; + position += consumed; + Self::User { user_id } + } + unknown => { + return Err(WireError::Validation(Cow::Owned(format!( + "unknown resource scope: {unknown}" + )))); + } + }; + Ok((target, position)) + } +} + +impl WireEncode for UpdateResourceRequest { + fn encoded_size(&self) -> usize { + self.target.encoded_size() + self.reset.encoded_size() + self.options.encoded_size() + } + + fn encode(&self, buf: &mut BytesMut) { + self.target.encode(buf); + self.reset.encode(buf); + self.options.encode(buf); + } +} + +impl WireDecode for UpdateResourceRequest { + fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> { + let (target, mut position) = ResourceTarget::decode(buf)?; + let (reset, consumed) = WireResetKeys::decode(&buf[position..])?; + position += consumed; + let options = WireOptions::from_slice(&buf[position..])?; + for entry in &options { + if reset + .iter() + .any(|name| name.as_str().as_bytes() == entry.key) + { + return Err(validation("a key is both set and reset")); + } + } + if reset.is_empty() && options.is_empty() { + return Err(validation("the patch sets and resets nothing")); + } + Ok(( + Self { + target, + reset, + options, + }, + buf.len(), + )) + } +} + +const fn validation(reason: &'static str) -> WireError { + WireError::Validation(Cow::Borrowed(reason)) +} + +/// Size of an optional name behind a presence byte. +pub(crate) fn optional_name_size(name: Option<&WireName>) -> usize { + 1 + name.map_or(0, WireEncode::encoded_size) +} + +/// Write an optional name behind a strict 0/1 presence byte. +pub(crate) fn encode_optional_name(name: Option<&WireName>, buf: &mut BytesMut) { + match name { + Some(name) => { + buf.put_u8(1); + name.encode(buf); + } + None => buf.put_u8(0), + } +} + +/// Read an optional name behind a strict 0/1 presence byte at `position`. +pub(crate) fn decode_optional_name( + buf: &[u8], + position: usize, +) -> Result<(Option<WireName>, usize), WireError> { + match read_u8(buf, position)? { + 0 => Ok((None, 1)), + 1 => { + let (name, consumed) = WireName::decode(&buf[position + 1..])?; + Ok((Some(name), 1 + consumed)) + } + _ => Err(validation("presence byte must be 0 or 1")), + } +} + +/// Size of the explicit and derived length-prefixed option blocks. +pub(crate) fn resolved_blocks_size(explicit: &WireOptions, derived: &WireOptions) -> usize { + options_prefixed_size(explicit) + options_prefixed_size(derived) +} + +/// Write the explicit then derived length-prefixed option blocks. +pub(crate) fn encode_resolved_blocks( + explicit: &WireOptions, + derived: &WireOptions, + buf: &mut BytesMut, +) { + encode_options_prefixed(explicit, buf); + encode_options_prefixed(derived, buf); +} + +/// Read the explicit then derived blocks at `position`, which must end the +/// body exactly. +pub(crate) fn decode_resolved_blocks( + buf: &[u8], + position: usize, +) -> Result<(WireOptions, WireOptions), WireError> { + let (explicit, consumed) = decode_options_prefixed(buf, position)?; + let (derived, derived_consumed) = decode_options_prefixed(buf, position + consumed)?; + if position + consumed + derived_consumed != buf.len() { + return Err(validation( + "trailing bytes after the resolved option blocks", + )); + } + Ok((explicit, derived)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::primitives::user_headers::encode_user_headers; + + const STRING: u8 = 2; + const UINT64: u8 = 12; + + #[test] + fn given_resource_patch_when_roundtripping_should_match_golden_bytes() { + let topic = UpdateResourceRequest { + target: ResourceTarget::Topic { + stream_id: WireIdentifier::numeric(1), + topic_id: WireIdentifier::named("orders").unwrap(), + }, + reset: reset_keys(&["max_topic_size"]), + options: options(&[ + ( + STRING, + b"message_expiry", + UINT64, + &86_400_000_000u64.to_le_bytes(), + ), + (STRING, b"name", STRING, b"orders-v2"), + ]), + }; + let bytes = topic.to_bytes(); + assert_eq!(&bytes[..1], &[OPTIONS_SCOPE_TOPIC]); + let (decoded, consumed) = UpdateResourceRequest::decode(&bytes).unwrap(); + assert_eq!(consumed, bytes.len()); + assert_eq!(decoded, topic); + assert_eq!(decoded.target.scope_code(), OPTIONS_SCOPE_TOPIC); + + let stream = UpdateResourceRequest { + target: ResourceTarget::Stream { + stream_id: WireIdentifier::numeric(3), + }, + reset: WireResetKeys::default(), + options: options(&[(STRING, b"name", STRING, b"renamed")]), + }; + let bytes = stream.to_bytes(); + let mut expected = vec![OPTIONS_SCOPE_STREAM]; + expected.extend_from_slice(&WireIdentifier::numeric(3).to_bytes()); + expected.push(0); + expected.extend_from_slice(stream.options.as_bytes()); + assert_eq!(&bytes[..], expected.as_slice()); + assert_eq!(UpdateResourceRequest::decode_from(&bytes).unwrap(), stream); + + let user = UpdateResourceRequest { + target: ResourceTarget::User { + user_id: WireIdentifier::named("alice").unwrap(), + }, + reset: WireResetKeys::default(), + options: options(&[ + (STRING, b"status", STRING, b"inactive"), + (STRING, b"username", STRING, b"alice2"), + ]), + }; + let bytes = user.to_bytes(); + assert_eq!(UpdateResourceRequest::decode_from(&bytes).unwrap(), user); + } + + #[test] + fn given_malformed_patch_when_decoding_should_reject_without_panicking() { + let valid = UpdateResourceRequest { + target: ResourceTarget::Topic { + stream_id: WireIdentifier::numeric(1), + topic_id: WireIdentifier::numeric(2), + }, + reset: reset_keys(&["max_topic_size"]), + options: options(&[(STRING, b"message_expiry", UINT64, &7u64.to_le_bytes())]), + }; + let bytes = valid.to_bytes(); + let options_start = bytes.len() - valid.options.encoded_size(); + // Every prefix that cuts the fixed part or splits a TLV entry fails. + // Cutting exactly at the start of the options block leaves a valid + // reset-only patch, so that one boundary decodes. + for length in 0..bytes.len() { + let outcome = UpdateResourceRequest::decode(&bytes[..length]); + if length == options_start { + assert!(outcome.is_ok(), "reset-only prefix must decode"); + } else { + assert!(outcome.is_err(), "prefix of {length} bytes must fail"); + } + } + + let mut unknown_scope = bytes.to_vec(); + unknown_scope[0] = 9; + assert!(UpdateResourceRequest::decode(&unknown_scope).is_err()); + + let duplicate_reset = UpdateResourceRequest { + reset: reset_keys(&["max_topic_size", "max_topic_size"]), + options: WireOptions::empty(), + ..valid.clone() + }; + assert!(UpdateResourceRequest::decode(&duplicate_reset.to_bytes()).is_err()); + + let cross_duplicate = UpdateResourceRequest { + reset: reset_keys(&["message_expiry"]), + ..valid.clone() + }; + assert!(UpdateResourceRequest::decode(&cross_duplicate.to_bytes()).is_err()); + + let empty = UpdateResourceRequest { + reset: WireResetKeys::default(), + options: WireOptions::empty(), + ..valid.clone() + }; + assert!(UpdateResourceRequest::decode(&empty.to_bytes()).is_err()); + + let mut duplicate_set = valid.target.to_bytes().to_vec(); + duplicate_set.push(0); + let mut entries = BytesMut::new(); + encode_user_headers( + &[ + (STRING, b"message_expiry", UINT64, &7u64.to_le_bytes()), + (STRING, b"message_expiry", UINT64, &8u64.to_le_bytes()), + ], + &mut entries, + ); + duplicate_set.extend_from_slice(&entries); + assert!(UpdateResourceRequest::decode(&duplicate_set).is_err()); + } + + #[test] + fn given_more_reset_names_than_the_count_byte_holds_when_building_should_refuse() { + let names = |count: usize| -> Vec<WireName> { + (0..count) + .map(|index| WireName::new(format!("key_{index}")).unwrap()) + .collect() + }; + assert!(WireResetKeys::try_from(names(MAX_RESET_KEYS + 1)).is_err()); + let request = UpdateResourceRequest { + target: ResourceTarget::Stream { + stream_id: WireIdentifier::numeric(1), + }, + reset: WireResetKeys::try_from(names(MAX_RESET_KEYS)).unwrap(), + options: WireOptions::empty(), + }; + assert_eq!( + UpdateResourceRequest::decode_from(&request.to_bytes()).unwrap(), + request + ); + } + + fn reset_keys(names: &[&str]) -> WireResetKeys { + WireResetKeys::try_from( + names + .iter() + .map(|name| WireName::new(*name).unwrap()) + .collect::<Vec<_>>(), + ) + .unwrap() + } + + fn options(entries: &[(u8, &[u8], u8, &[u8])]) -> WireOptions { + let mut buf = BytesMut::new(); + encode_user_headers(entries, &mut buf); + WireOptions::from_bytes(buf.freeze()).unwrap() + } +}
diff --git a/core/binary_protocol/src/requests/topics/mod.rs b/core/binary_protocol/src/requests/topics/mod.rs index c9708e0..6990998 100644 --- a/core/binary_protocol/src/requests/topics/mod.rs +++ b/core/binary_protocol/src/requests/topics/mod.rs
@@ -20,11 +20,11 @@ pub mod delete_topic; pub mod get_topic; pub mod get_topics; -pub mod update_topic; +pub mod update_topic_resolved; pub use create_topic::CreateTopicRequest; pub use create_topic_with_assignments::CreateTopicWithAssignmentsRequest; pub use delete_topic::DeleteTopicRequest; pub use get_topic::GetTopicRequest; pub use get_topics::GetTopicsRequest; -pub use update_topic::UpdateTopicRequest; +pub use update_topic_resolved::UpdateTopicResolvedRequest;
diff --git a/core/binary_protocol/src/requests/topics/update_topic.rs b/core/binary_protocol/src/requests/topics/update_topic.rs deleted file mode 100644 index 5f8865a..0000000 --- a/core/binary_protocol/src/requests/topics/update_topic.rs +++ /dev/null
@@ -1,158 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::WireError; -use crate::WireIdentifier; -use crate::codec::{WireDecode, WireEncode}; -use crate::primitives::identifier::WireName; -use crate::primitives::options::WireOptions; -use bytes::BytesMut; - -/// `UpdateTopic` request. -/// -/// Wire format: -/// `[stream_id:WireIdentifier][topic_id:WireIdentifier][name_len:u8][name:N] -/// [options TLV to end]` -/// -/// The options block shares `CreateTopic`'s encoding, but the server validates -/// which keys can be updated. Supporting a key at creation does not make it -/// mutable. Absent keys are left unchanged, so updating the name does not erase -/// settings introduced after the client was built. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct UpdateTopicRequest { - pub stream_id: WireIdentifier, - pub topic_id: WireIdentifier, - pub name: WireName, - pub options: WireOptions, -} - -impl WireEncode for UpdateTopicRequest { - fn encoded_size(&self) -> usize { - self.stream_id.encoded_size() - + self.topic_id.encoded_size() - + self.name.encoded_size() - + self.options.encoded_size() - } - - fn encode(&self, buf: &mut BytesMut) { - self.stream_id.encode(buf); - self.topic_id.encode(buf); - self.name.encode(buf); - self.options.encode(buf); - } -} - -impl WireDecode for UpdateTopicRequest { - fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> { - let (stream_id, mut pos) = WireIdentifier::decode(buf)?; - let (topic_id, consumed) = WireIdentifier::decode(&buf[pos..])?; - pos += consumed; - let (name, name_consumed) = WireName::decode(&buf[pos..])?; - pos += name_consumed; - let options = WireOptions::from_slice(&buf[pos..])?; - Ok(( - Self { - stream_id, - topic_id, - name, - options, - }, - buf.len(), - )) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::primitives::user_headers::encode_user_headers; - - fn sample_options() -> WireOptions { - let mut buf = BytesMut::new(); - encode_user_headers(&[(2, b"future_option", 9, &[3u8])], &mut buf); - WireOptions::from_bytes(buf.freeze()).unwrap() - } - - fn sample_request() -> UpdateTopicRequest { - UpdateTopicRequest { - stream_id: WireIdentifier::numeric(1), - topic_id: WireIdentifier::numeric(2), - name: WireName::new("updated-topic").unwrap(), - options: sample_options(), - } - } - - #[test] - fn roundtrip() { - let req = sample_request(); - let bytes = req.to_bytes(); - let (decoded, consumed) = UpdateTopicRequest::decode(&bytes).unwrap(); - assert_eq!(consumed, bytes.len()); - assert_eq!(decoded, req); - } - - #[test] - fn roundtrip_named_identifiers() { - let req = UpdateTopicRequest { - stream_id: WireIdentifier::named("stream-a").unwrap(), - topic_id: WireIdentifier::named("topic-b").unwrap(), - name: WireName::new("new-name").unwrap(), - options: WireOptions::empty(), - }; - let bytes = req.to_bytes(); - let (decoded, consumed) = UpdateTopicRequest::decode(&bytes).unwrap(); - assert_eq!(consumed, bytes.len()); - assert_eq!(decoded, req); - } - - #[test] - fn truncated_options_return_error() { - // One key-value pair, so any strict-interior truncation of the block - // is structurally invalid. Cutting at the block boundary is NOT an - // error: that is a legitimate update carrying no options. - let req = sample_request(); - let bytes = req.to_bytes(); - let fixed_end = bytes.len() - req.options.encoded_size(); - for i in fixed_end + 1..bytes.len() { - assert!( - UpdateTopicRequest::decode(&bytes[..i]).is_err(), - "expected error for truncation at byte {i}" - ); - } - } - - #[test] - fn truncated_fixed_fields_return_error() { - let req = UpdateTopicRequest { - options: WireOptions::empty(), - ..sample_request() - }; - let bytes = req.to_bytes(); - for i in 0..bytes.len() { - assert!( - UpdateTopicRequest::decode(&bytes[..i]).is_err(), - "expected error for truncation at byte {i}" - ); - } - } - - #[test] - fn encoded_size_matches_output() { - let req = sample_request(); - assert_eq!(req.encoded_size(), req.to_bytes().len()); - } -}
diff --git a/core/binary_protocol/src/requests/topics/update_topic_resolved.rs b/core/binary_protocol/src/requests/topics/update_topic_resolved.rs new file mode 100644 index 0000000..0fcfb52 --- /dev/null +++ b/core/binary_protocol/src/requests/topics/update_topic_resolved.rs
@@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::WireError; +use crate::WireIdentifier; +use crate::codec::{WireDecode, WireEncode}; +use crate::primitives::identifier::WireName; +use crate::primitives::options::WireOptions; +use crate::requests::system::update_resource::{ + decode_optional_name, decode_resolved_blocks, encode_optional_name, encode_resolved_blocks, + optional_name_size, resolved_blocks_size, +}; +use bytes::BytesMut; + +/// An `UpdateResource` on a topic after prepare: the rename extracted, the set +/// keys canonical in `explicit_options`, the reset keys resolved to their +/// defaults in `derived_options`. +/// +/// Wire format: +/// ```text +/// [stream_id][topic_id][has_name:u8][name?][explicit_len:u32][explicit][derived_len:u32][derived] +/// ``` +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UpdateTopicResolvedRequest { + pub stream_id: WireIdentifier, + pub topic_id: WireIdentifier, + pub name: Option<WireName>, + pub explicit_options: WireOptions, + pub derived_options: WireOptions, +} + +impl WireEncode for UpdateTopicResolvedRequest { + fn encoded_size(&self) -> usize { + self.stream_id.encoded_size() + + self.topic_id.encoded_size() + + optional_name_size(self.name.as_ref()) + + resolved_blocks_size(&self.explicit_options, &self.derived_options) + } + + fn encode(&self, buf: &mut BytesMut) { + self.stream_id.encode(buf); + self.topic_id.encode(buf); + encode_optional_name(self.name.as_ref(), buf); + encode_resolved_blocks(&self.explicit_options, &self.derived_options, buf); + } +} + +impl WireDecode for UpdateTopicResolvedRequest { + fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> { + let (stream_id, mut position) = WireIdentifier::decode(buf)?; + let (topic_id, consumed) = WireIdentifier::decode(&buf[position..])?; + position += consumed; + let (name, consumed) = decode_optional_name(buf, position)?; + position += consumed; + let (explicit_options, derived_options) = decode_resolved_blocks(buf, position)?; + Ok(( + Self { + stream_id, + topic_id, + name, + explicit_options, + derived_options, + }, + buf.len(), + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::primitives::user_headers::encode_user_headers; + + #[test] + fn given_resolved_patch_when_roundtripping_should_preserve_columns_and_provenance() { + let request = UpdateTopicResolvedRequest { + stream_id: WireIdentifier::numeric(1), + topic_id: WireIdentifier::named("orders").unwrap(), + name: Some(WireName::new("orders-v2").unwrap()), + explicit_options: options(b"message_expiry", &7u64.to_le_bytes()), + derived_options: options(b"max_topic_size", &u64::MAX.to_le_bytes()), + }; + let bytes = request.to_bytes(); + assert_eq!(bytes.len(), request.encoded_size()); + assert_eq!( + UpdateTopicResolvedRequest::decode_from(&bytes).unwrap(), + request + ); + for length in 0..bytes.len() { + assert!(UpdateTopicResolvedRequest::decode(&bytes[..length]).is_err()); + } + } + + fn options(key: &[u8], value: &[u8]) -> WireOptions { + let mut buf = BytesMut::new(); + encode_user_headers(&[(2, key, 12, value)], &mut buf); + WireOptions::from_bytes(buf.freeze()).unwrap() + } +}
diff --git a/core/binary_protocol/src/requests/users/mod.rs b/core/binary_protocol/src/requests/users/mod.rs index 59860b5..5e15e5f 100644 --- a/core/binary_protocol/src/requests/users/mod.rs +++ b/core/binary_protocol/src/requests/users/mod.rs
@@ -25,7 +25,7 @@ pub mod login_user; pub mod logout_user; pub mod update_permissions; -pub mod update_user; +pub mod update_user_resolved; pub use change_password::ChangePasswordRequest; pub use create_user::CreateUserRequest; @@ -37,4 +37,4 @@ pub use login_user::LoginUserRequest; pub use logout_user::LogoutUserRequest; pub use update_permissions::UpdatePermissionsRequest; -pub use update_user::UpdateUserRequest; +pub use update_user_resolved::UpdateUserResolvedRequest;
diff --git a/core/binary_protocol/src/requests/users/update_user.rs b/core/binary_protocol/src/requests/users/update_user.rs deleted file mode 100644 index 125e1df..0000000 --- a/core/binary_protocol/src/requests/users/update_user.rs +++ /dev/null
@@ -1,228 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::WireError; -use crate::WireIdentifier; -use crate::codec::{WireDecode, WireEncode, read_u8}; -use crate::primitives::identifier::WireName; -use crate::primitives::options::WireOptions; -use bytes::{BufMut, BytesMut}; - -/// `UpdateUser` request. -/// -/// Wire format: -/// `[user_id:WireIdentifier][has_username:u8][username_len:u8?][username:N?] -/// [has_status:u8][status:u8?][options TLV to end]` -/// -/// The options block mirrors `CreateUser`'s, so a user setting added to the -/// catalog is updatable without another layout change. Keys absent from the -/// block are left alone rather than reset: an update patches the stored map, -/// so a client built before a key existed cannot erase it. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct UpdateUserRequest { - pub user_id: WireIdentifier, - pub username: Option<WireName>, - pub status: Option<u8>, - pub options: WireOptions, -} - -impl WireEncode for UpdateUserRequest { - fn encoded_size(&self) -> usize { - self.user_id.encoded_size() - + 1 // has_username - + self.username.as_ref().map_or(0, WireEncode::encoded_size) - + 1 // has_status - + self.status.map_or(0, |_| 1) - + self.options.encoded_size() - } - - fn encode(&self, buf: &mut BytesMut) { - self.user_id.encode(buf); - match &self.username { - Some(name) => { - buf.put_u8(1); - name.encode(buf); - } - None => { - buf.put_u8(0); - } - } - match self.status { - Some(s) => { - buf.put_u8(1); - buf.put_u8(s); - } - None => { - buf.put_u8(0); - } - } - self.options.encode(buf); - } -} - -impl WireDecode for UpdateUserRequest { - fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> { - let (user_id, mut pos) = WireIdentifier::decode(buf)?; - - let has_username = read_u8(buf, pos)?; - pos += 1; - let username = if has_username == 1 { - let (name, consumed) = WireName::decode(&buf[pos..])?; - pos += consumed; - Some(name) - } else { - None - }; - - let has_status = read_u8(buf, pos)?; - pos += 1; - let status = if has_status == 1 { - let s = read_u8(buf, pos)?; - pos += 1; - Some(s) - } else { - None - }; - - let options = WireOptions::from_slice(&buf[pos..])?; - Ok(( - Self { - user_id, - username, - status, - options, - }, - buf.len(), - )) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::primitives::user_headers::encode_user_headers; - - fn sample_options() -> WireOptions { - let mut buf = BytesMut::new(); - encode_user_headers(&[(2, b"future_key", 2, b"value")], &mut buf); - WireOptions::from_bytes(buf.freeze()).unwrap() - } - - #[test] - fn roundtrip_both_present() { - let req = UpdateUserRequest { - user_id: WireIdentifier::numeric(1), - username: Some(WireName::new("new-name").unwrap()), - status: Some(2), - options: WireOptions::empty(), - }; - let bytes = req.to_bytes(); - let (decoded, consumed) = UpdateUserRequest::decode(&bytes).unwrap(); - assert_eq!(consumed, bytes.len()); - assert_eq!(decoded, req); - } - - #[test] - fn roundtrip_both_none() { - let req = UpdateUserRequest { - user_id: WireIdentifier::numeric(5), - username: None, - status: None, - options: WireOptions::empty(), - }; - let bytes = req.to_bytes(); - let (decoded, consumed) = UpdateUserRequest::decode(&bytes).unwrap(); - assert_eq!(consumed, bytes.len()); - assert_eq!(decoded, req); - } - - #[test] - fn roundtrip_username_only() { - let req = UpdateUserRequest { - user_id: WireIdentifier::named("admin").unwrap(), - username: Some(WireName::new("super-admin").unwrap()), - status: None, - options: WireOptions::empty(), - }; - let bytes = req.to_bytes(); - let (decoded, consumed) = UpdateUserRequest::decode(&bytes).unwrap(); - assert_eq!(consumed, bytes.len()); - assert_eq!(decoded, req); - } - - #[test] - fn roundtrip_status_only() { - let req = UpdateUserRequest { - user_id: WireIdentifier::numeric(10), - username: None, - status: Some(0), - options: WireOptions::empty(), - }; - let bytes = req.to_bytes(); - let (decoded, consumed) = UpdateUserRequest::decode(&bytes).unwrap(); - assert_eq!(consumed, bytes.len()); - assert_eq!(decoded, req); - } - - #[test] - fn encoded_size_matches_output() { - let req = UpdateUserRequest { - user_id: WireIdentifier::numeric(1), - username: Some(WireName::new("test").unwrap()), - status: Some(1), - options: WireOptions::empty(), - }; - assert_eq!(req.encoded_size(), req.to_bytes().len()); - } - - #[test] - fn truncated_options_return_error() { - // One pair, so any strict-interior truncation is invalid. Cutting at - // the block boundary is a legitimate update carrying no options. - let req = UpdateUserRequest { - user_id: WireIdentifier::numeric(1), - username: Some(WireName::new("name").unwrap()), - status: Some(1), - options: sample_options(), - }; - let bytes = req.to_bytes(); - let fixed_end = bytes.len() - req.options.encoded_size(); - for i in fixed_end + 1..bytes.len() { - assert!( - UpdateUserRequest::decode(&bytes[..i]).is_err(), - "expected error for truncation at byte {i}" - ); - } - } - - #[test] - fn truncated_returns_error() { - let req = UpdateUserRequest { - user_id: WireIdentifier::numeric(1), - username: Some(WireName::new("name").unwrap()), - status: Some(1), - options: WireOptions::empty(), - }; - let bytes = req.to_bytes(); - for i in 0..bytes.len() { - assert!( - UpdateUserRequest::decode(&bytes[..i]).is_err(), - "expected error for truncation at byte {i}" - ); - } - } -}
diff --git a/core/binary_protocol/src/requests/users/update_user_resolved.rs b/core/binary_protocol/src/requests/users/update_user_resolved.rs new file mode 100644 index 0000000..1a009d7 --- /dev/null +++ b/core/binary_protocol/src/requests/users/update_user_resolved.rs
@@ -0,0 +1,136 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::WireError; +use crate::WireIdentifier; +use crate::codec::{WireDecode, WireEncode, read_u8}; +use crate::primitives::identifier::WireName; +use crate::primitives::options::WireOptions; +use crate::requests::system::update_resource::{ + decode_optional_name, decode_resolved_blocks, encode_optional_name, encode_resolved_blocks, + optional_name_size, resolved_blocks_size, +}; +use bytes::{BufMut, BytesMut}; +use std::borrow::Cow; + +/// An `UpdateResource` on a user after prepare: the username and status +/// extracted, the set keys canonical in `explicit_options`, the reset keys +/// resolved to their defaults in `derived_options`. +/// +/// Wire format: +/// ```text +/// [user_id][has_username:u8][username?][has_status:u8][status:u8?] +/// [explicit_len:u32][explicit][derived_len:u32][derived] +/// ``` +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UpdateUserResolvedRequest { + pub user_id: WireIdentifier, + pub username: Option<WireName>, + pub status: Option<u8>, + pub explicit_options: WireOptions, + pub derived_options: WireOptions, +} + +impl WireEncode for UpdateUserResolvedRequest { + fn encoded_size(&self) -> usize { + self.user_id.encoded_size() + + optional_name_size(self.username.as_ref()) + + 1 + + usize::from(self.status.is_some()) + + resolved_blocks_size(&self.explicit_options, &self.derived_options) + } + + fn encode(&self, buf: &mut BytesMut) { + self.user_id.encode(buf); + encode_optional_name(self.username.as_ref(), buf); + match self.status { + Some(status) => { + buf.put_u8(1); + buf.put_u8(status); + } + None => buf.put_u8(0), + } + encode_resolved_blocks(&self.explicit_options, &self.derived_options, buf); + } +} + +impl WireDecode for UpdateUserResolvedRequest { + fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> { + let (user_id, mut position) = WireIdentifier::decode(buf)?; + let (username, consumed) = decode_optional_name(buf, position)?; + position += consumed; + let status = match read_u8(buf, position)? { + 0 => None, + 1 => { + position += 1; + Some(read_u8(buf, position)?) + } + _ => { + return Err(WireError::Validation(Cow::Borrowed( + "presence byte must be 0 or 1", + ))); + } + }; + position += 1; + let (explicit_options, derived_options) = decode_resolved_blocks(buf, position)?; + Ok(( + Self { + user_id, + username, + status, + explicit_options, + derived_options, + }, + buf.len(), + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn given_resolved_patch_when_roundtripping_should_preserve_columns_and_provenance() { + for (username, status) in [ + (None, None), + (Some(WireName::new("alice2").unwrap()), None), + (None, Some(2)), + (Some(WireName::new("alice2").unwrap()), Some(1)), + ] { + let request = UpdateUserResolvedRequest { + user_id: WireIdentifier::named("alice").unwrap(), + username, + status, + explicit_options: WireOptions::empty(), + derived_options: WireOptions::empty(), + }; + let bytes = request.to_bytes(); + assert_eq!(bytes.len(), request.encoded_size()); + assert_eq!( + UpdateUserResolvedRequest::decode_from(&bytes).unwrap(), + request + ); + for length in 0..bytes.len() { + assert!(UpdateUserResolvedRequest::decode(&bytes[..length]).is_err()); + } + } + let mut bad_status = WireIdentifier::numeric(1).to_bytes().to_vec(); + bad_status.extend_from_slice(&[0, 3]); + assert!(UpdateUserResolvedRequest::decode(&bad_status).is_err()); + } +}
diff --git a/core/binary_protocol/src/responses/streams/get_stream.rs b/core/binary_protocol/src/responses/streams/get_stream.rs index fd5b2ac..b8b6aa3 100644 --- a/core/binary_protocol/src/responses/streams/get_stream.rs +++ b/core/binary_protocol/src/responses/streams/get_stream.rs
@@ -16,7 +16,7 @@ // under the License. use crate::WireError; -use crate::codec::{WireDecode, WireEncode, read_u8, read_u32_le, read_u64_le}; +use crate::codec::{WireDecode, WireEncode, read_u32_le, read_u64_le}; use crate::primitives::identifier::WireName; use crate::primitives::options::{ WireOptions, decode_options_prefixed, encode_options_prefixed, options_prefixed_size, @@ -26,10 +26,9 @@ /// Topic header within a `GetStream` response. /// -/// Wire format (50 + `name_len` + options bytes): +/// Wire format (33 + `name_len` + options bytes): /// ```text -/// [id:4][created_at:8][partitions_count:4][message_expiry:8] -/// [compression_algorithm:1][max_topic_size:8] +/// [id:4][created_at:8][partitions_count:4] /// [size_bytes:8][messages_count:8][name_len:1][name:N] /// [explicit_options_len:4][explicit options TLV] /// [derived_options_len:4][derived options TLV] @@ -38,14 +37,12 @@ /// `options` are the keys the client pinned at create; `derived_options` are /// the defaults the admitting primary resolved. A round-tripping client /// re-sends only `options`; the effective configuration is the union. +/// Message expiry, compression and the size cap live in those blocks only. #[derive(Debug, Clone, PartialEq, Eq)] pub struct TopicHeader { pub id: u32, pub created_at: u64, pub partitions_count: u32, - pub message_expiry: u64, - pub compression_algorithm: u8, - pub max_topic_size: u64, pub size_bytes: u64, pub messages_count: u64, pub name: WireName, @@ -54,7 +51,7 @@ } impl TopicHeader { - pub(crate) const FIXED_SIZE: usize = 4 + 8 + 4 + 8 + 1 + 8 + 8 + 8 + 1; // 50 + pub(crate) const FIXED_SIZE: usize = 4 + 8 + 4 + 8 + 8 + 1; // 33 } impl WireEncode for TopicHeader { @@ -69,9 +66,6 @@ buf.put_u32_le(self.id); buf.put_u64_le(self.created_at); buf.put_u32_le(self.partitions_count); - buf.put_u64_le(self.message_expiry); - buf.put_u8(self.compression_algorithm); - buf.put_u64_le(self.max_topic_size); buf.put_u64_le(self.size_bytes); buf.put_u64_le(self.messages_count); self.name.encode(buf); @@ -85,13 +79,10 @@ let id = read_u32_le(buf, 0)?; let created_at = read_u64_le(buf, 4)?; let partitions_count = read_u32_le(buf, 12)?; - let message_expiry = read_u64_le(buf, 16)?; - let compression_algorithm = read_u8(buf, 24)?; - let max_topic_size = read_u64_le(buf, 25)?; - let size_bytes = read_u64_le(buf, 33)?; - let messages_count = read_u64_le(buf, 41)?; - let (name, name_consumed) = WireName::decode(&buf[49..])?; - let mut consumed = 49 + name_consumed; + let size_bytes = read_u64_le(buf, 16)?; + let messages_count = read_u64_le(buf, 24)?; + let (name, name_consumed) = WireName::decode(&buf[32..])?; + let mut consumed = 32 + name_consumed; let (options, options_consumed) = decode_options_prefixed(buf, consumed)?; consumed += options_consumed; let (derived_options, derived_consumed) = decode_options_prefixed(buf, consumed)?; @@ -102,9 +93,6 @@ id, created_at, partitions_count, - message_expiry, - compression_algorithm, - max_topic_size, size_bytes, messages_count, name, @@ -186,6 +174,7 @@ messages_count: 200, name: WireName::new("my-stream").unwrap(), options: WireOptions::empty(), + derived_options: WireOptions::empty(), } } @@ -194,9 +183,6 @@ id, created_at: 1_710_000_000_000, partitions_count: 3, - message_expiry: 0, - compression_algorithm: 1, - max_topic_size: 0, size_bytes: 1024, messages_count: 100, name: WireName::new(name).unwrap(),
diff --git a/core/binary_protocol/src/responses/streams/get_streams.rs b/core/binary_protocol/src/responses/streams/get_streams.rs index 11e30ab..33d3acd 100644 --- a/core/binary_protocol/src/responses/streams/get_streams.rs +++ b/core/binary_protocol/src/responses/streams/get_streams.rs
@@ -85,6 +85,7 @@ messages_count: 50, name: WireName::new("s1").unwrap(), options: WireOptions::empty(), + derived_options: WireOptions::empty(), }, StreamResponse { id: 2, @@ -94,6 +95,7 @@ messages_count: 0, name: WireName::new("stream-two").unwrap(), options: WireOptions::empty(), + derived_options: WireOptions::empty(), }, ], };
diff --git a/core/binary_protocol/src/responses/streams/mod.rs b/core/binary_protocol/src/responses/streams/mod.rs index 7b869d2..09fe7ec 100644 --- a/core/binary_protocol/src/responses/streams/mod.rs +++ b/core/binary_protocol/src/responses/streams/mod.rs
@@ -20,7 +20,6 @@ pub mod get_stream; pub mod get_streams; mod stream_response; -mod update_stream; pub use super::EmptyResponse; pub use create_stream::CreateStreamResponse; @@ -28,4 +27,3 @@ pub use get_stream::{GetStreamResponse, TopicHeader}; pub use get_streams::GetStreamsResponse; pub use stream_response::StreamResponse; -pub use update_stream::UpdateStreamResponse;
diff --git a/core/binary_protocol/src/responses/streams/stream_response.rs b/core/binary_protocol/src/responses/streams/stream_response.rs index 8beb76f..7253d20 100644 --- a/core/binary_protocol/src/responses/streams/stream_response.rs +++ b/core/binary_protocol/src/responses/streams/stream_response.rs
@@ -28,11 +28,12 @@ /// Wire format (33 + `name_len` + options bytes): /// ```text /// [id:4][created_at:8][topics_count:4][size_bytes:8][messages_count:8][name_len:1][name:N] -/// [options_len:4][options TLV] +/// [explicit_options_len:4][explicit options TLV] +/// [derived_options_len:4][derived options TLV] /// ``` /// -/// Streams have no server-derived options (no catalog keys resolve against -/// config yet), so a single client-explicit block suffices. +/// `options` are the keys the client set, `derived_options` the values the +/// server filled or computes, such as the encryption key source. #[derive(Debug, Clone, PartialEq, Eq)] pub struct StreamResponse { pub id: u32, @@ -42,6 +43,7 @@ pub messages_count: u64, pub name: WireName, pub options: WireOptions, + pub derived_options: WireOptions, } impl StreamResponse { @@ -50,7 +52,10 @@ impl WireEncode for StreamResponse { fn encoded_size(&self) -> usize { - Self::FIXED_SIZE + self.name.len() + options_prefixed_size(&self.options) + Self::FIXED_SIZE + + self.name.len() + + options_prefixed_size(&self.options) + + options_prefixed_size(&self.derived_options) } fn encode(&self, buf: &mut BytesMut) { @@ -61,6 +66,7 @@ buf.put_u64_le(self.messages_count); self.name.encode(buf); encode_options_prefixed(&self.options, buf); + encode_options_prefixed(&self.derived_options, buf); } } @@ -75,6 +81,8 @@ let mut consumed = 32 + name_consumed; let (options, options_consumed) = decode_options_prefixed(buf, consumed)?; consumed += options_consumed; + let (derived_options, derived_consumed) = decode_options_prefixed(buf, consumed)?; + consumed += derived_consumed; Ok(( Self { @@ -85,6 +93,7 @@ messages_count, name, options, + derived_options, }, consumed, )) @@ -104,6 +113,7 @@ messages_count: 100, name: WireName::new("test-stream").unwrap(), options: WireOptions::empty(), + derived_options: WireOptions::empty(), } } @@ -111,7 +121,7 @@ fn roundtrip() { let resp = sample(); let bytes = resp.to_bytes(); - assert_eq!(bytes.len(), StreamResponse::FIXED_SIZE + 11 + 4); + assert_eq!(bytes.len(), StreamResponse::FIXED_SIZE + 11 + 4 + 4); let (decoded, consumed) = StreamResponse::decode(&bytes).unwrap(); assert_eq!(consumed, bytes.len()); assert_eq!(decoded, resp);
diff --git a/core/binary_protocol/src/responses/streams/update_stream.rs b/core/binary_protocol/src/responses/streams/update_stream.rs deleted file mode 100644 index b002d9c..0000000 --- a/core/binary_protocol/src/responses/streams/update_stream.rs +++ /dev/null
@@ -1,19 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -/// `UpdateStream` response is empty. -pub type UpdateStreamResponse = super::EmptyResponse;
diff --git a/core/binary_protocol/src/responses/system/describe_options.rs b/core/binary_protocol/src/responses/system/describe_options.rs index 154196e..3b12d76 100644 --- a/core/binary_protocol/src/responses/system/describe_options.rs +++ b/core/binary_protocol/src/responses/system/describe_options.rs
@@ -24,33 +24,36 @@ /// /// Wire format: /// ```text -/// [key_len:u8][key:N][kind:u8] +/// [key_len:u8][key:N][kind:u8][flags:u8] /// [default_len:u32][default bytes][description_len:u32][description] /// ``` /// /// `kind` is this key's canonical header kind code: what the server encodes its -/// default under, and what a value set at create is stored as, since create -/// admission re-encodes the block from its own parse (a `String` value parsed by -/// the same rules as a config file value is accepted and canonicalized). An -/// update stores the client's bytes verbatim and is the exception. `default` is -/// that default in `kind`'s encoding, empty when the key has none, and is a -/// build constant rather than a per-node value. +/// default under, and what a value set at create or update is stored as, since +/// admission re-encodes every block from its own parse (a `String` value parsed +/// by the same rules as a config file value is accepted and canonicalized). +/// `flags` bits: 0 updatable, 1 resettable, 2 secret (write-only), 3 read-only +/// (server-computed), 4 critical. Higher bits are zero and readers ignore them. +/// `default` is that default in `kind`'s encoding, empty when the key has none, +/// and is a build constant rather than a per-node value. #[derive(Debug, Clone, PartialEq, Eq)] pub struct OptionDescriptor { pub key: WireName, pub kind: u8, + pub flags: u8, pub default_value: Bytes, pub description: String, } impl WireEncode for OptionDescriptor { fn encoded_size(&self) -> usize { - self.key.encoded_size() + 1 + 4 + self.default_value.len() + 4 + self.description.len() + self.key.encoded_size() + 1 + 1 + 4 + self.default_value.len() + 4 + self.description.len() } fn encode(&self, buf: &mut BytesMut) { self.key.encode(buf); buf.put_u8(self.kind); + buf.put_u8(self.flags); #[allow(clippy::cast_possible_truncation)] buf.put_u32_le(self.default_value.len() as u32); buf.put_slice(&self.default_value); @@ -65,6 +68,8 @@ let (key, mut pos) = WireName::decode(buf)?; let kind = read_u8(buf, pos)?; pos += 1; + let flags = read_u8(buf, pos)?; + pos += 1; let default_len = read_u32_le(buf, pos)? as usize; pos += 4; // `checked_add`: on a 32-bit target a wire-supplied length wraps, the @@ -108,6 +113,7 @@ Self { key, kind, + flags, default_value, description, }, @@ -143,9 +149,9 @@ } /// Smallest a descriptor can encode as: a one-character `WireName` (a length -/// byte plus at least one byte of name), a kind byte, and empty +/// byte plus at least one byte of name), a kind byte, a flags byte, and empty /// length-prefixed default and description. -const MIN_DESCRIPTOR_SIZE: usize = 2 + 1 + 4 + 4; +const MIN_DESCRIPTOR_SIZE: usize = 2 + 1 + 1 + 4 + 4; impl WireDecode for DescribeOptionsResponse { fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> { @@ -175,12 +181,14 @@ OptionDescriptor { key: WireName::new("partitions_count").unwrap(), kind: 11, + flags: 0, default_value: Bytes::copy_from_slice(&1u32.to_le_bytes()), description: "Number of partitions to create".to_string(), }, OptionDescriptor { key: WireName::new("max_topic_size").unwrap(), kind: 12, + flags: 0b11, default_value: Bytes::new(), description: String::new(), },
diff --git a/core/binary_protocol/src/responses/system/mod.rs b/core/binary_protocol/src/responses/system/mod.rs index 113eee3..a8e21d3 100644 --- a/core/binary_protocol/src/responses/system/mod.rs +++ b/core/binary_protocol/src/responses/system/mod.rs
@@ -21,6 +21,7 @@ pub mod get_snapshot; pub mod get_stats; mod ping; +mod update_resource; pub use super::EmptyResponse; pub use describe_options::{DescribeOptionsResponse, OptionDescriptor}; @@ -29,3 +30,4 @@ pub use get_snapshot::GetSnapshotResponse; pub use get_stats::StatsResponse; pub use ping::PingResponse; +pub use update_resource::UpdateResourceResponse;
diff --git a/core/binary_protocol/src/responses/users/update_user.rs b/core/binary_protocol/src/responses/system/update_resource.rs similarity index 88% rename from core/binary_protocol/src/responses/users/update_user.rs rename to core/binary_protocol/src/responses/system/update_resource.rs index 2b4a4e4..3645eb5 100644 --- a/core/binary_protocol/src/responses/users/update_user.rs +++ b/core/binary_protocol/src/responses/system/update_resource.rs
@@ -15,4 +15,5 @@ // specific language governing permissions and limitations // under the License. -pub type UpdateUserResponse = super::EmptyResponse; +/// `UpdateResource` response is empty. +pub type UpdateResourceResponse = super::EmptyResponse;
diff --git a/core/binary_protocol/src/responses/topics/get_topic.rs b/core/binary_protocol/src/responses/topics/get_topic.rs index 76d8672..96cd8e8 100644 --- a/core/binary_protocol/src/responses/topics/get_topic.rs +++ b/core/binary_protocol/src/responses/topics/get_topic.rs
@@ -147,9 +147,6 @@ id: 1, created_at: 1_710_000_000_000, partitions_count, - message_expiry: 0, - compression_algorithm: 1, - max_topic_size: 0, size_bytes: 2048, messages_count: 200, name: WireName::new("my-topic").unwrap(),
diff --git a/core/binary_protocol/src/responses/topics/get_topics.rs b/core/binary_protocol/src/responses/topics/get_topics.rs index 6852866..d8f5d9d 100644 --- a/core/binary_protocol/src/responses/topics/get_topics.rs +++ b/core/binary_protocol/src/responses/topics/get_topics.rs
@@ -89,9 +89,6 @@ id, created_at: 1_710_000_000_000, partitions_count: 3, - message_expiry: 0, - compression_algorithm: 1, - max_topic_size: 0, size_bytes: 1024, messages_count: 100, name: WireName::new(name).unwrap(),
diff --git a/core/binary_protocol/src/responses/topics/mod.rs b/core/binary_protocol/src/responses/topics/mod.rs index a6567d1..f1279ee 100644 --- a/core/binary_protocol/src/responses/topics/mod.rs +++ b/core/binary_protocol/src/responses/topics/mod.rs
@@ -19,11 +19,9 @@ mod delete_topic; pub mod get_topic; pub mod get_topics; -mod update_topic; pub use super::EmptyResponse; pub use create_topic::CreateTopicResponse; pub use delete_topic::DeleteTopicResponse; pub use get_topic::{GetTopicResponse, PartitionResponse}; pub use get_topics::GetTopicsResponse; -pub use update_topic::UpdateTopicResponse;
diff --git a/core/binary_protocol/src/responses/topics/update_topic.rs b/core/binary_protocol/src/responses/topics/update_topic.rs deleted file mode 100644 index 6df77c4..0000000 --- a/core/binary_protocol/src/responses/topics/update_topic.rs +++ /dev/null
@@ -1,19 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -/// `UpdateTopic` response is empty. -pub type UpdateTopicResponse = super::EmptyResponse;
diff --git a/core/binary_protocol/src/responses/users/get_user.rs b/core/binary_protocol/src/responses/users/get_user.rs index 6392e37..3628d23 100644 --- a/core/binary_protocol/src/responses/users/get_user.rs +++ b/core/binary_protocol/src/responses/users/get_user.rs
@@ -101,6 +101,7 @@ status: 1, username: WireName::new("admin").unwrap(), options: WireOptions::empty(), + derived_options: WireOptions::empty(), } }
diff --git a/core/binary_protocol/src/responses/users/get_users.rs b/core/binary_protocol/src/responses/users/get_users.rs index 87811f6..556f9ed 100644 --- a/core/binary_protocol/src/responses/users/get_users.rs +++ b/core/binary_protocol/src/responses/users/get_users.rs
@@ -81,6 +81,7 @@ status: 1, username: WireName::new("admin").unwrap(), options: WireOptions::empty(), + derived_options: WireOptions::empty(), }, UserResponse { id: 2, @@ -88,6 +89,7 @@ status: 2, username: WireName::new("alice").unwrap(), options: WireOptions::empty(), + derived_options: WireOptions::empty(), }, ], };
diff --git a/core/binary_protocol/src/responses/users/mod.rs b/core/binary_protocol/src/responses/users/mod.rs index cb95cdd..e7a8565 100644 --- a/core/binary_protocol/src/responses/users/mod.rs +++ b/core/binary_protocol/src/responses/users/mod.rs
@@ -24,7 +24,6 @@ pub mod login_user; mod logout_user; mod update_permissions; -mod update_user; pub mod user_response; pub use super::EmptyResponse; @@ -37,5 +36,4 @@ pub use login_user::IdentityResponse; pub use logout_user::LogoutUserResponse; pub use update_permissions::UpdatePermissionsResponse; -pub use update_user::UpdateUserResponse; pub use user_response::UserResponse;
diff --git a/core/binary_protocol/src/responses/users/user_response.rs b/core/binary_protocol/src/responses/users/user_response.rs index ef495a9..9b77774 100644 --- a/core/binary_protocol/src/responses/users/user_response.rs +++ b/core/binary_protocol/src/responses/users/user_response.rs
@@ -28,11 +28,12 @@ /// Wire format (13 + `username_len` + options bytes): /// ```text /// [id:4][created_at:8][status:1][username_len:1][username:N] -/// [options_len:4][options TLV] +/// [explicit_options_len:4][explicit options TLV] +/// [derived_options_len:4][derived options TLV] /// ``` /// -/// Users have no server-derived options (no catalog keys resolve against -/// config yet), so a single client-explicit block suffices. +/// `options` are the keys the client set, `derived_options` the values the +/// server filled. #[derive(Debug, Clone, PartialEq, Eq)] pub struct UserResponse { pub id: u32, @@ -40,6 +41,7 @@ pub status: u8, pub username: WireName, pub options: WireOptions, + pub derived_options: WireOptions, } impl UserResponse { @@ -48,7 +50,10 @@ impl WireEncode for UserResponse { fn encoded_size(&self) -> usize { - Self::FIXED_SIZE + self.username.encoded_size() + options_prefixed_size(&self.options) + Self::FIXED_SIZE + + self.username.encoded_size() + + options_prefixed_size(&self.options) + + options_prefixed_size(&self.derived_options) } fn encode(&self, buf: &mut BytesMut) { @@ -57,6 +62,7 @@ buf.put_u8(self.status); self.username.encode(buf); encode_options_prefixed(&self.options, buf); + encode_options_prefixed(&self.derived_options, buf); } } @@ -69,6 +75,8 @@ let mut consumed = 13 + name_consumed; let (options, options_consumed) = decode_options_prefixed(buf, consumed)?; consumed += options_consumed; + let (derived_options, derived_consumed) = decode_options_prefixed(buf, consumed)?; + consumed += derived_consumed; Ok(( Self { @@ -77,6 +85,7 @@ status, username, options, + derived_options, }, consumed, )) @@ -94,6 +103,7 @@ status: 1, username: WireName::new("admin").unwrap(), options: WireOptions::empty(), + derived_options: WireOptions::empty(), } } @@ -101,7 +111,7 @@ fn roundtrip() { let resp = sample(); let bytes = resp.to_bytes(); - assert_eq!(bytes.len(), UserResponse::FIXED_SIZE + 1 + 5 + 4); + assert_eq!(bytes.len(), UserResponse::FIXED_SIZE + 1 + 5 + 4 + 4); let (decoded, consumed) = UserResponse::decode(&bytes).unwrap(); assert_eq!(consumed, bytes.len()); assert_eq!(decoded, resp);
diff --git a/core/binary_protocol/src/version.rs b/core/binary_protocol/src/version.rs index adf1ba5..9dbe038 100644 --- a/core/binary_protocol/src/version.rs +++ b/core/binary_protocol/src/version.rs
@@ -102,17 +102,25 @@ /// Current development protocol version, independent of package versions. /// /// Version 0.11.1 requires the session-binding secret in login requests. +/// Version 0.12.0 replaces the per-resource update commands with the generic +/// `UpdateResource` patch, drops the message expiry, compression and size cap +/// fields from the topic header (50 to 33 fixed bytes, the values live in the +/// option blocks), adds derived option blocks to stream and user responses and +/// a flags byte to each `DescribeOptions` descriptor. /// Finalize this number and the minimum during stable release preparation. /// Released servers using protocol 0.11.0 accept every 0.11.x patch, so an /// incompatible release needs a new minor version for those servers to reject it. -pub const IGGY_PROTOCOL_VERSION: u32 = pack_protocol_version(0, 11, 1); +pub const IGGY_PROTOCOL_VERSION: u32 = pack_protocol_version(0, 12, 0); /// Oldest protocol version this build accepts at login. -/// Version 0.11.0 has no session-binding secret and is incompatible. +/// +/// Version 0.11.0 has no session-binding secret, and 0.11.1 still sends the +/// per-resource update commands, decodes the 50-byte topic header and reads +/// single-block stream and user responses, so both are incompatible. /// /// Keep this equal to the current protocol unless compatibility with older /// released protocols has been verified. Decide the range per stable release. -pub const IGGY_PROTOCOL_VERSION_MIN: u32 = pack_protocol_version(0, 11, 1); +pub const IGGY_PROTOCOL_VERSION_MIN: u32 = pack_protocol_version(0, 12, 0); const _: () = assert!(IGGY_PROTOCOL_VERSION_MIN > 0 && IGGY_PROTOCOL_VERSION_MIN <= IGGY_PROTOCOL_VERSION); @@ -255,11 +263,11 @@ } #[test] - fn login_without_binding_secret_is_outside_the_compatible_window() { + fn login_from_a_per_resource_update_protocol_is_outside_the_compatible_window() { assert!(!is_protocol_compatible(pack_protocol_version(0, 11, 0))); - assert!(is_protocol_compatible(pack_protocol_version(0, 11, 1))); - assert!(!is_protocol_compatible(pack_protocol_version(0, 11, 1023))); - assert!(!is_protocol_compatible(pack_protocol_version(0, 12, 0))); + assert!(!is_protocol_compatible(pack_protocol_version(0, 11, 1))); + assert!(is_protocol_compatible(pack_protocol_version(0, 12, 0))); + assert!(!is_protocol_compatible(pack_protocol_version(0, 12, 1))); } #[test] @@ -269,8 +277,8 @@ } #[test] - fn min_requires_the_session_binding_wire_layout() { - assert_eq!(IGGY_PROTOCOL_VERSION_MIN, pack_protocol_version(0, 11, 1)); + fn min_requires_the_generic_update_wire_layout() { + assert_eq!(IGGY_PROTOCOL_VERSION_MIN, pack_protocol_version(0, 12, 0)); } #[test]
diff --git a/core/cli/Cargo.toml b/core/cli/Cargo.toml index 5704e54..f8d4337 100644 --- a/core/cli/Cargo.toml +++ b/core/cli/Cargo.toml
@@ -17,7 +17,7 @@ [package] name = "iggy-cli" -version = "0.14.1-edge.1" +version = "0.14.1-edge.2" edition = "2024" rust-version.workspace = true authors = ["bartosz.ciesla@gmail.com"]
diff --git a/core/cli/src/args/common.rs b/core/cli/src/args/common.rs index 2accf01..08ae994 100644 --- a/core/cli/src/args/common.rs +++ b/core/cli/src/args/common.rs
@@ -16,6 +16,7 @@ // under the License. use clap::ValueEnum; +use iggy::prelude::OptionKey; use iggy_cli::commands::binary_client::get_clients::GetClientsOutput; use iggy_cli::commands::binary_cluster::get_cluster_metadata::GetClusterMetadataOutput; use iggy_cli::commands::binary_consumer_groups::get_consumer_groups::GetConsumerGroupsOutput; @@ -25,6 +26,7 @@ use iggy_cli::commands::binary_system::stats::GetStatsOutput; use iggy_cli::commands::binary_topics::get_topics::GetTopicsOutput; use iggy_cli::commands::binary_users::get_users::GetUsersOutput; +use std::collections::BTreeSet; #[derive(Debug, Clone, Copy, ValueEnum)] pub(crate) enum ListMode { @@ -152,3 +154,21 @@ } Ok((key.to_string(), value.to_string())) } + +/// Split repeated `--reset` names into keys this build checks before sending +/// and newer keys the server decides on. +pub(crate) fn split_reset(names: &[String]) -> (BTreeSet<OptionKey>, BTreeSet<String>) { + let mut known = BTreeSet::new(); + let mut newer = BTreeSet::new(); + for name in names { + match name.parse::<OptionKey>() { + Ok(key) => { + known.insert(key); + } + Err(_) => { + newer.insert(name.clone()); + } + } + } + (known, newer) +}
diff --git a/core/cli/src/args/mod.rs b/core/cli/src/args/mod.rs index a758284..ffe1145 100644 --- a/core/cli/src/args/mod.rs +++ b/core/cli/src/args/mod.rs
@@ -173,11 +173,13 @@ /// client address as seen by the server and protocol type. #[clap(verbatim_doc_comment)] Me, - /// list the options a resource's create command accepts + /// list the option catalog of a resource scope /// /// Prints the server's option catalog for one scope (topic, stream, user): - /// every key, the kind its default is reported in, that default, and the - /// bounds each value is checked against. These are the keys `--set` takes. + /// every key, the kind its default is reported in, that default, and what a + /// client may do with it (create only, updatable, resettable, read only, + /// secret, critical). Create-only keys go to `--set` on topic create, + /// updatable keys to `--set` and `--reset` on topic and stream update. #[clap(verbatim_doc_comment)] Options(OptionsArgs), /// get iggy server statistics
diff --git a/core/cli/src/args/stream.rs b/core/cli/src/args/stream.rs index 1609b93..993c729 100644 --- a/core/cli/src/args/stream.rs +++ b/core/cli/src/args/stream.rs
@@ -15,9 +15,10 @@ // specific language governing permissions and limitations // under the License. -use crate::args::common::ListMode; +use crate::args::common::{ListMode, parse_key_value, split_reset}; use clap::{Args, Subcommand}; -use iggy::prelude::Identifier; +use iggy::prelude::{Identifier, StreamUpdateOptions}; +use std::path::PathBuf; #[derive(Debug, Clone, Subcommand)] pub(crate) enum StreamAction { @@ -28,6 +29,7 @@ /// Examples: /// iggy stream create prod /// iggy stream create -s 1 test + /// iggy stream create --stream-key-file stream.key payments #[clap(verbatim_doc_comment, visible_alias = "c")] Create(StreamCreateArgs), /// Delete stream with given ID @@ -39,13 +41,14 @@ /// iggy stream delete test #[clap(verbatim_doc_comment, visible_alias = "d")] Delete(StreamDeleteArgs), - /// Update stream name for given stream ID + /// Update stream name or options for given stream ID /// /// Stream ID can be specified as a stream name or ID + /// Only the settings passed change, every other setting keeps its value. /// /// Examples: - /// iggy stream update 1 production - /// iggy stream update test development + /// iggy stream update 1 --name production + /// iggy stream update test --name development #[clap(verbatim_doc_comment, visible_alias = "u")] Update(StreamUpdateArgs), /// Get details of a single stream with given ID @@ -72,6 +75,15 @@ /// Legacy stream ID flag (ignored) #[clap(short, long)] pub(crate) stream_id: Option<u32>, + /// File holding the stream's own at-rest encryption key + /// + /// Standard base64 of exactly 32 random bytes from a secure source, such + /// as "head -c 32 /dev/urandom | base64". Use a new key for every stream. + /// Without this flag the server encryption setting applies. The server + /// stores the key wrapped and reads report only its fingerprint. Unrelated + /// to --encryption-key, which encrypts message payloads on the client. + #[clap(long, value_name = "PATH", verbatim_doc_comment)] + pub(crate) stream_key_file: Option<PathBuf>, /// Name of the stream pub(crate) name: String, } @@ -91,7 +103,31 @@ /// Stream ID can be specified as a stream name or ID pub(crate) stream_id: Identifier, /// New name for the stream - pub(crate) name: String, + #[arg(short, long)] + pub(crate) name: Option<String>, + /// Updatable stream option as key=value, repeatable + /// + /// Values are sent as strings and parsed server-side. + /// Run "iggy options stream" to list the keys and which ones can change. + #[arg(long = "set", value_name = "KEY=VALUE", value_parser = parse_key_value, verbatim_doc_comment)] + pub(crate) set: Vec<(String, String)>, + /// Setting to return to its default, repeatable + /// + /// The server refuses a key that cannot be reset. + #[arg(short, long, verbatim_doc_comment)] + pub(crate) reset: Vec<String>, +} + +impl From<&StreamUpdateArgs> for StreamUpdateOptions { + fn from(args: &StreamUpdateArgs) -> Self { + let (reset, raw_reset) = split_reset(&args.reset); + StreamUpdateOptions { + name: args.name.clone(), + reset, + raw_reset, + raw: args.set.iter().cloned().collect(), + } + } } #[derive(Debug, Clone, Args)] @@ -108,3 +144,39 @@ #[clap(short, long, value_enum, default_value_t = ListMode::Table)] pub(crate) list_mode: ListMode, } + +#[cfg(test)] +mod tests { + use super::StreamUpdateArgs; + use clap::Parser; + use iggy::prelude::StreamUpdateOptions; + use std::collections::{BTreeMap, BTreeSet}; + + #[derive(Parser)] + struct Update { + #[command(flatten)] + args: StreamUpdateArgs, + } + + #[test] + fn given_stream_update_without_name_when_parsing_should_accept_reset_and_set() { + let parsed = Update::try_parse_from([ + "iggy", + "orders", + "--reset", + "future_key", + "--set", + "newer_key=1", + ]) + .expect("an update without a name parses"); + + assert_eq!( + StreamUpdateOptions::from(&parsed.args), + StreamUpdateOptions { + raw_reset: BTreeSet::from(["future_key".to_owned()]), + raw: BTreeMap::from([("newer_key".to_owned(), "1".to_owned())]), + ..StreamUpdateOptions::default() + } + ); + } +}
diff --git a/core/cli/src/args/topic.rs b/core/cli/src/args/topic.rs index b798848..8e400fe 100644 --- a/core/cli/src/args/topic.rs +++ b/core/cli/src/args/topic.rs
@@ -15,9 +15,11 @@ // specific language governing permissions and limitations // under the License. -use crate::args::common::{ListMode, parse_key_value}; +use crate::args::common::{ListMode, parse_key_value, split_reset}; use clap::{Args, Subcommand}; -use iggy::prelude::{CompressionAlgorithm, Identifier, IggyExpiry, MaxTopicSize}; +use iggy::prelude::{ + CompressionAlgorithm, Identifier, IggyExpiry, MaxTopicSize, TopicUpdateOptions, +}; #[derive(Debug, Clone, Subcommand)] pub(crate) enum TopicAction { @@ -45,17 +47,19 @@ /// iggy topic delete 2 debugs #[clap(verbatim_doc_comment, visible_alias = "d")] Delete(TopicDeleteArgs), - /// Update topic name, compression algorithm and message expiry time for given topic ID in given stream ID + /// Update topic name, message expiry or max topic size for given topic ID in given stream ID /// /// Stream ID can be specified as a stream name or ID /// Topic ID can be specified as a topic name or ID + /// Only the settings passed change, every other setting keeps its value. + /// Compression, segment size, durability and the other create-time settings cannot change. /// /// Examples - /// iggy topic update 1 1 sensor3 none - /// iggy topic update prod sensor3 old-sensor none - /// iggy topic update test debugs ready gzip 15days - /// iggy topic update 1 1 new-name gzip - /// iggy topic update 1 2 new-name none 1day 1hour 1min 1sec + /// iggy topic update 1 1 --name sensor3 + /// iggy topic update prod sensor3 --name old-sensor --message-expiry 15days + /// iggy topic update test debugs --max-topic-size 15GB + /// iggy topic update 1 2 --message-expiry 1day 1hour 1min 1sec + /// iggy topic update 1 2 --reset message_expiry --reset max_topic_size #[clap(verbatim_doc_comment, visible_alias = "u")] Update(TopicUpdateArgs), /// Get topic detail for given topic ID and stream ID @@ -152,21 +156,42 @@ #[arg(value_parser = clap::value_parser!(Identifier))] pub(crate) topic_id: Identifier, /// New name for the topic - pub(crate) name: String, - /// Compression metadata (none or gzip). Payload compression is not implemented - #[arg(value_parser = clap::value_parser!(CompressionAlgorithm), verbatim_doc_comment)] - pub(crate) compression_algorithm: CompressionAlgorithm, + #[arg(short, long)] + pub(crate) name: Option<String>, /// New max topic size in human-readable format like "unlimited" or "15GB" /// - /// Skipping this parameter or using "server_default" preserves the current max topic size. /// A finite size cannot be lower than the topic segment size. - #[arg(short, long, default_value = "server_default", verbatim_doc_comment)] - pub(crate) max_topic_size: MaxTopicSize, + #[arg(short, long, verbatim_doc_comment)] + pub(crate) max_topic_size: Option<MaxTopicSize>, /// New message expiry time in human-readable format like "unlimited" or "15days 2min 2s" + #[arg(short = 'e', long, num_args = 1.., value_parser = clap::value_parser!(IggyExpiry), verbatim_doc_comment)] + pub(crate) message_expiry: Option<Vec<IggyExpiry>>, + /// Updatable topic option as key=value, repeatable /// - /// Skipping this parameter or using "server_default" preserves the current message expiry. - #[arg(default_value = "server_default", value_parser = clap::value_parser!(IggyExpiry), verbatim_doc_comment)] - pub(crate) message_expiry: Vec<IggyExpiry>, + /// Values are sent as strings and parsed server-side. + /// Run "iggy options topic" to list the keys and which ones can change. + #[arg(long = "set", value_name = "KEY=VALUE", value_parser = parse_key_value, verbatim_doc_comment)] + pub(crate) set: Vec<(String, String)>, + /// Setting to return to its default, like message_expiry or max_topic_size + /// + /// Repeat the flag to reset several settings. + /// The server refuses a key that cannot be reset. + #[arg(short, long, verbatim_doc_comment)] + pub(crate) reset: Vec<String>, +} + +impl From<&TopicUpdateArgs> for TopicUpdateOptions { + fn from(args: &TopicUpdateArgs) -> Self { + let (reset, raw_reset) = split_reset(&args.reset); + TopicUpdateOptions { + name: args.name.clone(), + message_expiry: args.message_expiry.clone().map(IggyExpiry::from), + max_topic_size: args.max_topic_size, + reset, + raw_reset, + raw: args.set.iter().cloned().collect(), + } + } } #[derive(Debug, Clone, Args)] @@ -197,6 +222,46 @@ } #[cfg(test)] +mod update_tests { + use super::TopicUpdateArgs; + use clap::Parser; + use iggy::prelude::{OptionKey, TopicUpdateOptions}; + use std::collections::{BTreeMap, BTreeSet}; + + #[derive(Parser)] + struct Update { + #[command(flatten)] + args: TopicUpdateArgs, + } + + #[test] + fn given_reset_name_newer_than_build_when_parsing_topic_update_should_send_it_raw() { + let parsed = Update::try_parse_from([ + "iggy", + "1", + "2", + "--reset", + "message_expiry", + "--reset", + "future_key", + "--set", + "newer_key=1", + ]) + .expect("a reset of a newer key parses"); + + assert_eq!( + TopicUpdateOptions::from(&parsed.args), + TopicUpdateOptions { + reset: BTreeSet::from([OptionKey::MessageExpiry]), + raw_reset: BTreeSet::from(["future_key".to_owned()]), + raw: BTreeMap::from([("newer_key".to_owned(), "1".to_owned())]), + ..TopicUpdateOptions::default() + } + ); + } +} + +#[cfg(test)] mod durability_tests { use super::TopicCreateArgs; use clap::Parser;
diff --git a/core/cli/src/commands/binary_streams/create_stream.rs b/core/cli/src/commands/binary_streams/create_stream.rs index 5ab9707..53e7ebe 100644 --- a/core/cli/src/commands/binary_streams/create_stream.rs +++ b/core/cli/src/commands/binary_streams/create_stream.rs
@@ -18,24 +18,52 @@ use crate::commands::cli_command::{CliCommand, PRINT_TARGET}; use anyhow::Context; use async_trait::async_trait; -use iggy_common::Client; -use iggy_common::create_stream::CreateStream; +use iggy_common::{ + Client, StreamCreateOptions, StreamDetails, StreamEncryptionKey, StreamEncryptionSource, +}; +use std::fs; +use std::path::PathBuf; +use std::str::FromStr; use tracing::{Level, event}; pub struct CreateStreamCmd { - create_stream: CreateStream, + name: String, + stream_key_file: Option<PathBuf>, } impl CreateStreamCmd { - pub fn new(name: String) -> Self { + pub fn new(name: String, stream_key_file: Option<PathBuf>) -> Self { Self { - create_stream: CreateStream { name }, + name, + stream_key_file, } } fn get_stream_id_info(&self) -> String { "ID auto incremented".to_string() } + + /// Only a file named on this call supplies the key, so no key spreads across streams. + fn options(&self) -> anyhow::Result<StreamCreateOptions> { + let encryption_key = self + .stream_key_file + .as_ref() + .map(|path| { + let text = fs::read_to_string(path).with_context(|| { + format!( + "Problem reading stream encryption key file: {}", + path.display() + ) + })?; + StreamEncryptionKey::from_str(text.trim_end()) + .context("Stream encryption key must be standard base64 of exactly 32 bytes") + }) + .transpose()?; + Ok(StreamCreateOptions { + encryption_key, + ..StreamCreateOptions::default() + }) + } } #[async_trait] @@ -43,29 +71,45 @@ fn explain(&self) -> String { format!( "create stream with name: {} and {}", - self.create_stream.name, + self.name, self.get_stream_id_info(), ) } async fn execute_cmd(&mut self, client: &dyn Client) -> anyhow::Result<(), anyhow::Error> { - client - .create_stream(&self.create_stream.name) + let options = self.options()?; + let stream = client + .create_stream(&self.name, &options) .await .with_context(|| { format!( "Problem creating stream (name: {} and {})", - self.create_stream.name, + self.name, self.get_stream_id_info(), ) })?; event!(target: PRINT_TARGET, Level::INFO, "Stream with name: {} and {} created", - self.create_stream.name, + self.name, self.get_stream_id_info(), ); + print_encryption(&stream); Ok(()) } } + +fn print_encryption(stream: &StreamDetails) { + let Some(source) = stream + .encryption_key_source() + .filter(|source| *source != StreamEncryptionSource::None) + else { + return; + }; + event!(target: PRINT_TARGET, Level::INFO, + "Stream encryption key source: {source}, fingerprint: {}, KEK fingerprints: {}", + stream.encryption_key_fingerprint().unwrap_or_default(), + stream.encryption_kek_fingerprints().join(", "), + ); +}
diff --git a/core/cli/src/commands/binary_streams/get_stream.rs b/core/cli/src/commands/binary_streams/get_stream.rs index 4f5998d..62b570b 100644 --- a/core/cli/src/commands/binary_streams/get_stream.rs +++ b/core/cli/src/commands/binary_streams/get_stream.rs
@@ -21,6 +21,7 @@ use comfy_table::Table; use iggy_common::Client; use iggy_common::Identifier; +use iggy_common::StreamEncryptionSource; use tracing::{Level, event}; pub struct GetStreamCmd { @@ -66,9 +67,43 @@ "Stream topics count", format!("{}", stream.topics_count).as_str(), ]); + table.add_row(vec![ + "Encryption source", + encryption_source(stream.encryption_key_source()), + ]); + if let Some(fingerprint) = stream.encryption_key_fingerprint() { + table.add_row(vec!["Key fingerprint", fingerprint.as_str()]); + } + let kek_fingerprints = stream.encryption_kek_fingerprints(); + if !kek_fingerprints.is_empty() { + table.add_row(vec![ + "KEK fingerprints", + kek_fingerprints.join(", ").as_str(), + ]); + } event!(target: PRINT_TARGET, Level::INFO, "{table}"); Ok(()) } } + +/// The server reports `none` for a plaintext stream, so a missing or unparsed +/// source is unknown, never plaintext. +pub(crate) fn encryption_source(source: Option<StreamEncryptionSource>) -> &'static str { + source.map_or("unknown", Into::into) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn given_missing_or_unparsed_key_source_when_rendering_should_print_unknown() { + assert_eq!(encryption_source(None), "unknown"); + assert_eq!( + encryption_source(Some(StreamEncryptionSource::None)), + "none" + ); + } +}
diff --git a/core/cli/src/commands/binary_streams/get_streams.rs b/core/cli/src/commands/binary_streams/get_streams.rs index cbc5257..113882c 100644 --- a/core/cli/src/commands/binary_streams/get_streams.rs +++ b/core/cli/src/commands/binary_streams/get_streams.rs
@@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use crate::commands::binary_streams::get_stream::encryption_source; use crate::commands::cli_command::{CliCommand, PRINT_TARGET}; use anyhow::Context; use async_trait::async_trait; @@ -71,7 +72,13 @@ let mut table = Table::new(); table.set_header(vec![ - "ID", "Created", "Name", "Size (B)", "Messages", "Topics", + "ID", + "Created", + "Name", + "Size (B)", + "Messages", + "Topics", + "Encryption", ]); streams.iter().for_each(|stream| { @@ -82,6 +89,7 @@ format!("{}", stream.size), format!("{}", stream.messages_count), format!("{}", stream.topics_count), + encryption_source(stream.encryption_key_source()).to_owned(), ]); }); @@ -90,13 +98,14 @@ GetStreamsOutput::List => { streams.iter().for_each(|stream| { event!(target: PRINT_TARGET, Level::INFO, - "{}|{}|{}|{}|{}|{}", + "{}|{}|{}|{}|{}|{}|{}", stream.id, stream.created_at, stream.name, stream.size, stream.messages_count, - stream.topics_count + stream.topics_count, + encryption_source(stream.encryption_key_source()) ); }); }
diff --git a/core/cli/src/commands/binary_streams/update_stream.rs b/core/cli/src/commands/binary_streams/update_stream.rs index 209c7af..15a0e54 100644 --- a/core/cli/src/commands/binary_streams/update_stream.rs +++ b/core/cli/src/commands/binary_streams/update_stream.rs
@@ -18,60 +18,75 @@ use crate::commands::cli_command::{CliCommand, PRINT_TARGET}; use anyhow::Context; use async_trait::async_trait; -use iggy_common::Client; -use iggy_common::Identifier; -use iggy_common::StreamUpdateOptions; -use iggy_common::update_stream::UpdateStream; -use std::collections::BTreeMap; +use core::fmt; +use iggy_common::{Client, Identifier, StreamUpdateOptions}; use tracing::{Level, event}; pub struct UpdateStreamCmd { - update_stream: UpdateStream, + stream_id: Identifier, options: StreamUpdateOptions, } impl UpdateStreamCmd { - pub fn new(stream_id: Identifier, name: String) -> Self { - UpdateStreamCmd { - update_stream: UpdateStream { - stream_id, - name, - options: BTreeMap::new(), - }, - options: StreamUpdateOptions::default(), + pub fn new(stream_id: Identifier, options: StreamUpdateOptions) -> Self { + Self { stream_id, options } + } + + /// The changes this command asks for, in a fixed order. + fn changes(&self) -> String { + let mut changes = Vec::new(); + if let Some(name) = &self.options.name { + changes.push(format!("name: {name}")); } + // A stream option value may be key material, so only the key is shown. + for key in self.options.raw.keys() { + changes.push(format!("set: {key}")); + } + for key in &self.options.reset { + changes.push(format!("reset: {key}")); + } + for key in &self.options.raw_reset { + changes.push(format!("reset: {key}")); + } + changes.join(", ") } } #[async_trait] impl CliCommand for UpdateStreamCmd { fn explain(&self) -> String { - format!( - "update stream with ID: {} and name: {}", - self.update_stream.stream_id, self.update_stream.name - ) + format!("{self}") } async fn execute_cmd(&mut self, client: &dyn Client) -> anyhow::Result<(), anyhow::Error> { client - .update_stream( - &self.update_stream.stream_id, - &self.update_stream.name, - &self.options, - ) + .update_stream(&self.stream_id, &self.options) .await .with_context(|| { format!( - "Problem updating stream with ID: {} and name: {}", - self.update_stream.stream_id, self.update_stream.name + "Problem updating stream with ID: {} ({})", + self.stream_id, + self.changes() ) })?; event!(target: PRINT_TARGET, Level::INFO, - "Stream with ID: {} updated name: {}", - self.update_stream.stream_id, self.update_stream.name + "Stream with ID: {} updated ({})", + self.stream_id, + self.changes(), ); Ok(()) } } + +impl fmt::Display for UpdateStreamCmd { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "update stream with ID: {} ({})", + self.stream_id, + self.changes() + ) + } +}
diff --git a/core/cli/src/commands/binary_system/options.rs b/core/cli/src/commands/binary_system/options.rs index 5fc7f6b..54cc0a7 100644 --- a/core/cli/src/commands/binary_system/options.rs +++ b/core/cli/src/commands/binary_system/options.rs
@@ -24,10 +24,10 @@ /// `iggy options <scope>`: print the server's option catalog. /// -/// This is the discovery surface for `--set`. A key outside the catalog is -/// refused at create, and the binary transports carry only the error code back, -/// so without this command a CLI user has no way to see which keys their server -/// knows or what each one is bounded by. +/// This is the discovery surface for `--set` and `--reset`. A key outside the +/// catalog is refused, and the binary transports carry only the error code +/// back, so without this command a CLI user has no way to see which keys their +/// server knows or what a client may do with each one. pub struct DescribeOptionsCmd { scope: OptionsScope, } @@ -52,17 +52,18 @@ if specs.is_empty() { event!(target: PRINT_TARGET, Level::INFO, - "{} resources accept no options yet", self.scope); + "{} resources accept no options", self.scope); return Ok(()); } let mut table = Table::new(); - table.set_header(vec!["Key", "Kind", "Default", "Description"]); + table.set_header(vec!["Key", "Kind", "Default", "Mutability", "Description"]); for spec in &specs { table.add_row(vec![ spec.key.as_str(), spec.kind.to_string().as_str(), render_default(spec).as_str(), + render_mutability(spec).as_str(), spec.description.as_str(), ]); } @@ -72,6 +73,26 @@ } } +/// What a client may do with the key after the resource exists. +fn render_mutability(spec: &OptionSpec) -> String { + let mutability = if spec.read_only { + "read only" + } else if spec.updatable && spec.resettable { + "updatable, resettable" + } else if spec.updatable { + "updatable" + } else { + "create only" + }; + let flags = [(spec.secret, "secret"), (spec.critical, "critical")]; + flags + .into_iter() + .filter(|(set, _)| *set) + .fold(mutability.to_owned(), |text, (_, flag)| { + format!("{text}, {flag}") + }) +} + /// The catalog's default in the string form `--set` takes, so the printed value /// can be handed straight back to a create. fn render_default(spec: &OptionSpec) -> String { @@ -84,3 +105,26 @@ Err(_) => format!("<{} bytes>", spec.default_value.len()), } } + +#[cfg(test)] +mod tests { + use super::*; + use iggy_common::HeaderKind; + + #[test] + fn given_critical_spec_when_rendering_mutability_should_print_critical() { + let spec = OptionSpec { + key: "critical_keys".to_owned(), + kind: HeaderKind::String, + updatable: false, + resettable: false, + secret: false, + read_only: true, + critical: true, + default_value: Vec::new(), + description: String::new(), + }; + + assert_eq!(render_mutability(&spec), "read only, critical"); + } +}
diff --git a/core/cli/src/commands/binary_topics/create_topic.rs b/core/cli/src/commands/binary_topics/create_topic.rs index f10d98d..7d11a7b 100644 --- a/core/cli/src/commands/binary_topics/create_topic.rs +++ b/core/cli/src/commands/binary_topics/create_topic.rs
@@ -19,19 +19,21 @@ use anyhow::Context; use async_trait::async_trait; use core::fmt; -use iggy_common::create_topic::CreateTopic; -use iggy_common::{Client, Durability}; -use iggy_common::{CompressionAlgorithm, Identifier, IggyExpiry, MaxTopicSize, TopicCreateOptions}; +use iggy_common::{ + Client, CompressionAlgorithm, Durability, Identifier, IggyExpiry, MaxTopicSize, + TopicCreateOptions, +}; use std::collections::BTreeMap; use tracing::{Level, event}; pub struct CreateTopicCmd { - create_topic: CreateTopic, + stream_id: Identifier, + name: String, + partitions_count: u32, + compression_algorithm: CompressionAlgorithm, message_expiry: IggyExpiry, max_topic_size: MaxTopicSize, - raw_options: BTreeMap<String, String>, - durability: Durability, - consumer_offset_durability: Durability, + options: TopicCreateOptions, } impl CreateTopicCmd { @@ -47,21 +49,26 @@ durability: Durability, consumer_offset_durability: Durability, ) -> Self { - Self { - create_topic: CreateTopic { - stream_id, - partitions_count, - compression_algorithm, - name, - message_expiry, - max_topic_size, - options: raw_options.clone(), - }, - message_expiry, - max_topic_size, - raw_options, + let options = TopicCreateOptions { + partitions_count: Some(partitions_count), + compression_algorithm: (compression_algorithm != CompressionAlgorithm::default()) + .then_some(compression_algorithm), + message_expiry: (message_expiry != IggyExpiry::ServerDefault).then_some(message_expiry), + max_topic_size: (max_topic_size != MaxTopicSize::ServerDefault) + .then_some(max_topic_size), + raw: raw_options, durability, consumer_offset_durability, + ..TopicCreateOptions::default() + }; + Self { + stream_id, + name, + partitions_count, + compression_algorithm, + message_expiry, + max_topic_size, + options, } } } @@ -74,43 +81,23 @@ async fn execute_cmd(&mut self, client: &dyn Client) -> anyhow::Result<(), anyhow::Error> { client - .create_topic( - &self.create_topic.stream_id, - &self.create_topic.name, - &TopicCreateOptions { - partitions_count: Some(self.create_topic.partitions_count), - compression_algorithm: (self.create_topic.compression_algorithm - != CompressionAlgorithm::default()) - .then_some(self.create_topic.compression_algorithm), - message_expiry: (self.create_topic.message_expiry != IggyExpiry::ServerDefault) - .then_some(self.create_topic.message_expiry), - max_topic_size: (self.create_topic.max_topic_size - != MaxTopicSize::ServerDefault) - .then_some(self.create_topic.max_topic_size), - raw: self.raw_options.clone(), - durability: self.durability, - consumer_offset_durability: self.consumer_offset_durability, - ..TopicCreateOptions::default() - }, - ) + .create_topic(&self.stream_id, &self.name, &self.options) .await .with_context(|| { format!( "Problem creating topic (name: {}, partitions count: {}) in stream with ID: {}", - self.create_topic.name, - self.create_topic.partitions_count, - self.create_topic.stream_id + self.name, self.partitions_count, self.stream_id ) })?; event!(target: PRINT_TARGET, Level::INFO, "Topic with name: {}, partitions count: {}, compression algorithm: {}, message expiry: {}, max topic size: {} created in stream with ID: {}", - self.create_topic.name, - self.create_topic.partitions_count, - self.create_topic.compression_algorithm, + self.name, + self.partitions_count, + self.compression_algorithm, self.message_expiry, self.max_topic_size, - self.create_topic.stream_id, + self.stream_id, ); Ok(()) @@ -119,11 +106,11 @@ impl fmt::Display for CreateTopicCmd { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let topic_name = &self.create_topic.name; - let compression_algorithm = &self.create_topic.compression_algorithm; + let topic_name = &self.name; + let compression_algorithm = &self.compression_algorithm; let message_expiry = &self.message_expiry; let max_topic_size = &self.max_topic_size; - let stream_id = &self.create_topic.stream_id; + let stream_id = &self.stream_id; write!( f,
diff --git a/core/cli/src/commands/binary_topics/get_topic.rs b/core/cli/src/commands/binary_topics/get_topic.rs index fe2d73b..717cff1 100644 --- a/core/cli/src/commands/binary_topics/get_topic.rs +++ b/core/cli/src/commands/binary_topics/get_topic.rs
@@ -77,6 +77,7 @@ let mut table = Table::new(); table.set_header(vec!["Property", "Value"]); + let resolved = topic.resolved_options(); table.add_row(vec!["Topic id", format!("{}", topic.id).as_str()]); table.add_row(vec![ "Created", @@ -86,11 +87,11 @@ table.add_row(vec!["Topic size", format!("{}", topic.size).as_str()]); table.add_row(vec![ "Compression", - topic.compression_algorithm.to_string().as_str(), + resolved.compression_algorithm.to_string().as_str(), ]); table.add_row(vec![ "Message expiry", - match topic.message_expiry { + match resolved.message_expiry { IggyExpiry::NeverExpire => String::from("unlimited"), IggyExpiry::ServerDefault => String::from("server_default"), IggyExpiry::ExpireDuration(value) => format!("{value}"), @@ -99,7 +100,7 @@ ]); table.add_row(vec![ "Max topic size", - format!("{}", topic.max_topic_size).as_str(), + format!("{}", resolved.max_topic_size).as_str(), ]); table.add_row(vec![ "Topic message count", @@ -161,8 +162,8 @@ mod tests { use super::topic_table; use iggy_common::{ - CompressionAlgorithm, HeaderKey, HeaderValue, IggyByteSize, IggyExpiry, IggyTimestamp, - MaxTopicSize, OptionValue, Partition, PartitionContext, ResourceOptions, TopicDetails, + HeaderKey, HeaderValue, IggyByteSize, IggyTimestamp, OptionValue, Partition, + PartitionContext, ResourceOptions, TopicDetails, }; fn topic() -> TopicDetails { @@ -171,9 +172,6 @@ created_at: IggyTimestamp::from(0_u64), name: "test-topic".to_owned(), size: IggyByteSize::from(0_u64), - message_expiry: IggyExpiry::NeverExpire, - compression_algorithm: CompressionAlgorithm::None, - max_topic_size: MaxTopicSize::Unlimited, messages_count: 0, partitions_count: 0, partitions: Vec::new(),
diff --git a/core/cli/src/commands/binary_topics/get_topics.rs b/core/cli/src/commands/binary_topics/get_topics.rs index 1b59c6e..53d881d 100644 --- a/core/cli/src/commands/binary_topics/get_topics.rs +++ b/core/cli/src/commands/binary_topics/get_topics.rs
@@ -83,14 +83,15 @@ ]); topics.iter().for_each(|topic| { + let resolved = topic.resolved_options(); table.add_row(vec![ format!("{}", topic.id), topic.created_at.to_utc_string("%Y-%m-%d %H:%M:%S"), topic.name.clone(), format!("{}", topic.size), - format!("{}", topic.max_topic_size), - topic.compression_algorithm.to_string(), - match topic.message_expiry { + format!("{}", resolved.max_topic_size), + resolved.compression_algorithm.to_string(), + match resolved.message_expiry { IggyExpiry::NeverExpire => String::from("unlimited"), IggyExpiry::ServerDefault => String::from("server_default"), IggyExpiry::ExpireDuration(value) => format!("{value}"), @@ -104,15 +105,16 @@ } GetTopicsOutput::List => { topics.iter().for_each(|topic| { + let resolved = topic.resolved_options(); event!(target: PRINT_TARGET, Level::INFO, "{}|{}|{}|{}|{}|{}|{}|{}|{}", topic.id, topic.created_at.to_utc_string("%Y-%m-%d %H:%M:%S"), topic.name, topic.size, - topic.max_topic_size, - topic.compression_algorithm.to_string(), - match topic.message_expiry { + resolved.max_topic_size, + resolved.compression_algorithm.to_string(), + match resolved.message_expiry { IggyExpiry::NeverExpire => String::from("unlimited"), IggyExpiry::ServerDefault => String::from("server_default"), IggyExpiry::ExpireDuration(value) => format!("{value}"),
diff --git a/core/cli/src/commands/binary_topics/update_topic.rs b/core/cli/src/commands/binary_topics/update_topic.rs index 04f3b82..0f885ca 100644 --- a/core/cli/src/commands/binary_topics/update_topic.rs +++ b/core/cli/src/commands/binary_topics/update_topic.rs
@@ -19,50 +19,47 @@ use anyhow::Context; use async_trait::async_trait; use core::fmt; -use iggy_common::Client; -use iggy_common::update_topic::UpdateTopic; -use iggy_common::{CompressionAlgorithm, Identifier, IggyExpiry, MaxTopicSize, TopicUpdateOptions}; -use std::collections::BTreeMap; +use iggy_common::{Client, Identifier, TopicUpdateOptions}; use tracing::{Level, event}; pub struct UpdateTopicCmd { - update_topic: UpdateTopic, - compression_algorithm: CompressionAlgorithm, - message_expiry: IggyExpiry, - max_topic_size: MaxTopicSize, + stream_id: Identifier, + topic_id: Identifier, options: TopicUpdateOptions, } impl UpdateTopicCmd { - pub fn new( - stream_id: Identifier, - topic_id: Identifier, - compression_algorithm: CompressionAlgorithm, - name: String, - message_expiry: IggyExpiry, - max_topic_size: MaxTopicSize, - ) -> Self { + pub fn new(stream_id: Identifier, topic_id: Identifier, options: TopicUpdateOptions) -> Self { Self { - update_topic: UpdateTopic { - stream_id, - topic_id, - name, - compression_algorithm: Some(compression_algorithm), - message_expiry: Some(message_expiry), - max_topic_size: Some(max_topic_size), - options: BTreeMap::new(), - }, - compression_algorithm, - message_expiry, - max_topic_size, - options: TopicUpdateOptions { - compression_algorithm: Some(compression_algorithm), - message_expiry: Some(message_expiry), - max_topic_size: Some(max_topic_size), - ..TopicUpdateOptions::default() - }, + stream_id, + topic_id, + options, } } + + /// The changes this command asks for, in a fixed order. + fn changes(&self) -> String { + let mut changes = Vec::new(); + if let Some(name) = &self.options.name { + changes.push(format!("name: {name}")); + } + if let Some(message_expiry) = &self.options.message_expiry { + changes.push(format!("message expiry: {message_expiry}")); + } + if let Some(max_topic_size) = &self.options.max_topic_size { + changes.push(format!("max topic size: {max_topic_size}")); + } + for (key, value) in &self.options.raw { + changes.push(format!("{key}: {value}")); + } + for key in &self.options.reset { + changes.push(format!("reset: {key}")); + } + for key in &self.options.raw_reset { + changes.push(format!("reset: {key}")); + } + changes.join(", ") + } } #[async_trait] @@ -73,26 +70,22 @@ async fn execute_cmd(&mut self, client: &dyn Client) -> anyhow::Result<(), anyhow::Error> { client - .update_topic(&self.update_topic.stream_id, &self.update_topic.topic_id, &self.update_topic.name, &self.options) + .update_topic(&self.stream_id, &self.topic_id, &self.options) .await .with_context(|| { format!( - "Problem updating topic (ID: {}, name: {}, message expiry: {}) in stream with ID: {}", - self.update_topic.topic_id, - self.update_topic.name, - self.message_expiry, - self.update_topic.stream_id + "Problem updating topic with ID: {} ({}) in stream with ID: {}", + self.topic_id, + self.changes(), + self.stream_id ) })?; event!(target: PRINT_TARGET, Level::INFO, - "Topic with ID: {} updated name: {}, updated message expiry: {}, updated compression algorithm: {}, updated max topic size: {} in stream with ID: {}", - self.update_topic.topic_id, - self.update_topic.name, - self.message_expiry, - self.compression_algorithm, - self.max_topic_size, - self.update_topic.stream_id, + "Topic with ID: {} updated ({}) in stream with ID: {}", + self.topic_id, + self.changes(), + self.stream_id, ); Ok(()) @@ -101,18 +94,12 @@ impl fmt::Display for UpdateTopicCmd { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let topic_id = &self.update_topic.topic_id; - let topic_name = &self.update_topic.name; - let compression_algorithm = &self.compression_algorithm; - let message_expiry = &self.message_expiry; - let max_topic_size = &self.max_topic_size; - let stream_id = &self.update_topic.stream_id; - write!( f, - "update topic with ID: {topic_id}, name: {topic_name}, message expiry: \ - {message_expiry}, compression algorithm: {compression_algorithm}, max topic size: \ - {max_topic_size}, in stream with ID: {stream_id}", + "update topic with ID: {} ({}) in stream with ID: {}", + self.topic_id, + self.changes(), + self.stream_id ) } }
diff --git a/core/cli/src/commands/binary_users/create_user.rs b/core/cli/src/commands/binary_users/create_user.rs index 58be7ed..7436353 100644 --- a/core/cli/src/commands/binary_users/create_user.rs +++ b/core/cli/src/commands/binary_users/create_user.rs
@@ -18,10 +18,8 @@ use crate::commands::cli_command::{CliCommand, PRINT_TARGET}; use anyhow::Context; use async_trait::async_trait; -use iggy_common::Client; -use iggy_common::Permissions; -use iggy_common::UserStatus; use iggy_common::create_user::CreateUser; +use iggy_common::{Client, OptionInputMap, Permissions, UserCreateOptions, UserStatus}; use secrecy::{ExposeSecret, SecretString}; use tracing::{Level, event}; @@ -42,6 +40,7 @@ password: SecretString::from(password), status, permissions, + options: OptionInputMap::default(), }, } } @@ -60,6 +59,7 @@ self.create_user.password.expose_secret(), self.create_user.status, self.create_user.permissions.clone(), + &UserCreateOptions::default(), ) .await .with_context(|| {
diff --git a/core/cli/src/commands/binary_users/update_user.rs b/core/cli/src/commands/binary_users/update_user.rs index 7366c6c..e9f76bc 100644 --- a/core/cli/src/commands/binary_users/update_user.rs +++ b/core/cli/src/commands/binary_users/update_user.rs
@@ -18,12 +18,7 @@ use crate::commands::cli_command::{CliCommand, PRINT_TARGET}; use anyhow::Context; use async_trait::async_trait; -use iggy_common::Client; -use iggy_common::Identifier; -use iggy_common::UserStatus; -use iggy_common::UserUpdateOptions; -use iggy_common::update_user::UpdateUser; -use std::collections::BTreeMap; +use iggy_common::{Client, Identifier, UserStatus, UserUpdateOptions}; use tracing::{Level, event}; #[derive(Debug, Clone)] @@ -33,25 +28,27 @@ } pub struct UpdateUserCmd { + user_id: Identifier, update_type: UpdateUserType, - update_user: UpdateUser, + options: UserUpdateOptions, } impl UpdateUserCmd { pub fn new(user_id: Identifier, update_type: UpdateUserType) -> Self { - let (username, status) = match update_type.clone() { - UpdateUserType::Name(username) => (Some(username), None), - UpdateUserType::Status(status) => (None, Some(status)), - }; - - UpdateUserCmd { - update_type, - update_user: UpdateUser { - user_id, - username, - status, - options: BTreeMap::new(), + let options = match update_type.clone() { + UpdateUserType::Name(username) => UserUpdateOptions { + username: Some(username), + ..UserUpdateOptions::default() }, + UpdateUserType::Status(status) => UserUpdateOptions { + status: Some(status), + ..UserUpdateOptions::default() + }, + }; + UpdateUserCmd { + user_id, + update_type, + options, } } @@ -68,33 +65,26 @@ fn explain(&self) -> String { format!( "update user with ID: {} with {}", - self.update_user.user_id, + self.user_id, self.get_message() ) } async fn execute_cmd(&mut self, client: &dyn Client) -> anyhow::Result<(), anyhow::Error> { client - .update_user( - &self.update_user.user_id, - self.update_user.username.as_deref(), - self.update_user.status, - // `iggy user name` / `iggy user status` are single-purpose - // commands; users have no option keys to set yet. - &UserUpdateOptions::default(), - ) + .update_user(&self.user_id, &self.options) .await .with_context(|| { format!( "Problem updating user with ID: {} with {}", - self.update_user.user_id, + self.user_id, self.get_message() ) })?; event!(target: PRINT_TARGET, Level::INFO, "User with ID: {} updated with {}", - self.update_user.user_id, self.get_message() + self.user_id, self.get_message() ); Ok(())
diff --git a/core/cli/src/main.rs b/core/cli/src/main.rs index e7c8ec7..a3ccad4 100644 --- a/core/cli/src/main.rs +++ b/core/cli/src/main.rs
@@ -38,7 +38,10 @@ use clap::Parser; use iggy::client_provider::{self, ClientProviderConfig}; use iggy::clients::client::IggyClient; -use iggy::prelude::{Aes256GcmEncryptor, ClientError, EncryptorKind, PersonalAccessTokenExpiry}; +use iggy::prelude::{ + Aes256GcmEncryptor, ClientError, EncryptorKind, PersonalAccessTokenExpiry, StreamUpdateOptions, + TopicUpdateOptions, +}; use iggy_cli::commands::binary_context::common::{ContextManager, validate_transport}; use iggy_cli::commands::binary_context::create_context::CreateContextCmd; use iggy_cli::commands::binary_context::delete_context::DeleteContextCmd; @@ -113,11 +116,14 @@ #[warn(clippy::let_and_return)] match command { Command::Stream(command) => match command { - StreamAction::Create(args) => Box::new(CreateStreamCmd::new(args.name.clone())), + StreamAction::Create(args) => Box::new(CreateStreamCmd::new( + args.name.clone(), + args.stream_key_file.clone(), + )), StreamAction::Delete(args) => Box::new(DeleteStreamCmd::new(args.stream_id.clone())), StreamAction::Update(args) => Box::new(UpdateStreamCmd::new( args.stream_id.clone(), - args.name.clone(), + StreamUpdateOptions::from(&args), )), StreamAction::Get(args) => Box::new(GetStreamCmd::new(args.stream_id.clone())), StreamAction::List(args) => Box::new(GetStreamsCmd::new(args.list_mode.into())), @@ -141,10 +147,7 @@ TopicAction::Update(args) => Box::new(UpdateTopicCmd::new( args.stream_id.clone(), args.topic_id.clone(), - args.compression_algorithm, - args.name.clone(), - args.message_expiry.clone().into(), - args.max_topic_size, + TopicUpdateOptions::from(&args), )), TopicAction::Get(args) => Box::new(GetTopicCmd::new( args.stream_id.clone(),
diff --git a/core/common/Cargo.toml b/core/common/Cargo.toml index 3f2d0ff..a48d622 100644 --- a/core/common/Cargo.toml +++ b/core/common/Cargo.toml
@@ -17,7 +17,7 @@ [package] name = "iggy_common" -version = "0.11.1-edge.1" +version = "0.11.1-edge.2" description = "Iggy is the persistent message streaming platform written in Rust, supporting QUIC, TCP and HTTP transport protocols, capable of processing millions of messages per second." edition = "2024" rust-version.workspace = true @@ -31,6 +31,7 @@ [dependencies] aes-gcm = { workspace = true } +aes-gcm-siv = { workspace = true } async-broadcast = { workspace = true } async-trait = { workspace = true } base64 = { workspace = true } @@ -60,5 +61,13 @@ url = { workspace = true } uuid = { workspace = true } +[dev-dependencies] +hex = { workspace = true } +xaes-256-gcm = { workspace = true } + +[[bench]] +name = "stream_encryption" +harness = false + [lints] workspace = true
diff --git a/core/common/benches/stream_encryption.rs b/core/common/benches/stream_encryption.rs new file mode 100644 index 0000000..00aec2d --- /dev/null +++ b/core/common/benches/stream_encryption.rs
@@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use aes_gcm::Aes256Gcm; +use aes_gcm::aead::{Aead, AeadCore, Generate, KeyInit, Nonce}; +use aes_gcm_siv::Aes256GcmSiv; +use iggy_common::{Aes256GcmEncryptor, Aes256GcmSivEncryptor, Encryptor}; +use std::hint::black_box; +use std::time::Instant; +use xaes_256_gcm::Xaes256Gcm; + +fn measure<C>(name: &str, plaintext: &[u8], iterations: usize) +where + C: Aead + AeadCore + KeyInit, + Nonce<C>: Generate, +{ + let cipher = C::new_from_slice(&[0x5a; 32]).expect("valid key"); + let sample_nonce = Nonce::<C>::generate(); + let sample = cipher + .encrypt(&sample_nonce, plaintext) + .expect("sample encrypts"); + assert_eq!( + cipher + .decrypt(&sample_nonce, sample.as_ref()) + .expect("sample decrypts"), + plaintext + ); + + let start = Instant::now(); + for _ in 0..iterations { + let nonce = Nonce::<C>::generate(); + black_box( + cipher + .encrypt(&nonce, black_box(plaintext)) + .expect("encrypts"), + ); + } + let encrypt = start.elapsed().as_nanos() as f64 / iterations as f64; + + let start = Instant::now(); + for _ in 0..iterations { + black_box( + cipher + .decrypt(&sample_nonce, black_box(sample.as_ref())) + .expect("decrypts"), + ); + } + let decrypt = start.elapsed().as_nanos() as f64 / iterations as f64; + + println!("{name}, {}, {encrypt:.1}, {decrypt:.1}", plaintext.len()); +} + +fn measure_wrapper<E: Encryptor>(name: &str, cipher: &E, plaintext: &[u8], iterations: usize) { + let sample = cipher.encrypt(plaintext).expect("sample encrypts"); + assert_eq!(cipher.decrypt(&sample).expect("sample decrypts"), plaintext); + + let start = Instant::now(); + for _ in 0..iterations { + black_box(cipher.encrypt(black_box(plaintext)).expect("encrypts")); + } + let encrypt = start.elapsed().as_nanos() as f64 / iterations as f64; + + let start = Instant::now(); + for _ in 0..iterations { + black_box(cipher.decrypt(black_box(&sample)).expect("decrypts")); + } + let decrypt = start.elapsed().as_nanos() as f64 / iterations as f64; + + println!("{name}, {}, {encrypt:.1}, {decrypt:.1}", plaintext.len()); +} + +fn main() { + println!("cipher, bytes, encrypt_ns_per_op, decrypt_ns_per_op"); + let current = Aes256GcmEncryptor::new(&[0x5a; 32]).expect("valid key"); + let siv = Aes256GcmSivEncryptor::new(&[0x5a; 32]).expect("valid key"); + for (size, iterations) in [ + (64, 100_000), + (1024, 50_000), + (16 * 1024, 5_000), + (1024 * 1024, 200), + ] { + let plaintext = vec![0x42; size]; + measure::<Aes256Gcm>("AES-256-GCM", &plaintext, iterations); + measure::<Aes256GcmSiv>("AES-256-GCM-SIV", &plaintext, iterations); + measure::<Xaes256Gcm>("XAES-256-GCM", &plaintext, iterations); + measure_wrapper("current-wrapper", ¤t, &plaintext, iterations); + measure_wrapper("siv-wrapper", &siv, &plaintext, iterations); + } +}
diff --git a/core/common/examples/resource_options_fixtures.rs b/core/common/examples/resource_options_fixtures.rs new file mode 100644 index 0000000..9017f82 --- /dev/null +++ b/core/common/examples/resource_options_fixtures.rs
@@ -0,0 +1,953 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#[path = "resource_options_fixtures/schema.rs"] +mod schema; + +use base64::Engine; +use base64::engine::general_purpose::STANDARD; +use iggy_binary_protocol::requests::system::DescribeOptionsRequest; +use iggy_binary_protocol::responses::streams::{GetStreamResponse, StreamResponse, TopicHeader}; +use iggy_binary_protocol::responses::system::DescribeOptionsResponse; +use iggy_binary_protocol::responses::topics::GetTopicResponse; +use iggy_binary_protocol::responses::users::{UserDetailsResponse, UserResponse}; +use iggy_binary_protocol::{WireDecode, WireEncode, WireName}; +use iggy_common::wire_conversions::{resource_options_from_wire, resource_options_to_wire_split}; +use iggy_common::{ + CATALOG, Durability, HeaderKey, HeaderValue, OptionColumns, OptionKey, OptionStorage, + OptionValue, OptionVisibility, OptionsScope, ResourceOptions, StreamDetails, + StreamEncryptionKey, StreamEncryptionSource, TopicDetails, UserInfoDetails, UserStatus, + descriptors_for, public_options, stream_key_fingerprint, +}; +use schema::{ + CANONICAL_PATH, CatalogRow, CreateStreamInput, CreateStreamVector, CreateTopicInput, + CreateTopicVector, CreateUserInput, CreateUserVector, DescribeOptionsVector, DescribedOption, + ErrorExpectation, GENERATOR, HttpUpdateVector, IdentifierInput, InvalidKeyVector, KeyInput, + LANGUAGE_COPIES, PatchInput, ResourceOptionsFixture, SCHEMA_VERSION, SCOPES, StreamKeyState, + StreamKeyVectors, StreamProjection, StreamResponseVector, TargetInput, TopicProjection, + TopicResponseVector, UpdateResourceErrorVector, UpdateResourceVector, UserProjection, + UserResponseVector, WireConstants, admit_create, admit_update, contains_bytes, + describe_options_response, evaluate_http_update, update_resource_request, +}; +use std::collections::BTreeMap; +use std::error::Error; +use std::fs; +use std::iter; +use std::path::Path; + +const FINGERPRINT_LABEL: &str = "iggy/stream-key-fingerprint/v1"; +const FIRST_KEK_CONFIG_KEY: &str = "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE="; +const SECOND_KEK_CONFIG_KEY: &str = "AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA="; +const CREATED_AT_MICROS: u64 = 1_710_000_000_000_000; +const SEVEN_DAYS_MICROS: &str = "604800000000"; +const ONE_DAY_MICROS: &str = "86400000000"; +const ABOVE_2_POW_53: u64 = (1 << 53) + 1; + +type GeneratorResult<T> = Result<T, Box<dyn Error>>; + +fn main() -> GeneratorResult<()> { + let fixture = ResourceOptionsFixture { + schema_version: SCHEMA_VERSION, + generated_by: GENERATOR.to_owned(), + conventions: BTreeMap::from( + [ + ( + "byte_strings", + "Every *_hex field is lowercase hex without separators.", + ), + ( + "u64_values", + "u64 option values and typed u64 inputs are decimal strings, so values above 2^53 keep their precision.", + ), + ( + "message_expiry", + "Microseconds. 18446744073709551615 means never expire.", + ), + ( + "max_topic_size", + "Bytes. 18446744073709551615 means unlimited.", + ), + ( + "option_maps", + "Option maps use the HTTP JSON form: value as reversible text, kind, and explicit provenance. Request blocks mark every entry explicit.", + ), + ( + "options_block_order", + "Options block entries are encoded in ascending key byte order.", + ), + ( + "reset_order", + "The typed encoders and the HTTP handler write reset keys in ascending key byte order.", + ), + ( + "request_bodies", + "body_hex and update_resource_hex are command payloads without framing. constants.command_codes maps each command to its code and VSR operation.", + ), + ( + "errors", + "name is the snake_case IggyError name and code its numeric code.", + ), + ( + "stream_state", + "state is the stored server state a stream response was projected from. SDKs need only response_hex and expected.", + ), + ( + "never_returned_keys", + "No response, decoded projection or accessor may contain these keys or their values.", + ), + ] + .map(|(name, text)| (name.to_owned(), text.to_owned())), + ), + constants: WireConstants::live()?, + catalog: CATALOG + .iter() + .map(CatalogRow::try_from) + .collect::<Result<_, _>>()?, + describe_options: describe_options_vectors()?, + stream_key: stream_key_vectors()?, + update_resource: update_resource_vectors()?, + update_resource_errors: update_resource_error_vectors()?, + create_stream: create_stream_vectors()?, + create_topic: create_topic_vectors()?, + create_user: create_user_vectors()?, + http_update: http_update_vectors()?, + stream_responses: stream_response_vectors()?, + topic_responses: topic_response_vectors()?, + user_responses: user_response_vectors()?, + }; + let mut json = serde_json::to_string_pretty(&fixture)?; + json.push('\n'); + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + for relative in iter::once(CANONICAL_PATH).chain(LANGUAGE_COPIES) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + fs::write(&path, &json)?; + println!("Wrote {}", path.display()); + } + Ok(()) +} + +fn describe_options_vectors() -> GeneratorResult<Vec<DescribeOptionsVector>> { + SCOPES + .into_iter() + .map(|scope| { + let bytes = describe_options_response(scope)?.to_bytes(); + let entries = DescribeOptionsResponse::decode_from(&bytes)? + .entries + .iter() + .map(DescribedOption::try_from) + .collect::<Result<_, _>>()?; + Ok(DescribeOptionsVector { + scope, + request_hex: hex::encode( + DescribeOptionsRequest { + scope: scope.as_code(), + } + .to_bytes(), + ), + response_hex: hex::encode(bytes), + entries, + }) + }) + .collect() +} + +fn stream_key_vectors() -> GeneratorResult<StreamKeyVectors> { + let key_bytes: Vec<u8> = (0u8..32).collect(); + let key = StreamEncryptionKey::try_from(key_bytes.as_slice())?; + let key_base64 = STANDARD.encode(&key_bytes); + let fingerprint = stream_key_fingerprint(&key_bytes)?; + if KeyInput::Base64(key_base64.clone()).parse()? != key || key.fingerprint() != fingerprint { + return Err("the raw and base64 forms of the fixed key disagree".into()); + } + let without_padding = key_base64.trim_end_matches('=').to_owned(); + let noncanonical_trailing_bits = format!("{}9=", &key_base64[..key_base64.len() - 2]); + let invalid_inputs = [ + ( + "raw_31_bytes", + KeyInput::RawHex(hex::encode(&key_bytes[..31])), + ), + ( + "raw_33_bytes", + KeyInput::RawHex(hex::encode([key_bytes.as_slice(), &[32]].concat())), + ), + ( + "base64_with_whitespace", + KeyInput::Base64(format!("{key_base64}\n")), + ), + ("base64_missing_padding", KeyInput::Base64(without_padding)), + ( + "base64_noncanonical_trailing_bits", + KeyInput::Base64(noncanonical_trailing_bits), + ), + ]; + let invalid_keys = invalid_inputs + .into_iter() + .map(|(case, input)| { + let Err(error) = input.parse() else { + return Err(format!("invalid key {case} was accepted").into()); + }; + Ok(InvalidKeyVector { + case: case.to_owned(), + error: ErrorExpectation::from(&error), + input, + }) + }) + .collect::<GeneratorResult<_>>()?; + Ok(StreamKeyVectors { + key_hex: hex::encode(&key_bytes), + key_base64, + fingerprint_label: FINGERPRINT_LABEL.to_owned(), + fingerprint, + invalid_keys, + }) +} + +fn update_resource_vectors() -> GeneratorResult<Vec<UpdateResourceVector>> { + let numeric_topic = TargetInput { + stream_id: Some(IdentifierInput::Numeric(1)), + topic_id: Some(IdentifierInput::Numeric(2)), + user_id: None, + }; + let cases = [ + ( + "topic_set_expiry_without_name_reset_max_topic_size", + OptionsScope::Topic, + numeric_topic.clone(), + PatchInput { + message_expiry: Some(SEVEN_DAYS_MICROS.to_owned()), + reset: vec![OptionKey::MaxTopicSize], + ..PatchInput::default() + }, + ), + ( + "topic_rename_only_named_identifiers", + OptionsScope::Topic, + TargetInput { + stream_id: Some(IdentifierInput::Named("orders".to_owned())), + topic_id: Some(IdentifierInput::Named("events".to_owned())), + user_id: None, + }, + PatchInput { + name: Some("events-v2".to_owned()), + ..PatchInput::default() + }, + ), + ( + "topic_rename_set_expiry_reset_max_topic_size", + OptionsScope::Topic, + TargetInput { + stream_id: Some(IdentifierInput::Numeric(7)), + topic_id: Some(IdentifierInput::Named("events".to_owned())), + user_id: None, + }, + PatchInput { + name: Some("events-v3".to_owned()), + message_expiry: Some(ONE_DAY_MICROS.to_owned()), + reset: vec![OptionKey::MaxTopicSize], + ..PatchInput::default() + }, + ), + ( + "topic_reset_both_defaults", + OptionsScope::Topic, + numeric_topic.clone(), + PatchInput { + reset: vec![OptionKey::MaxTopicSize, OptionKey::MessageExpiry], + ..PatchInput::default() + }, + ), + ( + "topic_max_topic_size_above_2_pow_53", + OptionsScope::Topic, + numeric_topic, + PatchInput { + max_topic_size: Some(ABOVE_2_POW_53.to_string()), + ..PatchInput::default() + }, + ), + ( + "stream_rename", + OptionsScope::Stream, + TargetInput { + stream_id: Some(IdentifierInput::Numeric(3)), + ..TargetInput::default() + }, + PatchInput { + name: Some("payments-v2".to_owned()), + ..PatchInput::default() + }, + ), + ( + "user_username_and_status", + OptionsScope::User, + TargetInput { + user_id: Some(IdentifierInput::Numeric(5)), + ..TargetInput::default() + }, + PatchInput { + username: Some("alice2".to_owned()), + status: Some(UserStatus::Inactive), + ..PatchInput::default() + }, + ), + ]; + cases + .into_iter() + .map(|(case, scope, target, input)| { + let request = update_resource_request(scope, &target, &input)?; + admit_update(&request)?; + Ok(UpdateResourceVector { + case: case.to_owned(), + scope, + wire_set: resource_options_from_wire(&request.options, true)?, + wire_reset: request + .reset + .iter() + .map(|name| name.as_str().to_owned()) + .collect(), + body_hex: hex::encode(request.to_bytes()), + target, + input, + }) + }) + .collect() +} + +fn update_resource_error_vectors() -> GeneratorResult<Vec<UpdateResourceErrorVector>> { + let numeric_topic = TargetInput { + stream_id: Some(IdentifierInput::Numeric(1)), + topic_id: Some(IdentifierInput::Numeric(2)), + user_id: None, + }; + let stream = TargetInput { + stream_id: Some(IdentifierInput::Numeric(3)), + ..TargetInput::default() + }; + let user = TargetInput { + user_id: Some(IdentifierInput::Numeric(5)), + ..TargetInput::default() + }; + let cases = [ + ( + "topic_empty_update", + OptionsScope::Topic, + numeric_topic.clone(), + PatchInput::default(), + ), + ( + "topic_set_and_reset_same_key", + OptionsScope::Topic, + numeric_topic.clone(), + PatchInput { + message_expiry: Some(SEVEN_DAYS_MICROS.to_owned()), + reset: vec![OptionKey::MessageExpiry], + ..PatchInput::default() + }, + ), + ( + "topic_reset_create_only_key", + OptionsScope::Topic, + numeric_topic.clone(), + PatchInput { + reset: vec![OptionKey::CompressionAlgorithm], + ..PatchInput::default() + }, + ), + ( + "stream_reset_name", + OptionsScope::Stream, + stream.clone(), + PatchInput { + reset: vec![OptionKey::Name], + ..PatchInput::default() + }, + ), + ( + "user_reset_status", + OptionsScope::User, + user.clone(), + PatchInput { + reset: vec![OptionKey::Status], + ..PatchInput::default() + }, + ), + ( + "stream_reset_read_only_key", + OptionsScope::Stream, + stream.clone(), + PatchInput { + reset: vec![OptionKey::EncryptionKeySource], + ..PatchInput::default() + }, + ), + ( + "user_reset_internal_key", + OptionsScope::User, + user.clone(), + PatchInput { + reset: vec![OptionKey::CriticalKeys], + ..PatchInput::default() + }, + ), + ( + "topic_empty_name", + OptionsScope::Topic, + numeric_topic.clone(), + PatchInput { + name: Some(String::new()), + ..PatchInput::default() + }, + ), + ( + "topic_empty_name_and_create_only_reset", + OptionsScope::Topic, + numeric_topic, + PatchInput { + name: Some(String::new()), + reset: vec![OptionKey::SegmentSize], + ..PatchInput::default() + }, + ), + ( + "stream_empty_name", + OptionsScope::Stream, + stream, + PatchInput { + name: Some(String::new()), + ..PatchInput::default() + }, + ), + ( + "user_short_username", + OptionsScope::User, + user.clone(), + PatchInput { + username: Some("a".to_owned()), + ..PatchInput::default() + }, + ), + ( + "user_short_username_and_status_reset", + OptionsScope::User, + user, + PatchInput { + username: Some("a".to_owned()), + reset: vec![OptionKey::Status], + ..PatchInput::default() + }, + ), + ]; + cases + .into_iter() + .map(|(case, scope, target, input)| { + let Err(error) = update_resource_request(scope, &target, &input) else { + return Err(format!("invalid update {case} was accepted").into()); + }; + Ok(UpdateResourceErrorVector { + case: case.to_owned(), + scope, + target, + input, + error: ErrorExpectation::from(&error), + }) + }) + .collect() +} + +fn create_stream_vectors() -> GeneratorResult<Vec<CreateStreamVector>> { + let cases = [ + ( + "stream_without_key", + CreateStreamInput { + name: "orders".to_owned(), + encryption_key: None, + raw: BTreeMap::new(), + }, + ), + ( + "stream_with_key", + CreateStreamInput { + name: "orders".to_owned(), + encryption_key: Some(KeyInput::RawHex(hex::encode((0u8..32).collect::<Vec<_>>()))), + raw: BTreeMap::new(), + }, + ), + ( + "stream_with_raw_extra_key", + CreateStreamInput { + name: "orders".to_owned(), + encryption_key: None, + raw: BTreeMap::from([("future_option".to_owned(), "enabled".to_owned())]), + }, + ), + ]; + cases + .into_iter() + .map(|(case, input)| { + let request = input.request()?; + let columns = OptionColumns { + name: Some(input.name.clone()), + ..OptionColumns::default() + }; + Ok(CreateStreamVector { + case: case.to_owned(), + wire_options: resource_options_from_wire(&request.options, true)?, + body_hex: hex::encode(request.to_bytes()), + http_body: input.http_body()?, + admission_error: admit_create(OptionsScope::Stream, columns, &request.options) + .err() + .map(|error| ErrorExpectation::from(&error)), + input, + }) + }) + .collect() +} + +fn create_topic_vectors() -> GeneratorResult<Vec<CreateTopicVector>> { + let input = CreateTopicInput { + stream_id: IdentifierInput::Numeric(1), + name: "events".to_owned(), + partitions_count: 3, + compression_algorithm: Some("gzip".to_owned()), + message_expiry: Some(SEVEN_DAYS_MICROS.to_owned()), + max_topic_size: Some(ABOVE_2_POW_53.to_string()), + segment_size: Some((128u64 * 1024 * 1024).to_string()), + durability: Durability::Persisted, + consumer_offset_durability: Durability::Replicated, + }; + let request = input.request()?; + let columns = OptionColumns { + name: Some(input.name.clone()), + ..OptionColumns::default() + }; + Ok(vec![CreateTopicVector { + case: "topic_with_options".to_owned(), + wire_options: resource_options_from_wire(&request.options, true)?, + body_hex: hex::encode(request.to_bytes()), + http_body: input.http_body()?, + admission_error: admit_create(OptionsScope::Topic, columns, &request.options) + .err() + .map(|error| ErrorExpectation::from(&error)), + input, + }]) +} + +fn create_user_vectors() -> GeneratorResult<Vec<CreateUserVector>> { + let input = CreateUserInput { + username: "alice".to_owned(), + password: "s3cret-pass".to_owned(), + status: UserStatus::Active, + }; + let request = input.request()?; + let columns = OptionColumns { + username: Some(input.username.clone()), + status: Some(input.status), + ..OptionColumns::default() + }; + Ok(vec![CreateUserVector { + case: "user_with_empty_options".to_owned(), + wire_options: resource_options_from_wire(&request.options, true)?, + body_hex: hex::encode(request.to_bytes()), + http_body: input.http_body()?, + admission_error: admit_create(OptionsScope::User, columns, &request.options) + .err() + .map(|error| ErrorExpectation::from(&error)), + input, + }]) +} + +fn http_update_vectors() -> GeneratorResult<Vec<HttpUpdateVector>> { + let topic = TargetInput { + stream_id: Some(IdentifierInput::Numeric(1)), + topic_id: Some(IdentifierInput::Numeric(2)), + user_id: None, + }; + let stream = TargetInput { + stream_id: Some(IdentifierInput::Numeric(3)), + ..TargetInput::default() + }; + let user = TargetInput { + user_id: Some(IdentifierInput::Numeric(5)), + ..TargetInput::default() + }; + let cases = [ + ( + "topic_flat_keys", + OptionsScope::Topic, + &topic, + r#"{"name":"events-v2","message_expiry":"7 days"}"#, + ), + ( + "topic_reset_array", + OptionsScope::Topic, + &topic, + r#"{"message_expiry":604800000000,"reset":["max_topic_size"]}"#, + ), + ( + "topic_null_ignored", + OptionsScope::Topic, + &topic, + r#"{"name":null,"max_topic_size":null,"message_expiry":"1 day"}"#, + ), + ( + "topic_u64_above_2_pow_53_as_string", + OptionsScope::Topic, + &topic, + r#"{"max_topic_size":"9007199254740993"}"#, + ), + ( + "stream_rename", + OptionsScope::Stream, + &stream, + r#"{"name":"payments-v2"}"#, + ), + ( + "user_username_and_status", + OptionsScope::User, + &user, + r#"{"username":"alice2","status":"inactive"}"#, + ), + ( + "topic_duplicate_key", + OptionsScope::Topic, + &topic, + r#"{"message_expiry":"1 day","message_expiry":"2 days"}"#, + ), + ( + "topic_nested_object", + OptionsScope::Topic, + &topic, + r#"{"message_expiry":{"value":"1 day"}}"#, + ), + ( + "topic_null_only", + OptionsScope::Topic, + &topic, + r#"{"max_topic_size":null}"#, + ), + ( + "topic_zero_sentinel_only", + OptionsScope::Topic, + &topic, + r#"{"message_expiry":0}"#, + ), + ( + "topic_reset_names_set_key", + OptionsScope::Topic, + &topic, + r#"{"message_expiry":"1 day","reset":["message_expiry"]}"#, + ), + ( + "topic_unknown_key", + OptionsScope::Topic, + &topic, + r#"{"retention":"1 day"}"#, + ), + ( + "topic_set_create_only_key", + OptionsScope::Topic, + &topic, + r#"{"compression_algorithm":"gzip"}"#, + ), + ( + "topic_reset_create_only_key", + OptionsScope::Topic, + &topic, + r#"{"reset":["compression_algorithm"]}"#, + ), + ( + "stream_reset_name", + OptionsScope::Stream, + &stream, + r#"{"reset":["name"]}"#, + ), + ( + "topic_name_as_number", + OptionsScope::Topic, + &topic, + r#"{"name":5}"#, + ), + ( + "stream_empty_name", + OptionsScope::Stream, + &stream, + r#"{"name":""}"#, + ), + ( + "user_status_as_bool", + OptionsScope::User, + &user, + r#"{"status":true}"#, + ), + ( + "topic_create_only_key_as_bool", + OptionsScope::Topic, + &topic, + r#"{"segment_size":true}"#, + ), + ( + "stream_read_only_key_as_number", + OptionsScope::Stream, + &stream, + r#"{"encryption_key_source":1}"#, + ), + ( + "topic_invalid_value", + OptionsScope::Topic, + &topic, + r#"{"message_expiry":"soon"}"#, + ), + ]; + cases + .into_iter() + .map(|(case, scope, target, body)| { + let evaluation = evaluate_http_update(scope, target, body)?; + Ok(HttpUpdateVector { + case: case.to_owned(), + scope, + target: target.clone(), + body: body.to_owned(), + update_resource_hex: evaluation.update_resource.map(hex::encode), + outcome: evaluation.outcome, + error: evaluation.error, + }) + }) + .collect() +} + +fn stream_response_vectors() -> GeneratorResult<Vec<StreamResponseVector>> { + let cases = [ + ( + "stream_source_none", + 1, + "plain-stream", + StreamKeyState { + encryption_key_source: StreamEncryptionSource::None, + data_key: None, + kek_config_keys: Vec::new(), + stored_extra: ResourceOptions::new(), + }, + ), + ( + "stream_source_stream_with_unknown_explicit_key", + 2, + "client-key-stream", + StreamKeyState { + encryption_key_source: StreamEncryptionSource::Stream, + data_key: Some(KeyInput::RawHex(hex::encode((0u8..32).collect::<Vec<_>>()))), + kek_config_keys: vec![FIRST_KEK_CONFIG_KEY.to_owned()], + stored_extra: ResourceOptions::from([( + HeaderKey::try_from("future_option")?, + OptionValue::explicit(HeaderValue::try_from("enabled")?), + )]), + }, + ), + ( + "stream_source_server_two_keks", + 3, + "server-key-stream", + StreamKeyState { + encryption_key_source: StreamEncryptionSource::Server, + data_key: Some(KeyInput::RawHex(hex::encode( + (32u8..64).collect::<Vec<_>>(), + ))), + kek_config_keys: vec![ + FIRST_KEK_CONFIG_KEY.to_owned(), + SECOND_KEK_CONFIG_KEY.to_owned(), + ], + stored_extra: ResourceOptions::new(), + }, + ), + ]; + cases + .into_iter() + .map(|(case, id, name, state)| { + let stored = state.stored_options()?; + let (options, derived_options) = + resource_options_to_wire_split(&public_options(OptionsScope::Stream, &stored)?)?; + let bytes = GetStreamResponse { + stream: StreamResponse { + id, + created_at: CREATED_AT_MICROS, + topics_count: 0, + size_bytes: 4096, + messages_count: 12, + name: WireName::new(name)?, + options, + derived_options, + }, + topics: Vec::new(), + } + .to_bytes(); + let expected = StreamProjection::from(&StreamDetails::try_from( + GetStreamResponse::decode_from(&bytes)?, + )?); + ensure_no_secrets(case, &bytes, &stored, &expected.options)?; + if let Some(data_key) = &state.data_key { + let key = data_key.parse()?; + if contains_bytes(&bytes, key.as_bytes()) + || contains_bytes(&bytes, STANDARD.encode(key.as_bytes()).as_bytes()) + { + return Err(format!("{case} response carries the stream key").into()); + } + } + Ok(StreamResponseVector { + case: case.to_owned(), + state, + response_hex: hex::encode(&bytes), + expected, + }) + }) + .collect() +} + +fn topic_response_vectors() -> GeneratorResult<Vec<TopicResponseVector>> { + let mut stored = ResourceOptions::from([ + ( + HeaderKey::from(OptionKey::CompressionAlgorithm), + OptionValue::explicit(HeaderValue::try_from("gzip")?), + ), + ( + HeaderKey::from(OptionKey::MessageExpiry), + OptionValue::explicit(HeaderValue::from(SEVEN_DAYS_MICROS.parse::<u64>()?)), + ), + ]); + for descriptor in descriptors_for(OptionsScope::Topic) + .filter(|descriptor| descriptor.storage == OptionStorage::Map) + { + if let Some(default) = descriptor.default_value()? { + stored + .entry(HeaderKey::from(descriptor.key)) + .or_insert_with(|| OptionValue::derived(default)); + } + } + let (options, derived_options) = + resource_options_to_wire_split(&public_options(OptionsScope::Topic, &stored)?)?; + let bytes = GetTopicResponse { + topic: TopicHeader { + id: 4, + created_at: CREATED_AT_MICROS, + partitions_count: 0, + size_bytes: 2048, + messages_count: 7, + name: WireName::new("events")?, + options, + derived_options, + }, + partitions: Vec::new(), + } + .to_bytes(); + let expected = TopicProjection::from(&TopicDetails::try_from(GetTopicResponse::decode_from( + &bytes, + )?)?); + ensure_no_secrets( + "topic_explicit_and_derived", + &bytes, + &stored, + &expected.options, + )?; + Ok(vec![TopicResponseVector { + case: "topic_explicit_and_derived".to_owned(), + response_hex: hex::encode(&bytes), + expected, + }]) +} + +fn user_response_vectors() -> GeneratorResult<Vec<UserResponseVector>> { + let cases = [ + ( + "user_empty_blocks", + 5, + "alice", + UserStatus::Active, + ResourceOptions::new(), + ), + ( + "user_populated_blocks", + 6, + "bob", + UserStatus::Inactive, + ResourceOptions::from([ + ( + HeaderKey::try_from("future_option")?, + OptionValue::explicit(HeaderValue::try_from("enabled")?), + ), + ( + HeaderKey::try_from("future_quota")?, + OptionValue::derived(HeaderValue::from(ABOVE_2_POW_53)), + ), + ]), + ), + ]; + cases + .into_iter() + .map(|(case, id, username, status, stored)| { + let (options, derived_options) = + resource_options_to_wire_split(&public_options(OptionsScope::User, &stored)?)?; + let bytes = UserDetailsResponse { + user: UserResponse { + id, + created_at: CREATED_AT_MICROS, + status: status.as_code(), + username: WireName::new(username)?, + options, + derived_options, + }, + permissions: None, + } + .to_bytes(); + let expected = UserProjection::from(&UserInfoDetails::try_from( + UserDetailsResponse::decode_from(&bytes)?, + )?); + ensure_no_secrets(case, &bytes, &stored, &expected.options)?; + Ok(UserResponseVector { + case: case.to_owned(), + response_hex: hex::encode(&bytes), + expected, + }) + }) + .collect() +} + +/// Refuse a response that echoes a secret or internal key or stored secret bytes. +fn ensure_no_secrets( + case: &str, + response: &[u8], + stored: &ResourceOptions, + projected: &ResourceOptions, +) -> GeneratorResult<()> { + for descriptor in CATALOG.iter().filter(|descriptor| { + matches!( + descriptor.visibility, + OptionVisibility::Secret | OptionVisibility::Internal + ) + }) { + let key = HeaderKey::from(descriptor.key); + if projected.contains_key(&key) { + return Err(format!("{case} projects {}", descriptor.key).into()); + } + if let Some(value) = stored.get(&key) + && contains_bytes(response, value.value.as_bytes()) + { + return Err(format!("{case} response carries {}", descriptor.key).into()); + } + } + Ok(()) +}
diff --git a/core/common/examples/resource_options_fixtures/schema.rs b/core/common/examples/resource_options_fixtures/schema.rs new file mode 100644 index 0000000..6fe4922 --- /dev/null +++ b/core/common/examples/resource_options_fixtures/schema.rs
@@ -0,0 +1,990 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use iggy_binary_protocol::codes::{ + CREATE_STREAM_CODE, CREATE_TOPIC_CODE, CREATE_USER_CODE, DESCRIBE_OPTIONS_CODE, + GET_STREAM_CODE, GET_STREAMS_CODE, GET_TOPIC_CODE, GET_TOPICS_CODE, GET_USER_CODE, + GET_USERS_CODE, UPDATE_RESOURCE_CODE, +}; +use iggy_binary_protocol::requests::streams::CreateStreamRequest; +use iggy_binary_protocol::requests::system::{ResourceTarget, UpdateResourceRequest}; +use iggy_binary_protocol::requests::topics::CreateTopicRequest; +use iggy_binary_protocol::requests::users::CreateUserRequest; +use iggy_binary_protocol::responses::system::OptionDescriptor as WireOptionDescriptor; +use iggy_binary_protocol::{ + IGGY_PROTOCOL_VERSION, IGGY_PROTOCOL_VERSION_MIN, Operation, ProtocolVersion, WireDecode, + WireEncode, WireIdentifier, WireName, WireOptions, lookup_command, +}; +use iggy_common::create_stream::CreateStream; +use iggy_common::create_topic::CreateTopic; +use iggy_common::create_user::CreateUser; +use iggy_common::resource_options_json::encode_value; +pub use iggy_common::wire_conversions::describe_options_response; +use iggy_common::wire_conversions::identifier_to_wire; +use iggy_common::{ + CATALOG, CompressionAlgorithm, DEFAULT_PARTITIONS_COUNT, Durability, HeaderKey, HeaderKind, + HeaderValue, Identifier, IggyByteSize, IggyError, IggyExpiry, MaxTopicSize, + OPTION_FLAG_CRITICAL, OPTION_FLAG_READ_ONLY, OPTION_FLAG_RESETTABLE, OPTION_FLAG_SECRET, + OPTION_FLAG_UPDATABLE, OptionAdmissionContext, OptionColumns, OptionDescriptor, OptionInputMap, + OptionKey, OptionMutability, OptionOperation, OptionStorage, OptionValue, OptionVisibility, + OptionsPatch, OptionsScope, OptionsUpdate, ResourceOptions, STREAM_KEY_LENGTH, + ServerKeyContext, StreamCreateOptions, StreamDetails, StreamEncryptionKey, + StreamEncryptionSource, StreamKeyOrigin, StreamUpdateOptions, TopicCreateOptions, TopicDetails, + TopicUpdateOptions, UserCreateOptions, UserInfoDetails, UserStatus, UserUpdateOptions, WrapSet, + admit, critical_keys_marker, +}; +use secrecy::SecretString; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use std::str::FromStr; + +pub const SCHEMA_VERSION: u32 = 1; +pub const GENERATOR: &str = "core/common/examples/resource_options_fixtures.rs"; +/// Relative to the `iggy_common` manifest directory. +pub const CANONICAL_PATH: &str = "fixtures/resource_options.json"; +/// Byte-identical copies each foreign SDK loads from its own test resources. +pub const LANGUAGE_COPIES: [&str; 8] = [ + "../../foreign/python/tests/fixtures/resource_options.json", + "../../foreign/cpp/tests/fixtures/resource_options.json", + "../../foreign/php/tests/fixtures/resource_options.json", + "../../foreign/node/src/wire/fixtures/resource_options.json", + "../../foreign/go/contracts/testdata/resource_options.json", + "../../foreign/java/java-sdk/src/test/resources/resource_options.json", + "../../foreign/csharp/Iggy_SDK_Tests/Fixtures/resource_options.json", + "../../foreign/swift/Tests/IggyTests/Fixtures/resource_options.json", +]; +pub const SCOPES: [OptionsScope; 3] = [ + OptionsScope::Topic, + OptionsScope::Stream, + OptionsScope::User, +]; + +const COMMANDS: [(&str, u32); 11] = [ + ("create_stream", CREATE_STREAM_CODE), + ("create_topic", CREATE_TOPIC_CODE), + ("create_user", CREATE_USER_CODE), + ("describe_options", DESCRIBE_OPTIONS_CODE), + ("get_stream", GET_STREAM_CODE), + ("get_streams", GET_STREAMS_CODE), + ("get_topic", GET_TOPIC_CODE), + ("get_topics", GET_TOPICS_CODE), + ("get_user", GET_USER_CODE), + ("get_users", GET_USERS_CODE), + ("update_resource", UPDATE_RESOURCE_CODE), +]; +const RETIRED_COMMANDS: [(&str, u32); 3] = [ + ("update_stream", 204), + ("update_topic", 304), + ("update_user", 35), +]; +const OPERATIONS: [(&str, Operation); 5] = [ + ("rewrap_stream_keys", Operation::RewrapStreamKeys), + ("update_resource", Operation::UpdateResource), + ("update_stream_resolved", Operation::UpdateStreamResolved), + ("update_topic_resolved", Operation::UpdateTopicResolved), + ("update_user_resolved", Operation::UpdateUserResolved), +]; +const RETIRED_OPERATIONS: [(&str, u8); 3] = [ + ("update_stream", 129), + ("update_topic", 133), + ("update_user", 142), +]; +const HEADER_KINDS: [HeaderKind; 5] = [ + HeaderKind::Raw, + HeaderKind::String, + HeaderKind::Bool, + HeaderKind::Uint32, + HeaderKind::Uint64, +]; +const OPTION_FLAGS: [(&str, u8); 5] = [ + ("critical", OPTION_FLAG_CRITICAL), + ("read_only", OPTION_FLAG_READ_ONLY), + ("resettable", OPTION_FLAG_RESETTABLE), + ("secret", OPTION_FLAG_SECRET), + ("updatable", OPTION_FLAG_UPDATABLE), +]; +const USER_STATUSES: [UserStatus; 2] = [UserStatus::Active, UserStatus::Inactive]; + +#[derive(Debug, Serialize, Deserialize)] +pub struct ResourceOptionsFixture { + pub schema_version: u32, + pub generated_by: String, + pub conventions: BTreeMap<String, String>, + pub constants: WireConstants, + pub catalog: Vec<CatalogRow>, + pub describe_options: Vec<DescribeOptionsVector>, + pub stream_key: StreamKeyVectors, + pub update_resource: Vec<UpdateResourceVector>, + pub update_resource_errors: Vec<UpdateResourceErrorVector>, + pub create_stream: Vec<CreateStreamVector>, + pub create_topic: Vec<CreateTopicVector>, + pub create_user: Vec<CreateUserVector>, + pub http_update: Vec<HttpUpdateVector>, + pub stream_responses: Vec<StreamResponseVector>, + pub topic_responses: Vec<TopicResponseVector>, + pub user_responses: Vec<UserResponseVector>, +} + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +pub struct WireConstants { + pub protocol_version: ProtocolVersionValue, + pub protocol_version_min: ProtocolVersionValue, + pub command_codes: BTreeMap<String, CommandCode>, + pub error_codes: BTreeMap<String, u32>, + pub retired_command_codes: BTreeMap<String, u32>, + pub operations: BTreeMap<String, OperationValue>, + pub retired_operations: BTreeMap<String, u8>, + pub scope_codes: BTreeMap<String, u8>, + pub header_kind_codes: BTreeMap<HeaderKind, u8>, + pub option_flags: BTreeMap<String, u8>, + pub user_status_codes: BTreeMap<String, u8>, + pub stream_key_length: usize, + pub never_returned_keys: Vec<OptionKey>, +} + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +pub struct ProtocolVersionValue { + pub packed: u32, + pub text: String, + pub major: u32, + pub minor: u32, + pub patch: u32, +} + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +pub struct CommandCode { + pub code: u32, + pub operation: Option<u8>, +} + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +pub struct OperationValue { + pub value: u8, + pub internal: bool, + pub request_operation: u8, +} + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +pub struct CatalogRow { + pub key: OptionKey, + pub scopes: Vec<OptionsScope>, + pub kind: HeaderKind, + pub kind_code: u8, + pub storage: String, + pub mutability: String, + pub visibility: String, + pub critical: bool, + pub flags: u8, + pub described: bool, + pub default: Option<String>, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct DescribeOptionsVector { + pub scope: OptionsScope, + pub request_hex: String, + pub response_hex: String, + pub entries: Vec<DescribedOption>, +} + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +pub struct DescribedOption { + pub key: String, + pub kind: HeaderKind, + pub flags: u8, + pub default: Option<String>, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct StreamKeyVectors { + pub key_hex: String, + pub key_base64: String, + pub fingerprint_label: String, + pub fingerprint: String, + pub invalid_keys: Vec<InvalidKeyVector>, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum KeyInput { + RawHex(String), + Base64(String), +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct InvalidKeyVector { + pub case: String, + pub input: KeyInput, + pub error: ErrorExpectation, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ErrorExpectation { + pub name: String, + pub code: u32, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum IdentifierInput { + Numeric(u32), + Named(String), +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct TargetInput { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stream_id: Option<IdentifierInput>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub topic_id: Option<IdentifierInput>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub user_id: Option<IdentifierInput>, +} + +/// Typed update input: absent fields keep their value, u64 values are decimal strings. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct PatchInput { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option<String>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub message_expiry: Option<String>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_topic_size: Option<String>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub username: Option<String>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub status: Option<UserStatus>, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub reset: Vec<OptionKey>, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct UpdateResourceVector { + pub case: String, + pub scope: OptionsScope, + pub target: TargetInput, + pub input: PatchInput, + #[serde(with = "iggy_common::resource_options_json")] + pub wire_set: ResourceOptions, + pub wire_reset: Vec<String>, + pub body_hex: String, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct UpdateResourceErrorVector { + pub case: String, + pub scope: OptionsScope, + pub target: TargetInput, + pub input: PatchInput, + pub error: ErrorExpectation, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct CreateStreamInput { + pub name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub encryption_key: Option<KeyInput>, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub raw: BTreeMap<String, String>, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct CreateStreamVector { + pub case: String, + pub input: CreateStreamInput, + #[serde(with = "iggy_common::resource_options_json")] + pub wire_options: ResourceOptions, + pub body_hex: String, + /// The JSON body the SDK's HTTP create sends for the same input. + pub http_body: String, + pub admission_error: Option<ErrorExpectation>, +} + +/// Typed topic create input. u64 values are decimal strings. +#[derive(Debug, Serialize, Deserialize)] +pub struct CreateTopicInput { + pub stream_id: IdentifierInput, + pub name: String, + pub partitions_count: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub compression_algorithm: Option<String>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub message_expiry: Option<String>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_topic_size: Option<String>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub segment_size: Option<String>, + pub durability: Durability, + pub consumer_offset_durability: Durability, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct CreateTopicVector { + pub case: String, + pub input: CreateTopicInput, + #[serde(with = "iggy_common::resource_options_json")] + pub wire_options: ResourceOptions, + pub body_hex: String, + /// The JSON body the SDK's HTTP create sends for the same input. + pub http_body: String, + pub admission_error: Option<ErrorExpectation>, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct CreateUserInput { + pub username: String, + pub password: String, + pub status: UserStatus, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct CreateUserVector { + pub case: String, + pub input: CreateUserInput, + #[serde(with = "iggy_common::resource_options_json")] + pub wire_options: ResourceOptions, + pub body_hex: String, + /// The JSON body the SDK's HTTP create sends for the same input. + pub http_body: String, + pub admission_error: Option<ErrorExpectation>, +} + +/// Where the server stops an HTTP update body. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum HttpOutcome { + Accepted, + JsonRefused, + DecodeRefused, + AdmissionRefused, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct HttpUpdateVector { + pub case: String, + pub scope: OptionsScope, + pub target: TargetInput, + pub body: String, + pub update_resource_hex: Option<String>, + pub outcome: HttpOutcome, + pub error: Option<ErrorExpectation>, +} + +#[derive(Debug, PartialEq)] +pub struct HttpEvaluation { + pub update_resource: Option<Vec<u8>>, + pub outcome: HttpOutcome, + pub error: Option<ErrorExpectation>, +} + +/// Stored state behind a stream response. Wrap nonces are random, so only projections are pinned. +#[derive(Debug, Serialize, Deserialize)] +pub struct StreamKeyState { + pub encryption_key_source: StreamEncryptionSource, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub data_key: Option<KeyInput>, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub kek_config_keys: Vec<String>, + #[serde(with = "iggy_common::resource_options_json")] + pub stored_extra: ResourceOptions, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct StreamResponseVector { + pub case: String, + pub state: StreamKeyState, + pub response_hex: String, + pub expected: StreamProjection, +} + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +pub struct StreamProjection { + pub id: u32, + pub created_at: u64, + pub name: String, + pub topics_count: u32, + pub size_bytes: u64, + pub messages_count: u64, + #[serde(with = "iggy_common::resource_options_json")] + pub options: ResourceOptions, + pub encryption_key_source: Option<StreamEncryptionSource>, + pub encryption_key_fingerprint: Option<String>, + pub encryption_kek_fingerprints: Vec<String>, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct TopicResponseVector { + pub case: String, + pub response_hex: String, + pub expected: TopicProjection, +} + +/// A topic read plus the three settings the header no longer carries, resolved from options. +#[derive(Debug, PartialEq, Serialize, Deserialize)] +pub struct TopicProjection { + pub id: u32, + pub created_at: u64, + pub name: String, + pub partitions_count: u32, + pub size_bytes: u64, + pub messages_count: u64, + #[serde(with = "iggy_common::resource_options_json")] + pub options: ResourceOptions, + pub compression_algorithm: String, + pub message_expiry: String, + pub max_topic_size: String, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct UserResponseVector { + pub case: String, + pub response_hex: String, + pub expected: UserProjection, +} + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +pub struct UserProjection { + pub id: u32, + pub created_at: u64, + pub status: UserStatus, + pub username: String, + #[serde(with = "iggy_common::resource_options_json")] + pub options: ResourceOptions, +} + +impl WireConstants { + /// The values the protocol crates define in this build. + pub fn live() -> Result<Self, IggyError> { + Ok(Self { + protocol_version: ProtocolVersionValue::try_from(IGGY_PROTOCOL_VERSION)?, + protocol_version_min: ProtocolVersionValue::try_from(IGGY_PROTOCOL_VERSION_MIN)?, + command_codes: COMMANDS + .iter() + .map(|(name, code)| { + let meta = lookup_command(*code).ok_or(IggyError::InvalidCommand)?; + Ok(( + (*name).to_owned(), + CommandCode { + code: *code, + operation: meta.operation.map(|operation| operation as u8), + }, + )) + }) + .collect::<Result<_, IggyError>>()?, + error_codes: BTreeMap::from([( + "cannot_modify_root_user".to_owned(), + IggyError::CannotModifyRootUser.as_code(), + )]), + retired_command_codes: RETIRED_COMMANDS + .iter() + .map(|(name, code)| ((*name).to_owned(), *code)) + .collect(), + operations: OPERATIONS + .iter() + .map(|(name, operation)| { + ( + (*name).to_owned(), + OperationValue { + value: *operation as u8, + internal: operation.is_internal(), + request_operation: operation.request_operation() as u8, + }, + ) + }) + .collect(), + retired_operations: RETIRED_OPERATIONS + .iter() + .map(|(name, value)| ((*name).to_owned(), *value)) + .collect(), + scope_codes: SCOPES + .iter() + .map(|scope| (scope.to_string(), scope.as_code())) + .collect(), + header_kind_codes: HEADER_KINDS + .iter() + .map(|kind| (*kind, kind.as_code())) + .collect(), + option_flags: OPTION_FLAGS + .iter() + .map(|(name, flag)| ((*name).to_owned(), *flag)) + .collect(), + user_status_codes: USER_STATUSES + .iter() + .map(|status| (status.to_string(), status.as_code())) + .collect(), + stream_key_length: STREAM_KEY_LENGTH, + never_returned_keys: CATALOG + .iter() + .filter(|descriptor| { + matches!( + descriptor.visibility, + OptionVisibility::Secret | OptionVisibility::Internal + ) + }) + .map(|descriptor| descriptor.key) + .collect(), + }) + } +} + +impl KeyInput { + /// The key this input names, parsed the way SDK callers parse it. + pub fn parse(&self) -> Result<StreamEncryptionKey, IggyError> { + match self { + Self::RawHex(text) => { + let bytes = hex::decode(text).map_err(|_| IggyError::InvalidEncryptionKey)?; + StreamEncryptionKey::try_from(bytes.as_slice()) + } + Self::Base64(text) => StreamEncryptionKey::from_str(text), + } + } +} + +impl TargetInput { + /// The wire target for `scope`, built the way the SDK clients build it. + pub fn wire(&self, scope: OptionsScope) -> Result<ResourceTarget, IggyError> { + let wire = |input: &Option<IdentifierInput>| -> Result<WireIdentifier, IggyError> { + let input = input.as_ref().ok_or(IggyError::InvalidIdentifier)?; + identifier_to_wire(&Identifier::try_from(input)?) + }; + Ok(match scope { + OptionsScope::Topic => ResourceTarget::Topic { + stream_id: wire(&self.stream_id)?, + topic_id: wire(&self.topic_id)?, + }, + OptionsScope::Stream => ResourceTarget::Stream { + stream_id: wire(&self.stream_id)?, + }, + OptionsScope::User => ResourceTarget::User { + user_id: wire(&self.user_id)?, + }, + }) + } +} + +impl PatchInput { + /// Encode through the scope's typed update options, the public SDK API. + pub fn patch(&self, scope: OptionsScope) -> Result<OptionsPatch, IggyError> { + let reset = self.reset.iter().copied().collect(); + match scope { + OptionsScope::Topic => TopicUpdateOptions { + name: self.name.clone(), + message_expiry: decimal(self.message_expiry.as_deref())?.map(IggyExpiry::from), + max_topic_size: decimal(self.max_topic_size.as_deref())?.map(MaxTopicSize::from), + reset, + ..TopicUpdateOptions::default() + } + .to_patch(), + OptionsScope::Stream => StreamUpdateOptions { + name: self.name.clone(), + reset, + ..StreamUpdateOptions::default() + } + .to_patch(), + OptionsScope::User => UserUpdateOptions { + username: self.username.clone(), + status: self.status, + reset, + ..UserUpdateOptions::default() + } + .to_patch(), + } + } +} + +impl CreateStreamInput { + /// The request the SDK's `create_stream` sends. + pub fn request(&self) -> Result<CreateStreamRequest, IggyError> { + Ok(CreateStreamRequest { + name: WireName::new(self.name.as_str()).map_err(|_| IggyError::InvalidFormat)?, + options: self.options()?.to_wire()?, + }) + } + + /// The body the SDK's HTTP `create_stream` posts. + pub fn http_body(&self) -> Result<String, IggyError> { + serde_json::to_string(&CreateStream { + name: self.name.clone(), + options: OptionInputMap(self.options()?.to_input_map()?), + }) + .map_err(|_| IggyError::InvalidFormat) + } + + fn options(&self) -> Result<StreamCreateOptions, IggyError> { + Ok(StreamCreateOptions { + encryption_key: self + .encryption_key + .as_ref() + .map(KeyInput::parse) + .transpose()?, + raw: self.raw.clone(), + }) + } +} + +impl CreateTopicInput { + /// The request the SDK's `create_topic` sends. + pub fn request(&self) -> Result<CreateTopicRequest, IggyError> { + let options = self.options()?; + Ok(CreateTopicRequest { + stream_id: identifier_to_wire(&Identifier::try_from(&self.stream_id)?)?, + partitions_count: options.partitions_count.unwrap_or(DEFAULT_PARTITIONS_COUNT), + name: WireName::new(self.name.as_str()).map_err(|_| IggyError::InvalidFormat)?, + options: options.to_wire()?, + }) + } + + /// The body the SDK's HTTP `create_topic` posts. + pub fn http_body(&self) -> Result<String, IggyError> { + let options = self.options()?; + serde_json::to_string(&CreateTopic { + stream_id: Identifier::try_from(&self.stream_id)?, + partitions_count: options.partitions_count.unwrap_or(DEFAULT_PARTITIONS_COUNT), + name: self.name.clone(), + options: OptionInputMap(options.to_input_map()?), + }) + .map_err(|_| IggyError::InvalidFormat) + } + + fn options(&self) -> Result<TopicCreateOptions, IggyError> { + Ok(TopicCreateOptions { + partitions_count: Some(self.partitions_count), + compression_algorithm: self + .compression_algorithm + .as_deref() + .map(CompressionAlgorithm::from_str) + .transpose() + .map_err(|_| IggyError::InvalidFormat)?, + message_expiry: decimal(self.message_expiry.as_deref())?.map(IggyExpiry::from), + max_topic_size: decimal(self.max_topic_size.as_deref())?.map(MaxTopicSize::from), + segment_size: decimal(self.segment_size.as_deref())?.map(IggyByteSize::from), + durability: self.durability, + consumer_offset_durability: self.consumer_offset_durability, + ..TopicCreateOptions::default() + }) + } +} + +impl CreateUserInput { + /// The request the SDK's `create_user` sends, without permissions. + pub fn request(&self) -> Result<CreateUserRequest, IggyError> { + Ok(CreateUserRequest { + username: WireName::new(self.username.as_str()) + .map_err(|_| IggyError::InvalidFormat)?, + password: self.password.clone(), + status: self.status.as_code(), + permissions: None, + options: UserCreateOptions::default().to_wire()?, + }) + } + + /// The body the SDK's HTTP `create_user` posts, without permissions. + pub fn http_body(&self) -> Result<String, IggyError> { + serde_json::to_string(&CreateUser { + username: self.username.clone(), + password: SecretString::from(self.password.as_str()), + status: self.status, + permissions: None, + options: OptionInputMap(UserCreateOptions::default().to_input_map()?), + }) + .map_err(|_| IggyError::InvalidFormat) + } +} + +impl StreamKeyState { + /// The options map the server stores: the extras plus the wrap set and its marker. + pub fn stored_options(&self) -> Result<ResourceOptions, IggyError> { + let mut stored = self.stored_extra.clone(); + let origin = match self.encryption_key_source { + StreamEncryptionSource::Stream => StreamKeyOrigin::Client, + StreamEncryptionSource::Server => StreamKeyOrigin::Server, + StreamEncryptionSource::None => return Ok(stored), + }; + let key = self + .data_key + .as_ref() + .ok_or(IggyError::InvalidEncryptionKey)? + .parse()?; + let envelopes = self + .kek_config_keys + .iter() + .map(|config_key| { + ServerKeyContext::from_config(config_key, &[], true) + .and_then(|context| context.wrap(&key, origin)) + }) + .collect::<Result<Vec<_>, _>>()?; + let set = WrapSet::new(envelopes)?; + stored.insert( + HeaderKey::from(OptionKey::EncryptionKeyWrapped), + OptionValue::explicit(HeaderValue::try_from(set.as_ref())?), + ); + stored.insert( + HeaderKey::from(OptionKey::CriticalKeys), + OptionValue::explicit(critical_keys_marker(&[OptionKey::EncryptionKeyWrapped])?), + ); + Ok(stored) + } +} + +impl TryFrom<u32> for ProtocolVersionValue { + type Error = IggyError; + + fn try_from(packed: u32) -> Result<Self, Self::Error> { + let text = ProtocolVersion(packed).to_string(); + let components = text + .split('.') + .map(u32::from_str) + .collect::<Result<Vec<_>, _>>() + .map_err(|_| IggyError::InvalidVersion(text.clone()))?; + let [major, minor, patch] = components[..] else { + return Err(IggyError::InvalidVersion(text)); + }; + Ok(Self { + packed, + text, + major, + minor, + patch, + }) + } +} + +impl TryFrom<&OptionDescriptor> for CatalogRow { + type Error = IggyError; + + fn try_from(descriptor: &OptionDescriptor) -> Result<Self, Self::Error> { + let storage = match descriptor.storage { + OptionStorage::Map => "map", + OptionStorage::Column => "column", + OptionStorage::Computed => "computed", + }; + let mutability = match descriptor.mutability { + OptionMutability::CreateOnly => "create_only", + OptionMutability::Updatable => "updatable", + OptionMutability::ReadOnly => "read_only", + }; + let visibility = match descriptor.visibility { + OptionVisibility::Public => "public", + OptionVisibility::Secret => "secret", + OptionVisibility::Internal => "internal", + OptionVisibility::ReadOnly => "read_only", + }; + Ok(Self { + key: descriptor.key, + scopes: descriptor.scopes.to_vec(), + kind: descriptor.kind, + kind_code: descriptor.kind.as_code(), + storage: storage.to_owned(), + mutability: mutability.to_owned(), + visibility: visibility.to_owned(), + critical: descriptor.critical, + flags: descriptor.flags(), + described: descriptor.is_described(), + default: descriptor + .default_value()? + .map(|value| encode_value(&value)) + .transpose()?, + }) + } +} + +impl TryFrom<&WireOptionDescriptor> for DescribedOption { + type Error = IggyError; + + fn try_from(entry: &WireOptionDescriptor) -> Result<Self, Self::Error> { + let kind = HeaderKind::from_code(entry.kind)?; + let default = (!entry.default_value.is_empty()) + .then(|| { + HeaderValue::from_raw(kind, &entry.default_value) + .and_then(|value| encode_value(&value)) + }) + .transpose()?; + Ok(Self { + key: entry.key.to_string(), + kind, + flags: entry.flags, + default, + }) + } +} + +impl TryFrom<&IdentifierInput> for Identifier { + type Error = IggyError; + + fn try_from(input: &IdentifierInput) -> Result<Self, Self::Error> { + match input { + IdentifierInput::Numeric(value) => Self::numeric(*value), + IdentifierInput::Named(name) => Self::named(name), + } + } +} + +impl From<&IggyError> for ErrorExpectation { + fn from(error: &IggyError) -> Self { + Self { + name: error.as_string().to_owned(), + code: error.as_code(), + } + } +} + +impl From<&StreamDetails> for StreamProjection { + fn from(details: &StreamDetails) -> Self { + Self { + id: details.id, + created_at: details.created_at.as_micros(), + name: details.name.clone(), + topics_count: details.topics_count, + size_bytes: details.size.as_bytes_u64(), + messages_count: details.messages_count, + options: details.options.clone(), + encryption_key_source: details.encryption_key_source(), + encryption_key_fingerprint: details.encryption_key_fingerprint(), + encryption_kek_fingerprints: details.encryption_kek_fingerprints(), + } + } +} + +impl From<&TopicDetails> for TopicProjection { + fn from(details: &TopicDetails) -> Self { + let resolved = details.resolved_options(); + Self { + id: details.id, + created_at: details.created_at.as_micros(), + name: details.name.clone(), + partitions_count: details.partitions_count, + size_bytes: details.size.as_bytes_u64(), + messages_count: details.messages_count, + options: details.options.clone(), + compression_algorithm: resolved.compression_algorithm.to_string(), + message_expiry: u64::from(resolved.message_expiry).to_string(), + max_topic_size: u64::from(resolved.max_topic_size).to_string(), + } + } +} + +impl From<&UserInfoDetails> for UserProjection { + fn from(details: &UserInfoDetails) -> Self { + Self { + id: details.id, + created_at: details.created_at.as_micros(), + status: details.status, + username: details.username.clone(), + options: details.options.clone(), + } + } +} + +/// The `UpdateResource` an SDK client sends for this typed input. +pub fn update_resource_request( + scope: OptionsScope, + target: &TargetInput, + input: &PatchInput, +) -> Result<UpdateResourceRequest, IggyError> { + let patch = input.patch(scope)?; + Ok(UpdateResourceRequest { + target: target.wire(scope)?, + reset: patch.reset, + options: patch.set, + }) +} + +/// The server's catalog admission of a decoded `UpdateResource`. +pub fn admit_update(request: &UpdateResourceRequest) -> Result<(), IggyError> { + let reset: Vec<String> = request + .reset + .iter() + .map(|name| name.as_str().to_owned()) + .collect(); + admit( + OptionsScope::from_code(request.target.scope_code())?, + OptionOperation::Update, + &request.options, + &reset, + &OptionAdmissionContext::default(), + ) + .map(|_| ()) +} + +/// The server's catalog admission of a create options block. +pub fn admit_create( + scope: OptionsScope, + create_columns: OptionColumns, + options: &WireOptions, +) -> Result<(), IggyError> { + admit( + scope, + OptionOperation::Create, + options, + &[], + &OptionAdmissionContext { + current: None, + create_columns, + }, + ) + .map(|_| ()) +} + +/// An HTTP update body followed through JSON parsing, the handler, the decoder and admission. +pub fn evaluate_http_update( + scope: OptionsScope, + target: &TargetInput, + body: &str, +) -> Result<HttpEvaluation, IggyError> { + let Ok(update) = serde_json::from_str::<OptionsUpdate>(body) else { + return Ok(HttpEvaluation { + update_resource: None, + outcome: HttpOutcome::JsonRefused, + error: None, + }); + }; + let patch = update.to_patch()?; + let bytes = UpdateResourceRequest { + target: target.wire(scope)?, + reset: patch.reset, + options: patch.set, + } + .to_bytes() + .to_vec(); + let verdict = UpdateResourceRequest::decode_from(&bytes) + .map_err(|_| (HttpOutcome::DecodeRefused, IggyError::InvalidCommand)) + .and_then(|request| { + admit_update(&request).map_err(|error| (HttpOutcome::AdmissionRefused, error)) + }); + let (outcome, error) = match verdict { + Ok(()) => (HttpOutcome::Accepted, None), + Err((outcome, error)) => (outcome, Some(ErrorExpectation::from(&error))), + }; + Ok(HttpEvaluation { + update_resource: Some(bytes), + outcome, + error, + }) +} + +pub fn contains_bytes(haystack: &[u8], needle: &[u8]) -> bool { + !needle.is_empty() + && haystack + .windows(needle.len()) + .any(|window| window == needle) +} + +fn decimal(text: Option<&str>) -> Result<Option<u64>, IggyError> { + text.map(u64::from_str) + .transpose() + .map_err(|_| IggyError::InvalidFormat) +}
diff --git a/core/common/fixtures/resource_options.json b/core/common/fixtures/resource_options.json new file mode 100644 index 0000000..c5cc2ac --- /dev/null +++ b/core/common/fixtures/resource_options.json
@@ -0,0 +1,1619 @@ +{ + "schema_version": 1, + "generated_by": "core/common/examples/resource_options_fixtures.rs", + "conventions": { + "byte_strings": "Every *_hex field is lowercase hex without separators.", + "errors": "name is the snake_case IggyError name and code its numeric code.", + "max_topic_size": "Bytes. 18446744073709551615 means unlimited.", + "message_expiry": "Microseconds. 18446744073709551615 means never expire.", + "never_returned_keys": "No response, decoded projection or accessor may contain these keys or their values.", + "option_maps": "Option maps use the HTTP JSON form: value as reversible text, kind, and explicit provenance. Request blocks mark every entry explicit.", + "options_block_order": "Options block entries are encoded in ascending key byte order.", + "request_bodies": "body_hex and update_resource_hex are command payloads without framing. constants.command_codes maps each command to its code and VSR operation.", + "reset_order": "The typed encoders and the HTTP handler write reset keys in ascending key byte order.", + "stream_state": "state is the stored server state a stream response was projected from. SDKs need only response_hex and expected.", + "u64_values": "u64 option values and typed u64 inputs are decimal strings, so values above 2^53 keep their precision." + }, + "constants": { + "protocol_version": { + "packed": 12288, + "text": "0.12.0", + "major": 0, + "minor": 12, + "patch": 0 + }, + "protocol_version_min": { + "packed": 12288, + "text": "0.12.0", + "major": 0, + "minor": 12, + "patch": 0 + }, + "command_codes": { + "create_stream": { + "code": 202, + "operation": 128 + }, + "create_topic": { + "code": 302, + "operation": 132 + }, + "create_user": { + "code": 33, + "operation": 141 + }, + "describe_options": { + "code": 13, + "operation": null + }, + "get_stream": { + "code": 200, + "operation": null + }, + "get_streams": { + "code": 201, + "operation": null + }, + "get_topic": { + "code": 300, + "operation": null + }, + "get_topics": { + "code": 301, + "operation": null + }, + "get_user": { + "code": 31, + "operation": null + }, + "get_users": { + "code": 32, + "operation": null + }, + "update_resource": { + "code": 16, + "operation": 150 + } + }, + "error_codes": { + "cannot_modify_root_user": 4108 + }, + "retired_command_codes": { + "update_stream": 204, + "update_topic": 304, + "update_user": 35 + }, + "operations": { + "rewrap_stream_keys": { + "value": 75, + "internal": true, + "request_operation": 75 + }, + "update_resource": { + "value": 150, + "internal": false, + "request_operation": 150 + }, + "update_stream_resolved": { + "value": 72, + "internal": true, + "request_operation": 150 + }, + "update_topic_resolved": { + "value": 73, + "internal": true, + "request_operation": 150 + }, + "update_user_resolved": { + "value": 74, + "internal": true, + "request_operation": 150 + } + }, + "retired_operations": { + "update_stream": 129, + "update_topic": 133, + "update_user": 142 + }, + "scope_codes": { + "stream": 2, + "topic": 1, + "user": 3 + }, + "header_kind_codes": { + "raw": 1, + "string": 2, + "bool": 3, + "uint32": 11, + "uint64": 12 + }, + "option_flags": { + "critical": 16, + "read_only": 8, + "resettable": 2, + "secret": 4, + "updatable": 1 + }, + "user_status_codes": { + "active": 1, + "inactive": 2 + }, + "stream_key_length": 32, + "never_returned_keys": [ + "encryption_key", + "encryption_key_wrapped", + "critical_keys" + ] + }, + "catalog": [ + { + "key": "name", + "scopes": [ + "stream", + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "column", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 1, + "described": true, + "default": null + }, + { + "key": "username", + "scopes": [ + "user" + ], + "kind": "string", + "kind_code": 2, + "storage": "column", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 1, + "described": true, + "default": null + }, + { + "key": "status", + "scopes": [ + "user" + ], + "kind": "string", + "kind_code": 2, + "storage": "column", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 1, + "described": true, + "default": null + }, + { + "key": "compression_algorithm", + "scopes": [ + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "none" + }, + { + "key": "message_expiry", + "scopes": [ + "topic" + ], + "kind": "uint64", + "kind_code": 12, + "storage": "map", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 3, + "described": true, + "default": "18446744073709551615" + }, + { + "key": "max_topic_size", + "scopes": [ + "topic" + ], + "kind": "uint64", + "kind_code": 12, + "storage": "map", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 3, + "described": true, + "default": "18446744073709551615" + }, + { + "key": "segment_size", + "scopes": [ + "topic" + ], + "kind": "uint64", + "kind_code": 12, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "1073741824" + }, + { + "key": "durability", + "scopes": [ + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "replicated" + }, + { + "key": "consumer_offset_durability", + "scopes": [ + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "replicated" + }, + { + "key": "partition_resize_policy", + "scopes": [ + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "mutable" + }, + { + "key": "messages_required_to_save", + "scopes": [ + "topic" + ], + "kind": "uint32", + "kind_code": 11, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "1024" + }, + { + "key": "size_of_messages_required_to_save", + "scopes": [ + "topic" + ], + "kind": "uint64", + "kind_code": 12, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "1048576" + }, + { + "key": "preallocate_segments", + "scopes": [ + "topic" + ], + "kind": "bool", + "kind_code": 3, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "false" + }, + { + "key": "encryption_key", + "scopes": [ + "stream" + ], + "kind": "raw", + "kind_code": 1, + "storage": "map", + "mutability": "create_only", + "visibility": "secret", + "critical": true, + "flags": 20, + "described": true, + "default": null + }, + { + "key": "encryption_key_wrapped", + "scopes": [ + "stream" + ], + "kind": "raw", + "kind_code": 1, + "storage": "map", + "mutability": "create_only", + "visibility": "internal", + "critical": true, + "flags": 16, + "described": false, + "default": null + }, + { + "key": "encryption_key_source", + "scopes": [ + "stream" + ], + "kind": "string", + "kind_code": 2, + "storage": "computed", + "mutability": "read_only", + "visibility": "read_only", + "critical": false, + "flags": 8, + "described": true, + "default": null + }, + { + "key": "encryption_key_fingerprint", + "scopes": [ + "stream" + ], + "kind": "string", + "kind_code": 2, + "storage": "computed", + "mutability": "read_only", + "visibility": "read_only", + "critical": false, + "flags": 8, + "described": true, + "default": null + }, + { + "key": "encryption_kek_fingerprint", + "scopes": [ + "stream" + ], + "kind": "string", + "kind_code": 2, + "storage": "computed", + "mutability": "read_only", + "visibility": "read_only", + "critical": false, + "flags": 8, + "described": true, + "default": null + }, + { + "key": "critical_keys", + "scopes": [ + "stream", + "topic", + "user" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "internal", + "critical": false, + "flags": 0, + "described": false, + "default": null + } + ], + "describe_options": [ + { + "scope": "topic", + "request_hex": "01", + "response_hex": "0b000000046e616d650201000000007e0000005265736f75726365206e616d652c203120746f203235352062797465732c20756e697175652077697468696e2069747320706172656e742e20536574206279207468652063726561746520636f6d6d616e642773206e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e15636f6d7072657373696f6e5f616c676f726974686d0200040000006e6f6e6550000000436f6d7072657373696f6e20616c676f726974686d20286e6f6e652c20677a6970292e2053746f72656420616e64207265706f727465642c206e6f74206170706c69656420746f206d657373616765730e6d6573736167655f6578706972790c0308000000ffffffffffffffff430000004d6573736167652065787069727920696e206d6963726f7365636f6e64732c206f7220612068756d616e74696d6520737472696e672028652e672e20372064617973290e6d61785f746f7069635f73697a650c0308000000ffffffffffffffff9d000000546f7069632d77696465207365616c65642d7365676d656e742073697a65206361702c2073706c6974206163726f737320616c6c20706172746974696f6e732c20696e206279746573206f72206120627974652d73697a6520737472696e672028652e672e203120476942293b2066696e6974652076616c756573206d757374206265206174206c6561737420746865207365676d656e742073697a650c7365676d656e745f73697a650c000800000000000040000000006c0000005365676d656e742073697a6520696e2062797465732c206f72206120627974652d73697a6520737472696e672028652e672e20313238204d6942293b2061203531322d62797465206d756c7469706c652077697468696e20313034383537362e2e3d313037333734313832340a6475726162696c69747902000a0000007265706c6963617465643e0100004d65737361676520636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20412073696e676c65746f6e2071756f72756d20686173206f6e6520636f70792e20496e207265706c6963617465642067726f7570732c20706572736973746564206d65737361676573207573652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c2057414c207265636c616d6174696f6e2e205468652066756c6c20626f64792073697a65207374696c6c20636f756e747320616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e1a636f6e73756d65725f6f66667365745f6475726162696c69747902000a0000007265706c6963617465646e0100004578706c69636974206f666673657420636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20506f6c6c206175746f2d636f6d6d69742072656d61696e73206173796e6368726f6e6f75732e20496e207265706c6963617465642067726f7570732c20706572736973746564206f66667365747320616c736f20656e61626c652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c207265636c616d6174696f6e2c206576656e2077697468207265706c696361746564206d657373616765206475726162696c6974792e2046756c6c20626f64792073697a657320636f756e7420616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e17706172746974696f6e5f726573697a655f706f6c6963790200070000006d757461626c65c8000000496d6d757461626c65206166746572206372656174696f6e3a206d757461626c65207065726d69747320706172746974696f6e206164646974696f6e2f64656c6574696f6e20776974686f7574206120626f756e646564206375746f766572206f72207065722d6b6579206f72646572696e67206163726f7373206c61796f7574733b2066697865642072656a6563747320626f74682e20526574656e74696f6e20616e6420746f7069632064656c6574696f6e2072656d61696e2064657374727563746976652e196d657373616765735f72657175697265645f746f5f736176650b000400000000040000870000004f7264696e617279206d6573736167652d636f756e7420666c75736820747269676765723b20312e2e3d31363737373231362e2052657175697265642070657273697374656e63652c2063617061636974792070726573737572652c20616e64206c6966656379636c65206f7065726174696f6e732063616e20666c757368206561726c6965722173697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c0008000000000010000000000083000000466c75736820746865206a6f75726e616c206f6e636520697420686f6c64732074686973206d616e792062797465732c206f72206120627974652d73697a6520737472696e673b20776869636865766572207468726573686f6c6420747269707320666972737420666c75736865732e204174206d6f7374203130373337343138323414707265616c6c6f636174655f7365676d656e7473030001000000001201000052657175657374207365676d656e745f73697a65206279746573206f66206469736b207265736572766174696f6e207768656e2065616368207365676d656e74206f70656e732e20556e737570706f72746564206f72206661696c6564207265736572766174696f6e732066616c6c206261636b20746f206f7264696e61727920616c6c6f636174696f6e20776974682061207761726e696e672e205265736572766174696f6e2072756e7320696e6c696e65206f6e20746865206f776e696e672073686172642e207365676d656e745f73697a65202a20706172746974696f6e735f636f756e74206973206361707065642061742036383731393437363733362062797465732070657220637265617465", + "entries": [ + { + "key": "name", + "kind": "string", + "flags": 1, + "default": null + }, + { + "key": "compression_algorithm", + "kind": "string", + "flags": 0, + "default": "none" + }, + { + "key": "message_expiry", + "kind": "uint64", + "flags": 3, + "default": "18446744073709551615" + }, + { + "key": "max_topic_size", + "kind": "uint64", + "flags": 3, + "default": "18446744073709551615" + }, + { + "key": "segment_size", + "kind": "uint64", + "flags": 0, + "default": "1073741824" + }, + { + "key": "durability", + "kind": "string", + "flags": 0, + "default": "replicated" + }, + { + "key": "consumer_offset_durability", + "kind": "string", + "flags": 0, + "default": "replicated" + }, + { + "key": "partition_resize_policy", + "kind": "string", + "flags": 0, + "default": "mutable" + }, + { + "key": "messages_required_to_save", + "kind": "uint32", + "flags": 0, + "default": "1024" + }, + { + "key": "size_of_messages_required_to_save", + "kind": "uint64", + "flags": 0, + "default": "1048576" + }, + { + "key": "preallocate_segments", + "kind": "bool", + "flags": 0, + "default": "false" + } + ] + }, + { + "scope": "stream", + "request_hex": "02", + "response_hex": "05000000046e616d650201000000007e0000005265736f75726365206e616d652c203120746f203235352062797465732c20756e697175652077697468696e2069747320706172656e742e20536574206279207468652063726561746520636f6d6d616e642773206e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e0e656e6372797074696f6e5f6b6579011400000000a30100005468652073747265616d2773206f776e206461746120656e6372797074696f6e206b65793a2033322072616e646f6d2062797465732c20726177206f72207374616e6461726420706164646564206261736536342e204372656174652d6f6e6c7920616e642077726974652d6f6e6c793a20746865207365727665722073746f726573206974207772617070656420756e6465722061206b657920646572697665642066726f6d207468652073657276657220656e6372797074696f6e206b657920616e64206e657665722072657475726e732069742e205265717569726573207468652073657276657220656e6372797074696f6e206b657920746f20626520636f6e666967757265642e20576974686f75742069742c206120736572766572207769746820656e6372797074696f6e20656e61626c65642067656e6572617465732061206b657920666f72207468652073747265616d2e20546865206b65792074726176656c7320696e20746865206372656174652072657175657374206f76657220746865207472616e73706f72742074686520636c69656e7420757365732e15656e6372797074696f6e5f6b65795f736f757263650208000000009f00000057686f2063686f7365207468652073747265616d27732061742d72657374206b65793a2073747265616d207768656e2074686520636c69656e742073656e742069742c20736572766572207768656e20746865207365727665722067656e6572617465642069742c206e6f6e65207768656e207468652073747265616d2773206d65737361676573206172652073746f72656420756e656e637279707465641a656e6372797074696f6e5f6b65795f66696e6765727072696e740208000000009f0000005468652066697273742038206279746573206f66205348412d323536206f76657220746865206c6162656c20696767792f73747265616d2d6b65792d66696e6765727072696e742f763120616e64207468652073747265616d2773206b65792c206173203136206c6f776572636173652068657820636861726163746572732e20416273656e74207768656e2074686520736f75726365206973206e6f6e651a656e6372797074696f6e5f6b656b5f66696e6765727072696e74020800000000b800000046696e6765727072696e7473206f6620746865206b65792d656e6372797074696f6e206b65797320746861742077726170207468652073747265616d2773206b65792c20636f6d6d612d7365706172617465643a206f6e65206f7574736964652061206b657920726f746174696f6e2c206f6e652070657220636f6e6669677572656420736572766572206b657920647572696e672069742e20416273656e74207768656e2074686520736f75726365206973206e6f6e65", + "entries": [ + { + "key": "name", + "kind": "string", + "flags": 1, + "default": null + }, + { + "key": "encryption_key", + "kind": "raw", + "flags": 20, + "default": null + }, + { + "key": "encryption_key_source", + "kind": "string", + "flags": 8, + "default": null + }, + { + "key": "encryption_key_fingerprint", + "kind": "string", + "flags": 8, + "default": null + }, + { + "key": "encryption_kek_fingerprint", + "kind": "string", + "flags": 8, + "default": null + } + ] + }, + { + "scope": "user", + "request_hex": "03", + "response_hex": "0200000008757365726e616d650201000000006b00000055736572206e616d652c203320746f2035302062797465732c20756e697175652e20536574206279207468652063726561746520636f6d6d616e64277320757365726e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e067374617475730201000000008f00000055736572207374617475732c20616374697665206f7220696e6163746976652e20536574206279207468652063726561746520636f6d6d616e64277320737461747573206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e20446561637469766174696e672061207573657220656e6473206974732073657373696f6e732e", + "entries": [ + { + "key": "username", + "kind": "string", + "flags": 1, + "default": null + }, + { + "key": "status", + "kind": "string", + "flags": 1, + "default": null + } + ] + } + ], + "stream_key": { + "key_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "key_base64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", + "fingerprint_label": "iggy/stream-key-fingerprint/v1", + "fingerprint": "8441e38538cbfa57", + "invalid_keys": [ + { + "case": "raw_31_bytes", + "input": { + "raw_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + }, + { + "case": "raw_33_bytes", + "input": { + "raw_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + }, + { + "case": "base64_with_whitespace", + "input": { + "base64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=\n" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + }, + { + "case": "base64_missing_padding", + "input": { + "base64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + }, + { + "case": "base64_noncanonical_trailing_bits", + "input": { + "base64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh9=" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + } + ] + }, + "update_resource": [ + { + "case": "topic_set_expiry_without_name_reset_max_topic_size", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "message_expiry": "604800000000", + "reset": [ + "max_topic_size" + ] + }, + "wire_set": { + "message_expiry": { + "value": "604800000000", + "kind": "uint64", + "explicit": true + } + }, + "wire_reset": [ + "max_topic_size" + ], + "body_hex": "01010401000000010402000000010e6d61785f746f7069635f73697a65020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000" + }, + { + "case": "topic_rename_only_named_identifiers", + "scope": "topic", + "target": { + "stream_id": "orders", + "topic_id": "events" + }, + "input": { + "name": "events-v2" + }, + "wire_set": { + "name": { + "value": "events-v2", + "kind": "string", + "explicit": true + } + }, + "wire_reset": [], + "body_hex": "0102066f726465727302066576656e74730002040000006e616d6502090000006576656e74732d7632" + }, + { + "case": "topic_rename_set_expiry_reset_max_topic_size", + "scope": "topic", + "target": { + "stream_id": 7, + "topic_id": "events" + }, + "input": { + "name": "events-v3", + "message_expiry": "86400000000", + "reset": [ + "max_topic_size" + ] + }, + "wire_set": { + "message_expiry": { + "value": "86400000000", + "kind": "uint64", + "explicit": true + }, + "name": { + "value": "events-v3", + "kind": "string", + "explicit": true + } + }, + "wire_reset": [ + "max_topic_size" + ], + "body_hex": "0101040700000002066576656e7473010e6d61785f746f7069635f73697a65020e0000006d6573736167655f6578706972790c080000000060d71d1400000002040000006e616d6502090000006576656e74732d7633" + }, + { + "case": "topic_reset_both_defaults", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "reset": [ + "max_topic_size", + "message_expiry" + ] + }, + "wire_set": {}, + "wire_reset": [ + "max_topic_size", + "message_expiry" + ], + "body_hex": "01010401000000010402000000020e6d61785f746f7069635f73697a650e6d6573736167655f657870697279" + }, + { + "case": "topic_max_topic_size_above_2_pow_53", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "max_topic_size": "9007199254740993" + }, + "wire_set": { + "max_topic_size": { + "value": "9007199254740993", + "kind": "uint64", + "explicit": true + } + }, + "wire_reset": [], + "body_hex": "0101040100000001040200000000020e0000006d61785f746f7069635f73697a650c080000000100000000002000" + }, + { + "case": "stream_rename", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "input": { + "name": "payments-v2" + }, + "wire_set": { + "name": { + "value": "payments-v2", + "kind": "string", + "explicit": true + } + }, + "wire_reset": [], + "body_hex": "020104030000000002040000006e616d65020b0000007061796d656e74732d7632" + }, + { + "case": "user_username_and_status", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "username": "alice2", + "status": "inactive" + }, + "wire_set": { + "status": { + "value": "inactive", + "kind": "string", + "explicit": true + }, + "username": { + "value": "alice2", + "kind": "string", + "explicit": true + } + }, + "wire_reset": [], + "body_hex": "030104050000000002060000007374617475730208000000696e6163746976650208000000757365726e616d650206000000616c69636532" + } + ], + "update_resource_errors": [ + { + "case": "topic_empty_update", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": {}, + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_set_and_reset_same_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "message_expiry": "604800000000", + "reset": [ + "message_expiry" + ] + }, + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_reset_create_only_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "reset": [ + "compression_algorithm" + ] + }, + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "stream_reset_name", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "input": { + "reset": [ + "name" + ] + }, + "error": { + "name": "option_not_resettable", + "code": 4104 + } + }, + { + "case": "user_reset_status", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "reset": [ + "status" + ] + }, + "error": { + "name": "option_not_resettable", + "code": 4104 + } + }, + { + "case": "stream_reset_read_only_key", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "input": { + "reset": [ + "encryption_key_source" + ] + }, + "error": { + "name": "read_only_option", + "code": 4105 + } + }, + { + "case": "user_reset_internal_key", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "reset": [ + "critical_keys" + ] + }, + "error": { + "name": "read_only_option", + "code": 4105 + } + }, + { + "case": "topic_empty_name", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "name": "" + }, + "error": { + "name": "invalid_topic_name", + "code": 2014 + } + }, + { + "case": "topic_empty_name_and_create_only_reset", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "name": "", + "reset": [ + "segment_size" + ] + }, + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "stream_empty_name", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "input": { + "name": "" + }, + "error": { + "name": "invalid_stream_name", + "code": 1013 + } + }, + { + "case": "user_short_username", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "username": "a" + }, + "error": { + "name": "invalid_username", + "code": 43 + } + }, + { + "case": "user_short_username_and_status_reset", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "username": "a", + "reset": [ + "status" + ] + }, + "error": { + "name": "option_not_resettable", + "code": 4104 + } + } + ], + "create_stream": [ + { + "case": "stream_without_key", + "input": { + "name": "orders" + }, + "wire_options": {}, + "body_hex": "066f7264657273", + "http_body": "{\"name\":\"orders\"}", + "admission_error": null + }, + { + "case": "stream_with_key", + "input": { + "name": "orders", + "encryption_key": { + "raw_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f" + } + }, + "wire_options": { + "encryption_key": { + "value": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", + "kind": "raw", + "explicit": true + } + }, + "body_hex": "066f7264657273020e000000656e6372797074696f6e5f6b65790120000000000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "http_body": "{\"name\":\"orders\",\"options\":{\"encryption_key\":\"AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=\"}}", + "admission_error": null + }, + { + "case": "stream_with_raw_extra_key", + "input": { + "name": "orders", + "raw": { + "future_option": "enabled" + } + }, + "wire_options": { + "future_option": { + "value": "enabled", + "kind": "string", + "explicit": true + } + }, + "body_hex": "066f7264657273020d0000006675747572655f6f7074696f6e0207000000656e61626c6564", + "http_body": "{\"name\":\"orders\",\"options\":{\"future_option\":\"enabled\"}}", + "admission_error": { + "name": "unsupported_option_key", + "code": 4041 + } + } + ], + "create_topic": [ + { + "case": "topic_with_options", + "input": { + "stream_id": 1, + "name": "events", + "partitions_count": 3, + "compression_algorithm": "gzip", + "message_expiry": "604800000000", + "max_topic_size": "9007199254740993", + "segment_size": "134217728", + "durability": "persisted", + "consumer_offset_durability": "replicated" + }, + "wire_options": { + "compression_algorithm": { + "value": "gzip", + "kind": "string", + "explicit": true + }, + "consumer_offset_durability": { + "value": "replicated", + "kind": "string", + "explicit": true + }, + "durability": { + "value": "persisted", + "kind": "string", + "explicit": true + }, + "max_topic_size": { + "value": "9007199254740993", + "kind": "uint64", + "explicit": true + }, + "message_expiry": { + "value": "604800000000", + "kind": "uint64", + "explicit": true + }, + "segment_size": { + "value": "134217728", + "kind": "uint64", + "explicit": true + } + }, + "body_hex": "01040100000003000000066576656e74730215000000636f6d7072657373696f6e5f616c676f726974686d0204000000677a6970021a000000636f6e73756d65725f6f66667365745f6475726162696c697479020a0000007265706c696361746564020a0000006475726162696c6974790209000000706572736973746564020e0000006d61785f746f7069635f73697a650c080000000100000000002000020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000020c0000007365676d656e745f73697a650c080000000000000800000000", + "http_body": "{\"partitions_count\":3,\"name\":\"events\",\"options\":{\"compression_algorithm\":\"gzip\",\"consumer_offset_durability\":\"replicated\",\"durability\":\"persisted\",\"max_topic_size\":\"9007199254740993\",\"message_expiry\":604800000000,\"segment_size\":134217728}}", + "admission_error": null + } + ], + "create_user": [ + { + "case": "user_with_empty_options", + "input": { + "username": "alice", + "password": "s3cret-pass", + "status": "active" + }, + "wire_options": {}, + "body_hex": "05616c6963650b7333637265742d706173730100", + "http_body": "{\"username\":\"alice\",\"password\":\"s3cret-pass\",\"status\":\"active\",\"permissions\":null}", + "admission_error": null + } + ], + "http_update": [ + { + "case": "topic_flat_keys", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"name\":\"events-v2\",\"message_expiry\":\"7 days\"}", + "update_resource_hex": "0101040100000001040200000000020e0000006d6573736167655f657870697279020600000037206461797302040000006e616d6502090000006576656e74732d7632", + "outcome": "accepted", + "error": null + }, + { + "case": "topic_reset_array", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":604800000000,\"reset\":[\"max_topic_size\"]}", + "update_resource_hex": "01010401000000010402000000010e6d61785f746f7069635f73697a65020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000", + "outcome": "accepted", + "error": null + }, + { + "case": "topic_null_ignored", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"name\":null,\"max_topic_size\":null,\"message_expiry\":\"1 day\"}", + "update_resource_hex": "0101040100000001040200000000020e0000006d6573736167655f65787069727902050000003120646179", + "outcome": "accepted", + "error": null + }, + { + "case": "topic_u64_above_2_pow_53_as_string", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"max_topic_size\":\"9007199254740993\"}", + "update_resource_hex": "0101040100000001040200000000020e0000006d61785f746f7069635f73697a65021000000039303037313939323534373430393933", + "outcome": "accepted", + "error": null + }, + { + "case": "stream_rename", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "body": "{\"name\":\"payments-v2\"}", + "update_resource_hex": "020104030000000002040000006e616d65020b0000007061796d656e74732d7632", + "outcome": "accepted", + "error": null + }, + { + "case": "user_username_and_status", + "scope": "user", + "target": { + "user_id": 5 + }, + "body": "{\"username\":\"alice2\",\"status\":\"inactive\"}", + "update_resource_hex": "030104050000000002060000007374617475730208000000696e6163746976650208000000757365726e616d650206000000616c69636532", + "outcome": "accepted", + "error": null + }, + { + "case": "topic_duplicate_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":\"1 day\",\"message_expiry\":\"2 days\"}", + "update_resource_hex": null, + "outcome": "json_refused", + "error": null + }, + { + "case": "topic_nested_object", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":{\"value\":\"1 day\"}}", + "update_resource_hex": null, + "outcome": "json_refused", + "error": null + }, + { + "case": "topic_null_only", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"max_topic_size\":null}", + "update_resource_hex": "0101040100000001040200000000", + "outcome": "decode_refused", + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_zero_sentinel_only", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":0}", + "update_resource_hex": "0101040100000001040200000000020e0000006d6573736167655f6578706972790c080000000000000000000000", + "outcome": "admission_refused", + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_reset_names_set_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":\"1 day\",\"reset\":[\"message_expiry\"]}", + "update_resource_hex": "01010401000000010402000000010e6d6573736167655f657870697279020e0000006d6573736167655f65787069727902050000003120646179", + "outcome": "decode_refused", + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_unknown_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"retention\":\"1 day\"}", + "update_resource_hex": "01010401000000010402000000000209000000726574656e74696f6e02050000003120646179", + "outcome": "admission_refused", + "error": { + "name": "unsupported_option_key", + "code": 4041 + } + }, + { + "case": "topic_set_create_only_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"compression_algorithm\":\"gzip\"}", + "update_resource_hex": "01010401000000010402000000000215000000636f6d7072657373696f6e5f616c676f726974686d0204000000677a6970", + "outcome": "admission_refused", + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "topic_reset_create_only_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"reset\":[\"compression_algorithm\"]}", + "update_resource_hex": "010104010000000104020000000115636f6d7072657373696f6e5f616c676f726974686d", + "outcome": "admission_refused", + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "stream_reset_name", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "body": "{\"reset\":[\"name\"]}", + "update_resource_hex": "0201040300000001046e616d65", + "outcome": "admission_refused", + "error": { + "name": "option_not_resettable", + "code": 4104 + } + }, + { + "case": "topic_name_as_number", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"name\":5}", + "update_resource_hex": "010104010000000104020000000002040000006e616d650c080000000500000000000000", + "outcome": "admission_refused", + "error": { + "name": "invalid_topic_name", + "code": 2014 + } + }, + { + "case": "stream_empty_name", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "body": "{\"name\":\"\"}", + "update_resource_hex": "020104030000000002040000006e616d65030100000000", + "outcome": "admission_refused", + "error": { + "name": "invalid_stream_name", + "code": 1013 + } + }, + { + "case": "user_status_as_bool", + "scope": "user", + "target": { + "user_id": 5 + }, + "body": "{\"status\":true}", + "update_resource_hex": "03010405000000000206000000737461747573030100000001", + "outcome": "admission_refused", + "error": { + "name": "invalid_user_status", + "code": 45 + } + }, + { + "case": "topic_create_only_key_as_bool", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"segment_size\":true}", + "update_resource_hex": "0101040100000001040200000000020c0000007365676d656e745f73697a65030100000001", + "outcome": "admission_refused", + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "stream_read_only_key_as_number", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "body": "{\"encryption_key_source\":1}", + "update_resource_hex": "02010403000000000215000000656e6372797074696f6e5f6b65795f736f757263650c080000000100000000000000", + "outcome": "admission_refused", + "error": { + "name": "read_only_option", + "code": 4105 + } + }, + { + "case": "topic_invalid_value", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":\"soon\"}", + "update_resource_hex": "0101040100000001040200000000020e0000006d6573736167655f6578706972790204000000736f6f6e", + "outcome": "admission_refused", + "error": { + "name": "invalid_option_value", + "code": 4042 + } + } + ], + "stream_responses": [ + { + "case": "stream_source_none", + "state": { + "encryption_key_source": "none", + "stored_extra": {} + }, + "response_hex": "0100000000e090663c1306000000000000100000000000000c000000000000000c706c61696e2d73747265616d00000000230000000215000000656e6372797074696f6e5f6b65795f736f7572636502040000006e6f6e65", + "expected": { + "id": 1, + "created_at": 1710000000000000, + "name": "plain-stream", + "topics_count": 0, + "size_bytes": 4096, + "messages_count": 12, + "options": { + "encryption_key_source": { + "value": "none", + "kind": "string", + "explicit": false + } + }, + "encryption_key_source": "none", + "encryption_key_fingerprint": null, + "encryption_kek_fingerprints": [] + } + }, + { + "case": "stream_source_stream_with_unknown_explicit_key", + "state": { + "encryption_key_source": "stream", + "data_key": { + "raw_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f" + }, + "kek_config_keys": [ + "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=" + ], + "stored_extra": { + "future_option": { + "value": "enabled", + "kind": "string", + "explicit": true + } + } + }, + "response_hex": "0200000000e090663c1306000000000000100000000000000c0000000000000011636c69656e742d6b65792d73747265616d1e000000020d0000006675747572655f6f7074696f6e0207000000656e61626c65648d000000021a000000656e6372797074696f6e5f6b656b5f66696e6765727072696e74021000000033396534663066393563333639336337021a000000656e6372797074696f6e5f6b65795f66696e6765727072696e740210000000383434316533383533386362666135370215000000656e6372797074696f6e5f6b65795f736f75726365020600000073747265616d", + "expected": { + "id": 2, + "created_at": 1710000000000000, + "name": "client-key-stream", + "topics_count": 0, + "size_bytes": 4096, + "messages_count": 12, + "options": { + "encryption_kek_fingerprint": { + "value": "39e4f0f95c3693c7", + "kind": "string", + "explicit": false + }, + "encryption_key_fingerprint": { + "value": "8441e38538cbfa57", + "kind": "string", + "explicit": false + }, + "encryption_key_source": { + "value": "stream", + "kind": "string", + "explicit": false + }, + "future_option": { + "value": "enabled", + "kind": "string", + "explicit": true + } + }, + "encryption_key_source": "stream", + "encryption_key_fingerprint": "8441e38538cbfa57", + "encryption_kek_fingerprints": [ + "39e4f0f95c3693c7" + ] + } + }, + { + "case": "stream_source_server_two_keks", + "state": { + "encryption_key_source": "server", + "data_key": { + "raw_hex": "202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f" + }, + "kek_config_keys": [ + "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=", + "AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA=" + ], + "stored_extra": {} + }, + "response_hex": "0300000000e090663c1306000000000000100000000000000c00000000000000117365727665722d6b65792d73747265616d000000009e000000021a000000656e6372797074696f6e5f6b656b5f66696e6765727072696e740221000000333965346630663935633336393363372c64353564343739383239626332393336021a000000656e6372797074696f6e5f6b65795f66696e6765727072696e740210000000306563616666316261323931666530640215000000656e6372797074696f6e5f6b65795f736f757263650206000000736572766572", + "expected": { + "id": 3, + "created_at": 1710000000000000, + "name": "server-key-stream", + "topics_count": 0, + "size_bytes": 4096, + "messages_count": 12, + "options": { + "encryption_kek_fingerprint": { + "value": "39e4f0f95c3693c7,d55d479829bc2936", + "kind": "string", + "explicit": false + }, + "encryption_key_fingerprint": { + "value": "0ecaff1ba291fe0d", + "kind": "string", + "explicit": false + }, + "encryption_key_source": { + "value": "server", + "kind": "string", + "explicit": false + } + }, + "encryption_key_source": "server", + "encryption_key_fingerprint": "0ecaff1ba291fe0d", + "encryption_kek_fingerprints": [ + "39e4f0f95c3693c7", + "d55d479829bc2936" + ] + } + } + ], + "topic_responses": [ + { + "case": "topic_explicit_and_derived", + "response_hex": "0400000000e090663c1306000000000000080000000000000700000000000000066576656e7473430000000215000000636f6d7072657373696f6e5f616c676f726974686d0204000000677a6970020e0000006d6573736167655f6578706972790c0800000000a0e3d08c0000002b010000021a000000636f6e73756d65725f6f66667365745f6475726162696c697479020a0000007265706c696361746564020a0000006475726162696c697479020a0000007265706c696361746564020e0000006d61785f746f7069635f73697a650c08000000ffffffffffffffff02190000006d657373616765735f72657175697265645f746f5f736176650b04000000000400000217000000706172746974696f6e5f726573697a655f706f6c69637902070000006d757461626c650214000000707265616c6c6f636174655f7365676d656e7473030100000000020c0000007365676d656e745f73697a650c080000000000004000000000022100000073697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c080000000000100000000000", + "expected": { + "id": 4, + "created_at": 1710000000000000, + "name": "events", + "partitions_count": 0, + "size_bytes": 2048, + "messages_count": 7, + "options": { + "compression_algorithm": { + "value": "gzip", + "kind": "string", + "explicit": true + }, + "consumer_offset_durability": { + "value": "replicated", + "kind": "string", + "explicit": false + }, + "durability": { + "value": "replicated", + "kind": "string", + "explicit": false + }, + "max_topic_size": { + "value": "18446744073709551615", + "kind": "uint64", + "explicit": false + }, + "message_expiry": { + "value": "604800000000", + "kind": "uint64", + "explicit": true + }, + "messages_required_to_save": { + "value": "1024", + "kind": "uint32", + "explicit": false + }, + "partition_resize_policy": { + "value": "mutable", + "kind": "string", + "explicit": false + }, + "preallocate_segments": { + "value": "false", + "kind": "bool", + "explicit": false + }, + "segment_size": { + "value": "1073741824", + "kind": "uint64", + "explicit": false + }, + "size_of_messages_required_to_save": { + "value": "1048576", + "kind": "uint64", + "explicit": false + } + }, + "compression_algorithm": "gzip", + "message_expiry": "604800000000", + "max_topic_size": "18446744073709551615" + } + } + ], + "user_responses": [ + { + "case": "user_empty_blocks", + "response_hex": "0500000000e090663c1306000105616c696365000000000000000000000000", + "expected": { + "id": 5, + "created_at": 1710000000000000, + "status": "active", + "username": "alice", + "options": {} + } + }, + { + "case": "user_populated_blocks", + "response_hex": "0600000000e090663c1306000203626f621e000000020d0000006675747572655f6f7074696f6e0207000000656e61626c65641e000000020c0000006675747572655f71756f74610c08000000010000000000200000000000", + "expected": { + "id": 6, + "created_at": 1710000000000000, + "status": "inactive", + "username": "bob", + "options": { + "future_option": { + "value": "enabled", + "kind": "string", + "explicit": true + }, + "future_quota": { + "value": "9007199254740993", + "kind": "uint64", + "explicit": false + } + } + } + } + ] +}
diff --git a/core/common/src/error/iggy_error.rs b/core/common/src/error/iggy_error.rs index 49357b2..580f7de 100644 --- a/core/common/src/error/iggy_error.rs +++ b/core/common/src/error/iggy_error.rs
@@ -496,6 +496,25 @@ /// open fails instead. Field order: `(on_disk, expected)`. #[error("Segment file size on disk: {0} does not match expected size: {1}")] SegmentSizeMismatchAtOpen(u64, u64) = 4102, + /// A create-only option key in an update's set or reset list. + #[error("Option cannot be updated after creation: {0}")] + OptionNotUpdatable(String) = 4103, + /// A reset of a key without a default, or of a fixed resource column. + #[error("Option cannot be reset: {0}")] + OptionNotResettable(String) = 4104, + /// A client sent a server-maintained or read-only option key. + #[error("Option is read-only: {0}")] + ReadOnlyOption(String) = 4105, + /// A stream brought its own encryption key but the server has no + /// encryption key configured to wrap it with. + #[error("Stream encryption key requires a server encryption key")] + StreamEncryptionUnavailable = 4106, + /// A resource holds a critical option this build does not understand, so + /// serving it could ignore a security setting. + #[error("Unsupported critical option: {0}")] + UnsupportedCriticalOption(String) = 4107, + #[error("Cannot modify root user")] + CannotModifyRootUser = 4108, #[error("Consumer group with ID: {0} for topic with ID: {1} was not found.")] ConsumerGroupIdNotFound(Identifier, Identifier) = 5000, #[error("Invalid consumer group ID")]
diff --git a/core/common/src/http/mod.rs b/core/common/src/http/mod.rs index ebd5574..aabaa34 100644 --- a/core/common/src/http/mod.rs +++ b/core/common/src/http/mod.rs
@@ -18,6 +18,7 @@ pub(crate) mod consumer_groups; pub(crate) mod consumer_offsets; pub(crate) mod messages; +pub(crate) mod options; pub(crate) mod partitions; pub(crate) mod personal_access_tokens; pub(crate) mod segments;
diff --git a/core/common/src/http/options.rs b/core/common/src/http/options.rs new file mode 100644 index 0000000..75816e2 --- /dev/null +++ b/core/common/src/http/options.rs
@@ -0,0 +1,706 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::IggyError; +use crate::types::message::{HeaderKey, HeaderKind, HeaderValue}; +use crate::types::options::{ + OptionKey, OptionValue, OptionsPatch, OptionsProvenance, ResourceOptions, erase_options, + key_preview, wire_reset_names, +}; +use crate::wire_conversions::resource_options_to_wire; +use base64::Engine; +use base64::engine::general_purpose::STANDARD; +use iggy_binary_protocol::WireOptions; +use secrecy::zeroize::Zeroizing; +use serde::de::{Error as _, MapAccess, Visitor}; +use serde::ser::{Error as _, SerializeMap}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt::{Debug, Formatter}; +use std::str::FromStr; + +/// The update body member that lists the keys to reset. No option is ever +/// named `reset`. +pub const RESET_FIELD: &str = "reset"; + +/// The largest integer a JSON number carries exactly in every client, 2^53 - 1. +const MAX_EXACT_JSON_INTEGER: u64 = (1 << 53) - 1; + +/// One option value in an HTTP body: a JSON boolean, a JSON integer, or a +/// string in the same form a config file uses (`"7 days"`, `"1 GiB"`). A +/// 64-bit value above 2^53 travels as a decimal string so JavaScript clients +/// keep its precision. Text is erased on drop, since it may be the base64 +/// stream key. +#[derive(Clone, PartialEq, Eq)] +pub enum OptionInputValue { + Bool(bool), + U64(u64), + String(Zeroizing<String>), +} + +/// Option values for a create body, keyed by option name. A name repeated in +/// the JSON object is refused rather than silently collapsed. +#[derive(Clone, Default, PartialEq, Eq)] +pub struct OptionInputMap(pub BTreeMap<String, OptionInputValue>); + +/// An HTTP update body: one flat object of option keys plus an optional +/// `reset` array. A value sets the key, an absent key or `null` changes +/// nothing, a key named in `reset` returns to its default. `null` never resets, +/// because many JSON serializers write unset optional fields as `null`. A body +/// that changes nothing, with no keys, only nulls or only zero sentinels such +/// as `"message_expiry": 0`, is refused with `InvalidCommand` on every +/// transport. +#[derive(Clone, Default, PartialEq, Eq)] +pub struct OptionsUpdate { + pub set: BTreeMap<String, OptionInputValue>, + pub reset: BTreeSet<String>, +} + +struct InputValueVisitor; + +struct UniqueKeysVisitor; + +struct OptionsUpdateVisitor; + +impl OptionInputMap { + /// Encode the values into a create options block, each in the kind its + /// catalog entry expects where the JSON value fits it. Admission validates + /// the keys and values, so a mismatch fails in the same category as on + /// the binary transports. The map built on the way may hold the stream + /// key, so it is erased once encoded. + /// + /// # Errors + /// + /// `UnsupportedOptionKey` for a key that is not a valid option name. + pub fn to_wire(&self) -> Result<WireOptions, IggyError> { + let options = input_options(&self.0)?; + let wire = resource_options_to_wire(&options, OptionsProvenance::All); + erase_options(options); + wire + } + + #[must_use] + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +impl OptionsUpdate { + /// Split the body into the keys to set and the keys to reset, in wire + /// form. Admission validates the keys and values. + /// + /// # Errors + /// + /// As [`OptionInputMap::to_wire`], and `UnsupportedOptionKey` for a reset + /// name that is not a valid option name or a reset list longer than the + /// wire carries. + pub fn to_patch(&self) -> Result<OptionsPatch, IggyError> { + let set = input_options(&self.set)?; + let patch = wire_reset_names(self.reset.iter().map(String::as_str)).and_then(|reset| { + Ok(OptionsPatch { + set: resource_options_to_wire(&set, OptionsProvenance::All)?, + reset, + }) + }); + erase_options(set); + patch + } +} + +impl TryFrom<&HeaderValue> for OptionInputValue { + type Error = IggyError; + + /// The HTTP form of a canonical option value. Raw bytes, the stream + /// encryption key, travel as standard base64, and a kind without a JSON + /// form travels as the text a config file uses. + fn try_from(value: &HeaderValue) -> Result<Self, Self::Error> { + match value.kind() { + HeaderKind::String => Ok(Self::String(Zeroizing::new(value.as_str()?.to_owned()))), + HeaderKind::Bool => Ok(Self::Bool(value.as_bool()?)), + HeaderKind::Uint8 => Ok(Self::U64(u64::from(value.as_uint8()?))), + HeaderKind::Uint16 => Ok(Self::U64(u64::from(value.as_uint16()?))), + HeaderKind::Uint32 => Ok(Self::U64(u64::from(value.as_uint32()?))), + HeaderKind::Uint64 => { + let number = value.as_uint64()?; + Ok(if number > MAX_EXACT_JSON_INTEGER { + Self::String(Zeroizing::new(number.to_string())) + } else { + Self::U64(number) + }) + } + HeaderKind::Raw => Ok(Self::String(Zeroizing::new( + STANDARD.encode(value.as_bytes()), + ))), + _ => Ok(Self::String(Zeroizing::new(value.to_string_value()))), + } + } +} + +impl Debug for OptionInputValue { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Self::Bool(value) => write!(formatter, "{value}"), + Self::U64(value) => write!(formatter, "{value}"), + Self::String(value) => write!(formatter, "{:?}", value.as_str()), + } + } +} + +impl Debug for OptionInputMap { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + formatter + .debug_map() + .entries( + self.0 + .iter() + .map(|(key, value)| (key, redacted(key, Some(value)))), + ) + .finish() + } +} + +impl Debug for OptionsUpdate { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("OptionsUpdate") + .field( + "set", + &self + .set + .iter() + .map(|(key, value)| (key, redacted(key, Some(value)))) + .collect::<BTreeMap<_, _>>(), + ) + .field("reset", &self.reset) + .finish() + } +} + +impl Serialize for OptionInputValue { + fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> { + match self { + Self::Bool(value) => serializer.serialize_bool(*value), + Self::U64(value) => serializer.serialize_u64(*value), + Self::String(value) => serializer.serialize_str(value), + } + } +} + +impl<'de> Deserialize<'de> for OptionInputValue { + fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> { + deserializer.deserialize_any(InputValueVisitor) + } +} + +impl Serialize for OptionInputMap { + fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> { + self.0.serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for OptionInputMap { + fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> { + deserializer.deserialize_map(UniqueKeysVisitor) + } +} + +impl Serialize for OptionsUpdate { + fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> { + let reset_entries = usize::from(!self.reset.is_empty()); + if self.set.contains_key(RESET_FIELD) { + return Err(S::Error::custom("no option is named reset")); + } + let mut map = serializer.serialize_map(Some(self.set.len() + reset_entries))?; + for (key, value) in &self.set { + map.serialize_entry(key, value)?; + } + if !self.reset.is_empty() { + map.serialize_entry(RESET_FIELD, &self.reset)?; + } + map.end() + } +} + +impl<'de> Deserialize<'de> for OptionsUpdate { + fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> { + deserializer.deserialize_map(OptionsUpdateVisitor) + } +} + +impl Visitor<'_> for InputValueVisitor { + type Value = OptionInputValue; + + fn expecting(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + formatter.write_str("a boolean, a non-negative integer or a string") + } + + fn visit_bool<E: serde::de::Error>(self, value: bool) -> Result<Self::Value, E> { + Ok(OptionInputValue::Bool(value)) + } + + fn visit_u64<E: serde::de::Error>(self, value: u64) -> Result<Self::Value, E> { + Ok(OptionInputValue::U64(value)) + } + + fn visit_i64<E: serde::de::Error>(self, value: i64) -> Result<Self::Value, E> { + u64::try_from(value) + .map(OptionInputValue::U64) + .map_err(|_| E::custom("a negative integer is not an option value")) + } + + fn visit_str<E: serde::de::Error>(self, value: &str) -> Result<Self::Value, E> { + Ok(OptionInputValue::String(Zeroizing::new(value.to_owned()))) + } + + fn visit_string<E: serde::de::Error>(self, value: String) -> Result<Self::Value, E> { + Ok(OptionInputValue::String(Zeroizing::new(value))) + } +} + +impl<'de> Visitor<'de> for UniqueKeysVisitor { + type Value = OptionInputMap; + + fn expecting(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + formatter.write_str("an object of option names to scalar values") + } + + fn visit_map<A: MapAccess<'de>>(self, mut access: A) -> Result<Self::Value, A::Error> { + let mut values = OptionInputMap::default(); + while let Some((key, value)) = access.next_entry::<String, OptionInputValue>()? { + if values.0.contains_key(&key) { + return Err(A::Error::custom(format!("duplicate option key: {key}"))); + } + values.0.insert(key, value); + } + Ok(values) + } +} + +impl<'de> Visitor<'de> for OptionsUpdateVisitor { + type Value = OptionsUpdate; + + fn expecting(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + formatter + .write_str("an object of option names to scalar values and an optional reset array") + } + + fn visit_map<A: MapAccess<'de>>(self, mut access: A) -> Result<Self::Value, A::Error> { + let mut update = OptionsUpdate::default(); + let mut seen = BTreeSet::new(); + while let Some(key) = access.next_key::<String>()? { + if !seen.insert(key.clone()) { + return Err(A::Error::custom(format!("duplicate option key: {key}"))); + } + if key == RESET_FIELD { + for name in access.next_value::<Vec<String>>()? { + if !update.reset.insert(name.clone()) { + return Err(A::Error::custom(format!("duplicate reset key: {name}"))); + } + } + continue; + } + if let Some(value) = access.next_value::<Option<OptionInputValue>>()? { + update.set.insert(key, value); + } + } + Ok(update) + } +} + +fn input_options( + values: &BTreeMap<String, OptionInputValue>, +) -> Result<ResourceOptions, IggyError> { + let mut options = ResourceOptions::new(); + for (key, value) in values { + let Ok(header_key) = HeaderKey::from_str(key) else { + // An entry taken before this one may hold the stream key. + erase_options(options); + return Err(IggyError::UnsupportedOptionKey(key_preview(key))); + }; + options.insert( + header_key, + OptionValue::explicit(input_header_value(key, value)), + ); + } + Ok(options) +} + +/// The header value an HTTP option value encodes to: the kind its catalog +/// entry expects where the JSON value fits it, otherwise the JSON value's own +/// kind. Admission then refuses a mismatch in its own category order. +/// +/// Text that no header value holds (empty, or over 255 bytes) travels as a +/// value of a kind its key never accepts, so admission still reports the +/// key's own error, a name error for a name and `InvalidOptionValue` for a +/// map key, after every check that comes before it. +fn input_header_value(key: &str, value: &OptionInputValue) -> HeaderValue { + let kind = OptionKey::from_str(key) + .ok() + .map(|option| option.descriptor().kind); + match value { + OptionInputValue::String(text) => { + HeaderValue::try_from(text.as_str()).unwrap_or_else(|_| match kind { + Some(HeaderKind::Bool) => HeaderValue::from(0u64), + _ => HeaderValue::from(false), + }) + } + OptionInputValue::Bool(flag) => HeaderValue::from(*flag), + OptionInputValue::U64(number) => { + number_in_kind(kind, *number).unwrap_or_else(|| HeaderValue::from(*number)) + } + } +} + +fn number_in_kind(kind: Option<HeaderKind>, number: u64) -> Option<HeaderValue> { + match kind? { + HeaderKind::Uint8 => u8::try_from(number).ok().map(HeaderValue::from), + HeaderKind::Uint16 => u16::try_from(number).ok().map(HeaderValue::from), + HeaderKind::Uint32 => u32::try_from(number).ok().map(HeaderValue::from), + HeaderKind::Uint64 => Some(HeaderValue::from(number)), + HeaderKind::Uint128 => Some(HeaderValue::from(u128::from(number))), + HeaderKind::Int8 => i8::try_from(number).ok().map(HeaderValue::from), + HeaderKind::Int16 => i16::try_from(number).ok().map(HeaderValue::from), + HeaderKind::Int32 => i32::try_from(number).ok().map(HeaderValue::from), + HeaderKind::Int64 => i64::try_from(number).ok().map(HeaderValue::from), + HeaderKind::Int128 => Some(HeaderValue::from(i128::from(number))), + _ => None, + } +} + +fn redacted<'a>(key: &str, value: Option<&'a OptionInputValue>) -> Option<&'a dyn Debug> { + if key == <&'static str>::from(OptionKey::EncryptionKey) { + return Some(&"<redacted>"); + } + value.map(|value| value as &dyn Debug) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::options::{ + AdmittedOptions, OptionAdmissionContext, OptionOperation, OptionsScope, admit, + descriptors_for, + }; + use iggy_binary_protocol::WireName; + use secrecy::zeroize::ZeroizeOnDrop; + + #[test] + fn given_update_body_when_parsing_json_should_skip_nulls_and_separate_resets() { + let update: OptionsUpdate = serde_json::from_str( + r#"{"message_expiry":"1 day","max_topic_size":null,"messages_required_to_save":64,"preallocate_segments":true,"reset":["segment_size"]}"#, + ) + .expect("valid update body"); + assert!(!update.set.contains_key("max_topic_size")); + let parts = update.to_patch().expect("encodes"); + assert_eq!(*parts.reset, [WireName::new("segment_size").unwrap()]); + let kinds: BTreeMap<String, u8> = (&parts.set) + .into_iter() + .map(|entry| { + ( + String::from_utf8_lossy(entry.key).into_owned(), + entry.value_kind.0, + ) + }) + .collect(); + assert_eq!( + kinds, + BTreeMap::from([ + ("message_expiry".to_owned(), HeaderKind::String.as_code()), + ( + "messages_required_to_save".to_owned(), + HeaderKind::Uint32.as_code() + ), + ( + "preallocate_segments".to_owned(), + HeaderKind::Bool.as_code() + ), + ]) + ); + } + + #[test] + fn given_update_body_when_serializing_should_omit_unset_keys_and_empty_reset() { + let update = OptionsUpdate { + set: BTreeMap::from([( + "message_expiry".to_owned(), + OptionInputValue::String(Zeroizing::new("7 days".to_owned())), + )]), + reset: BTreeSet::new(), + }; + assert_eq!( + serde_json::to_string(&update).unwrap(), + r#"{"message_expiry":"7 days"}"# + ); + let with_reset = OptionsUpdate { + reset: BTreeSet::from(["max_topic_size".to_owned()]), + ..update + }; + let json = serde_json::to_string(&with_reset).unwrap(); + assert_eq!( + json, + r#"{"message_expiry":"7 days","reset":["max_topic_size"]}"# + ); + assert_eq!( + serde_json::from_str::<OptionsUpdate>(&json).unwrap(), + with_reset + ); + } + + #[test] + fn given_invalid_json_shapes_when_parsing_should_refuse() { + for body in [ + r#"{"message_expiry":1,"message_expiry":2}"#, + r#"{"message_expiry":{"nested":1}}"#, + r#"{"message_expiry":[1]}"#, + r#"[1]"#, + r#""text""#, + r#"{"message_expiry":-1}"#, + r#"{"message_expiry":1.5}"#, + r#"{"reset":"message_expiry"}"#, + r#"{"reset":[1]}"#, + r#"{"reset":["message_expiry","message_expiry"]}"#, + r#"{"reset":null}"#, + ] { + assert!( + serde_json::from_str::<OptionsUpdate>(body).is_err(), + "{body} must be refused" + ); + } + assert!(serde_json::from_str::<OptionInputMap>(r#"{"a":"x","a":"y"}"#).is_err()); + } + + #[test] + fn given_json_value_that_does_not_fit_its_key_when_admitting_should_report_the_binary_category() + { + let long_name = format!(r#"{{"name":"{}"}}"#, "n".repeat(256)); + for (scope, body, expected) in [ + ( + OptionsScope::Topic, + r#"{"name":5}"#, + IggyError::InvalidTopicName, + ), + ( + OptionsScope::Stream, + r#"{"name":""}"#, + IggyError::InvalidStreamName, + ), + ( + OptionsScope::Topic, + long_name.as_str(), + IggyError::InvalidTopicName, + ), + ( + OptionsScope::User, + r#"{"status":true}"#, + IggyError::InvalidUserStatus, + ), + ( + OptionsScope::User, + r#"{"username":""}"#, + IggyError::InvalidUsername, + ), + ( + OptionsScope::User, + r#"{"name":""}"#, + IggyError::UnsupportedOptionKey("name".to_owned()), + ), + ( + OptionsScope::Topic, + r#"{"segment_size":true}"#, + IggyError::OptionNotUpdatable("segment_size".to_owned()), + ), + ( + OptionsScope::Stream, + r#"{"encryption_key_source":1}"#, + IggyError::ReadOnlyOption("encryption_key_source".to_owned()), + ), + ( + OptionsScope::Topic, + r#"{"message_expiry":true}"#, + IggyError::InvalidOptionValue("message_expiry".to_owned()), + ), + ] { + assert_eq!(admit_update(scope, body).err(), Some(expected), "{body}"); + } + for (scope, body, expected) in [ + ( + OptionsScope::Topic, + r#"{"messages_required_to_save":4294967296}"#, + IggyError::InvalidOptionValue("messages_required_to_save".to_owned()), + ), + ( + OptionsScope::Topic, + r#"{"preallocate_segments":""}"#, + IggyError::InvalidOptionValue("preallocate_segments".to_owned()), + ), + ( + OptionsScope::Stream, + r#"{"encryption_key":""}"#, + IggyError::InvalidEncryptionKey, + ), + ] { + assert_eq!(admit_create(scope, body).err(), Some(expected), "{body}"); + } + } + + #[test] + fn given_text_no_header_value_holds_for_any_key_when_admitting_should_refuse() { + let long_text = "v".repeat(256); + for scope in [ + OptionsScope::Topic, + OptionsScope::Stream, + OptionsScope::User, + ] { + for descriptor in descriptors_for(scope) { + for text in ["", long_text.as_str()] { + let body = format!(r#"{{"{}":"{text}"}}"#, descriptor.key); + assert!( + admit_update(scope, &body).is_err(), + "{scope:?} update {}", + descriptor.key + ); + assert!( + admit_create(scope, &body).is_err(), + "{scope:?} create {}", + descriptor.key + ); + } + } + } + } + + #[test] + fn given_oversized_key_when_converting_should_echo_a_bounded_preview() { + let key = "k".repeat(300); + let echoed = |error: Option<IggyError>| match error { + Some(IggyError::UnsupportedOptionKey(shown)) => shown, + other => panic!("expected an unsupported key, got {other:?}"), + }; + let set = OptionsUpdate { + set: BTreeMap::from([(key.clone(), OptionInputValue::U64(1))]), + reset: BTreeSet::new(), + }; + assert_eq!(echoed(set.to_patch().err()), "k".repeat(64)); + let reset = OptionsUpdate { + set: BTreeMap::new(), + reset: BTreeSet::from([key.clone()]), + }; + assert_eq!(echoed(reset.to_patch().err()), "k".repeat(64)); + let create = OptionInputMap(BTreeMap::from([(key, OptionInputValue::U64(1))])); + assert_eq!(echoed(create.to_wire().err()), "k".repeat(64)); + } + + #[test] + fn given_more_resets_than_the_wire_carries_when_converting_should_refuse_unsupported_key() { + let update = OptionsUpdate { + set: BTreeMap::new(), + reset: (0..256).map(|index| format!("key_{index:03}")).collect(), + }; + assert_eq!( + update.to_patch(), + Err(IggyError::UnsupportedOptionKey("key_000".to_owned())) + ); + } + + #[test] + fn given_u64_above_2_pow_53_when_encoding_for_http_should_send_a_decimal_string() { + let exact = (1u64 << 53) - 1; + assert_eq!( + OptionInputValue::try_from(&HeaderValue::from(exact)), + Ok(OptionInputValue::U64(exact)) + ); + assert_eq!( + OptionInputValue::try_from(&HeaderValue::from(exact + 2)), + Ok(OptionInputValue::String(Zeroizing::new( + (exact + 2).to_string() + ))) + ); + } + + #[test] + fn given_set_key_named_reset_when_serializing_should_refuse() { + let update = OptionsUpdate { + set: BTreeMap::from([( + RESET_FIELD.to_owned(), + OptionInputValue::String(Zeroizing::new("x".to_owned())), + )]), + reset: BTreeSet::new(), + }; + assert!(serde_json::to_string(&update).is_err()); + } + + #[test] + fn given_secret_input_when_debugging_should_redact() { + let input: OptionInputMap = + serde_json::from_str(r#"{"encryption_key":"c2VjcmV0LXNlY3JldA=="}"#).unwrap(); + let rendered = format!("{input:?}"); + assert!(!rendered.contains("c2VjcmV0"), "{rendered}"); + let update: OptionsUpdate = + serde_json::from_str(r#"{"encryption_key":"c2VjcmV0LXNlY3JldA=="}"#).unwrap(); + assert!(!format!("{update:?}").contains("c2VjcmV0")); + } + + #[test] + fn given_parsed_bodies_with_stream_key_when_moving_values_out_should_keep_them_erased_on_drop() + { + const BODY: &str = r#"{"encryption_key":"c2VjcmV0LXNlY3JldA==","message_expiry":7}"#; + let input: OptionInputMap = serde_json::from_str(BODY).expect("valid create options"); + let OptionsUpdate { set, reset } = serde_json::from_str(BODY).expect("valid update body"); + assert!(reset.is_empty()); + for mut values in [input.0, set] { + let key = values + .remove(<&str>::from(OptionKey::EncryptionKey)) + .expect("key parsed"); + let OptionInputValue::String(text) = key else { + panic!("the key parses as text"); + }; + assert_erased_on_drop(&text); + assert_eq!(text.as_str(), "c2VjcmV0LXNlY3JldA=="); + assert_eq!(values["message_expiry"], OptionInputValue::U64(7)); + } + } + + fn admit_update(scope: OptionsScope, body: &str) -> Result<AdmittedOptions, IggyError> { + let update: OptionsUpdate = serde_json::from_str(body).expect("valid update body"); + let patch = update.to_patch()?; + let reset: Vec<String> = patch + .reset + .iter() + .map(|name| name.as_str().to_owned()) + .collect(); + admit( + scope, + OptionOperation::Update, + &patch.set, + &reset, + &OptionAdmissionContext::default(), + ) + } + + fn admit_create(scope: OptionsScope, body: &str) -> Result<AdmittedOptions, IggyError> { + let input: OptionInputMap = serde_json::from_str(body).expect("valid create options"); + admit( + scope, + OptionOperation::Create, + &input.to_wire()?, + &[], + &OptionAdmissionContext::default(), + ) + } + + fn assert_erased_on_drop<T: ZeroizeOnDrop>(_: &T) {} +}
diff --git a/core/common/src/http/streams/create_stream.rs b/core/common/src/http/streams/create_stream.rs index 0326b11..9b49dfa 100644 --- a/core/common/src/http/streams/create_stream.rs +++ b/core/common/src/http/streams/create_stream.rs
@@ -16,6 +16,7 @@ // under the License. use super::MAX_NAME_LENGTH; +use crate::OptionInputMap; use crate::Validatable; use crate::error::IggyError; use serde::{Deserialize, Serialize}; @@ -23,16 +24,22 @@ /// `CreateStream` command is used to create a new stream. /// It has additional payload: /// - `name` - unique stream name (string), max length is 255 characters. +/// - `options` - stream options, such as the stream's own `encryption_key` as +/// canonical standard base64. #[derive(Debug, Serialize, Deserialize, PartialEq, Clone)] pub struct CreateStream { /// Unique stream name (string), max length is 255 characters. pub name: String, + /// Stream options keyed by option name. Unknown keys are rejected. + #[serde(default, skip_serializing_if = "OptionInputMap::is_empty")] + pub options: OptionInputMap, } impl Default for CreateStream { fn default() -> Self { CreateStream { name: "stream".to_string(), + options: OptionInputMap::default(), } } }
diff --git a/core/common/src/http/streams/mod.rs b/core/common/src/http/streams/mod.rs index 61ee5e0..34d7c42 100644 --- a/core/common/src/http/streams/mod.rs +++ b/core/common/src/http/streams/mod.rs
@@ -17,6 +17,5 @@ pub mod create_stream; pub mod delete_stream; -pub mod update_stream; use crate::MAX_NAME_LENGTH;
diff --git a/core/common/src/http/streams/update_stream.rs b/core/common/src/http/streams/update_stream.rs deleted file mode 100644 index 1c92741..0000000 --- a/core/common/src/http/streams/update_stream.rs +++ /dev/null
@@ -1,60 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use super::MAX_NAME_LENGTH; -use crate::Identifier; -use crate::Validatable; -use crate::error::IggyError; -use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; - -/// `UpdateStream` command is used to update an existing stream. -/// It has additional payload: -/// - `stream_id` - unique stream ID (numeric or name). -/// - `name` - unique stream name (string), max length is 255 characters. -#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)] -pub struct UpdateStream { - /// Unique stream ID (numeric or name). - #[serde(skip)] - pub stream_id: Identifier, - /// Unique stream name (string), max length is 255 characters. - pub name: String, - /// Additional stream options as string key-values. Restricted to the keys - /// an update may change; anything else is rejected. - #[serde(default)] - pub options: BTreeMap<String, String>, -} - -impl Default for UpdateStream { - fn default() -> Self { - UpdateStream { - stream_id: Identifier::default(), - name: "stream".to_string(), - options: BTreeMap::new(), - } - } -} - -impl Validatable<IggyError> for UpdateStream { - fn validate(&self) -> Result<(), IggyError> { - if self.name.is_empty() || self.name.len() > MAX_NAME_LENGTH { - return Err(IggyError::InvalidStreamName); - } - - Ok(()) - } -}
diff --git a/core/common/src/http/topics/create_topic.rs b/core/common/src/http/topics/create_topic.rs index 99a18f7..08675f3 100644 --- a/core/common/src/http/topics/create_topic.rs +++ b/core/common/src/http/topics/create_topic.rs
@@ -16,23 +16,19 @@ // under the License. use super::{MAX_NAME_LENGTH, MAX_PARTITIONS_COUNT}; -use crate::CompressionAlgorithm; use crate::Identifier; +use crate::OptionInputMap; use crate::Validatable; use crate::error::IggyError; -use crate::utils::expiry::IggyExpiry; -use crate::utils::topic_size::MaxTopicSize; use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; /// `CreateTopic` command is used to create a new topic in a stream. /// It has additional payload: /// - `stream_id` - unique stream ID (numeric or name). /// - `partitions_count` - number of partitions in the topic, max value is 1000. -/// - `message_expiry` - message expiry, if `NeverExpire` then messages will never expire. -/// - `max_topic_size` - maximum size of the topic, if `Unlimited` then topic size is unlimited. -/// Can't be lower than segment size in the config. /// - `name` - unique topic name, max length is 255 characters. +/// - `options` - topic options keyed by option name, including +/// `compression_algorithm`, `message_expiry` and `max_topic_size`. #[derive(Debug, Serialize, Deserialize, PartialEq, Clone)] pub struct CreateTopic { /// Unique stream ID (numeric or name). @@ -40,23 +36,13 @@ pub stream_id: Identifier, /// Number of partitions in the topic, max value is 1000. pub partitions_count: u32, - /// Compression algorithm for the topic. - pub compression_algorithm: CompressionAlgorithm, - /// Message expiry, if `NeverExpire` then messages will never expire. - pub message_expiry: IggyExpiry, - /// Max topic size, if `Unlimited` then topic size is unlimited. - /// Can't be lower than segment size in the config. - pub max_topic_size: MaxTopicSize, /// Unique topic name, max length is 255 characters. pub name: String, - /// Additional topic options as string key-values, parsed by the server - /// with the same rules as its config file. Unknown keys are rejected. - /// - /// A response renders each option as `{"value": "<string>", "explicit": - /// bool}`, in this same string form, so the values a `GET` reports can be - /// sent straight back here. - #[serde(default)] - pub options: BTreeMap<String, String>, + /// Topic options keyed by option name: a JSON string in the config-file + /// form (`"7 days"`, `"1 GiB"`), a JSON integer, or a boolean. Unknown keys + /// are rejected. + #[serde(default, skip_serializing_if = "OptionInputMap::is_empty")] + pub options: OptionInputMap, } impl Default for CreateTopic { @@ -64,11 +50,8 @@ CreateTopic { stream_id: Identifier::default(), partitions_count: 1, - compression_algorithm: CompressionAlgorithm::None, - message_expiry: IggyExpiry::NeverExpire, - max_topic_size: MaxTopicSize::ServerDefault, name: "topic".to_string(), - options: BTreeMap::new(), + options: OptionInputMap::default(), } } }
diff --git a/core/common/src/http/topics/mod.rs b/core/common/src/http/topics/mod.rs index 2d34b55..75c868f 100644 --- a/core/common/src/http/topics/mod.rs +++ b/core/common/src/http/topics/mod.rs
@@ -17,6 +17,5 @@ pub mod create_topic; pub mod delete_topic; -pub mod update_topic; use crate::{MAX_NAME_LENGTH, MAX_PARTITIONS_COUNT};
diff --git a/core/common/src/http/topics/update_topic.rs b/core/common/src/http/topics/update_topic.rs deleted file mode 100644 index 7208c88..0000000 --- a/core/common/src/http/topics/update_topic.rs +++ /dev/null
@@ -1,86 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use super::MAX_NAME_LENGTH; -use crate::CompressionAlgorithm; -use crate::Identifier; -use crate::Validatable; -use crate::error::IggyError; -use crate::utils::expiry::IggyExpiry; -use crate::utils::topic_size::MaxTopicSize; -use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; - -/// `UpdateTopic` command is used to update a topic in a stream. -/// It has additional payload: -/// - `stream_id` - unique stream ID (numeric or name). -/// - `topic_id` - unique topic ID (numeric or name). -/// - `name` - unique topic name, max length is 255 characters. -/// - `compression_algorithm`, `message_expiry`, `max_topic_size` - omit a field -/// to leave the topic's current value alone. Named here for REST ergonomics; -/// the server folds them into the same option keys the binary protocol uses, -/// so there is still one source per setting. -/// - `options` - additional option keys as strings; only keys the update path -/// accepts are allowed. -#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)] -pub struct UpdateTopic { - /// Unique stream ID (numeric or name). - #[serde(skip)] - pub stream_id: Identifier, - /// Unique topic ID (numeric or name). - #[serde(skip)] - pub topic_id: Identifier, - /// Compression algorithm; omit to leave the current one alone. - #[serde(default)] - pub compression_algorithm: Option<CompressionAlgorithm>, - /// Message expiry; omit to leave the current one alone. - #[serde(default)] - pub message_expiry: Option<IggyExpiry>, - /// Max topic size; omit to leave the current one alone. - #[serde(default)] - pub max_topic_size: Option<MaxTopicSize>, - /// Unique topic name, max length is 255 characters. - pub name: String, - /// Additional topic options as string key-values. Restricted to the keys - /// an update may change; anything else is rejected. - #[serde(default)] - pub options: BTreeMap<String, String>, -} - -impl Default for UpdateTopic { - fn default() -> Self { - UpdateTopic { - stream_id: Identifier::default(), - topic_id: Identifier::default(), - compression_algorithm: None, - message_expiry: None, - max_topic_size: None, - name: "topic".to_string(), - options: BTreeMap::new(), - } - } -} - -impl Validatable<IggyError> for UpdateTopic { - fn validate(&self) -> Result<(), IggyError> { - if self.name.is_empty() || self.name.len() > MAX_NAME_LENGTH { - return Err(IggyError::InvalidTopicName); - } - - Ok(()) - } -}
diff --git a/core/common/src/http/users/create_user.rs b/core/common/src/http/users/create_user.rs index 7a6fe95..16d884d 100644 --- a/core/common/src/http/users/create_user.rs +++ b/core/common/src/http/users/create_user.rs
@@ -16,6 +16,7 @@ // under the License. use super::defaults::*; +use crate::OptionInputMap; use crate::Permissions; use crate::UserStatus; use crate::Validatable; @@ -40,6 +41,9 @@ pub status: UserStatus, /// Optional permissions of the user. If not provided, user will have no permissions. pub permissions: Option<Permissions>, + /// User options keyed by option name. Unknown keys are rejected. + #[serde(default, skip_serializing_if = "OptionInputMap::is_empty")] + pub options: OptionInputMap, } impl Default for CreateUser { @@ -49,6 +53,7 @@ password: SecretString::from("secret"), status: UserStatus::Active, permissions: None, + options: OptionInputMap::default(), } } }
diff --git a/core/common/src/http/users/mod.rs b/core/common/src/http/users/mod.rs index ec51fd0..d91faa9 100644 --- a/core/common/src/http/users/mod.rs +++ b/core/common/src/http/users/mod.rs
@@ -21,4 +21,3 @@ pub mod delete_user; pub mod login_user; pub mod update_permissions; -pub mod update_user;
diff --git a/core/common/src/http/users/update_user.rs b/core/common/src/http/users/update_user.rs deleted file mode 100644 index f29711c..0000000 --- a/core/common/src/http/users/update_user.rs +++ /dev/null
@@ -1,59 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use super::defaults::*; -use crate::Identifier; -use crate::UserStatus; -use crate::Validatable; -use crate::error::IggyError; -use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; - -/// `UpdateUser` command is used to update a user's username and status. -/// It has additional payload: -/// - `user_id` - unique user ID (numeric or name). -/// - `username` - new username (optional), if provided, must be between 3 and 50 characters long. -/// - `status` - new status (optional) -#[derive(Debug, Serialize, Deserialize, PartialEq, Default, Clone)] -pub struct UpdateUser { - #[serde(skip)] - pub user_id: Identifier, - pub username: Option<String>, - pub status: Option<UserStatus>, - /// Additional user options as string key-values. Restricted to the keys - /// an update may change; anything else is rejected. - #[serde(default)] - pub options: BTreeMap<String, String>, -} - -impl Validatable<IggyError> for UpdateUser { - fn validate(&self) -> Result<(), IggyError> { - if self.username.is_none() { - return Ok(()); - } - - let username = self.username.as_ref().unwrap(); - if username.is_empty() - || username.len() > MAX_USERNAME_LENGTH - || username.len() < MIN_USERNAME_LENGTH - { - return Err(IggyError::InvalidUsername); - } - - Ok(()) - } -}
diff --git a/core/common/src/lib.rs b/core/common/src/lib.rs index 300614b..4568ebb 100644 --- a/core/common/src/lib.rs +++ b/core/common/src/lib.rs
@@ -69,6 +69,7 @@ pub use http::consumer_groups::*; pub use http::consumer_offsets::*; pub use http::messages::*; +pub use http::options::{OptionInputMap, OptionInputValue, OptionsUpdate, RESET_FIELD}; pub use http::partitions::*; pub use http::personal_access_tokens::*; pub use http::segments::*; @@ -169,6 +170,11 @@ pub use utils::personal_access_token_expiry::PersonalAccessTokenExpiry; pub use utils::random_id; pub use utils::serde_secret; +pub use utils::stream_key::{ + MAX_PREVIOUS_KEYS, MAX_WRAP_SET_ENVELOPES, RewrapPlanError, STREAM_KEY_LENGTH, + ServerKeyContext, StreamEncryptionKey, StreamKeyOrigin, WRAPPED_STREAM_KEY_LENGTH, WrapSet, + WrappedStreamKey, stream_key_fingerprint, +}; pub use utils::text; pub use utils::timestamp::*; pub use utils::topic_size::MaxTopicSize;
diff --git a/core/common/src/traits/binary_impls/mod.rs b/core/common/src/traits/binary_impls/mod.rs index 80723f0..627bbca 100644 --- a/core/common/src/traits/binary_impls/mod.rs +++ b/core/common/src/traits/binary_impls/mod.rs
@@ -30,13 +30,17 @@ pub use messages::{BinarySendPartitioning, decode_send_confirmations, send_binary_messages}; use crate::IggyError; +use crate::OptionsPatch; use crate::http::users::defaults::{ MAX_PASSWORD_LENGTH, MAX_USERNAME_LENGTH, MIN_PASSWORD_LENGTH, MIN_USERNAME_LENGTH, }; use crate::{BinaryClient, ClientState, Credentials, IdentityInfo}; use bytes::Bytes; use iggy_binary_protocol::codec::WireEncode; -use iggy_binary_protocol::codes::{LOGIN_REGISTER_CODE, LOGIN_REGISTER_WITH_PAT_CODE}; +use iggy_binary_protocol::codes::{ + LOGIN_REGISTER_CODE, LOGIN_REGISTER_WITH_PAT_CODE, UPDATE_RESOURCE_CODE, +}; +use iggy_binary_protocol::requests::system::{ResourceTarget, UpdateResourceRequest}; use iggy_binary_protocol::requests::users::login_register::BindSecret; use iggy_binary_protocol::requests::users::{LoginRegisterRequest, LoginRegisterWithPatRequest}; use iggy_binary_protocol::responses::users::LoginRegisterResponse; @@ -46,6 +50,26 @@ use secrecy::ExposeSecret; use tokio::time::timeout_at; +/// Send one `UpdateResource` patch for `target`. +async fn update_resource<B: BinaryClient + ?Sized>( + client: &B, + target: ResourceTarget, + patch: OptionsPatch, +) -> Result<(), IggyError> { + client + .send_raw_with_response( + UPDATE_RESOURCE_CODE, + UpdateResourceRequest { + target, + reset: patch.reset, + options: patch.set, + } + .to_bytes(), + ) + .await?; + Ok(()) +} + /// SDK identifier sent in the login-register version prefix. Foreign SDKs /// send their own (e.g. `go-sdk`) once they adopt VSR framing. pub(crate) const RUST_SDK_NAME: &str = "rust-sdk";
diff --git a/core/common/src/traits/binary_impls/streams.rs b/core/common/src/traits/binary_impls/streams.rs index 2667341..c070b03 100644 --- a/core/common/src/traits/binary_impls/streams.rs +++ b/core/common/src/traits/binary_impls/streams.rs
@@ -18,19 +18,20 @@ use crate::traits::binary_auth::fail_if_not_authenticated; use crate::wire_conversions::{identifier_to_wire, streams_from_wire}; use crate::{ - BinaryClient, Identifier, IggyError, Stream, StreamClient, StreamDetails, StreamUpdateOptions, + BinaryClient, Identifier, IggyError, Stream, StreamClient, StreamCreateOptions, StreamDetails, + StreamUpdateOptions, }; +use iggy_binary_protocol::WireName; use iggy_binary_protocol::codec::WireEncode; use iggy_binary_protocol::codes::{ - CREATE_STREAM_CODE, DELETE_STREAM_CODE, GET_STREAM_CODE, GET_STREAMS_CODE, UPDATE_STREAM_CODE, + CREATE_STREAM_CODE, DELETE_STREAM_CODE, GET_STREAM_CODE, GET_STREAMS_CODE, }; use iggy_binary_protocol::requests::streams::{ CreateStreamRequest, DeleteStreamRequest, GetStreamRequest, GetStreamsRequest, - UpdateStreamRequest, }; +use iggy_binary_protocol::requests::system::ResourceTarget; use iggy_binary_protocol::responses::streams::get_stream::GetStreamResponse; use iggy_binary_protocol::responses::streams::get_streams::GetStreamsResponse; -use iggy_binary_protocol::{WireName, WireOptions}; #[async_trait::async_trait] impl<B: BinaryClient> StreamClient for B { @@ -62,15 +63,20 @@ Ok(streams_from_wire(wire_resp)?) } - async fn create_stream(&self, name: &str) -> Result<StreamDetails, IggyError> { + async fn create_stream( + &self, + name: &str, + options: &StreamCreateOptions, + ) -> Result<StreamDetails, IggyError> { fail_if_not_authenticated(self).await?; let wire_name = WireName::new(name).map_err(|_| IggyError::InvalidFormat)?; + // The transport owns the encoded request, so a plaintext stream key in it is not erased. let response = self .send_raw_with_response( CREATE_STREAM_CODE, CreateStreamRequest { name: wire_name, - options: WireOptions::empty(), + options: options.to_wire()?, } .to_bytes(), ) @@ -82,23 +88,14 @@ async fn update_stream( &self, stream_id: &Identifier, - name: &str, options: &StreamUpdateOptions, ) -> Result<(), IggyError> { fail_if_not_authenticated(self).await?; - let wire_id = identifier_to_wire(stream_id)?; - let wire_name = WireName::new(name).map_err(|_| IggyError::InvalidFormat)?; - self.send_raw_with_response( - UPDATE_STREAM_CODE, - UpdateStreamRequest { - stream_id: wire_id, - name: wire_name, - options: options.to_wire()?, - } - .to_bytes(), - ) - .await?; - Ok(()) + let patch = options.to_patch()?; + let target = ResourceTarget::Stream { + stream_id: identifier_to_wire(stream_id)?, + }; + super::update_resource(self, target, patch).await } async fn delete_stream(&self, stream_id: &Identifier) -> Result<(), IggyError> {
diff --git a/core/common/src/traits/binary_impls/topics.rs b/core/common/src/traits/binary_impls/topics.rs index 7ce2f88..c0358b7 100644 --- a/core/common/src/traits/binary_impls/topics.rs +++ b/core/common/src/traits/binary_impls/topics.rs
@@ -24,10 +24,11 @@ use iggy_binary_protocol::WireName; use iggy_binary_protocol::codec::WireEncode; use iggy_binary_protocol::codes::{ - CREATE_TOPIC_CODE, DELETE_TOPIC_CODE, GET_TOPIC_CODE, GET_TOPICS_CODE, UPDATE_TOPIC_CODE, + CREATE_TOPIC_CODE, DELETE_TOPIC_CODE, GET_TOPIC_CODE, GET_TOPICS_CODE, }; +use iggy_binary_protocol::requests::system::ResourceTarget; use iggy_binary_protocol::requests::topics::{ - CreateTopicRequest, DeleteTopicRequest, GetTopicRequest, GetTopicsRequest, UpdateTopicRequest, + CreateTopicRequest, DeleteTopicRequest, GetTopicRequest, GetTopicsRequest, }; use iggy_binary_protocol::responses::topics::get_topic::GetTopicResponse; use iggy_binary_protocol::responses::topics::get_topics::GetTopicsResponse; @@ -107,25 +108,15 @@ &self, stream_id: &Identifier, topic_id: &Identifier, - name: &str, options: &TopicUpdateOptions, ) -> Result<(), IggyError> { fail_if_not_authenticated(self).await?; - let wire_stream_id = identifier_to_wire(stream_id)?; - let wire_topic_id = identifier_to_wire(topic_id)?; - let wire_name = WireName::new(name).map_err(|_| IggyError::InvalidFormat)?; - self.send_raw_with_response( - UPDATE_TOPIC_CODE, - UpdateTopicRequest { - stream_id: wire_stream_id, - topic_id: wire_topic_id, - name: wire_name, - options: options.to_wire()?, - } - .to_bytes(), - ) - .await?; - Ok(()) + let patch = options.to_patch()?; + let target = ResourceTarget::Topic { + stream_id: identifier_to_wire(stream_id)?, + topic_id: identifier_to_wire(topic_id)?, + }; + super::update_resource(self, target, patch).await } async fn delete_topic(
diff --git a/core/common/src/traits/binary_impls/users.rs b/core/common/src/traits/binary_impls/users.rs index 9d87319..7f9dd91 100644 --- a/core/common/src/traits/binary_impls/users.rs +++ b/core/common/src/traits/binary_impls/users.rs
@@ -19,19 +19,20 @@ use crate::wire_conversions::{identifier_to_wire, permissions_to_wire, users_from_wire}; use crate::{ BinaryClient, Credentials, Identifier, IdentityInfo, IggyError, Permissions, UserClient, - UserInfo, UserInfoDetails, UserStatus, UserUpdateOptions, + UserCreateOptions, UserInfo, UserInfoDetails, UserStatus, UserUpdateOptions, }; +use iggy_binary_protocol::WireName; use iggy_binary_protocol::codec::WireEncode; use iggy_binary_protocol::codes::{ CHANGE_PASSWORD_CODE, CREATE_USER_CODE, DELETE_USER_CODE, GET_USER_CODE, GET_USERS_CODE, - LOGOUT_USER_CODE, UPDATE_PERMISSIONS_CODE, UPDATE_USER_CODE, + LOGOUT_USER_CODE, UPDATE_PERMISSIONS_CODE, }; +use iggy_binary_protocol::requests::system::ResourceTarget; use iggy_binary_protocol::requests::users::{ ChangePasswordRequest, CreateUserRequest, DeleteUserRequest, GetUserRequest, GetUsersRequest, - LogoutUserRequest, UpdatePermissionsRequest, UpdateUserRequest, + LogoutUserRequest, UpdatePermissionsRequest, }; use iggy_binary_protocol::responses::users::{GetUsersResponse, UserDetailsResponse}; -use iggy_binary_protocol::{WireName, WireOptions}; use secrecy::SecretString; #[async_trait::async_trait] @@ -70,6 +71,7 @@ password: &str, status: UserStatus, permissions: Option<Permissions>, + options: &UserCreateOptions, ) -> Result<UserInfoDetails, IggyError> { fail_if_not_authenticated(self).await?; super::validate_username(username)?; @@ -84,7 +86,7 @@ password: password.to_string(), status: status.as_code(), permissions: wire_perms, - options: WireOptions::empty(), + options: options.to_wire()?, } .to_bytes(), ) @@ -107,28 +109,14 @@ async fn update_user( &self, user_id: &Identifier, - username: Option<&str>, - status: Option<UserStatus>, options: &UserUpdateOptions, ) -> Result<(), IggyError> { fail_if_not_authenticated(self).await?; - let wire_id = identifier_to_wire(user_id)?; - let wire_username = username - .map(WireName::new) - .transpose() - .map_err(|_| IggyError::InvalidFormat)?; - self.send_raw_with_response( - UPDATE_USER_CODE, - UpdateUserRequest { - user_id: wire_id, - username: wire_username, - status: status.map(|s| s.as_code()), - options: options.to_wire()?, - } - .to_bytes(), - ) - .await?; - Ok(()) + let patch = options.to_patch()?; + let target = ResourceTarget::User { + user_id: identifier_to_wire(user_id)?, + }; + super::update_resource(self, target, patch).await } async fn update_permissions( @@ -201,3 +189,151 @@ .await?; client.finish_sign_out(signing_in).await } + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + BinaryTransport, Client, ClientState, ConsumerGroupClientState, DiagnosticEvent, + NonZeroIggyDuration, OptionKey, PartitionContext, VsrSessionControl, VsrSessionSealed, + }; + use async_broadcast::Receiver; + use bytes::Bytes; + use iggy_binary_protocol::requests::messages::PollMessagesRequest; + use iggy_binary_protocol::requests::system::SessionIdentity; + use iggy_binary_protocol::requests::users::login_register::BindSecret; + use std::collections::BTreeSet; + use std::sync::{Arc, Mutex}; + + #[tokio::test] + async fn given_short_username_and_status_reset_when_updating_user_should_report_admission_category() + { + let client = RecordingClient::default(); + let outcome = client + .update_user( + &Identifier::numeric(5).expect("numeric id"), + &UserUpdateOptions { + username: Some("ab".to_owned()), + reset: BTreeSet::from([OptionKey::Status]), + ..UserUpdateOptions::default() + }, + ) + .await; + assert_eq!( + outcome, + Err(IggyError::OptionNotResettable( + OptionKey::Status.to_string() + )) + ); + assert!(client.sent_codes().is_empty()); + } + + /// An authenticated binary client that records the codes it sends. + #[derive(Debug, Default)] + struct RecordingClient { + sent: Mutex<Vec<u32>>, + consumer_groups: Arc<ConsumerGroupClientState>, + } + + impl RecordingClient { + fn sent_codes(&self) -> Vec<u32> { + self.sent.lock().expect("lock").clone() + } + + fn record(&self, code: u32) -> Result<Bytes, IggyError> { + self.sent.lock().expect("lock").push(code); + Ok(Bytes::new()) + } + } + + #[async_trait::async_trait] + impl BinaryTransport for RecordingClient { + async fn get_state(&self) -> ClientState { + ClientState::Authenticated + } + + async fn set_state(&self, _state: ClientState) {} + + async fn publish_event(&self, _event: DiagnosticEvent) {} + + async fn send_raw_with_context( + &self, + code: u32, + _payload: Bytes, + _context: PartitionContext, + ) -> Result<Bytes, IggyError> { + self.record(code) + } + + async fn send_offset_write_with_response( + &self, + code: u32, + _payload: Bytes, + _context: Option<PartitionContext>, + ) -> Result<Bytes, IggyError> { + self.record(code) + } + + async fn send_poll_with_response( + &self, + _request: &PollMessagesRequest, + _context: Option<PartitionContext>, + ) -> Result<Bytes, IggyError> { + Err(IggyError::FeatureUnavailable) + } + + fn get_heartbeat_interval(&self) -> NonZeroIggyDuration { + NonZeroIggyDuration::ONE_SECOND + } + + fn consumer_group_state(&self) -> Arc<ConsumerGroupClientState> { + Arc::clone(&self.consumer_groups) + } + } + + #[async_trait::async_trait] + impl Client for RecordingClient { + async fn connect(&self) -> Result<(), IggyError> { + Ok(()) + } + + async fn disconnect(&self) -> Result<(), IggyError> { + Ok(()) + } + + async fn shutdown(&self) -> Result<(), IggyError> { + Ok(()) + } + + async fn subscribe_events(&self) -> Receiver<DiagnosticEvent> { + async_broadcast::broadcast(1).1 + } + } + + impl VsrSessionSealed for RecordingClient {} + + #[async_trait::async_trait] + impl VsrSessionControl for RecordingClient { + async fn session_bind_secret(&self) -> Result<BindSecret, IggyError> { + Err(IggyError::FeatureUnavailable) + } + + async fn session_identity(&self) -> Result<SessionIdentity, IggyError> { + Err(IggyError::FeatureUnavailable) + } + + async fn bind_vsr_session(&self, _session: u64) -> Result<(), IggyError> { + Ok(()) + } + + async fn reset_vsr_session(&self) -> Result<(), IggyError> { + Ok(()) + } + + fn sdk_version(&self) -> &'static str { + env!("CARGO_PKG_VERSION") + } + } + + impl BinaryClient for RecordingClient {} +}
diff --git a/core/common/src/traits/stream_client.rs b/core/common/src/traits/stream_client.rs index 4958423..03af9cc 100644 --- a/core/common/src/traits/stream_client.rs +++ b/core/common/src/traits/stream_client.rs
@@ -15,7 +15,9 @@ // specific language governing permissions and limitations // under the License. -use crate::{Identifier, IggyError, Stream, StreamDetails, StreamUpdateOptions}; +use crate::{ + Identifier, IggyError, Stream, StreamCreateOptions, StreamDetails, StreamUpdateOptions, +}; use async_trait::async_trait; /// This trait defines the methods to interact with the stream module. @@ -31,15 +33,21 @@ async fn get_streams(&self) -> Result<Vec<Stream>, IggyError>; /// Create a new stream. /// - /// Authentication is required, and the permission to manage the streams. - async fn create_stream(&self, name: &str) -> Result<StreamDetails, IggyError>; - /// Update a stream by unique ID or name. + /// `options` may carry the stream's own encryption key. The server stores + /// it wrapped and returns only its source and fingerprint. Authentication + /// is required, and the permission to manage the streams. + async fn create_stream( + &self, + name: &str, + options: &StreamCreateOptions, + ) -> Result<StreamDetails, IggyError>; + /// Patch a stream by unique ID or name: rename it or change its options. + /// Keys `options` leaves out keep their values. /// /// Authentication is required, and the permission to manage the streams. async fn update_stream( &self, stream_id: &Identifier, - name: &str, options: &StreamUpdateOptions, ) -> Result<(), IggyError>; /// Delete a stream by unique ID or name.
diff --git a/core/common/src/traits/topic_client.rs b/core/common/src/traits/topic_client.rs index d88125d..c93221f 100644 --- a/core/common/src/traits/topic_client.rs +++ b/core/common/src/traits/topic_client.rs
@@ -51,14 +51,15 @@ name: &str, options: &TopicCreateOptions, ) -> Result<TopicDetails, IggyError>; - /// Update a topic by unique ID or name. + /// Patch a topic by unique ID or name: rename it, change its updatable + /// options, or reset them to their defaults. Keys `options` leaves out keep + /// their values. /// /// Authentication is required, and the permission to manage the topics. async fn update_topic( &self, stream_id: &Identifier, topic_id: &Identifier, - name: &str, options: &TopicUpdateOptions, ) -> Result<(), IggyError>; /// Delete a topic by unique ID or name.
diff --git a/core/common/src/traits/user_client.rs b/core/common/src/traits/user_client.rs index b2509c8..e420c07 100644 --- a/core/common/src/traits/user_client.rs +++ b/core/common/src/traits/user_client.rs
@@ -16,8 +16,8 @@ // under the License. use crate::{ - Identifier, IdentityInfo, IggyError, Permissions, UserInfo, UserInfoDetails, UserStatus, - UserUpdateOptions, + Identifier, IdentityInfo, IggyError, Permissions, UserCreateOptions, UserInfo, UserInfoDetails, + UserStatus, UserUpdateOptions, }; use async_trait::async_trait; @@ -48,19 +48,19 @@ password: &str, status: UserStatus, permissions: Option<Permissions>, + options: &UserCreateOptions, ) -> Result<UserInfoDetails, IggyError>; /// Delete a user by unique ID or username. /// /// Authentication is required, and the permission to manage the users. async fn delete_user(&self, user_id: &Identifier) -> Result<(), IggyError>; - /// Update a user by unique ID or username. + /// Patch a user by unique ID or username: rename it, change its status or + /// its options. Keys `options` leaves out keep their values. /// /// Authentication is required, and the permission to manage the users. async fn update_user( &self, user_id: &Identifier, - username: Option<&str>, - status: Option<UserStatus>, options: &UserUpdateOptions, ) -> Result<(), IggyError>; /// Update the permissions of a user by unique ID or username.
diff --git a/core/common/src/types/options/catalog.rs b/core/common/src/types/options/catalog.rs new file mode 100644 index 0000000..d6edbba --- /dev/null +++ b/core/common/src/types/options/catalog.rs
@@ -0,0 +1,1798 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::codec::{OptionValueCodec, invalid}; +use super::key::OptionKey; +use super::{ + DEFAULT_MAX_TOPIC_SIZE, DEFAULT_MESSAGE_EXPIRY, DEFAULT_MESSAGES_REQUIRED_TO_SAVE, + DEFAULT_PREALLOCATE_SEGMENTS, DEFAULT_SEGMENT_SIZE, DEFAULT_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE, + Durability, MAX_MESSAGES_REQUIRED_TO_SAVE, MAX_PREALLOCATED_TOPIC_BYTES, + MAX_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE, MAX_TOPIC_SEGMENT_SIZE, MIN_TOPIC_SEGMENT_SIZE, + OptionValue, OptionsScope, PartitionResizePolicy, ResourceOptions, erase_options, +}; +use crate::http::users::defaults::{MAX_USERNAME_LENGTH, MIN_USERNAME_LENGTH}; +use crate::types::compression::compression_algorithm::CompressionAlgorithm; +use crate::types::message::{HeaderKey, HeaderKind, HeaderValue}; +use crate::utils::stream_key::{StreamEncryptionKey, StreamKeyOrigin, WrapSet}; +use crate::wire_conversions::options_block_size; +use crate::{IggyByteSize, IggyError, IggyExpiry, MaxTopicSize, UserStatus}; +use iggy_binary_protocol::{WireHeaderKind, WireOptions, WireUserHeaderEntry}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt::{Debug, Formatter}; +use strum::{Display, EnumString, IntoStaticStr}; + +/// Bit set in a `DescribeOptions` entry's flags when an update may set the key. +pub const OPTION_FLAG_UPDATABLE: u8 = 1; +/// Bit set when an update may reset the key to its default. +pub const OPTION_FLAG_RESETTABLE: u8 = 1 << 1; +/// Bit set when the key is write-only: accepted, never returned. +pub const OPTION_FLAG_SECRET: u8 = 1 << 2; +/// Bit set when the server computes the key and refuses it from clients. +pub const OPTION_FLAG_READ_ONLY: u8 = 1 << 3; +/// Bit set when a node that does not understand the key must not serve the +/// resource. +pub const OPTION_FLAG_CRITICAL: u8 = 1 << 4; + +/// Longest marker a resource may carry. Every critical key name fits many +/// times over, and the bound keeps the marker a single header field. +const CRITICAL_KEYS_MAX_LENGTH: usize = 255; +/// Longest key echoed back in an option error. A rejected key is +/// attacker-sized, and over HTTP the error text rides the response body. +const ERROR_KEY_PREVIEW_LENGTH: usize = 64; +const MAX_RESOURCE_NAME_LENGTH: usize = 255; + +const ALL_SCOPES: &[OptionsScope] = &[ + OptionsScope::Stream, + OptionsScope::Topic, + OptionsScope::User, +]; +const STREAM_AND_TOPIC: &[OptionsScope] = &[OptionsScope::Stream, OptionsScope::Topic]; +const STREAM_ONLY: &[OptionsScope] = &[OptionsScope::Stream]; +const TOPIC_ONLY: &[OptionsScope] = &[OptionsScope::Topic]; +const USER_ONLY: &[OptionsScope] = &[OptionsScope::User]; + +/// Every option key, indexed by its `OptionKey` discriminant. +pub static CATALOG: [OptionDescriptor; 19] = [ + OptionDescriptor { + key: OptionKey::Name, + scopes: STREAM_AND_TOPIC, + kind: HeaderKind::String, + storage: OptionStorage::Column, + mutability: OptionMutability::Updatable, + default: OptionDefault::None, + visibility: OptionVisibility::Public, + critical: false, + description: || { + "Resource name, 1 to 255 bytes, unique within its parent. Set by the create \ + command's name field and changed through an update." + .to_owned() + }, + }, + OptionDescriptor { + key: OptionKey::Username, + scopes: USER_ONLY, + kind: HeaderKind::String, + storage: OptionStorage::Column, + mutability: OptionMutability::Updatable, + default: OptionDefault::None, + visibility: OptionVisibility::Public, + critical: false, + description: || { + format!( + "User name, {MIN_USERNAME_LENGTH} to {MAX_USERNAME_LENGTH} bytes, unique. Set by \ + the create command's username field and changed through an update." + ) + }, + }, + OptionDescriptor { + key: OptionKey::Status, + scopes: USER_ONLY, + kind: HeaderKind::String, + storage: OptionStorage::Column, + mutability: OptionMutability::Updatable, + default: OptionDefault::None, + visibility: OptionVisibility::Public, + critical: false, + description: || { + "User status, active or inactive. Set by the create command's status field and \ + changed through an update. Deactivating a user ends its sessions." + .to_owned() + }, + }, + OptionDescriptor { + key: OptionKey::CompressionAlgorithm, + scopes: TOPIC_ONLY, + kind: HeaderKind::String, + storage: OptionStorage::Map, + mutability: OptionMutability::CreateOnly, + default: OptionDefault::Const(|| CompressionAlgorithm::default().encode()), + visibility: OptionVisibility::Public, + critical: false, + description: || { + "Compression algorithm (none, gzip). Stored and reported, not applied to messages" + .to_owned() + }, + }, + OptionDescriptor { + key: OptionKey::MessageExpiry, + scopes: TOPIC_ONLY, + kind: HeaderKind::Uint64, + storage: OptionStorage::Map, + mutability: OptionMutability::Updatable, + default: OptionDefault::Const(|| Ok(HeaderValue::from(DEFAULT_MESSAGE_EXPIRY))), + visibility: OptionVisibility::Public, + critical: false, + description: || { + "Message expiry in microseconds, or a humantime string (e.g. 7 days)".to_owned() + }, + }, + OptionDescriptor { + key: OptionKey::MaxTopicSize, + scopes: TOPIC_ONLY, + kind: HeaderKind::Uint64, + storage: OptionStorage::Map, + mutability: OptionMutability::Updatable, + default: OptionDefault::Const(|| Ok(HeaderValue::from(DEFAULT_MAX_TOPIC_SIZE))), + visibility: OptionVisibility::Public, + critical: false, + description: || { + "Topic-wide sealed-segment size cap, split across all partitions, in bytes or a \ + byte-size string (e.g. 1 GiB); finite values must be at least the segment size" + .to_owned() + }, + }, + OptionDescriptor { + key: OptionKey::SegmentSize, + scopes: TOPIC_ONLY, + kind: HeaderKind::Uint64, + storage: OptionStorage::Map, + mutability: OptionMutability::CreateOnly, + default: OptionDefault::Const(|| Ok(HeaderValue::from(DEFAULT_SEGMENT_SIZE))), + visibility: OptionVisibility::Public, + critical: false, + description: || { + format!( + "Segment size in bytes, or a byte-size string (e.g. 128 MiB); a 512-byte \ + multiple within {MIN_TOPIC_SEGMENT_SIZE}..={MAX_TOPIC_SEGMENT_SIZE}" + ) + }, + }, + OptionDescriptor { + key: OptionKey::Durability, + scopes: TOPIC_ONLY, + kind: HeaderKind::String, + storage: OptionStorage::Map, + mutability: OptionMutability::CreateOnly, + default: OptionDefault::Const(|| Durability::default().encode()), + visibility: OptionVisibility::Public, + critical: false, + description: || { + "Message completion: replicated or persisted. Independently defaults to \ + replicated. A singleton quorum has one copy. In replicated groups, persisted \ + messages use WAL references to segment bodies, retaining their inodes by hard \ + link until WAL reclamation. The full body size still counts against \ + partition.wal_bytes_max." + .to_owned() + }, + }, + OptionDescriptor { + key: OptionKey::ConsumerOffsetDurability, + scopes: TOPIC_ONLY, + kind: HeaderKind::String, + storage: OptionStorage::Map, + mutability: OptionMutability::CreateOnly, + default: OptionDefault::Const(|| Durability::default().encode()), + visibility: OptionVisibility::Public, + critical: false, + description: || { + "Explicit offset completion: replicated or persisted. Independently defaults to \ + replicated. Poll auto-commit remains asynchronous. In replicated groups, persisted \ + offsets also enable WAL references to segment bodies, retaining their inodes by \ + hard link until reclamation, even with replicated message durability. Full body \ + sizes count against partition.wal_bytes_max." + .to_owned() + }, + }, + OptionDescriptor { + key: OptionKey::PartitionResizePolicy, + scopes: TOPIC_ONLY, + kind: HeaderKind::String, + storage: OptionStorage::Map, + mutability: OptionMutability::CreateOnly, + default: OptionDefault::Const(|| PartitionResizePolicy::default().encode()), + visibility: OptionVisibility::Public, + critical: false, + description: || { + "Immutable after creation: mutable permits partition addition/deletion without a \ + bounded cutover or per-key ordering across layouts; fixed rejects both. Retention \ + and topic deletion remain destructive." + .to_owned() + }, + }, + OptionDescriptor { + key: OptionKey::MessagesRequiredToSave, + scopes: TOPIC_ONLY, + kind: HeaderKind::Uint32, + storage: OptionStorage::Map, + mutability: OptionMutability::CreateOnly, + default: OptionDefault::Const(|| Ok(HeaderValue::from(DEFAULT_MESSAGES_REQUIRED_TO_SAVE))), + visibility: OptionVisibility::Public, + critical: false, + description: || { + format!( + "Ordinary message-count flush trigger; 1..={MAX_MESSAGES_REQUIRED_TO_SAVE}. \ + Required persistence, capacity pressure, and lifecycle operations can flush \ + earlier" + ) + }, + }, + OptionDescriptor { + key: OptionKey::SizeOfMessagesRequiredToSave, + scopes: TOPIC_ONLY, + kind: HeaderKind::Uint64, + storage: OptionStorage::Map, + mutability: OptionMutability::CreateOnly, + default: OptionDefault::Const(|| { + Ok(HeaderValue::from(DEFAULT_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE)) + }), + visibility: OptionVisibility::Public, + critical: false, + description: || { + format!( + "Flush the journal once it holds this many bytes, or a byte-size string; \ + whichever threshold trips first flushes. At most \ + {MAX_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE}" + ) + }, + }, + OptionDescriptor { + key: OptionKey::PreallocateSegments, + scopes: TOPIC_ONLY, + kind: HeaderKind::Bool, + storage: OptionStorage::Map, + mutability: OptionMutability::CreateOnly, + default: OptionDefault::Const(|| Ok(HeaderValue::from(DEFAULT_PREALLOCATE_SEGMENTS))), + visibility: OptionVisibility::Public, + critical: false, + description: || { + format!( + "Request segment_size bytes of disk reservation when each segment opens. \ + Unsupported or failed reservations fall back to ordinary allocation with a \ + warning. Reservation runs inline on the owning shard. segment_size * \ + partitions_count is capped at {MAX_PREALLOCATED_TOPIC_BYTES} bytes per create" + ) + }, + }, + OptionDescriptor { + key: OptionKey::EncryptionKey, + scopes: STREAM_ONLY, + kind: HeaderKind::Raw, + storage: OptionStorage::Map, + mutability: OptionMutability::CreateOnly, + default: OptionDefault::None, + visibility: OptionVisibility::Secret, + critical: true, + description: || { + "The stream's own data encryption key: 32 random bytes, raw or standard padded \ + base64. Create-only and write-only: the server stores it wrapped under a key \ + derived from the server encryption key and never returns it. Requires the server \ + encryption key to be configured. Without it, a server with encryption enabled \ + generates a key for the stream. The key travels in the create request over the \ + transport the client uses." + .to_owned() + }, + }, + OptionDescriptor { + key: OptionKey::EncryptionKeyWrapped, + scopes: STREAM_ONLY, + kind: HeaderKind::Raw, + storage: OptionStorage::Map, + mutability: OptionMutability::CreateOnly, + default: OptionDefault::None, + visibility: OptionVisibility::Internal, + critical: true, + description: || { + "The stream key wrapped by the server under one or more key-encryption keys, \ + never returned" + .to_owned() + }, + }, + OptionDescriptor { + key: OptionKey::EncryptionKeySource, + scopes: STREAM_ONLY, + kind: HeaderKind::String, + storage: OptionStorage::Computed, + mutability: OptionMutability::ReadOnly, + default: OptionDefault::None, + visibility: OptionVisibility::ReadOnly, + critical: false, + description: || { + "Who chose the stream's at-rest key: stream when the client sent it, server when \ + the server generated it, none when the stream's messages are stored unencrypted" + .to_owned() + }, + }, + OptionDescriptor { + key: OptionKey::EncryptionKeyFingerprint, + scopes: STREAM_ONLY, + kind: HeaderKind::String, + storage: OptionStorage::Computed, + mutability: OptionMutability::ReadOnly, + default: OptionDefault::None, + visibility: OptionVisibility::ReadOnly, + critical: false, + description: || { + "The first 8 bytes of SHA-256 over the label iggy/stream-key-fingerprint/v1 and \ + the stream's key, as 16 lowercase hex characters. Absent when the source is none" + .to_owned() + }, + }, + OptionDescriptor { + key: OptionKey::EncryptionKekFingerprint, + scopes: STREAM_ONLY, + kind: HeaderKind::String, + storage: OptionStorage::Computed, + mutability: OptionMutability::ReadOnly, + default: OptionDefault::None, + visibility: OptionVisibility::ReadOnly, + critical: false, + description: || { + "Fingerprints of the key-encryption keys that wrap the stream's key, \ + comma-separated: one outside a key rotation, one per configured server key \ + during it. Absent when the source is none" + .to_owned() + }, + }, + OptionDescriptor { + key: OptionKey::CriticalKeys, + scopes: ALL_SCOPES, + kind: HeaderKind::String, + storage: OptionStorage::Map, + mutability: OptionMutability::CreateOnly, + default: OptionDefault::None, + visibility: OptionVisibility::Internal, + critical: false, + description: || { + "Sorted comma-separated names of stored keys a node must understand before \ + serving the resource" + .to_owned() + }, + }, +]; + +/// One option key's catalog entry. +pub struct OptionDescriptor { + pub key: OptionKey, + pub scopes: &'static [OptionsScope], + /// Canonical kind: what admission stores whatever kind the client sent. + pub kind: HeaderKind, + pub storage: OptionStorage, + pub mutability: OptionMutability, + pub default: OptionDefault, + pub visibility: OptionVisibility, + /// A node that does not understand the key must not serve the resource. + pub critical: bool, + pub description: fn() -> String, +} + +/// Where a key's value lives. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OptionStorage { + /// In the resource's options map. + Map, + /// In a typed, indexed field of the resource, such as its name. + Column, + /// Nowhere: computed when the resource is read. + Computed, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OptionMutability { + CreateOnly, + Updatable, + ReadOnly, +} + +#[derive(Clone, Copy)] +pub enum OptionDefault { + None, + Const(fn() -> Result<HeaderValue, IggyError>), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OptionVisibility { + /// Accepted from clients and returned on reads. + Public, + /// Accepted from clients, never returned. + Secret, + /// Written by the server, never accepted from clients, never returned. + Internal, + /// Computed by the server on reads, never accepted from clients. + ReadOnly, +} + +/// Which command an options block arrived on. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OptionOperation { + Create, + Update, +} + +/// Who chose a stream's at-rest key, as reported on reads. +#[derive( + Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Display, EnumString, IntoStaticStr, +)] +#[serde(rename_all = "snake_case")] +#[strum(serialize_all = "snake_case")] +pub enum StreamEncryptionSource { + /// The client sent the key. + Stream, + /// The server generated the key. + Server, + /// Stored unencrypted. + None, +} + +/// The fixed columns a create carries or an update patches. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct OptionColumns { + pub name: Option<String>, + pub username: Option<String>, + pub status: Option<UserStatus>, +} + +/// What admission may know about the resource beyond the request itself. +#[derive(Debug, Clone, Default)] +pub struct OptionAdmissionContext { + /// The resource's stored options, for an update's capacity check. + pub current: Option<ResourceOptions>, + /// The create command's own fixed fields. + pub create_columns: OptionColumns, +} + +/// An options block that passed admission, in canonical kinds. +#[derive(Clone, Default, PartialEq)] +pub struct AdmittedOptions { + /// Map keys to set, explicit and canonical. Zero sentinels that mean "use + /// the default" are dropped, so they leave the stored value alone. A secret + /// key holds the secret itself, so a caller done with the map erases it + /// with [`super::erase_options`]. + pub explicit: ResourceOptions, + /// Map keys to reset to their defaults. + pub reset: Vec<OptionKey>, + /// Column values the update sets, or the create's validated columns. + pub columns: OptionColumns, +} + +impl Debug for AdmittedOptions { + /// Prints secret values as `<redacted>`. + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + let explicit: BTreeMap<String, &dyn Debug> = self + .explicit + .iter() + .map(|(key, option)| { + let secret = OptionKey::try_from(key) + .is_ok_and(|known| known.descriptor().visibility != OptionVisibility::Public); + let shown: &dyn Debug = if secret { &"<redacted>" } else { &option.value }; + (String::from_utf8_lossy(key.as_bytes()).into_owned(), shown) + }) + .collect(); + formatter + .debug_struct("AdmittedOptions") + .field("explicit", &explicit) + .field("reset", &self.reset) + .field("columns", &self.columns) + .finish() + } +} + +impl From<StreamKeyOrigin> for StreamEncryptionSource { + fn from(origin: StreamKeyOrigin) -> Self { + match origin { + StreamKeyOrigin::Client => Self::Stream, + StreamKeyOrigin::Server => Self::Server, + } + } +} + +impl OptionKey { + /// This key's catalog entry. + #[must_use] + pub fn descriptor(self) -> &'static OptionDescriptor { + &CATALOG[self as usize] + } +} + +impl OptionDescriptor { + #[must_use] + pub fn applies_to(&self, scope: OptionsScope) -> bool { + self.scopes.contains(&scope) + } + + #[must_use] + pub fn is_updatable(&self) -> bool { + self.mutability == OptionMutability::Updatable + } + + /// An update may reset only a mutable map key with a default. A default + /// alone does not make a create-only setting resettable. + #[must_use] + pub fn is_resettable(&self) -> bool { + self.is_updatable() + && self.storage == OptionStorage::Map + && matches!(self.default, OptionDefault::Const(_)) + } + + /// Whether `DescribeOptions` lists this key. Internal keys are the + /// server's bookkeeping and stay out of the catalog clients see. + #[must_use] + pub fn is_described(&self) -> bool { + self.visibility != OptionVisibility::Internal + } + + /// The `DescribeOptions` flag byte. + #[must_use] + pub fn flags(&self) -> u8 { + let mut flags = 0; + if self.is_updatable() { + flags |= OPTION_FLAG_UPDATABLE; + } + if self.is_resettable() { + flags |= OPTION_FLAG_RESETTABLE; + } + if self.visibility == OptionVisibility::Secret { + flags |= OPTION_FLAG_SECRET; + } + if self.visibility == OptionVisibility::ReadOnly { + flags |= OPTION_FLAG_READ_ONLY; + } + if self.critical { + flags |= OPTION_FLAG_CRITICAL; + } + flags + } + + /// The key's default in its canonical kind, or `None` without one. + /// + /// # Errors + /// + /// Propagates a default that does not fit a header value, which the + /// catalog tests rule out. + pub fn default_value(&self) -> Result<Option<HeaderValue>, IggyError> { + match self.default { + OptionDefault::None => Ok(None), + OptionDefault::Const(default) => default().map(Some), + } + } +} + +/// Catalog entries valid in `scope`, in catalog order. +pub fn descriptors_for(scope: OptionsScope) -> impl Iterator<Item = &'static OptionDescriptor> { + CATALOG + .iter() + .filter(move |descriptor| descriptor.applies_to(scope)) +} + +/// The catalog default of every map key in `scope` that `explicit` does not +/// set, as derived entries, so a create stores a concrete value for each. +/// +/// # Errors +/// +/// Propagates a default that does not fit a header value, which the catalog +/// tests rule out. +pub fn derived_defaults( + scope: OptionsScope, + explicit: &ResourceOptions, +) -> Result<ResourceOptions, IggyError> { + let mut derived = ResourceOptions::new(); + for descriptor in + descriptors_for(scope).filter(|descriptor| descriptor.storage == OptionStorage::Map) + { + let key = HeaderKey::from(descriptor.key); + if explicit.contains_key(&key) { + continue; + } + if let Some(default) = descriptor.default_value()? { + derived.insert(key, OptionValue::derived(default)); + } + } + Ok(derived) +} + +/// Validate and canonicalize a create or update options block. +/// +/// Checks run in a fixed order so every transport and SDK observes the same +/// error category for the same request: +/// +/// 1. `InvalidCommand`: a name repeated across set and reset, a repeated reset +/// name, an update with nothing to change, a reset on a create. +/// 2. `UnsupportedOptionKey`: a key outside the catalog or outside `scope`. +/// 3. `ReadOnlyOption`: an internal or read-only key. +/// 4. `OptionNotUpdatable`: a create-only key in an update's set or reset. +/// 5. `OptionNotResettable`: a reset of a column or of a key without default. +/// 6. Value errors: a column inside a create block is `InvalidOptionValue`, +/// column values keep their own errors, other keys `InvalidOptionValue`. +/// 7. `InvalidCommand`: an update that changes nothing once its zero +/// sentinels are dropped. +/// 8. `OptionsBlockTooLarge`: the resulting map exceeds the block limits. +/// +/// # Errors +/// +/// As listed above. +pub fn admit( + scope: OptionsScope, + operation: OptionOperation, + block: &WireOptions, + reset: &[String], + context: &OptionAdmissionContext, +) -> Result<AdmittedOptions, IggyError> { + let mut reset_names = BTreeSet::new(); + for name in reset { + if !reset_names.insert(name.as_str()) { + return Err(IggyError::InvalidCommand); + } + } + let entries: Vec<WireUserHeaderEntry<'_>> = block.into_iter().collect(); + let mut set_names = Vec::with_capacity(entries.len()); + for entry in &entries { + // Wire validation already enforced UTF-8 string keys. + let name = std::str::from_utf8(entry.key).map_err(|_| IggyError::InvalidCommand)?; + if reset_names.contains(name) { + return Err(IggyError::InvalidCommand); + } + set_names.push(name); + } + match operation { + OptionOperation::Create if !reset.is_empty() => return Err(IggyError::InvalidCommand), + OptionOperation::Update if entries.is_empty() && reset.is_empty() => { + return Err(IggyError::InvalidCommand); + } + _ => {} + } + + let set_keys = set_names + .iter() + .map(|name| scoped_key(scope, name)) + .collect::<Result<Vec<_>, _>>()?; + let reset_keys = reset + .iter() + .map(|name| scoped_key(scope, name)) + .collect::<Result<Vec<_>, _>>()?; + let all_keys = || set_keys.iter().chain(reset_keys.iter()).copied(); + + if let Some(key) = all_keys().find(|key| { + matches!( + key.descriptor().visibility, + OptionVisibility::Internal | OptionVisibility::ReadOnly + ) + }) { + return Err(IggyError::ReadOnlyOption(key.to_string())); + } + if operation == OptionOperation::Update + && let Some(key) = all_keys().find(|key| !key.descriptor().is_updatable()) + { + return Err(IggyError::OptionNotUpdatable(key.to_string())); + } + if let Some(key) = reset_keys + .iter() + .find(|key| !key.descriptor().is_resettable()) + { + return Err(IggyError::OptionNotResettable(key.to_string())); + } + if operation == OptionOperation::Create + && let Some(key) = set_keys + .iter() + .find(|key| key.descriptor().storage == OptionStorage::Column) + { + return Err(invalid(*key)); + } + + let mut columns = match operation { + OptionOperation::Create => validated_create_columns(scope, &context.create_columns)?, + OptionOperation::Update => OptionColumns::default(), + }; + let mut explicit = ResourceOptions::new(); + for (key, entry) in set_keys.iter().zip(&entries) { + let decoded = match key.descriptor().storage { + OptionStorage::Column => decode_column(scope, *key, entry, &mut columns), + OptionStorage::Map => canonical_value(*key, entry).map(|value| { + if let Some(value) = value { + explicit.insert(HeaderKey::from(*key), OptionValue::explicit(value)); + } + }), + OptionStorage::Computed => Err(IggyError::ReadOnlyOption(key.to_string())), + }; + if let Err(error) = decoded { + erase_options(explicit); + return Err(error); + } + } + if operation == OptionOperation::Update + && explicit.is_empty() + && reset_keys.is_empty() + && columns == OptionColumns::default() + { + return Err(IggyError::InvalidCommand); + } + if let Err(error) = check_block_size(context.current.as_ref(), &reset_keys, &explicit) { + erase_options(explicit); + return Err(error); + } + + Ok(AdmittedOptions { + explicit, + reset: reset_keys, + columns, + }) +} + +/// The canonical value of one map key, or `None` for a zero sentinel that +/// means "use the default" and therefore sets nothing. A secret key's value +/// is the secret itself, for the caller to erase. +/// +/// # Errors +/// +/// `InvalidOptionValue` naming `key` when the value does not decode or +/// breaks a per-key bound. `InvalidEncryptionKey` for a malformed stream key. +pub fn canonical_value( + key: OptionKey, + entry: &WireUserHeaderEntry<'_>, +) -> Result<Option<HeaderValue>, IggyError> { + let value = match key { + OptionKey::CompressionAlgorithm => CompressionAlgorithm::decode(key, entry)?.encode()?, + OptionKey::MessageExpiry => { + let expiry = IggyExpiry::decode(key, entry)?; + if expiry == IggyExpiry::ServerDefault { + return Ok(None); + } + expiry.encode()? + } + OptionKey::MaxTopicSize => { + let size = MaxTopicSize::decode(key, entry)?; + if size == MaxTopicSize::ServerDefault { + return Ok(None); + } + size.encode()? + } + OptionKey::SegmentSize | OptionKey::SizeOfMessagesRequiredToSave => { + let size = IggyByteSize::decode(key, entry)?; + if size.as_bytes_u64() == 0 { + return Ok(None); + } + size.encode()? + } + OptionKey::Durability | OptionKey::ConsumerOffsetDurability => { + Durability::decode(key, entry)?.encode()? + } + OptionKey::PartitionResizePolicy => PartitionResizePolicy::decode(key, entry)?.encode()?, + OptionKey::MessagesRequiredToSave => u32::decode(key, entry)?.encode()?, + OptionKey::PreallocateSegments => bool::decode(key, entry)?.encode()?, + OptionKey::EncryptionKey => { + HeaderValue::try_from(stream_key_from_entry(entry)?.as_bytes().as_slice())? + } + OptionKey::Name + | OptionKey::Username + | OptionKey::Status + | OptionKey::EncryptionKeyWrapped + | OptionKey::EncryptionKeySource + | OptionKey::EncryptionKeyFingerprint + | OptionKey::EncryptionKekFingerprint + | OptionKey::CriticalKeys => return Err(invalid(key)), + }; + Ok(Some(value)) +} + +/// The stream key a create block carries: 32 raw bytes, or their canonical +/// standard padded base64 as a string. +/// +/// # Errors +/// +/// `InvalidEncryptionKey` for any other kind, length or encoding. +pub fn stream_key_from_entry( + entry: &WireUserHeaderEntry<'_>, +) -> Result<StreamEncryptionKey, IggyError> { + match HeaderKind::from_code(entry.value_kind.0) { + Ok(HeaderKind::Raw) => StreamEncryptionKey::try_from(entry.value), + Ok(HeaderKind::String) => std::str::from_utf8(entry.value) + .map_err(|_| IggyError::InvalidEncryptionKey)? + .parse(), + _ => Err(IggyError::InvalidEncryptionKey), + } +} + +/// The options a read returns: public map keys by provenance, unknown keys +/// this build stores without understanding, and the stream's encryption +/// projections as derived entries. +/// +/// The projections come from the stored wrap set alone, so every node answers +/// the same for the same stream state. A cached create reply keeps the +/// create-time projections, so its KEK fingerprints predate any later rewrap. +/// +/// # Errors +/// +/// `UnsupportedCriticalOption` when the resource holds critical state this +/// build cannot interpret. Its values are never echoed then. +pub fn public_options( + scope: OptionsScope, + stored: &ResourceOptions, +) -> Result<ResourceOptions, IggyError> { + let wrap_set = critical_wrap_set(stored)?; + let mut public = ResourceOptions::new(); + for (key, value) in stored { + let returned = OptionKey::try_from(key).map_or(true, |known| { + let descriptor = known.descriptor(); + descriptor.visibility == OptionVisibility::Public + && descriptor.storage == OptionStorage::Map + }); + if returned { + public.insert(key.clone(), value.clone()); + } + } + if scope == OptionsScope::Stream { + let mut project = |key: OptionKey, text: &str| -> Result<(), IggyError> { + public.insert( + HeaderKey::from(key), + OptionValue::derived(HeaderValue::try_from(text)?), + ); + Ok(()) + }; + match wrap_set { + Some(set) => { + let source: &'static str = StreamEncryptionSource::from(set.origin()).into(); + project(OptionKey::EncryptionKeySource, source)?; + project(OptionKey::EncryptionKeyFingerprint, &set.key_fingerprint())?; + project( + OptionKey::EncryptionKekFingerprint, + &set.kek_fingerprints().join(","), + )?; + } + None => { + let source: &'static str = StreamEncryptionSource::None.into(); + project(OptionKey::EncryptionKeySource, source)?; + } + } + } + Ok(public) +} + +/// The stream's stored wrap set, if the stream is encrypted. +/// +/// # Errors +/// +/// `UnsupportedCriticalOption` for a set this build cannot read. +pub fn stream_wrap_set(stored: &ResourceOptions) -> Result<Option<WrapSet>, IggyError> { + stored + .get(&HeaderKey::from(OptionKey::EncryptionKeyWrapped)) + .map(|option| WrapSet::try_from(option.value.as_bytes())) + .transpose() +} + +/// The canonical `critical_keys` marker naming `keys`. +/// +/// # Errors +/// +/// `InvalidOptionValue` when the list is empty or does not fit the marker. +pub fn critical_keys_marker(keys: &[OptionKey]) -> Result<HeaderValue, IggyError> { + let names: BTreeSet<&'static str> = keys.iter().map(|key| (*key).into()).collect(); + let marker = names.into_iter().collect::<Vec<_>>().join(","); + if marker.is_empty() || marker.len() > CRITICAL_KEYS_MAX_LENGTH { + return Err(invalid(OptionKey::CriticalKeys)); + } + HeaderValue::try_from(marker.as_str()) +} + +/// Refuse stored state this build must understand but cannot. +/// +/// A malformed marker, a listed key this build does not know as critical, a +/// listed key missing from the map, and a wrapped key in an unknown envelope +/// version all fail. A wrapped key is critical even when no marker names it. +/// +/// # Errors +/// +/// `UnsupportedCriticalOption` naming the offending key. +pub fn check_critical_options(stored: &ResourceOptions) -> Result<(), IggyError> { + critical_wrap_set(stored).map(|_| ()) +} + +/// [`check_critical_options`], returning the stream's wrap set it parsed so a +/// caller that needs the set reads it once. +/// +/// # Errors +/// +/// As [`check_critical_options`]. +pub(super) fn critical_wrap_set(stored: &ResourceOptions) -> Result<Option<WrapSet>, IggyError> { + if let Some(marker) = stored.get(&HeaderKey::from(OptionKey::CriticalKeys)) { + let names = parse_critical_marker(&marker.value)?; + for name in names { + let known = name + .parse::<OptionKey>() + .ok() + .filter(|key| key.descriptor().critical) + .ok_or_else(|| IggyError::UnsupportedCriticalOption(key_preview(name)))?; + if !stored.contains_key(&HeaderKey::from(known)) { + return Err(IggyError::UnsupportedCriticalOption(name.to_owned())); + } + } + } + stream_wrap_set(stored) +} + +/// A string key in `scope`, or `UnsupportedOptionKey`. +fn scoped_key(scope: OptionsScope, name: &str) -> Result<OptionKey, IggyError> { + name.parse::<OptionKey>() + .ok() + .filter(|key| key.descriptor().applies_to(scope)) + .ok_or_else(|| IggyError::UnsupportedOptionKey(key_preview(name))) +} + +/// Refuse a resulting map past the block limits: `current` with `reset_keys` +/// at their defaults and `explicit` set. The size is summed over the three +/// maps, so no value is cloned or encoded to measure it. +fn check_block_size( + current: Option<&ResourceOptions>, + reset_keys: &[OptionKey], + explicit: &ResourceOptions, +) -> Result<(), IggyError> { + let mut defaults = ResourceOptions::new(); + for key in reset_keys { + if let Some(default) = key.descriptor().default_value()? { + defaults.insert(HeaderKey::from(*key), OptionValue::derived(default)); + } + } + let kept = current + .into_iter() + .flatten() + .filter(|(key, _)| !defaults.contains_key(*key) && !explicit.contains_key(*key)); + let reset = defaults + .iter() + .filter(|(key, _)| !explicit.contains_key(*key)); + options_block_size(kept.chain(reset).chain(explicit)).map(|_| ()) +} + +fn validated_create_columns( + scope: OptionsScope, + columns: &OptionColumns, +) -> Result<OptionColumns, IggyError> { + if let Some(name) = &columns.name { + validate_name(scope, name)?; + } + if let Some(username) = &columns.username { + validate_username(username)?; + } + Ok(columns.clone()) +} + +fn decode_column( + scope: OptionsScope, + key: OptionKey, + entry: &WireUserHeaderEntry<'_>, + columns: &mut OptionColumns, +) -> Result<(), IggyError> { + let text = (entry.value_kind == WireHeaderKind(HeaderKind::String.as_code())) + .then(|| std::str::from_utf8(entry.value).ok()) + .flatten(); + match key { + OptionKey::Name => { + let name = text.ok_or_else(|| name_error(scope))?; + validate_name(scope, name)?; + columns.name = Some(name.to_owned()); + } + OptionKey::Username => { + let username = text.ok_or(IggyError::InvalidUsername)?; + validate_username(username)?; + columns.username = Some(username.to_owned()); + } + OptionKey::Status => { + columns.status = Some( + text.ok_or(IggyError::InvalidUserStatus)? + .parse() + .map_err(|_| IggyError::InvalidUserStatus)?, + ); + } + _ => return Err(invalid(key)), + } + Ok(()) +} + +fn validate_name(scope: OptionsScope, name: &str) -> Result<(), IggyError> { + if name.is_empty() || name.len() > MAX_RESOURCE_NAME_LENGTH { + return Err(name_error(scope)); + } + Ok(()) +} + +fn validate_username(username: &str) -> Result<(), IggyError> { + if !(MIN_USERNAME_LENGTH..=MAX_USERNAME_LENGTH).contains(&username.len()) { + return Err(IggyError::InvalidUsername); + } + Ok(()) +} + +const fn name_error(scope: OptionsScope) -> IggyError { + match scope { + OptionsScope::Topic => IggyError::InvalidTopicName, + OptionsScope::Stream | OptionsScope::User => IggyError::InvalidStreamName, + } +} + +fn parse_critical_marker(value: &HeaderValue) -> Result<Vec<&str>, IggyError> { + let malformed = || IggyError::UnsupportedCriticalOption(OptionKey::CriticalKeys.to_string()); + if value.kind() != HeaderKind::String || value.as_bytes().len() > CRITICAL_KEYS_MAX_LENGTH { + return Err(malformed()); + } + let text = std::str::from_utf8(value.as_bytes()).map_err(|_| malformed())?; + let names: Vec<&str> = text.split(',').collect(); + let well_formed = names.iter().all(|name| { + !name.is_empty() + && name + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_') + }); + if !well_formed || names.windows(2).any(|pair| pair[0] >= pair[1]) { + return Err(malformed()); + } + Ok(names) +} + +fn key_preview(key: &str) -> String { + key.chars().take(ERROR_KEY_PREVIEW_LENGTH).collect() +} + +#[cfg(test)] +mod tests { + use super::super::{OptionsProvenance, topic_option_keys}; + use super::*; + use crate::wire_conversions::resource_options_to_wire; + use std::str::FromStr; + use strum::IntoEnumIterator; + + #[test] + fn given_catalog_when_listing_keys_should_never_name_the_reset_field() { + assert!( + OptionKey::iter().all(|key| key.to_string() != crate::RESET_FIELD), + "an option named `reset` would collide with the HTTP update body" + ); + } + + #[test] + fn given_secret_catalog_key_when_describing_should_be_create_only() { + // No update path wraps a secret, so an updatable one would be stored in + // the clear. + for descriptor in CATALOG + .iter() + .filter(|descriptor| descriptor.visibility == OptionVisibility::Secret) + { + assert_eq!( + descriptor.mutability, + OptionMutability::CreateOnly, + "{} is a secret", + descriptor.key + ); + } + } + + #[test] + fn given_catalog_when_iterating_should_have_unique_scoped_descriptors() { + assert_eq!(CATALOG.len(), OptionKey::iter().count()); + for (index, key) in OptionKey::iter().enumerate() { + let descriptor = key.descriptor(); + assert_eq!(descriptor.key, key, "catalog row {index} out of order"); + assert!(!descriptor.scopes.is_empty(), "{key} has no scope"); + assert!( + !(descriptor.description)().is_empty(), + "{key} has no description" + ); + if let Some(default) = descriptor.default_value().expect("default encodes") { + assert_eq!(default.kind(), descriptor.kind, "{key} default kind"); + } + match descriptor.visibility { + OptionVisibility::ReadOnly => { + assert_eq!(descriptor.storage, OptionStorage::Computed, "{key}"); + } + OptionVisibility::Public + | OptionVisibility::Secret + | OptionVisibility::Internal => { + assert_ne!(descriptor.storage, OptionStorage::Computed, "{key}"); + } + } + let header = HeaderKey::from(key); + assert_eq!(OptionKey::try_from(&header), Ok(key)); + assert_eq!(OptionKey::from_str(&key.to_string()), Ok(key)); + } + for (constant, key) in [ + ( + topic_option_keys::COMPRESSION_ALGORITHM, + OptionKey::CompressionAlgorithm, + ), + (topic_option_keys::MESSAGE_EXPIRY, OptionKey::MessageExpiry), + (topic_option_keys::MAX_TOPIC_SIZE, OptionKey::MaxTopicSize), + (topic_option_keys::SEGMENT_SIZE, OptionKey::SegmentSize), + (topic_option_keys::DURABILITY, OptionKey::Durability), + ( + topic_option_keys::CONSUMER_OFFSET_DURABILITY, + OptionKey::ConsumerOffsetDurability, + ), + ( + topic_option_keys::PARTITION_RESIZE_POLICY, + OptionKey::PartitionResizePolicy, + ), + ( + topic_option_keys::MESSAGES_REQUIRED_TO_SAVE, + OptionKey::MessagesRequiredToSave, + ), + ( + topic_option_keys::SIZE_OF_MESSAGES_REQUIRED_TO_SAVE, + OptionKey::SizeOfMessagesRequiredToSave, + ), + ( + topic_option_keys::PREALLOCATE_SEGMENTS, + OptionKey::PreallocateSegments, + ), + ] { + assert_eq!(constant, key.to_string()); + } + let resettable: Vec<OptionKey> = OptionKey::iter() + .filter(|key| key.descriptor().is_resettable()) + .collect(); + assert_eq!( + resettable, + [OptionKey::MessageExpiry, OptionKey::MaxTopicSize] + ); + } + + #[test] + fn given_canonical_and_text_values_when_decoding_should_match_typed_values() { + let cases: Vec<(OptionKey, HeaderValue, HeaderValue)> = vec![ + ( + OptionKey::CompressionAlgorithm, + HeaderValue::from(2u8), + HeaderValue::try_from("gzip").unwrap(), + ), + ( + OptionKey::CompressionAlgorithm, + HeaderValue::try_from("gzip").unwrap(), + HeaderValue::try_from("gzip").unwrap(), + ), + ( + OptionKey::MessageExpiry, + HeaderValue::try_from("5s").unwrap(), + HeaderValue::from(5_000_000u64), + ), + ( + OptionKey::MessageExpiry, + HeaderValue::try_from("18446744073709551614").unwrap(), + HeaderValue::from(u64::MAX - 1), + ), + ( + OptionKey::MaxTopicSize, + HeaderValue::try_from("2 GiB").unwrap(), + HeaderValue::from(2u64 * 1024 * 1024 * 1024), + ), + ( + OptionKey::SegmentSize, + HeaderValue::try_from("128MiB").unwrap(), + HeaderValue::from(134_217_728u64), + ), + ( + OptionKey::Durability, + HeaderValue::try_from("persisted").unwrap(), + HeaderValue::try_from("persisted").unwrap(), + ), + ( + OptionKey::ConsumerOffsetDurability, + HeaderValue::try_from("replicated").unwrap(), + HeaderValue::try_from("replicated").unwrap(), + ), + ( + OptionKey::PartitionResizePolicy, + HeaderValue::try_from("fixed").unwrap(), + HeaderValue::try_from("fixed").unwrap(), + ), + ( + OptionKey::MessagesRequiredToSave, + HeaderValue::try_from("64").unwrap(), + HeaderValue::from(64u32), + ), + ( + OptionKey::MessagesRequiredToSave, + HeaderValue::from(64u32), + HeaderValue::from(64u32), + ), + ( + OptionKey::SizeOfMessagesRequiredToSave, + HeaderValue::try_from("4KiB").unwrap(), + HeaderValue::from(4096u64), + ), + ( + OptionKey::PreallocateSegments, + HeaderValue::try_from("true").unwrap(), + HeaderValue::from(true), + ), + ( + OptionKey::PreallocateSegments, + HeaderValue::from(false), + HeaderValue::from(false), + ), + ]; + for (key, input, expected) in cases { + let canonical = canonical_value(key, &entry(key, &input)) + .unwrap_or_else(|error| panic!("{key} {input:?}: {error}")) + .expect("not a sentinel"); + assert_eq!(canonical, expected, "{key}"); + } + let invalid_cases: Vec<(OptionKey, HeaderValue)> = vec![ + (OptionKey::CompressionAlgorithm, HeaderValue::from(9u8)), + (OptionKey::MessageExpiry, HeaderValue::from(7u32)), + ( + OptionKey::MessageExpiry, + HeaderValue::try_from("99999999999999999999").unwrap(), + ), + (OptionKey::MessagesRequiredToSave, HeaderValue::from(0u32)), + (OptionKey::MessagesRequiredToSave, HeaderValue::from(7u64)), + ( + OptionKey::MessagesRequiredToSave, + HeaderValue::from(MAX_MESSAGES_REQUIRED_TO_SAVE + 1), + ), + ( + OptionKey::SizeOfMessagesRequiredToSave, + HeaderValue::from(MAX_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE + 1), + ), + (OptionKey::PreallocateSegments, HeaderValue::from(2u8)), + ( + OptionKey::Durability, + HeaderValue::try_from("eventually").unwrap(), + ), + ]; + for (key, input) in invalid_cases { + assert_eq!( + canonical_value(key, &entry(key, &input)), + Err(IggyError::InvalidOptionValue(key.to_string())), + "{key} {input:?}" + ); + } + } + + #[test] + fn given_create_only_option_when_resetting_should_reject_not_updatable() { + for key in [OptionKey::SegmentSize, OptionKey::Durability] { + assert_eq!( + admit( + OptionsScope::Topic, + OptionOperation::Update, + &WireOptions::empty(), + &[key.to_string()], + &OptionAdmissionContext::default(), + ), + Err(IggyError::OptionNotUpdatable(key.to_string())) + ); + } + assert_eq!( + admit( + OptionsScope::Topic, + OptionOperation::Update, + &WireOptions::empty(), + &[OptionKey::Name.to_string()], + &OptionAdmissionContext::default(), + ), + Err(IggyError::OptionNotResettable(OptionKey::Name.to_string())) + ); + let admitted = admit( + OptionsScope::Topic, + OptionOperation::Update, + &WireOptions::empty(), + &[OptionKey::MaxTopicSize.to_string()], + &OptionAdmissionContext::default(), + ) + .expect("a mutable key with a default resets"); + assert_eq!(admitted.reset, [OptionKey::MaxTopicSize]); + for scope in [OptionsScope::Stream, OptionsScope::User] { + for key in descriptors_for(scope).map(|descriptor| descriptor.key) { + assert!( + admit( + scope, + OptionOperation::Update, + &WireOptions::empty(), + &[key.to_string()], + &OptionAdmissionContext::default(), + ) + .is_err(), + "{key} must not reset in {scope}" + ); + } + } + } + + #[test] + fn given_fixed_create_columns_when_repeated_in_options_should_reject() { + assert_eq!( + admit( + OptionsScope::Stream, + OptionOperation::Create, + &block(&[(OptionKey::Name, HeaderValue::try_from("shadow").unwrap())]), + &[], + &OptionAdmissionContext::default(), + ), + Err(IggyError::InvalidOptionValue(OptionKey::Name.to_string())) + ); + assert_eq!( + admit( + OptionsScope::User, + OptionOperation::Create, + &block(&[(OptionKey::Status, HeaderValue::try_from("active").unwrap())]), + &[], + &OptionAdmissionContext::default(), + ), + Err(IggyError::InvalidOptionValue(OptionKey::Status.to_string())) + ); + let admitted = admit( + OptionsScope::User, + OptionOperation::Update, + &block(&[ + ( + OptionKey::Status, + HeaderValue::try_from("inactive").unwrap(), + ), + ( + OptionKey::Username, + HeaderValue::try_from("renamed").unwrap(), + ), + ]), + &[], + &OptionAdmissionContext::default(), + ) + .expect("columns patch through an update"); + assert_eq!(admitted.columns.status, Some(UserStatus::Inactive)); + assert_eq!(admitted.columns.username.as_deref(), Some("renamed")); + assert!(admitted.explicit.is_empty()); + assert_eq!( + admit( + OptionsScope::User, + OptionOperation::Update, + &block(&[(OptionKey::Username, HeaderValue::try_from("ab").unwrap())]), + &[], + &OptionAdmissionContext::default(), + ), + Err(IggyError::InvalidUsername) + ); + assert_eq!( + admit( + OptionsScope::Topic, + OptionOperation::Update, + &block(&[(OptionKey::Name, HeaderValue::from(7u32))]), + &[], + &OptionAdmissionContext::default(), + ), + Err(IggyError::InvalidTopicName) + ); + } + + #[test] + fn given_update_of_only_zero_sentinels_when_admitting_should_refuse_it_as_empty() { + for entries in [ + vec![(OptionKey::MessageExpiry, HeaderValue::from(0u64))], + vec![ + (OptionKey::MaxTopicSize, HeaderValue::from(0u64)), + (OptionKey::MessageExpiry, HeaderValue::from(0u64)), + ], + ] { + assert_eq!( + admit( + OptionsScope::Topic, + OptionOperation::Update, + &block(&entries), + &[], + &OptionAdmissionContext::default(), + ) + .err(), + Some(IggyError::InvalidCommand), + "{entries:?}" + ); + } + } + + #[test] + fn given_topic_sentinels_when_canonicalizing_should_preserve_existing_provenance_rules() { + let admitted = admit( + OptionsScope::Topic, + OptionOperation::Update, + &block(&[ + (OptionKey::MessageExpiry, HeaderValue::from(0u64)), + (OptionKey::MaxTopicSize, HeaderValue::from(0u64)), + (OptionKey::Name, HeaderValue::try_from("orders").unwrap()), + ]), + &[], + &OptionAdmissionContext::default(), + ) + .expect("zero sentinels next to a rename are accepted"); + assert!( + admitted.explicit.is_empty(), + "a zero sentinel leaves the stored value and its provenance alone" + ); + let admitted = admit( + OptionsScope::Topic, + OptionOperation::Update, + &block(&[( + OptionKey::MessageExpiry, + HeaderValue::try_from("1 day").unwrap(), + )]), + &[OptionKey::MaxTopicSize.to_string()], + &OptionAdmissionContext::default(), + ) + .expect("set and reset"); + assert_eq!( + admitted.explicit[&HeaderKey::from(OptionKey::MessageExpiry)], + OptionValue::explicit(HeaderValue::from(86_400_000_000u64)) + ); + assert_eq!(admitted.reset, [OptionKey::MaxTopicSize]); + } + + #[test] + fn given_oversized_patch_result_when_admitting_should_reject_without_mutation() { + let mut current = ResourceOptions::new(); + for index in 0..iggy_binary_protocol::MAX_OPTIONS { + current.insert( + HeaderKey::try_from(format!("future_key_{index:05}").as_str()).unwrap(), + OptionValue::explicit(HeaderValue::from(1u8)), + ); + } + let context = OptionAdmissionContext { + current: Some(current.clone()), + ..OptionAdmissionContext::default() + }; + assert!(matches!( + admit( + OptionsScope::Topic, + OptionOperation::Update, + &block(&[(OptionKey::MessageExpiry, HeaderValue::from(9u64))]), + &[], + &context, + ), + Err(IggyError::OptionsBlockTooLarge(_)) + )); + assert_eq!(context.current, Some(current)); + } + + #[test] + fn given_patch_result_at_the_byte_cap_when_admitting_should_admit_it_but_not_one_byte_more() { + let filler_key = |index: usize| { + HeaderKey::try_from(format!("future_key_{index:05}").as_str()).expect("valid key") + }; + let filler_value = + |length: usize| HeaderValue::try_from(vec![1u8; length].as_slice()).expect("fits"); + let mut current = ResourceOptions::new(); + for key in [OptionKey::MessageExpiry, OptionKey::MaxTopicSize] { + current.insert( + HeaderKey::from(key), + OptionValue::explicit(HeaderValue::from(1u8)), + ); + } + for index in 0..355 { + current.insert(filler_key(index), OptionValue::explicit(filler_value(255))); + } + let patch = block(&[(OptionKey::MessageExpiry, HeaderValue::from(9u64))]); + let reset = [OptionKey::MaxTopicSize.to_string()]; + let mut resulting = current.clone(); + resulting.insert( + HeaderKey::from(OptionKey::MessageExpiry), + OptionValue::explicit(HeaderValue::from(9u64)), + ); + resulting.insert( + HeaderKey::from(OptionKey::MaxTopicSize), + OptionValue::derived( + OptionKey::MaxTopicSize + .descriptor() + .default_value() + .expect("default encodes") + .expect("has a default"), + ), + ); + let resulting_size = resource_options_to_wire(&resulting, OptionsProvenance::All) + .expect("under the cap") + .as_bytes() + .len(); + // The last entry's kinds, lengths and 16-byte key. + let free = iggy_binary_protocol::MAX_OPTIONS_BYTES - resulting_size - 26; + assert!( + (1..255).contains(&free), + "the last value fits a header value" + ); + let admit_with_last_value = |length: usize| { + let mut current = current.clone(); + current.insert(filler_key(355), OptionValue::explicit(filler_value(length))); + let context = OptionAdmissionContext { + current: Some(current), + ..OptionAdmissionContext::default() + }; + admit( + OptionsScope::Topic, + OptionOperation::Update, + &patch, + &reset, + &context, + ) + }; + + assert!(admit_with_last_value(free).is_ok(), "exactly at the cap"); + assert!(matches!( + admit_with_last_value(free + 1), + Err(IggyError::OptionsBlockTooLarge(_)) + )); + } + + #[test] + fn given_unknown_critical_option_when_projecting_should_refuse_without_echoing_value() { + let mut stored = ResourceOptions::new(); + stored.insert( + HeaderKey::try_from("future_secret").unwrap(), + OptionValue::explicit(HeaderValue::try_from("do-not-echo").unwrap()), + ); + stored.insert( + HeaderKey::from(OptionKey::CriticalKeys), + OptionValue::derived(HeaderValue::try_from("future_secret").unwrap()), + ); + assert_eq!( + public_options(OptionsScope::Stream, &stored), + Err(IggyError::UnsupportedCriticalOption( + "future_secret".to_owned() + )) + ); + + let mut stored = ResourceOptions::new(); + stored.insert( + HeaderKey::try_from("future_tuning").unwrap(), + OptionValue::explicit(HeaderValue::from(5u64)), + ); + stored.insert( + HeaderKey::from(OptionKey::MessageExpiry), + OptionValue::derived(HeaderValue::from(9u64)), + ); + let public = public_options(OptionsScope::Stream, &stored) + .expect("non-critical unknown keys round-trip"); + assert_eq!( + public[&HeaderKey::try_from("future_tuning").unwrap()], + OptionValue::explicit(HeaderValue::from(5u64)) + ); + assert_eq!( + public[&HeaderKey::from(OptionKey::EncryptionKeySource)], + OptionValue::derived(HeaderValue::try_from("none").unwrap()) + ); + assert!(!public.contains_key(&HeaderKey::from(OptionKey::EncryptionKeyFingerprint))); + assert!(!public.contains_key(&HeaderKey::from(OptionKey::EncryptionKekFingerprint))); + } + + #[test] + fn given_wrap_set_when_projecting_stream_should_report_origin_and_both_fingerprints() { + let context = crate::ServerKeyContext::from_config( + "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=", + &[], + true, + ) + .unwrap(); + let key = StreamEncryptionKey::try_from([5u8; 32].as_slice()).unwrap(); + for (origin, source) in [ + (StreamKeyOrigin::Client, "stream"), + (StreamKeyOrigin::Server, "server"), + ] { + let set = WrapSet::from(context.wrap(&key, origin).unwrap()); + let mut stored = ResourceOptions::new(); + stored.insert( + HeaderKey::from(OptionKey::EncryptionKeyWrapped), + OptionValue::explicit(HeaderValue::try_from(set.as_ref()).unwrap()), + ); + stored.insert( + HeaderKey::from(OptionKey::CriticalKeys), + OptionValue::explicit( + critical_keys_marker(&[OptionKey::EncryptionKeyWrapped]).unwrap(), + ), + ); + let public = public_options(OptionsScope::Stream, &stored).unwrap(); + let text = |key: OptionKey| { + public[&HeaderKey::from(key)] + .value + .as_str() + .unwrap() + .to_owned() + }; + assert_eq!(text(OptionKey::EncryptionKeySource), source); + assert_eq!(text(OptionKey::EncryptionKeyFingerprint), key.fingerprint()); + assert_eq!( + text(OptionKey::EncryptionKekFingerprint), + context.kek_fingerprint().unwrap() + ); + assert!(!public.contains_key(&HeaderKey::from(OptionKey::EncryptionKeyWrapped))); + assert!(!public.contains_key(&HeaderKey::from(OptionKey::CriticalKeys))); + } + } + + #[test] + fn given_malformed_critical_marker_when_resolving_should_mark_unavailable() { + let wrapped = crate::ServerKeyContext::from_config( + "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=", + &[], + false, + ) + .unwrap() + .wrap( + &StreamEncryptionKey::try_from([5u8; 32].as_slice()).unwrap(), + StreamKeyOrigin::Client, + ) + .unwrap(); + let with_wrapped = |marker: Option<&str>| { + let mut stored = ResourceOptions::new(); + stored.insert( + HeaderKey::from(OptionKey::EncryptionKeyWrapped), + OptionValue::derived(HeaderValue::try_from(wrapped.as_ref()).unwrap()), + ); + if let Some(marker) = marker { + stored.insert( + HeaderKey::from(OptionKey::CriticalKeys), + OptionValue::derived(HeaderValue::try_from(marker).unwrap()), + ); + } + stored + }; + assert_eq!( + check_critical_options(&with_wrapped(Some("encryption_key_wrapped"))), + Ok(()) + ); + assert_eq!(check_critical_options(&with_wrapped(None)), Ok(())); + for marker in [ + "encryption_key_wrapped,encryption_key_wrapped", + "encryption_key_wrapped,", + "Encryption_key_wrapped", + "zz_unknown,encryption_key_wrapped", + "encryption_key_wrapped,message_expiry", + ] { + assert!( + check_critical_options(&with_wrapped(Some(marker))).is_err(), + "{marker} must be refused" + ); + } + let mut stored = with_wrapped(Some("encryption_key_wrapped")); + stored.insert( + HeaderKey::from(OptionKey::EncryptionKeyWrapped), + OptionValue::derived(HeaderValue::try_from([2u8; 78].as_slice()).unwrap()), + ); + assert!(check_critical_options(&stored).is_err()); + let mut topic = ResourceOptions::new(); + topic.insert( + HeaderKey::from(OptionKey::SegmentSize), + OptionValue::derived(HeaderValue::from(DEFAULT_SEGMENT_SIZE)), + ); + assert_eq!(check_critical_options(&topic), Ok(())); + assert_eq!( + critical_keys_marker(&[OptionKey::EncryptionKeyWrapped]) + .unwrap() + .as_bytes(), + b"encryption_key_wrapped" + ); + } + + #[test] + fn given_admitted_stream_key_when_debugging_should_redact_it() { + let key = HeaderValue::try_from([7u8; 32].as_slice()).unwrap(); + let admitted = admit( + OptionsScope::Stream, + OptionOperation::Create, + &block(&[(OptionKey::EncryptionKey, key)]), + &[], + &OptionAdmissionContext::default(), + ) + .expect("a 32-byte key is admitted"); + let rendered = format!("{admitted:?}"); + assert!(!rendered.contains("7, 7"), "{rendered}"); + assert!(rendered.contains("<redacted>"), "{rendered}"); + } + + #[test] + fn given_invalid_requests_when_admitting_should_report_categories_in_order() { + let context = OptionAdmissionContext::default(); + let set_and_reset = admit( + OptionsScope::Topic, + OptionOperation::Update, + &block(&[(OptionKey::MessageExpiry, HeaderValue::from(9u64))]), + &[OptionKey::MessageExpiry.to_string()], + &context, + ); + assert_eq!(set_and_reset, Err(IggyError::InvalidCommand)); + assert_eq!( + admit( + OptionsScope::Topic, + OptionOperation::Update, + &WireOptions::empty(), + &[], + &context + ), + Err(IggyError::InvalidCommand) + ); + assert_eq!( + admit( + OptionsScope::Stream, + OptionOperation::Update, + &block(&[(OptionKey::MessageExpiry, HeaderValue::from(9u64))]), + &[], + &context + ), + Err(IggyError::UnsupportedOptionKey( + OptionKey::MessageExpiry.to_string() + )) + ); + assert_eq!( + admit( + OptionsScope::Stream, + OptionOperation::Create, + &block(&[( + OptionKey::EncryptionKeyWrapped, + HeaderValue::try_from([1u8; 77].as_slice()).unwrap() + )]), + &[], + &context + ), + Err(IggyError::ReadOnlyOption( + OptionKey::EncryptionKeyWrapped.to_string() + )) + ); + assert_eq!( + admit( + OptionsScope::Stream, + OptionOperation::Update, + &block(&[( + OptionKey::EncryptionKey, + HeaderValue::try_from([1u8; 32].as_slice()).unwrap() + )]), + &[], + &context + ), + Err(IggyError::OptionNotUpdatable( + OptionKey::EncryptionKey.to_string() + )) + ); + let created = admit( + OptionsScope::Stream, + OptionOperation::Create, + &block(&[( + OptionKey::EncryptionKey, + HeaderValue::try_from("AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=").unwrap(), + )]), + &[], + &context, + ) + .expect("a canonical base64 stream key is admitted"); + let stored = &created.explicit[&HeaderKey::from(OptionKey::EncryptionKey)]; + assert_eq!(stored.value.kind(), HeaderKind::Raw); + assert_eq!(stored.value.as_bytes(), (0u8..32).collect::<Vec<_>>()); + } + + fn entry<'a>(key: OptionKey, value: &'a HeaderValue) -> WireUserHeaderEntry<'a> { + let name: &'static str = key.into(); + WireUserHeaderEntry { + key_kind: WireHeaderKind(HeaderKind::String.as_code()), + key: name.as_bytes(), + value_kind: WireHeaderKind(value.kind().as_code()), + value: value.as_bytes(), + } + } + + fn block(entries: &[(OptionKey, HeaderValue)]) -> WireOptions { + let options: ResourceOptions = entries + .iter() + .map(|(key, value)| (HeaderKey::from(*key), OptionValue::explicit(value.clone()))) + .collect(); + resource_options_to_wire(&options, OptionsProvenance::All).expect("encodes") + } +}
diff --git a/core/common/src/types/options/codec.rs b/core/common/src/types/options/codec.rs new file mode 100644 index 0000000..897dd68 --- /dev/null +++ b/core/common/src/types/options/codec.rs
@@ -0,0 +1,273 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::key::OptionKey; +use super::{ + Durability, MAX_MESSAGES_REQUIRED_TO_SAVE, MAX_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE, + PartitionResizePolicy, +}; +use crate::types::compression::compression_algorithm::CompressionAlgorithm; +use crate::types::message::{HeaderKind, HeaderValue}; +use crate::{IggyByteSize, IggyError, IggyExpiry, MaxTopicSize, UserStatus}; +use iggy_binary_protocol::WireUserHeaderEntry; +use std::str::FromStr; + +/// One value type an option key carries: how it reads from a wire entry, +/// its canonical encoding and its string form. +/// +/// `decode` accepts the canonical kind and a `String` in the same form a +/// config file or an HTTP body carries, and applies the bounds that belong to +/// `key` itself. Bounds that depend on another key or on stored state are +/// checked by the caller. +pub trait OptionValueCodec: Sized { + const KIND: HeaderKind; + + /// # Errors + /// + /// `InvalidOptionValue` naming `key` when the kind, the payload or a + /// per-key bound does not hold. + fn decode(key: OptionKey, entry: &WireUserHeaderEntry<'_>) -> Result<Self, IggyError>; + + /// # Errors + /// + /// `InvalidHeaderValue` when the canonical form does not fit a header value. + fn encode(&self) -> Result<HeaderValue, IggyError>; + + fn to_option_string(&self) -> String; +} + +impl OptionValueCodec for CompressionAlgorithm { + const KIND: HeaderKind = HeaderKind::String; + + fn decode(key: OptionKey, entry: &WireUserHeaderEntry<'_>) -> Result<Self, IggyError> { + if entry.value_kind.0 == HeaderKind::Uint8.as_code() { + // Exactly one byte, for the same reason as the bool decoder. + let [code] = entry.value else { + return Err(invalid(key)); + }; + return Self::from_code(*code).map_err(|_| invalid(key)); + } + parse_text(key, entry, |text| Self::from_str(text).ok()) + } + + fn encode(&self) -> Result<HeaderValue, IggyError> { + HeaderValue::try_from(self.to_string().as_str()) + } + + fn to_option_string(&self) -> String { + self.to_string() + } +} + +impl OptionValueCodec for IggyExpiry { + const KIND: HeaderKind = HeaderKind::Uint64; + + fn decode(key: OptionKey, entry: &WireUserHeaderEntry<'_>) -> Result<Self, IggyError> { + decode_u64_or(key, entry, |text| Self::from_str(text).ok().map(u64::from)).map(Self::from) + } + + fn encode(&self) -> Result<HeaderValue, IggyError> { + Ok(HeaderValue::from(u64::from(*self))) + } + + fn to_option_string(&self) -> String { + u64::from(*self).to_string() + } +} + +impl OptionValueCodec for MaxTopicSize { + const KIND: HeaderKind = HeaderKind::Uint64; + + fn decode(key: OptionKey, entry: &WireUserHeaderEntry<'_>) -> Result<Self, IggyError> { + decode_u64_or(key, entry, |text| Self::from_str(text).ok().map(u64::from)).map(Self::from) + } + + fn encode(&self) -> Result<HeaderValue, IggyError> { + Ok(HeaderValue::from(u64::from(*self))) + } + + fn to_option_string(&self) -> String { + u64::from(*self).to_string() + } +} + +impl OptionValueCodec for IggyByteSize { + const KIND: HeaderKind = HeaderKind::Uint64; + + fn decode(key: OptionKey, entry: &WireUserHeaderEntry<'_>) -> Result<Self, IggyError> { + let bytes = decode_u64_or(key, entry, |text| { + Self::from_str(text).ok().map(|size| size.as_bytes_u64()) + })?; + if key == OptionKey::SizeOfMessagesRequiredToSave + && bytes > MAX_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE + { + return Err(invalid(key)); + } + Ok(Self::from(bytes)) + } + + fn encode(&self) -> Result<HeaderValue, IggyError> { + Ok(HeaderValue::from(self.as_bytes_u64())) + } + + fn to_option_string(&self) -> String { + self.as_bytes_u64().to_string() + } +} + +impl OptionValueCodec for Durability { + const KIND: HeaderKind = HeaderKind::String; + + fn decode(key: OptionKey, entry: &WireUserHeaderEntry<'_>) -> Result<Self, IggyError> { + parse_text(key, entry, |text| text.parse().ok()) + } + + fn encode(&self) -> Result<HeaderValue, IggyError> { + HeaderValue::from_str(self.as_ref()) + } + + fn to_option_string(&self) -> String { + self.to_string() + } +} + +impl OptionValueCodec for PartitionResizePolicy { + const KIND: HeaderKind = HeaderKind::String; + + fn decode(key: OptionKey, entry: &WireUserHeaderEntry<'_>) -> Result<Self, IggyError> { + parse_text(key, entry, |text| text.parse().ok()) + } + + fn encode(&self) -> Result<HeaderValue, IggyError> { + HeaderValue::from_str(self.as_ref()) + } + + fn to_option_string(&self) -> String { + self.to_string() + } +} + +impl OptionValueCodec for UserStatus { + const KIND: HeaderKind = HeaderKind::String; + + fn decode(key: OptionKey, entry: &WireUserHeaderEntry<'_>) -> Result<Self, IggyError> { + parse_text(key, entry, |text| text.parse().ok()) + } + + fn encode(&self) -> Result<HeaderValue, IggyError> { + HeaderValue::try_from(self.to_string().as_str()) + } + + fn to_option_string(&self) -> String { + self.to_string() + } +} + +impl OptionValueCodec for bool { + const KIND: HeaderKind = HeaderKind::Bool; + + fn decode(key: OptionKey, entry: &WireUserHeaderEntry<'_>) -> Result<Self, IggyError> { + if entry.value_kind.0 == HeaderKind::Bool.as_code() { + // Exactly one byte of 0 or 1. Anything looser admits a value that + // `HeaderValue::as_bool` later refuses to read, so the stored map + // would hold an entry its own public accessor rejects. + return match entry.value { + [0] => Ok(false), + [1] => Ok(true), + _ => Err(invalid(key)), + }; + } + parse_text(key, entry, |text| text.parse().ok()) + } + + fn encode(&self) -> Result<HeaderValue, IggyError> { + Ok(HeaderValue::from(*self)) + } + + fn to_option_string(&self) -> String { + self.to_string() + } +} + +impl OptionValueCodec for u32 { + const KIND: HeaderKind = HeaderKind::Uint32; + + fn decode(key: OptionKey, entry: &WireUserHeaderEntry<'_>) -> Result<Self, IggyError> { + let value = if entry.value_kind.0 == HeaderKind::Uint32.as_code() { + let bytes: [u8; 4] = entry.value.try_into().map_err(|_| invalid(key))?; + Self::from_le_bytes(bytes) + } else { + parse_text(key, entry, |text| text.parse().ok())? + }; + if key == OptionKey::MessagesRequiredToSave + && (value == 0 || value > MAX_MESSAGES_REQUIRED_TO_SAVE) + { + return Err(invalid(key)); + } + Ok(value) + } + + fn encode(&self) -> Result<HeaderValue, IggyError> { + Ok(HeaderValue::from(*self)) + } + + fn to_option_string(&self) -> String { + self.to_string() + } +} + +/// `InvalidOptionValue` naming `key`. +pub(super) fn invalid(key: OptionKey) -> IggyError { + IggyError::InvalidOptionValue(key.to_string()) +} + +/// `Uint64` verbatim, or a `String` holding a checked decimal `u64` or the +/// type's own text form. +/// +/// The decimal form comes first so a client that cannot carry a 64-bit integer +/// natively (a JSON number above 2^53) can still send the exact value. +fn decode_u64_or( + key: OptionKey, + entry: &WireUserHeaderEntry<'_>, + from_text: impl Fn(&str) -> Option<u64>, +) -> Result<u64, IggyError> { + if entry.value_kind.0 == HeaderKind::Uint64.as_code() { + let bytes: [u8; 8] = entry.value.try_into().map_err(|_| invalid(key))?; + return Ok(u64::from_le_bytes(bytes)); + } + parse_text(key, entry, |text| { + if !text.is_empty() && text.bytes().all(|byte| byte.is_ascii_digit()) { + return text.parse().ok(); + } + from_text(text) + }) +} + +/// The value of a `String` entry through `from_text`, refusing every other kind. +fn parse_text<T>( + key: OptionKey, + entry: &WireUserHeaderEntry<'_>, + from_text: impl Fn(&str) -> Option<T>, +) -> Result<T, IggyError> { + if entry.value_kind.0 != HeaderKind::String.as_code() { + return Err(invalid(key)); + } + std::str::from_utf8(entry.value) + .ok() + .and_then(from_text) + .ok_or_else(|| invalid(key)) +}
diff --git a/core/common/src/types/options/key.rs b/core/common/src/types/options/key.rs new file mode 100644 index 0000000..fe99018 --- /dev/null +++ b/core/common/src/types/options/key.rs
@@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::types::message::{HeaderKey, HeaderKind}; +use serde::{Deserialize, Serialize}; +use strum::{Display, EnumIter, EnumString, IntoStaticStr}; + +/// Every option key the catalog knows, spelled as it travels on the wire. +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Serialize, + Deserialize, + Display, + EnumString, + IntoStaticStr, + EnumIter, +)] +#[serde(rename_all = "snake_case")] +#[strum(serialize_all = "snake_case")] +pub enum OptionKey { + Name, + Username, + Status, + CompressionAlgorithm, + MessageExpiry, + MaxTopicSize, + SegmentSize, + Durability, + ConsumerOffsetDurability, + PartitionResizePolicy, + MessagesRequiredToSave, + SizeOfMessagesRequiredToSave, + PreallocateSegments, + EncryptionKey, + EncryptionKeyWrapped, + EncryptionKeySource, + EncryptionKeyFingerprint, + EncryptionKekFingerprint, + CriticalKeys, +} + +impl From<OptionKey> for HeaderKey { + fn from(key: OptionKey) -> Self { + let name: &'static str = key.into(); + // Catalog names are short ASCII, always a valid string header key. + Self::new_unchecked(HeaderKind::String, name.as_bytes()) + } +} + +impl TryFrom<&HeaderKey> for OptionKey { + type Error = (); + + fn try_from(key: &HeaderKey) -> Result<Self, Self::Error> { + if key.kind() != HeaderKind::String { + return Err(()); + } + std::str::from_utf8(key.as_bytes()) + .ok() + .and_then(|name| name.parse().ok()) + .ok_or(()) + } +}
diff --git a/core/common/src/types/options/mod.rs b/core/common/src/types/options/mod.rs index d64a49d..284dd33 100644 --- a/core/common/src/types/options/mod.rs +++ b/core/common/src/types/options/mod.rs
@@ -15,7 +15,8 @@ // specific language governing permissions and limitations // under the License. -//! Key-value options attached to streams, topics, and users at creation. +//! Key-value options of streams, topics, and users: set at creation, changed +//! later with an `UpdateResource` patch. //! //! Options reuse the typed [`HeaderKey`] / [`HeaderValue`] machinery from //! message user headers. The map MUST stay a `BTreeMap`: metadata snapshots @@ -24,50 +25,69 @@ //! //! # SDK parity //! -//! Every SDK can set any option key, read both blocks back, and ask the server -//! which keys it accepts. What differs is only how each language spells the -//! argument, since each reuses the type it already had for message user headers: +//! Every SDK sends catalog keys through typed create and update options, names +//! resets with a typed key enum, reads both option blocks back and asks the +//! server which keys it accepts with `describe_options`. A key newer than the +//! build travels through the raw escape hatch, string-keyed entries and reset +//! names that only server admission judges: //! -//! | SDK | Arbitrary keys on create/update | Reads the blocks | `describe_options` | -//! |--------|--------------------------------------|------------------|--------------------| -//! | Rust | `TopicCreateOptions::raw` | yes | yes | -//! | Node | `options?: OptionEntry[]` | yes | yes | -//! | Python | `options=` string dict | yes | yes | -//! | Go | trailing `options ...HeaderEntry` | yes | yes | -//! | Java | `Map<String, HeaderValue>` overload | yes | yes | -//! | C# | optional `IReadOnlyDictionary` | yes | yes | -//! | C++ | trailing `Vec<HeaderEntry>` | yes | yes | +//! | SDK | Typed options | Raw entries and resets | +//! |--------|---------------------------------------------------------|---------------------------------| +//! | Rust | `StreamCreateOptions`, `TopicUpdateOptions` and kin | `raw`, `raw_reset` | +//! | Node | `StreamOptions`, `UpdateTopic` and kin | `options`, `rawReset` | +//! | Python | `StreamOptions`, `TopicOptions`, `update_*` keywords | `raw` or `options`, `raw_reset` | +//! | Go | `StreamOptions`, `TopicUpdate` and kin | `Raw` or `Options`, `RawReset` | +//! | Java | `StreamOptions`, `TopicUpdate` and kin | `raw`, `rawReset` | +//! | C# | `StreamOptions`, `TopicUpdate` and kin | `Options`, `RawReset` | +//! | C++ | `StreamCreateOptions`, `TopicUpdateOptions` and kin | `SetRawEntries()`, `RawReset()` | +//! | Swift | `StreamCreateOptions`, `TopicUpdateOptions` and kin | `raw`, `rawReset` | +//! | PHP | named arguments of `createTopic`, `updateTopic` and kin | `$raw`, `$raw_reset` | //! -//! One transport caveat everywhere: REST renders option values as readable -//! strings rather than the typed TLV, so a value read back over HTTP is -//! String-kinded while the same value over a binary transport carries the kind -//! the server stored. REST also reports both provenances in one map with a -//! per-entry flag, which each SDK splits into the two maps its binary path -//! produces. +//! REST carries each value as reversible text with its kind, so an HTTP reader +//! rebuilds the value a binary reader gets. It reports both provenances in one +//! map with a per-entry flag, which each SDK splits into the two maps its +//! binary path produces. //! -//! Beyond the eight typed keys, Go, Java, C# and C++ ship typed constructors or -//! builders for the five keys that have no named parameter of their own, so no -//! caller has to spell an option key as a bare string. -//! -//! The block's byte layout is pinned by a golden vector that Rust, Node, Go and -//! Java each assert independently, so a new encoder has a fixture to match -//! rather than a description to interpret. +//! The byte layout of every option-carrying request and response is pinned by +//! the `resource_options.json` fixture, which Rust and all eight foreign SDKs +//! assert independently, so a new encoder has a fixture to match rather than +//! a description to interpret. +mod catalog; +mod codec; mod durability; +mod key; mod partition_resize_policy; +pub use catalog::{ + AdmittedOptions, CATALOG, OPTION_FLAG_CRITICAL, OPTION_FLAG_READ_ONLY, OPTION_FLAG_RESETTABLE, + OPTION_FLAG_SECRET, OPTION_FLAG_UPDATABLE, OptionAdmissionContext, OptionColumns, + OptionDefault, OptionDescriptor, OptionMutability, OptionOperation, OptionStorage, + OptionVisibility, StreamEncryptionSource, admit, canonical_value, check_critical_options, + critical_keys_marker, derived_defaults, descriptors_for, public_options, stream_key_from_entry, + stream_wrap_set, +}; +pub use codec::OptionValueCodec; pub use durability::Durability; +pub use key::OptionKey; pub use partition_resize_policy::PartitionResizePolicy; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt::{Debug, Formatter}; use std::str::FromStr; -use iggy_binary_protocol::{WireOptions, WireUserHeaderEntry}; +use bytes::Bytes; +use iggy_binary_protocol::requests::system::{MAX_RESET_KEYS, WireResetKeys}; +use iggy_binary_protocol::{WireName, WireOptions, WireUserHeaderEntry}; +use secrecy::zeroize::Zeroize; use serde::{Deserialize, Serialize}; +use self::catalog::critical_wrap_set; +use crate::http::options::{OptionInputValue, OptionsUpdate, RESET_FIELD}; use crate::types::compression::compression_algorithm::CompressionAlgorithm; use crate::types::message::{HeaderKey, HeaderKind, HeaderValue}; -use crate::{IggyByteSize, IggyError, IggyExpiry, MaxTopicSize}; +use crate::utils::stream_key::{StreamEncryptionKey, WrapSet}; +use crate::{IggyByteSize, IggyError, IggyExpiry, MaxTopicSize, UserStatus}; /// A single resolved option entry. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -120,6 +140,14 @@ explicit: false, } } + + /// Zero the value's bytes when this is their only handle, for a value that + /// may be a secret. Shared bytes are left to their other owners. + pub fn erase(self) { + let bytes = self.value.value(); + drop(self); + erase_buffer(bytes); + } } /// Options attached to a stream, topic, or user, keyed by option name. @@ -133,52 +161,56 @@ /// carrying options goes through here instead, rendering the key as its plain /// text. Binary transports are unaffected: they never touch serde. pub mod resource_options_json { - use super::{HeaderKey, HeaderValue, OptionValue, ResourceOptions}; + use super::{HeaderKey, HeaderKind, HeaderValue, OptionValue, ResourceOptions}; + use crate::IggyError; + use base64::Engine; + use base64::engine::general_purpose::STANDARD; use serde::de::Error as _; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use std::collections::BTreeMap; use std::str::FromStr; - /// One entry's JSON form: the value in the same string form a config file - /// or a `POST` body would carry it in, plus its provenance. + /// One entry's JSON form: the value as text, its kind, and its provenance. /// - /// The stored value is a typed [`HeaderValue`], and serializing that gives - /// `{"kind":"uint64","value":"<base64>"}` - which no operator can read and - /// no client can feed back, since the create body takes options as a plain - /// string map. Rendering the string form makes the "re-send only the - /// explicit options" round trip actually expressible over HTTP. + /// The text is reversible for every kind, so an HTTP reader rebuilds the + /// exact value a binary reader gets: UTF-8 for strings, `true`/`false`, + /// base-10 integers, standard padded base64 for raw bytes, and `0x` + /// followed by the IEEE-754 bits in fixed-width lowercase hex for floats. #[derive(Serialize, Deserialize)] struct OptionValueJson { value: String, + kind: HeaderKind, explicit: bool, } /// # Errors /// - /// Propagates the serializer's own errors. + /// Propagates the serializer's own errors, and a stored value whose bytes + /// do not match its kind. pub fn serialize<S: Serializer>( options: &ResourceOptions, serializer: S, ) -> Result<S::Ok, S::Error> { - let readable: BTreeMap<String, OptionValueJson> = options + let readable = options .iter() .map(|(key, option)| { - ( + Ok(( String::from_utf8_lossy(key.as_bytes()).into_owned(), OptionValueJson { - value: option.value.to_string_value(), + value: encode_value(&option.value).map_err(serde::ser::Error::custom)?, + kind: option.value.kind(), explicit: option.explicit, }, - ) + )) }) - .collect(); + .collect::<Result<BTreeMap<String, OptionValueJson>, S::Error>>()?; readable.serialize(serializer) } /// # Errors /// /// Returns a deserializer error when a key is not a valid option name or a - /// value does not fit a header value. + /// value's text does not decode as its kind. pub fn deserialize<'de, D: Deserializer<'de>>( deserializer: D, ) -> Result<ResourceOptions, D::Error> { @@ -187,7 +219,7 @@ .into_iter() .map(|(key, option)| { let key = HeaderKey::from_str(&key).map_err(D::Error::custom)?; - let value = HeaderValue::from_str(&option.value).map_err(D::Error::custom)?; + let value = decode_value(option.kind, &option.value).map_err(D::Error::custom)?; Ok(( key, OptionValue { @@ -198,6 +230,85 @@ }) .collect() } + + /// The reversible text of `value` for its kind. + /// + /// # Errors + /// + /// `InvalidHeaderValue` when the bytes do not match the kind's size or + /// encoding. + pub fn encode_value(value: &HeaderValue) -> Result<String, IggyError> { + let bytes = value.as_bytes(); + Ok(match value.kind() { + HeaderKind::Raw => STANDARD.encode(bytes), + HeaderKind::String => value.as_str()?.to_owned(), + HeaderKind::Bool => value.as_bool()?.to_string(), + HeaderKind::Int8 => value.as_int8()?.to_string(), + HeaderKind::Int16 => value.as_int16()?.to_string(), + HeaderKind::Int32 => value.as_int32()?.to_string(), + HeaderKind::Int64 => value.as_int64()?.to_string(), + HeaderKind::Int128 => value.as_int128()?.to_string(), + HeaderKind::Uint8 => value.as_uint8()?.to_string(), + HeaderKind::Uint16 => value.as_uint16()?.to_string(), + HeaderKind::Uint32 => value.as_uint32()?.to_string(), + HeaderKind::Uint64 => value.as_uint64()?.to_string(), + HeaderKind::Uint128 => value.as_uint128()?.to_string(), + HeaderKind::Float32 => format!("0x{:08x}", value.as_float32()?.to_bits()), + HeaderKind::Float64 => format!("0x{:016x}", value.as_float64()?.to_bits()), + }) + } + + /// The value `text` encodes for `kind`, the inverse of [`encode_value`]. + /// + /// # Errors + /// + /// `InvalidHeaderValue` when the text is not the canonical form of a value + /// of that kind. + pub fn decode_value(kind: HeaderKind, text: &str) -> Result<HeaderValue, IggyError> { + let value = match kind { + HeaderKind::Raw => { + HeaderValue::from_raw(kind, &STANDARD.decode(text).map_err(invalid)?)? + } + HeaderKind::String => HeaderValue::try_from(text)?, + HeaderKind::Bool => match text { + "true" => HeaderValue::from(true), + "false" => HeaderValue::from(false), + _ => return Err(IggyError::InvalidHeaderValue), + }, + HeaderKind::Int8 => HeaderValue::from(text.parse::<i8>().map_err(invalid)?), + HeaderKind::Int16 => HeaderValue::from(text.parse::<i16>().map_err(invalid)?), + HeaderKind::Int32 => HeaderValue::from(text.parse::<i32>().map_err(invalid)?), + HeaderKind::Int64 => HeaderValue::from(text.parse::<i64>().map_err(invalid)?), + HeaderKind::Int128 => HeaderValue::from(text.parse::<i128>().map_err(invalid)?), + HeaderKind::Uint8 => HeaderValue::from(text.parse::<u8>().map_err(invalid)?), + HeaderKind::Uint16 => HeaderValue::from(text.parse::<u16>().map_err(invalid)?), + HeaderKind::Uint32 => HeaderValue::from(text.parse::<u32>().map_err(invalid)?), + HeaderKind::Uint64 => HeaderValue::from(text.parse::<u64>().map_err(invalid)?), + HeaderKind::Uint128 => HeaderValue::from(text.parse::<u128>().map_err(invalid)?), + HeaderKind::Float32 => HeaderValue::from(f32::from_bits( + u32::try_from(float_bits(text, 8)?).map_err(invalid)?, + )), + HeaderKind::Float64 => HeaderValue::from(f64::from_bits(float_bits(text, 16)?)), + }; + // Reject non-canonical text (a leading `+`, padding variants, leading + // zeros) so one value has one JSON form. + if encode_value(&value)? != text { + return Err(IggyError::InvalidHeaderValue); + } + Ok(value) + } + + fn invalid<E>(_: E) -> IggyError { + IggyError::InvalidHeaderValue + } + + fn float_bits(text: &str, digits: usize) -> Result<u64, IggyError> { + let hex = text + .strip_prefix("0x") + .filter(|hex| hex.len() == digits) + .ok_or(IggyError::InvalidHeaderValue)?; + u64::from_str_radix(hex, 16).map_err(|_| IggyError::InvalidHeaderValue) + } } /// Topic option catalog: the keys `CreateTopic` accepts. @@ -255,23 +366,6 @@ /// 1 MiB (was `[partition] size_of_messages_required_to_save`). pub const DEFAULT_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE: u64 = 1024 * 1024; -/// Every runtime knob at its default, for a partition built with no resolved -/// topic options (simulator, unit tests). -impl Default for TopicRuntimeDefaults { - fn default() -> Self { - Self { - segment_size: IggyByteSize::from(DEFAULT_SEGMENT_SIZE), - durability: Durability::default(), - consumer_offset_durability: Durability::default(), - messages_required_to_save: DEFAULT_MESSAGES_REQUIRED_TO_SAVE, - size_of_messages_required_to_save: IggyByteSize::from( - DEFAULT_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE, - ), - preallocate_segments: DEFAULT_PREALLOCATE_SEGMENTS, - } - } -} - /// Ceiling for `size_of_messages_required_to_save`. /// /// A threshold above the largest a segment may be can never trip, so every @@ -362,6 +456,20 @@ Ok(()) } +/// Zero every value of `options` this map holds the only handle to, for a map +/// that may hold a secret value. A value another handle shares is left to it. +pub fn erase_options(options: ResourceOptions) { + options.into_values().for_each(OptionValue::erase); +} + +/// Zero `buffer` when this is its only handle, for a buffer that may hold a +/// secret. A shared buffer is left to its other owners. +pub fn erase_buffer(buffer: Bytes) { + if let Ok(mut owned) = buffer.try_into_mut() { + owned.as_mut().zeroize(); + } +} + /// Resource whose option catalog `DescribeOptions` serves. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -427,9 +535,23 @@ /// it as. Create admission re-encodes the block from its own parse, so a /// `--set segment_size=128MiB` string lands as a `Uint64`. /// - /// An UPDATE is the exception: it stores the client's bytes verbatim, so a - /// key set that way keeps the kind it arrived in. + /// Updates canonicalize the same way. pub kind: HeaderKind, + /// An update may set the key. + #[serde(default)] + pub updatable: bool, + /// An update may reset the key to its default. + #[serde(default)] + pub resettable: bool, + /// Write-only: accepted, never returned. + #[serde(default)] + pub secret: bool, + /// Computed by the server and returned on reads, never accepted. + #[serde(default)] + pub read_only: bool, + /// A node that does not understand the key must not serve the resource. + #[serde(default)] + pub critical: bool, /// The key's default in `kind`'s encoding; empty when the key has no /// default. A build constant rather than a per-node value: these defaults /// stopped being config-derived when the `[system.*]` keys became options. @@ -440,53 +562,6 @@ pub description: String, } -/// Every key `CreateTopic` currently accepts. Unknown keys are rejected at -/// the edge, never skipped: a silently ignored knob would hand the client -/// server defaults without it ever learning. -pub const TOPIC_OPTION_KEYS: &[&str] = &[ - topic_option_keys::COMPRESSION_ALGORITHM, - topic_option_keys::MESSAGE_EXPIRY, - topic_option_keys::MAX_TOPIC_SIZE, - topic_option_keys::SEGMENT_SIZE, - topic_option_keys::DURABILITY, - topic_option_keys::CONSUMER_OFFSET_DURABILITY, - topic_option_keys::PARTITION_RESIZE_POLICY, - topic_option_keys::MESSAGES_REQUIRED_TO_SAVE, - topic_option_keys::SIZE_OF_MESSAGES_REQUIRED_TO_SAVE, - topic_option_keys::PREALLOCATE_SEGMENTS, -]; - -/// The subset of [`TOPIC_OPTION_KEYS`] an `UpdateTopic` options block may -/// carry. -/// -/// These three used to be fixed fields of the update command, which meant one -/// setting had two homes and an update always rewrote all three whether the -/// caller meant to or not. As options they are patched: a key the client did -/// not send keeps its current value. -/// -/// The partition runtime knobs (`segment_size`, both durability policies, both flush -/// thresholds, `preallocate_segments`) stay out, and not only because nothing -/// re-pushes them to a live partition. They describe how a partition's storage -/// was laid down: changing `segment_size` mid-segment leaves one segment sized -/// by the old cap and the next by the new one, and `preallocate_segments` can -/// only act on a file not yet opened. A topic gets them at creation and keeps -/// them, so its segments stay uniform. -pub const UPDATABLE_TOPIC_OPTION_KEYS: &[&str] = &[ - topic_option_keys::COMPRESSION_ALGORITHM, - topic_option_keys::MESSAGE_EXPIRY, - topic_option_keys::MAX_TOPIC_SIZE, -]; - -/// Keys an `UpdateStream` options block may carry. Empty because streams have -/// no catalog keys yet: the block exists so the first one costs a catalog -/// entry instead of another wire change, and until then every key is rejected -/// by name rather than stored and ignored. -pub const UPDATABLE_STREAM_OPTION_KEYS: &[&str] = &[]; - -/// Keys an `UpdateUser` options block may carry. Empty for the same reason as -/// [`UPDATABLE_STREAM_OPTION_KEYS`]. -pub const UPDATABLE_USER_OPTION_KEYS: &[&str] = &[]; - /// Longest key echoed back in an option error. A rejected key is attacker-sized /// by definition, and over HTTP the error text rides the response body back to /// the client. @@ -497,16 +572,41 @@ /// there - see the note on that variant. const ERROR_KEY_PREVIEW_LEN: usize = 64; -fn key_preview(key: &str) -> String { +pub(crate) fn key_preview(key: &str) -> String { key.chars().take(ERROR_KEY_PREVIEW_LEN).collect() } +/// Reset names in wire form. +/// +/// # Errors +/// +/// `UnsupportedOptionKey` for a name no option can have, and for a list longer +/// than the wire carries, naming one of its names outside the catalog. +pub(crate) fn wire_reset_names<'a>( + names: impl Iterator<Item = &'a str>, +) -> Result<WireResetKeys, IggyError> { + let reset = names + .map(|name| { + WireName::new(name).map_err(|_| IggyError::UnsupportedOptionKey(key_preview(name))) + }) + .collect::<Result<Vec<_>, _>>()?; + if reset.len() > MAX_RESET_KEYS { + // The catalog is far smaller than the limit, so such a list always + // names a key admission would refuse first. + return Err(reset + .iter() + .find(|name| OptionKey::from_str(name.as_str()).is_err()) + .map_or(IggyError::InvalidCommand, |name| { + IggyError::UnsupportedOptionKey(key_preview(name.as_str())) + })); + } + WireResetKeys::try_from(reset).map_err(|_| IggyError::InvalidCommand) +} + /// Build an options map from string key-values, all marked explicit. /// -/// Shared by the update-options types: their keys are all client-sent, so none -/// of the derived-provenance handling that create needs applies. Callers that -/// also have typed fields insert those afterwards, so a typed value wins on -/// collision and keeps its canonical kind. +/// Callers that also have typed fields insert those afterwards, so a typed +/// value wins on collision and keeps its canonical kind. /// /// # Errors /// @@ -516,83 +616,501 @@ fn raw_options_map(raw: &BTreeMap<String, String>) -> Result<ResourceOptions, IggyError> { let mut options = ResourceOptions::new(); for (key, value) in raw { - let header_key = HeaderKey::from_str(key) - .map_err(|_| IggyError::UnsupportedOptionKey(key_preview(key)))?; - let header_value = HeaderValue::try_from(value.as_str()) - .map_err(|_| IggyError::InvalidOptionValue(key_preview(key)))?; - options.insert(header_key, OptionValue::explicit(header_value)); + let entry = HeaderKey::from_str(key) + .map_err(|_| IggyError::UnsupportedOptionKey(key_preview(key))) + .and_then(|header_key| { + HeaderValue::try_from(value.as_str()) + .map(|header_value| (header_key, header_value)) + .map_err(|_| IggyError::InvalidOptionValue(key_preview(key))) + }); + match entry { + Ok((header_key, header_value)) => { + options.insert(header_key, OptionValue::explicit(header_value)); + } + Err(error) => { + // A raw value may be a secret, the stream key among them. + erase_options(options); + return Err(error); + } + } } Ok(options) } -/// Encode string key-values into an options block, all marked explicit. -/// -/// # Errors -/// -/// See [`raw_options_map`]. -fn raw_options_to_wire(raw: &BTreeMap<String, String>) -> Result<WireOptions, IggyError> { - crate::wire_conversions::resource_options_to_wire( - &raw_options_map(raw)?, - OptionsProvenance::All, - ) +/// An update's options in wire form: the keys to set and the keys to reset +/// to their defaults. Every other key keeps its stored value. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OptionsPatch { + pub set: WireOptions, + pub reset: WireResetKeys, } -/// The options an `UpdateStream` may carry. -/// -/// Absent keys leave the stream's existing values alone -- an update patches -/// the option map, it does not replace it. See [`TopicUpdateOptions`] for why. +/// The options a `CreateStream` may carry. +#[derive(Clone, Default, PartialEq)] +pub struct StreamCreateOptions { + /// The stream's own data encryption key. `None` follows the server + /// encryption setting. The server stores the key wrapped and never + /// returns it, reads report only its source and fingerprint. + pub encryption_key: Option<StreamEncryptionKey>, + /// Keys sent as `String` values, for keys newer than this build. A typed + /// field wins on collision. + pub raw: BTreeMap<String, String>, +} + +/// The options an update may change on a stream. Absent keys keep their +/// stored values. #[derive(Debug, Clone, Default, PartialEq)] pub struct StreamUpdateOptions { - /// Keys sent as `String` values, checked against - /// [`UPDATABLE_STREAM_OPTION_KEYS`] server-side. That list is empty, so - /// every key is currently refused by name; the field exists so the first - /// updatable stream key costs a catalog entry, not a wire change. + /// `None` keeps the stream's name. + pub name: Option<String>, + /// Keys to reset to their defaults. No stream key is resettable yet, so a + /// non-empty set is refused before anything is sent. + pub reset: BTreeSet<OptionKey>, + /// Keys to reset by name, for keys newer than this build. The server + /// decides whether each one may be reset. + pub raw_reset: BTreeSet<String>, + /// Keys sent as `String` values, for updatable keys newer than this build. pub raw: BTreeMap<String, String>, } +/// The options an update may change on a topic. Absent keys keep their stored +/// values. Create-only settings (compression, `segment_size`, durability, +/// flush thresholds, preallocation, resize policy) have no field here, so a +/// caller cannot send one only to learn at the server that it was refused. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct TopicUpdateOptions { + /// `None` keeps the topic's name. + pub name: Option<String>, + /// `None` keeps the topic's current expiry. `ServerDefault` changes + /// nothing, use `reset` to return to the default. + pub message_expiry: Option<IggyExpiry>, + /// `None` keeps the topic's current cap. `ServerDefault` changes nothing, + /// use `reset` to return to the default. + pub max_topic_size: Option<MaxTopicSize>, + /// Keys to reset to their defaults: `message_expiry` or `max_topic_size`. + pub reset: BTreeSet<OptionKey>, + /// Keys to reset by name, for keys newer than this build. The server + /// decides whether each one may be reset. + pub raw_reset: BTreeSet<String>, + /// Keys sent as `String` values, for updatable keys newer than this build. + pub raw: BTreeMap<String, String>, +} + +/// The options a `CreateUser` may carry. No user key exists yet, the type +/// exists so the first one costs a field, not a signature change. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct UserCreateOptions { + /// Keys sent as `String` values, for keys newer than this build. + pub raw: BTreeMap<String, String>, +} + +/// The options an update may change on a user. Absent keys keep their stored +/// values. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct UserUpdateOptions { + /// `None` keeps the username. + pub username: Option<String>, + /// `None` keeps the status. Deactivating a user ends its sessions. + pub status: Option<UserStatus>, + /// Keys to reset to their defaults. No user key is resettable yet, so a + /// non-empty set is refused before anything is sent. + pub reset: BTreeSet<OptionKey>, + /// Keys to reset by name, for keys newer than this build. The server + /// decides whether each one may be reset. + pub raw_reset: BTreeSet<String>, + /// Keys sent as `String` values, for updatable keys newer than this build. + pub raw: BTreeMap<String, String>, +} + +/// How a stream's messages are stored, read from its committed options. +#[derive(Debug, Clone, PartialEq)] +pub enum StreamEncryption { + /// No wrap set: the stream is plaintext for life. + Plaintext, + /// Encrypted under the key the wrap set holds. + Wrapped(WrapSet), + /// Critical state this build cannot read. The stream is never served as + /// plaintext then. + Unavailable(IggyError), +} + +/// A topic's effective settings, every one resolved: the stored value, or the +/// catalog default for a key the map does not hold. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct TopicResolvedOptions { + pub compression_algorithm: CompressionAlgorithm, + pub message_expiry: IggyExpiry, + pub max_topic_size: MaxTopicSize, + pub partition_resize_policy: PartitionResizePolicy, + /// The partition-facing knobs, with the shard-default semantics of + /// [`TopicRuntimeOptions`]. + pub runtime: TopicRuntimeOptions, +} + +impl StreamCreateOptions { + /// Encode the present keys into a create options block. + /// + /// # Errors + /// + /// See `raw_options_map`. + pub fn to_wire(&self) -> Result<WireOptions, IggyError> { + let options = self.to_option_map()?; + let wire = + crate::wire_conversions::resource_options_to_wire(&options, OptionsProvenance::All); + erase_options(options); + wire + } + + /// The present keys as string values for an HTTP body. The key travels as + /// canonical standard base64. + /// + /// # Errors + /// + /// See `raw_options_map`. + pub fn to_input_map(&self) -> Result<BTreeMap<String, OptionInputValue>, IggyError> { + let options = self.to_option_map()?; + let input = option_input_map(&options); + erase_options(options); + input + } + + /// The present keys as a map that holds the key itself, so callers erase it. + fn to_option_map(&self) -> Result<ResourceOptions, IggyError> { + let mut options = raw_options_map(&self.raw)?; + if let Some(key) = &self.encryption_key { + match HeaderValue::try_from(key.as_bytes().as_slice()) { + Ok(value) => { + options.insert( + HeaderKey::from(OptionKey::EncryptionKey), + OptionValue::explicit(value), + ); + } + Err(error) => { + erase_options(options); + return Err(error); + } + } + } + Ok(options) + } +} + impl StreamUpdateOptions { - /// Encode the present keys into an options block. + /// Encode the patch for `UpdateResource`. /// /// # Errors /// - /// See `raw_options_to_wire`. - pub fn to_wire(&self) -> Result<WireOptions, IggyError> { - raw_options_to_wire(&self.raw) + /// `InvalidCommand` when the patch changes nothing or names a key both + /// set and reset, the server's admission error for a key that cannot be + /// reset, `InvalidStreamName` for a name that does not fit. + pub fn to_patch(&self) -> Result<OptionsPatch, IggyError> { + options_patch( + OptionsScope::Stream, + self.to_option_map()?, + &self.reset, + &self.raw_reset, + !self.raw.is_empty(), + ) + } + + /// The patch as an HTTP update body: set keys with their values, reset keys + /// in the `reset` array. + /// + /// # Errors + /// + /// See [`Self::to_patch`]. + pub fn to_update_body(&self) -> Result<OptionsUpdate, IggyError> { + update_body( + OptionsScope::Stream, + self.to_option_map()?, + &self.reset, + &self.raw_reset, + !self.raw.is_empty(), + ) + } + + fn to_option_map(&self) -> Result<ResourceOptions, IggyError> { + let mut options = raw_options_map(&self.raw)?; + if let Some(name) = &self.name { + insert_text(&mut options, OptionKey::Name, name); + } + Ok(options) } } -/// The options an `UpdateUser` may carry. Same patch semantics as -/// [`StreamUpdateOptions`]. -#[derive(Debug, Clone, Default, PartialEq)] -pub struct UserUpdateOptions { - /// Keys sent as `String` values, checked against - /// [`UPDATABLE_USER_OPTION_KEYS`] server-side. That list is empty, so every - /// key is currently refused by name; the field exists so the first - /// updatable user key costs a catalog entry, not a wire change. - pub raw: BTreeMap<String, String>, +impl TopicUpdateOptions { + /// Encode the patch for `UpdateResource`, in canonical kinds. + /// + /// # Errors + /// + /// `InvalidCommand` when the patch changes nothing or names a key both + /// set and reset, `OptionNotUpdatable` for a create-only key, + /// `OptionNotResettable` for a key without a default, `InvalidTopicName` + /// for a name that does not fit. + pub fn to_patch(&self) -> Result<OptionsPatch, IggyError> { + options_patch( + OptionsScope::Topic, + self.to_option_map()?, + &self.reset, + &self.raw_reset, + !self.raw.is_empty(), + ) + } + + /// The patch as an HTTP update body. + /// + /// # Errors + /// + /// See [`Self::to_patch`]. + pub fn to_update_body(&self) -> Result<OptionsUpdate, IggyError> { + update_body( + OptionsScope::Topic, + self.to_option_map()?, + &self.reset, + &self.raw_reset, + !self.raw.is_empty(), + ) + } + + fn to_option_map(&self) -> Result<ResourceOptions, IggyError> { + let mut options = raw_options_map(&self.raw)?; + if let Some(name) = &self.name { + insert_text(&mut options, OptionKey::Name, name); + } + if let Some(message_expiry) = self.message_expiry + && message_expiry != IggyExpiry::ServerDefault + { + options.insert( + HeaderKey::from(OptionKey::MessageExpiry), + OptionValue::explicit(message_expiry.encode()?), + ); + } + if let Some(max_topic_size) = self.max_topic_size + && max_topic_size != MaxTopicSize::ServerDefault + { + options.insert( + HeaderKey::from(OptionKey::MaxTopicSize), + OptionValue::explicit(max_topic_size.encode()?), + ); + } + Ok(options) + } +} + +impl UserCreateOptions { + /// Encode the present keys into a create options block. + /// + /// # Errors + /// + /// See `raw_options_map`. + pub fn to_wire(&self) -> Result<WireOptions, IggyError> { + crate::wire_conversions::resource_options_to_wire( + &raw_options_map(&self.raw)?, + OptionsProvenance::All, + ) + } + + /// The present keys as values for an HTTP body. + /// + /// # Errors + /// + /// See `raw_options_map`. + pub fn to_input_map(&self) -> Result<BTreeMap<String, OptionInputValue>, IggyError> { + option_input_map(&raw_options_map(&self.raw)?) + } } impl UserUpdateOptions { - /// Encode the present keys into an options block. + /// Encode the patch for `UpdateResource`. /// /// # Errors /// - /// See `raw_options_to_wire`. - pub fn to_wire(&self) -> Result<WireOptions, IggyError> { - raw_options_to_wire(&self.raw) + /// `InvalidCommand` when the patch changes nothing or names a key both + /// set and reset, the server's admission error for a key that cannot be + /// reset, `InvalidUsername` for a username that does not fit. + pub fn to_patch(&self) -> Result<OptionsPatch, IggyError> { + options_patch( + OptionsScope::User, + self.to_option_map()?, + &self.reset, + &self.raw_reset, + !self.raw.is_empty(), + ) + } + + /// The patch as an HTTP update body. + /// + /// # Errors + /// + /// See [`Self::to_patch`]. + pub fn to_update_body(&self) -> Result<OptionsUpdate, IggyError> { + update_body( + OptionsScope::User, + self.to_option_map()?, + &self.reset, + &self.raw_reset, + !self.raw.is_empty(), + ) + } + + fn to_option_map(&self) -> Result<ResourceOptions, IggyError> { + let mut options = raw_options_map(&self.raw)?; + if let Some(username) = &self.username { + insert_text(&mut options, OptionKey::Username, username); + } + if let Some(status) = self.status { + options.insert( + HeaderKey::from(OptionKey::Status), + OptionValue::explicit(status.encode()?), + ); + } + Ok(options) } } -/// This node's configured fallbacks for the runtime knobs, used to fill the -/// derived-options block for keys a client did not send. -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct TopicRuntimeDefaults { - pub segment_size: IggyByteSize, - pub durability: Durability, - pub consumer_offset_durability: Durability, - pub messages_required_to_save: u32, - pub size_of_messages_required_to_save: IggyByteSize, - pub preallocate_segments: bool, +impl From<&ResourceOptions> for StreamEncryption { + fn from(options: &ResourceOptions) -> Self { + match critical_wrap_set(options) { + Ok(Some(set)) => Self::Wrapped(set), + Ok(None) => Self::Plaintext, + Err(error) => Self::Unavailable(error), + } + } +} + +impl From<&ResourceOptions> for TopicResolvedOptions { + /// Degrades per key: an entry this build cannot read resolves to its + /// catalog default and leaves every other setting intact. + fn from(options: &ResourceOptions) -> Self { + let parsed = TopicCreateOptions::from_resource_options(options); + Self { + compression_algorithm: parsed.compression_algorithm.unwrap_or_default(), + message_expiry: parsed + .message_expiry + .unwrap_or_else(|| IggyExpiry::from(DEFAULT_MESSAGE_EXPIRY)), + max_topic_size: parsed + .max_topic_size + .unwrap_or_else(|| MaxTopicSize::from(DEFAULT_MAX_TOPIC_SIZE)), + partition_resize_policy: parsed.partition_resize_policy.unwrap_or_default(), + runtime: TopicRuntimeOptions::from(&parsed), + } + } +} + +impl Debug for StreamCreateOptions { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + let raw: BTreeMap<&str, &str> = self + .raw + .iter() + .map(|(key, value)| { + let shown = if key.as_str() == <&str>::from(OptionKey::EncryptionKey) { + "<redacted>" + } else { + value.as_str() + }; + (key.as_str(), shown) + }) + .collect(); + formatter + .debug_struct("StreamCreateOptions") + .field("encryption_key", &self.encryption_key) + .field("raw", &raw) + .finish() + } +} + +/// Insert a text column value. Text that no header value holds travels as a +/// value of a kind the column never accepts, so admission reports the +/// column's own error after every check that comes before it. +fn insert_text(options: &mut ResourceOptions, key: OptionKey, text: &str) { + let value = HeaderValue::try_from(text).unwrap_or_else(|_| HeaderValue::from(false)); + options.insert(HeaderKey::from(key), OptionValue::explicit(value)); +} + +/// Validate a patch for `scope` and encode its set and reset parts. +fn options_patch( + scope: OptionsScope, + set: ResourceOptions, + reset: &BTreeSet<OptionKey>, + raw_reset: &BTreeSet<String>, + has_raw_set: bool, +) -> Result<OptionsPatch, IggyError> { + let reset = validate_patch(scope, &set, reset, raw_reset, has_raw_set)?; + Ok(OptionsPatch { + set: crate::wire_conversions::resource_options_to_wire(&set, OptionsProvenance::All)?, + reset: wire_reset_names(reset.iter().map(String::as_str))?, + }) +} + +fn update_body( + scope: OptionsScope, + set: ResourceOptions, + reset: &BTreeSet<OptionKey>, + raw_reset: &BTreeSet<String>, + has_raw_set: bool, +) -> Result<OptionsUpdate, IggyError> { + let reset = validate_patch(scope, &set, reset, raw_reset, has_raw_set)?; + let set = option_input_map(&set)?; + if set.contains_key(RESET_FIELD) { + return Err(IggyError::UnsupportedOptionKey(RESET_FIELD.to_owned())); + } + Ok(OptionsUpdate { + set, + reset: reset.into_iter().collect(), + }) +} + +/// Check a patch the way server admission does and return its reset names. +/// +/// A patch with raw entries gets only the structural checks, because the +/// server's catalog may know keys or flags this build does not. The server +/// decides the rest, so a refusal never reads differently on either side. +fn validate_patch( + scope: OptionsScope, + set: &ResourceOptions, + reset: &BTreeSet<OptionKey>, + raw_reset: &BTreeSet<String>, + has_raw_set: bool, +) -> Result<Vec<String>, IggyError> { + let names: Vec<String> = reset + .iter() + .map(ToString::to_string) + .chain(raw_reset.iter().cloned()) + .collect::<BTreeSet<_>>() + .into_iter() + .collect(); + if has_raw_set || !raw_reset.is_empty() { + let overlaps = names + .iter() + .any(|name| set.keys().any(|key| key.as_bytes() == name.as_bytes())); + if overlaps || (set.is_empty() && names.is_empty()) { + return Err(IggyError::InvalidCommand); + } + return Ok(names); + } + let block = crate::wire_conversions::resource_options_to_wire(set, OptionsProvenance::All)?; + admit( + scope, + OptionOperation::Update, + &block, + &names, + &OptionAdmissionContext::default(), + )?; + Ok(names) +} + +fn option_input_map( + options: &ResourceOptions, +) -> Result<BTreeMap<String, OptionInputValue>, IggyError> { + options + .iter() + .map(|(key, option)| { + Ok(( + String::from_utf8_lossy(key.as_bytes()).into_owned(), + OptionInputValue::try_from(&option.value)?, + )) + }) + .collect() } /// A topic's resolved runtime knobs, as carried from the metadata plane to @@ -624,7 +1142,12 @@ /// reset its segment size along with it. #[must_use] pub fn from_resource_options(options: &ResourceOptions) -> Self { - let parsed = TopicCreateOptions::from_resource_options(options); + Self::from(&TopicCreateOptions::from_resource_options(options)) + } +} + +impl From<&TopicCreateOptions> for TopicRuntimeOptions { + fn from(parsed: &TopicCreateOptions) -> Self { Self { segment_size: parsed.segment_size, durability: parsed.durability, @@ -636,66 +1159,6 @@ } } -/// The options an `UpdateTopic` may carry, as a type that cannot express a -/// key the update path rejects (see [`UPDATABLE_TOPIC_OPTION_KEYS`]). -/// -/// Separate from [`TopicCreateOptions`] on purpose: reusing that struct would -/// let a caller set `segment_size` on an update, encode it, and only learn at -/// the server that it was refused. Absent keys leave the topic's existing -/// value alone -- an update patches the option map, it does not replace it. -#[derive(Debug, Clone, Default, PartialEq)] -pub struct TopicUpdateOptions { - /// `None` leaves the topic's current algorithm alone. - pub compression_algorithm: Option<CompressionAlgorithm>, - /// `None` leaves the topic's current expiry alone. - pub message_expiry: Option<IggyExpiry>, - /// `None` leaves the topic's current cap alone. - pub max_topic_size: Option<MaxTopicSize>, - /// Keys sent as `String` values, checked against - /// [`UPDATABLE_TOPIC_OPTION_KEYS`] server-side. Lets a client reach an - /// updatable key added to the catalog after this build shipped. - pub raw: BTreeMap<String, String>, -} - -impl TopicUpdateOptions { - /// Encode the present keys into an options block, in canonical kinds. - /// - /// A typed field is inserted after the raw entries, so it wins on collision - /// and keeps its canonical kind. - /// - /// # Errors - /// - /// See `raw_options_map`. - pub fn to_wire(&self) -> Result<WireOptions, IggyError> { - let mut options = raw_options_map(&self.raw)?; - if let Some(compression_algorithm) = self.compression_algorithm { - options.insert( - HeaderKey::from_str(topic_option_keys::COMPRESSION_ALGORITHM) - .expect("catalog key is a valid header key"), - OptionValue::explicit( - HeaderValue::try_from(compression_algorithm.to_string().as_str()) - .expect("compression name fits a header value"), - ), - ); - } - if let Some(message_expiry) = self.message_expiry { - options.insert( - HeaderKey::from_str(topic_option_keys::MESSAGE_EXPIRY) - .expect("catalog key is a valid header key"), - OptionValue::explicit(HeaderValue::from(u64::from(message_expiry))), - ); - } - if let Some(max_topic_size) = self.max_topic_size { - options.insert( - HeaderKey::from_str(topic_option_keys::MAX_TOPIC_SIZE) - .expect("catalog key is a valid header key"), - OptionValue::explicit(HeaderValue::from(u64::from(max_topic_size))), - ); - } - crate::wire_conversions::resource_options_to_wire(&options, OptionsProvenance::All) - } -} - /// Typed view of the known topic option keys, parsed from a wire block. /// /// Optional fields left as `None` resolve from server defaults at admission. @@ -760,7 +1223,7 @@ /// /// # Errors /// - /// `UnsupportedOptionKey` for a key outside [`TOPIC_OPTION_KEYS`]; + /// `UnsupportedOptionKey` for a key outside the topic catalog, /// `InvalidOptionValue` when a value has the wrong kind or fails its /// type's `FromStr`. pub fn parse(options: &WireOptions) -> Result<Self, IggyError> { @@ -793,7 +1256,7 @@ } /// Fold one catalog entry into `self`, refusing an entry this build cannot - /// interpret: a key outside [`TOPIC_OPTION_KEYS`], or a catalog key whose + /// interpret: a key outside the topic catalog, or a catalog key whose /// value kind or payload does not parse. Shared by [`Self::parse`] (wire /// block) and [`Self::from_resource_options`] (persisted map) so both read /// the identical key set, kinds and value bounds. @@ -806,54 +1269,58 @@ entry: &WireUserHeaderEntry<'_>, key: &str, ) -> Result<(), IggyError> { - let parsed = self; - { - match key { - topic_option_keys::COMPRESSION_ALGORITHM => { - let value = parse_compression(entry, key)?; - parsed.compression_algorithm = Some(value); - } - topic_option_keys::MESSAGE_EXPIRY => { - let expiry = IggyExpiry::from(parse_u64_or(entry, key, IggyExpiry::from_str)?); - parsed.message_expiry = (expiry != IggyExpiry::ServerDefault).then_some(expiry); - } - topic_option_keys::MAX_TOPIC_SIZE => { - let size = - MaxTopicSize::from(parse_u64_or(entry, key, MaxTopicSize::from_str)?); - parsed.max_topic_size = (size != MaxTopicSize::ServerDefault).then_some(size); - } - topic_option_keys::SEGMENT_SIZE => { - let size = parse_byte_size(entry, key)?; - parsed.segment_size = (size != 0).then_some(IggyByteSize::from(size)); - } - topic_option_keys::DURABILITY => { - parsed.durability = Self::string_value(entry, key)?; - } - topic_option_keys::CONSUMER_OFFSET_DURABILITY => { - parsed.consumer_offset_durability = Self::string_value(entry, key)?; - } - topic_option_keys::PARTITION_RESIZE_POLICY => { - parsed.partition_resize_policy = Some(Self::string_value(entry, key)?); - } - topic_option_keys::MESSAGES_REQUIRED_TO_SAVE => { - let messages = parse_u32(entry, key)?; - if messages == 0 || messages > MAX_MESSAGES_REQUIRED_TO_SAVE { - return Err(IggyError::InvalidOptionValue(key.to_string())); - } - parsed.messages_required_to_save = Some(messages); - } - topic_option_keys::SIZE_OF_MESSAGES_REQUIRED_TO_SAVE => { - let size = parse_byte_size(entry, key)?; - if size > MAX_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE { - return Err(IggyError::InvalidOptionValue(key.to_string())); - } - parsed.size_of_messages_required_to_save = - (size != 0).then_some(IggyByteSize::from(size)); - } - topic_option_keys::PREALLOCATE_SEGMENTS => { - parsed.preallocate_segments = Some(parse_bool(entry, key)?); - } - _ => return Err(IggyError::UnsupportedOptionKey(key.to_string())), + let option = key + .parse::<OptionKey>() + .ok() + .filter(|option| { + let descriptor = option.descriptor(); + descriptor.applies_to(OptionsScope::Topic) + && descriptor.storage == OptionStorage::Map + }) + .ok_or_else(|| IggyError::UnsupportedOptionKey(key_preview(key)))?; + match option { + OptionKey::CompressionAlgorithm => { + self.compression_algorithm = Some(CompressionAlgorithm::decode(option, entry)?); + } + OptionKey::MessageExpiry => { + let expiry = IggyExpiry::decode(option, entry)?; + self.message_expiry = (expiry != IggyExpiry::ServerDefault).then_some(expiry); + } + OptionKey::MaxTopicSize => { + let size = MaxTopicSize::decode(option, entry)?; + self.max_topic_size = (size != MaxTopicSize::ServerDefault).then_some(size); + } + OptionKey::SegmentSize => { + let size = IggyByteSize::decode(option, entry)?; + self.segment_size = (size.as_bytes_u64() != 0).then_some(size); + } + OptionKey::Durability => self.durability = Durability::decode(option, entry)?, + OptionKey::ConsumerOffsetDurability => { + self.consumer_offset_durability = Durability::decode(option, entry)?; + } + OptionKey::PartitionResizePolicy => { + self.partition_resize_policy = Some(PartitionResizePolicy::decode(option, entry)?); + } + OptionKey::MessagesRequiredToSave => { + self.messages_required_to_save = Some(u32::decode(option, entry)?); + } + OptionKey::SizeOfMessagesRequiredToSave => { + let size = IggyByteSize::decode(option, entry)?; + self.size_of_messages_required_to_save = (size.as_bytes_u64() != 0).then_some(size); + } + OptionKey::PreallocateSegments => { + self.preallocate_segments = Some(bool::decode(option, entry)?); + } + OptionKey::Name + | OptionKey::Username + | OptionKey::Status + | OptionKey::EncryptionKey + | OptionKey::EncryptionKeyWrapped + | OptionKey::EncryptionKeySource + | OptionKey::EncryptionKeyFingerprint + | OptionKey::EncryptionKekFingerprint + | OptionKey::CriticalKeys => { + return Err(IggyError::UnsupportedOptionKey(key_preview(key))); } } Ok(()) @@ -997,170 +1464,14 @@ Ok(options) } - /// Render the present keys as string key-values, for a transport that - /// carries options as a JSON object instead of a TLV block. - /// - /// `compression_algorithm`, `message_expiry` and `max_topic_size` are left - /// out: the JSON body has dedicated fields for those three, and sending - /// both would be the two-sources-for-one-setting the update path refuses. - /// The rest ride as strings and are parsed server-side by the same - /// `FromStr` rules a config file value goes through, so a typed field - /// survives the round trip rather than being silently dropped. - pub fn to_string_options(&self) -> Result<BTreeMap<String, String>, IggyError> { - let resolved = self.to_option_map()?; - // Typed fields are inserted over the raw entries, matching the - // collision rule `to_wire` applies. - let mut options = self.raw.clone(); - if let Some(policy) = self.partition_resize_policy { - options.insert( - topic_option_keys::PARTITION_RESIZE_POLICY.to_owned(), - policy.to_string(), - ); - } - if let Some(segment_size) = self.segment_size { - options.insert( - topic_option_keys::SEGMENT_SIZE.to_owned(), - segment_size.as_bytes_u64().to_string(), - ); - } - options.insert( - topic_option_keys::DURABILITY.to_owned(), - resolved[&HeaderKey::from_str(topic_option_keys::DURABILITY) - .expect("catalog key is valid")] - .value - .to_string_value(), - ); - options.insert( - topic_option_keys::CONSUMER_OFFSET_DURABILITY.to_owned(), - resolved[&HeaderKey::from_str(topic_option_keys::CONSUMER_OFFSET_DURABILITY) - .expect("catalog key is valid")] - .value - .to_string_value(), - ); - if let Some(messages_required_to_save) = self.messages_required_to_save { - options.insert( - topic_option_keys::MESSAGES_REQUIRED_TO_SAVE.to_owned(), - messages_required_to_save.to_string(), - ); - } - if let Some(size_of_messages) = self.size_of_messages_required_to_save { - options.insert( - topic_option_keys::SIZE_OF_MESSAGES_REQUIRED_TO_SAVE.to_owned(), - size_of_messages.as_bytes_u64().to_string(), - ); - } - if let Some(preallocate_segments) = self.preallocate_segments { - options.insert( - topic_option_keys::PREALLOCATE_SEGMENTS.to_owned(), - preallocate_segments.to_string(), - ); - } - Ok(options) - } - - /// Encode the resolved values for every key the client did NOT send into - /// a derived-options wire block, in canonical kinds. `partitions_count` - /// is never included: it is consumed at admission. + /// The present keys as HTTP body values, the same set and canonical + /// values [`Self::to_wire`] encodes. /// /// # Errors /// - /// See [`crate::wire_conversions::resource_options_to_wire`]. - pub fn derived_block( - &self, - compression_algorithm: CompressionAlgorithm, - message_expiry: IggyExpiry, - max_topic_size: MaxTopicSize, - runtime_defaults: TopicRuntimeDefaults, - supplied: &WireOptions, - ) -> Result<WireOptions, IggyError> { - let mut derived = ResourceOptions::new(); - if self.partition_resize_policy.is_none() { - derived.insert( - HeaderKey::from_str(topic_option_keys::PARTITION_RESIZE_POLICY) - .expect("catalog key is a valid header key"), - OptionValue::derived(HeaderValue::from_str( - PartitionResizePolicy::default().as_ref(), - )?), - ); - } - if self.compression_algorithm.is_none() { - derived.insert( - HeaderKey::from_str(topic_option_keys::COMPRESSION_ALGORITHM) - .expect("catalog key is a valid header key"), - OptionValue::derived( - HeaderValue::try_from(compression_algorithm.to_string().as_str()) - .expect("compression name fits a header value"), - ), - ); - } - if self.message_expiry.is_none() { - derived.insert( - HeaderKey::from_str(topic_option_keys::MESSAGE_EXPIRY) - .expect("catalog key is a valid header key"), - OptionValue::derived(HeaderValue::from(u64::from(message_expiry))), - ); - } - if self.max_topic_size.is_none() { - derived.insert( - HeaderKey::from_str(topic_option_keys::MAX_TOPIC_SIZE) - .expect("catalog key is a valid header key"), - OptionValue::derived(HeaderValue::from(u64::from(max_topic_size))), - ); - } - if self.segment_size.is_none() { - derived.insert( - HeaderKey::from_str(topic_option_keys::SEGMENT_SIZE) - .expect("catalog key is a valid header key"), - OptionValue::derived(HeaderValue::from( - runtime_defaults.segment_size.as_bytes_u64(), - )), - ); - } - for (key, policy) in [ - (topic_option_keys::DURABILITY, runtime_defaults.durability), - ( - topic_option_keys::CONSUMER_OFFSET_DURABILITY, - runtime_defaults.consumer_offset_durability, - ), - ] { - if !supplied - .into_iter() - .any(|entry| entry.key == key.as_bytes()) - { - derived.insert( - HeaderKey::from_str(key).expect("catalog key is valid"), - OptionValue::derived(HeaderValue::from_str(policy.as_ref())?), - ); - } - } - if self.messages_required_to_save.is_none() { - derived.insert( - HeaderKey::from_str(topic_option_keys::MESSAGES_REQUIRED_TO_SAVE) - .expect("catalog key is a valid header key"), - OptionValue::derived(HeaderValue::from( - runtime_defaults.messages_required_to_save, - )), - ); - } - if self.size_of_messages_required_to_save.is_none() { - derived.insert( - HeaderKey::from_str(topic_option_keys::SIZE_OF_MESSAGES_REQUIRED_TO_SAVE) - .expect("catalog key is a valid header key"), - OptionValue::derived(HeaderValue::from( - runtime_defaults - .size_of_messages_required_to_save - .as_bytes_u64(), - )), - ); - } - if self.preallocate_segments.is_none() { - derived.insert( - HeaderKey::from_str(topic_option_keys::PREALLOCATE_SEGMENTS) - .expect("catalog key is a valid header key"), - OptionValue::derived(HeaderValue::from(runtime_defaults.preallocate_segments)), - ); - } - crate::wire_conversions::resource_options_to_wire(&derived, OptionsProvenance::All) + /// See [`Self::to_option_map`]. + pub fn to_input_map(&self) -> Result<BTreeMap<String, OptionInputValue>, IggyError> { + option_input_map(&self.to_option_map()?) } /// Parse a topic's PERSISTED options map (explicit plus admission-derived @@ -1191,117 +1502,224 @@ } parsed } - fn string_value<T: FromStr>( - entry: &WireUserHeaderEntry<'_>, - key: &str, - ) -> Result<T, IggyError> { - if entry.value_kind.0 != HeaderKind::String.as_code() { - return Err(IggyError::InvalidOptionValue(key.to_owned())); - } - std::str::from_utf8(entry.value) - .ok() - .and_then(|value| value.parse().ok()) - .ok_or_else(|| IggyError::InvalidOptionValue(key.to_owned())) - } -} - -fn parse_u32(entry: &WireUserHeaderEntry<'_>, key: &str) -> Result<u32, IggyError> { - if entry.value_kind.0 == HeaderKind::Uint32.as_code() { - let bytes: [u8; 4] = entry - .value - .try_into() - .map_err(|_| IggyError::InvalidOptionValue(key.to_string()))?; - return Ok(u32::from_le_bytes(bytes)); - } - if entry.value_kind.0 == HeaderKind::String.as_code() { - return std::str::from_utf8(entry.value) - .ok() - .and_then(|text| text.parse().ok()) - .ok_or_else(|| IggyError::InvalidOptionValue(key.to_string())); - } - Err(IggyError::InvalidOptionValue(key.to_string())) -} - -/// `Uint64` verbatim, or a `String` routed through `from_str` and converted -/// back to the type's `u64` sentinel encoding. -fn parse_u64_or<T, E>( - entry: &WireUserHeaderEntry<'_>, - key: &str, - from_str: impl Fn(&str) -> Result<T, E>, -) -> Result<u64, IggyError> -where - u64: From<T>, -{ - if entry.value_kind.0 == HeaderKind::Uint64.as_code() { - let bytes: [u8; 8] = entry - .value - .try_into() - .map_err(|_| IggyError::InvalidOptionValue(key.to_string()))?; - return Ok(u64::from_le_bytes(bytes)); - } - if entry.value_kind.0 == HeaderKind::String.as_code() { - return std::str::from_utf8(entry.value) - .ok() - .and_then(|text| from_str(text).ok()) - .map(u64::from) - .ok_or_else(|| IggyError::InvalidOptionValue(key.to_string())); - } - Err(IggyError::InvalidOptionValue(key.to_string())) -} - -/// `Uint64` verbatim, or a byte-size `String` (`128MiB`). -fn parse_byte_size(entry: &WireUserHeaderEntry<'_>, key: &str) -> Result<u64, IggyError> { - parse_u64_or(entry, key, |text| { - IggyByteSize::from_str(text).map(|size| size.as_bytes_u64()) - }) -} - -fn parse_bool(entry: &WireUserHeaderEntry<'_>, key: &str) -> Result<bool, IggyError> { - if entry.value_kind.0 == HeaderKind::Bool.as_code() { - // Exactly one byte of 0 or 1. Anything looser admits a value that - // `HeaderValue::as_bool` later refuses to read, so the stored map would - // hold an entry its own public accessor rejects; a multi-byte payload - // would pass this gate and only fail at apply, turning a client - // mistake into a committed state-machine rejection. - return match entry.value { - [0] => Ok(false), - [1] => Ok(true), - _ => Err(IggyError::InvalidOptionValue(key.to_string())), - }; - } - if entry.value_kind.0 == HeaderKind::String.as_code() { - return std::str::from_utf8(entry.value) - .ok() - .and_then(|text| text.parse().ok()) - .ok_or_else(|| IggyError::InvalidOptionValue(key.to_string())); - } - Err(IggyError::InvalidOptionValue(key.to_string())) -} - -fn parse_compression( - entry: &WireUserHeaderEntry<'_>, - key: &str, -) -> Result<CompressionAlgorithm, IggyError> { - if entry.value_kind.0 == HeaderKind::Uint8.as_code() { - // Exactly one byte, for the same reason as `parse_bool`. - let [code] = entry.value else { - return Err(IggyError::InvalidOptionValue(key.to_string())); - }; - return CompressionAlgorithm::from_code(*code) - .map_err(|_| IggyError::InvalidOptionValue(key.to_string())); - } - if entry.value_kind.0 == HeaderKind::String.as_code() { - return std::str::from_utf8(entry.value) - .ok() - .and_then(|text| CompressionAlgorithm::from_str(text).ok()) - .ok_or_else(|| IggyError::InvalidOptionValue(key.to_string())); - } - Err(IggyError::InvalidOptionValue(key.to_string())) } #[cfg(test)] mod tests { use super::*; + use strum::IntoEnumIterator; + + #[test] + fn given_any_single_reset_when_patching_should_match_server_admission() { + let context = OptionAdmissionContext::default(); + for scope in [ + OptionsScope::Topic, + OptionsScope::Stream, + OptionsScope::User, + ] { + for key in OptionKey::iter() { + let typed = options_patch( + scope, + ResourceOptions::new(), + &BTreeSet::from([key]), + &BTreeSet::new(), + false, + ); + let server = admit( + scope, + OptionOperation::Update, + &WireOptions::empty(), + &[key.to_string()], + &context, + ); + assert_eq!( + typed.as_ref().err().map(IggyError::as_code), + server.as_ref().err().map(IggyError::as_code), + "{scope:?} reset of {key}" + ); + } + } + } + + #[test] + fn given_raw_key_named_reset_when_building_update_body_should_refuse_unsupported_key() { + let update = TopicUpdateOptions { + raw: BTreeMap::from([(RESET_FIELD.to_owned(), "x".to_owned())]), + ..TopicUpdateOptions::default() + }; + assert_eq!( + update.to_update_body(), + Err(IggyError::UnsupportedOptionKey(RESET_FIELD.to_owned())) + ); + } + + #[test] + fn given_raw_reset_when_patching_should_send_it_for_the_server_to_decide() { + let update = TopicUpdateOptions { + reset: BTreeSet::from([OptionKey::MessageExpiry]), + raw_reset: BTreeSet::from(["future_key".to_owned(), "segment_size".to_owned()]), + ..TopicUpdateOptions::default() + }; + let names: Vec<String> = update + .to_patch() + .expect("the server decides raw resets") + .reset + .iter() + .map(|name| name.as_str().to_owned()) + .collect(); + assert_eq!(names, ["future_key", "message_expiry", "segment_size"]); + assert_eq!( + update.to_update_body().expect("body builds").reset, + BTreeSet::from([ + "future_key".to_owned(), + "message_expiry".to_owned(), + "segment_size".to_owned() + ]) + ); + let overlapping = TopicUpdateOptions { + raw: BTreeMap::from([("future_key".to_owned(), "1".to_owned())]), + raw_reset: BTreeSet::from(["future_key".to_owned()]), + ..TopicUpdateOptions::default() + }; + assert_eq!(overlapping.to_patch(), Err(IggyError::InvalidCommand)); + } + + #[test] + fn given_patch_failing_several_checks_when_patching_should_report_the_admission_category() { + let read_only_raw = StreamUpdateOptions { + reset: BTreeSet::from([OptionKey::Name]), + raw: BTreeMap::from([( + OptionKey::EncryptionKeySource.to_string(), + "none".to_owned(), + )]), + ..StreamUpdateOptions::default() + }; + let patch = read_only_raw + .to_patch() + .expect("the server decides raw keys"); + let reset: Vec<String> = patch + .reset + .iter() + .map(|name| name.as_str().to_owned()) + .collect(); + assert_eq!( + admit( + OptionsScope::Stream, + OptionOperation::Update, + &patch.set, + &reset, + &OptionAdmissionContext::default(), + ) + .err(), + Some(IggyError::ReadOnlyOption( + OptionKey::EncryptionKeySource.to_string() + )) + ); + let bad_name_and_reset = TopicUpdateOptions { + name: Some(String::new()), + reset: BTreeSet::from([OptionKey::SegmentSize]), + ..TopicUpdateOptions::default() + }; + assert_eq!( + bad_name_and_reset.to_patch(), + Err(IggyError::OptionNotUpdatable( + OptionKey::SegmentSize.to_string() + )) + ); + let bad_name = TopicUpdateOptions { + name: Some("n".repeat(256)), + ..TopicUpdateOptions::default() + }; + assert_eq!(bad_name.to_patch(), Err(IggyError::InvalidTopicName)); + let short_username = UserUpdateOptions { + username: Some("a".to_owned()), + ..UserUpdateOptions::default() + }; + assert_eq!(short_username.to_patch(), Err(IggyError::InvalidUsername)); + } + + #[test] + fn given_every_topic_catalog_row_when_creating_should_parse_store_and_default_it() { + let derived = derived_defaults(OptionsScope::Topic, &ResourceOptions::new()) + .expect("derived defaults build"); + let unset = TopicResolvedOptions::from(&ResourceOptions::new()); + for descriptor in descriptors_for(OptionsScope::Topic).filter(|descriptor| { + descriptor.storage == OptionStorage::Map + && descriptor.visibility == OptionVisibility::Public + }) { + let key = HeaderKey::from(descriptor.key); + let default = descriptor + .default_value() + .expect("default encodes") + .expect("every topic map key has a default"); + let stored = + ResourceOptions::from([(key.clone(), OptionValue::explicit(default.clone()))]); + let block = + crate::wire_conversions::resource_options_to_wire(&stored, OptionsProvenance::All) + .expect("block encodes"); + let parsed = TopicCreateOptions::parse(&block).expect("create parses every topic key"); + let explicit = crate::wire_conversions::resource_options_from_wire( + &parsed + .to_explicit_wire(|_| true) + .expect("explicit block encodes"), + true, + ) + .expect("explicit block decodes"); + assert_eq!( + explicit.get(&key).map(|option| &option.value), + Some(&default), + "create stores {}", + descriptor.key + ); + assert_eq!( + derived.get(&key).map(|option| &option.value), + Some(&default), + "create derives the catalog default of {}", + descriptor.key + ); + let resolved = TopicResolvedOptions::from(&stored); + assert_eq!( + ( + resolved.compression_algorithm, + resolved.message_expiry, + resolved.max_topic_size, + resolved.partition_resize_policy, + resolved.runtime.durability, + resolved.runtime.consumer_offset_durability, + resolved.runtime.effective_segment_size(), + ), + ( + unset.compression_algorithm, + unset.message_expiry, + unset.max_topic_size, + unset.partition_resize_policy, + unset.runtime.durability, + unset.runtime.consumer_offset_durability, + unset.runtime.effective_segment_size(), + ), + "reads resolve an absent {} to its catalog default", + descriptor.key + ); + } + } + + #[test] + fn given_stream_create_options_with_keys_when_debugging_should_redact_them() { + let options = StreamCreateOptions { + encryption_key: Some( + StreamEncryptionKey::try_from([7u8; 32].as_slice()).expect("32-byte key"), + ), + raw: BTreeMap::from([( + OptionKey::EncryptionKey.to_string(), + "c2VjcmV0LXNlY3JldA==".to_owned(), + )]), + }; + let rendered = format!("{options:?}"); + assert!( + !rendered.contains("c2VjcmV0") && !rendered.contains("7, 7"), + "{rendered}" + ); + } #[test] fn to_wire_parse_roundtrip_preserves_typed_fields() { @@ -1327,16 +1745,11 @@ fn given_omitted_resize_policy_when_resolving_defaults_should_persist_mutable_as_derived() { let options = TopicCreateOptions::default(); let supplied = options.to_wire().unwrap(); - let derived = options - .derived_block( - CompressionAlgorithm::default(), - IggyExpiry::default(), - MaxTopicSize::default(), - TopicRuntimeDefaults::default(), - &supplied, - ) - .unwrap(); - let stored = crate::wire_conversions::resource_options_from_wire(&derived, false).unwrap(); + let stored = derived_defaults( + OptionsScope::Topic, + &crate::wire_conversions::resource_options_from_wire(&supplied, true).unwrap(), + ) + .unwrap(); let policy = &stored[&HeaderKey::from_str(topic_option_keys::PARTITION_RESIZE_POLICY).unwrap()]; assert_eq!(policy.value.as_bytes(), b"mutable"); @@ -1368,8 +1781,8 @@ Some(PartitionResizePolicy::Fixed), ); assert_eq!( - options.to_string_options().unwrap()[topic_option_keys::PARTITION_RESIZE_POLICY], - "fixed" + options.to_input_map().unwrap()[topic_option_keys::PARTITION_RESIZE_POLICY], + OptionInputValue::String("fixed".to_owned().into()) ); options.partition_resize_policy = Some(PartitionResizePolicy::Mutable); @@ -1380,8 +1793,8 @@ Some(PartitionResizePolicy::Mutable), ); assert_eq!( - options.to_string_options().unwrap()[topic_option_keys::PARTITION_RESIZE_POLICY], - "mutable" + options.to_input_map().unwrap()[topic_option_keys::PARTITION_RESIZE_POLICY], + OptionInputValue::String("mutable".to_owned().into()) ); } @@ -1435,10 +1848,9 @@ fn resource_options_render_as_readable_json_and_round_trip() { // `HeaderKey` serializes as a struct, and JSON object keys must be // strings, so the derived impl fails with "key must be a string" for - // any non-empty map -- which 500'd every HTTP response carrying - // options. Keys must render as their plain text, and values in the - // string form the create body takes them in: a base64 payload behind a - // kind tag is neither readable nor re-sendable. + // any non-empty map. Keys render as their plain text, values as + // readable text plus their kind, so an HTTP reader rebuilds the exact + // value a binary reader gets. #[derive(Debug, serde::Serialize, serde::Deserialize, PartialEq)] struct Holder { #[serde(default, with = "super::resource_options_json")] @@ -1457,32 +1869,75 @@ "the key must render as plain text, got: {json}" ); assert!( - json.contains("\"value\":\"1048576\""), - "the value must render as its string form, got: {json}" + json.contains("\"value\":\"1048576\"") && json.contains("\"kind\":\"uint64\""), + "the value must render as text with its kind, got: {json}" ); let decoded: Holder = serde_json::from_str(&json).expect("options must round-trip"); - let (key, option) = decoded.options.iter().next().unwrap(); - assert_eq!(key.as_bytes(), topic_option_keys::SEGMENT_SIZE.as_bytes()); - assert!(!option.explicit, "provenance survives the round trip"); - // Re-read as a string-kinded value, which is exactly what the create - // body's string map produces, so the server parses it the same way. - assert_eq!(option.value.kind(), HeaderKind::String); - assert_eq!(option.value.as_bytes(), b"1048576"); - assert!( - TopicCreateOptions::parse( - &crate::wire_conversions::resource_options_to_wire( - &decoded.options, - OptionsProvenance::All - ) - .unwrap() - ) - .is_ok(), - "the rendered value must parse back through the catalog" + assert_eq!( + decoded, holder, + "kind and provenance survive the round trip" ); } #[test] + fn given_all_header_kinds_when_roundtripping_http_should_preserve_bytes_and_provenance() { + #[derive(Debug, serde::Serialize, serde::Deserialize, PartialEq)] + struct Holder { + #[serde(default, with = "super::resource_options_json")] + options: ResourceOptions, + } + let values = [ + HeaderValue::try_from([0u8, 255, 7].as_slice()).unwrap(), + HeaderValue::try_from("text with spaces").unwrap(), + HeaderValue::from(true), + HeaderValue::from(-5i8), + HeaderValue::from(-300i16), + HeaderValue::from(-70_000i32), + HeaderValue::from(i64::MIN), + HeaderValue::from(i128::MAX), + HeaderValue::from(250u8), + HeaderValue::from(65_000u16), + HeaderValue::from(u32::MAX), + HeaderValue::from(u64::MAX - 1), + HeaderValue::from(u128::MAX), + HeaderValue::from(f32::from_bits(0x7fc0_0001)), + HeaderValue::from(-0.0f64), + HeaderValue::from(f64::from_bits(0x7ff8_0000_0000_0abc)), + ]; + let options: ResourceOptions = values + .iter() + .enumerate() + .map(|(index, value)| { + ( + HeaderKey::try_from(format!("future_key_{index:02}").as_str()).unwrap(), + OptionValue { + value: value.clone(), + explicit: index % 2 == 0, + }, + ) + }) + .collect(); + let holder = Holder { options }; + let json = serde_json::to_string(&holder).expect("serializes"); + let decoded: Holder = serde_json::from_str(&json).expect("deserializes"); + assert_eq!(decoded, holder, "{json}"); + for (kind, text) in [ + (HeaderKind::Uint64, "+5"), + (HeaderKind::Uint64, "05"), + (HeaderKind::Bool, "True"), + (HeaderKind::Raw, "AAF="), + (HeaderKind::Float32, "0x7fc00001a"), + (HeaderKind::Raw, "not base64"), + ] { + assert!( + resource_options_json::decode_value(kind, text).is_err(), + "{kind:?} {text} must be refused" + ); + } + } + + #[test] fn zero_valued_sentinels_normalize_to_unspecified() { // 0 is the ServerDefault sentinel for both expiry and size on the // wire, so a client sending it means "resolve from the default" @@ -1568,8 +2023,8 @@ assert_eq!(parsed.durability, Durability::Persisted); assert_eq!(parsed.consumer_offset_durability, Durability::Replicated); assert_eq!( - options.to_string_options().unwrap()["durability"], - "persisted" + options.to_input_map().unwrap()["durability"], + OptionInputValue::String("persisted".to_owned().into()) ); } @@ -1581,7 +2036,7 @@ ..TopicCreateOptions::default() }; assert!(options.to_wire().is_err()); - assert!(options.to_string_options().is_err()); + assert!(options.to_input_map().is_err()); } #[test] @@ -1613,20 +2068,18 @@ }) .unwrap(); assert_eq!(explicit.into_iter().count(), 1); - let derived = parsed - .derived_block( - CompressionAlgorithm::default(), - IggyExpiry::default(), - MaxTopicSize::default(), - TopicRuntimeDefaults::default(), - &supplied, - ) - .unwrap(); - let entry = derived - .into_iter() - .find(|entry| entry.key == other.as_bytes()) - .unwrap(); - assert_eq!(entry.value, b"replicated"); + let derived = derived_defaults( + OptionsScope::Topic, + &crate::wire_conversions::resource_options_from_wire(&explicit, true).unwrap(), + ) + .unwrap(); + assert!(!derived.contains_key(&HeaderKey::from_str(key).unwrap())); + assert_eq!( + derived[&HeaderKey::from_str(other).unwrap()] + .value + .as_bytes(), + b"replicated" + ); } }
diff --git a/core/common/src/types/personal_access_tokens/mod.rs b/core/common/src/types/personal_access_tokens/mod.rs index 662fb74..18d4bc2 100644 --- a/core/common/src/types/personal_access_tokens/mod.rs +++ b/core/common/src/types/personal_access_tokens/mod.rs
@@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use crate::IggyError; use crate::IggyExpiry; use crate::IggyTimestamp; use crate::UserId; @@ -40,17 +41,18 @@ name: &str, now: IggyTimestamp, expiry: IggyExpiry, - ) -> (Self, String) { + ) -> Result<(Self, String), IggyError> { + let expiry_at = Self::calculate_expiry_at(now, expiry)?; let (token, token_hash) = Self::mint_raw_and_hash(); - ( + Ok(( Self { user_id, name: Arc::from(name), token: Arc::from(token_hash), - expiry_at: Self::calculate_expiry_at(now, expiry), + expiry_at, }, token, - ) + )) } /// Mint a fresh raw token and its hash without constructing a @@ -94,13 +96,18 @@ hash::calculate_256(token.as_bytes()) } - pub fn calculate_expiry_at(now: IggyTimestamp, expiry: IggyExpiry) -> Option<IggyTimestamp> { + /// `InvalidPersonalAccessTokenExpiry` when the expiry ends past the last representable timestamp. + pub fn calculate_expiry_at( + now: IggyTimestamp, + expiry: IggyExpiry, + ) -> Result<Option<IggyTimestamp>, IggyError> { match expiry { - IggyExpiry::ExpireDuration(expiry) => { - Some(IggyTimestamp::from(now.as_micros() + expiry.as_micros())) - } - IggyExpiry::NeverExpire => None, - IggyExpiry::ServerDefault => None, // This will + IggyExpiry::ExpireDuration(expiry) => now + .as_micros() + .checked_add(expiry.as_micros()) + .map(|expiry_at| Some(IggyTimestamp::from(expiry_at))) + .ok_or(IggyError::InvalidPersonalAccessTokenExpiry), + IggyExpiry::NeverExpire | IggyExpiry::ServerDefault => Ok(None), } } } @@ -117,7 +124,8 @@ let now = IggyTimestamp::now(); let name = "test_token"; let (personal_access_token, raw_token) = - PersonalAccessToken::new(user_id, name, now, IggyExpiry::NeverExpire); + PersonalAccessToken::new(user_id, name, now, IggyExpiry::NeverExpire) + .expect("a never-expiring token has no expiry to overflow"); assert_eq!(&*personal_access_token.name, name); assert!(!personal_access_token.token.is_empty()); assert!(!raw_token.is_empty()); @@ -135,8 +143,25 @@ let expiry_ms = 10; let expiry = IggyExpiry::ExpireDuration(IggyDuration::from(expiry_ms)); let name = "test_token"; - let (personal_access_token, _) = PersonalAccessToken::new(user_id, name, now, expiry); + let (personal_access_token, _) = PersonalAccessToken::new(user_id, name, now, expiry) + .expect("a 10 microsecond expiry fits the clock"); let later = IggyTimestamp::from(now.as_micros() + expiry_ms + 1); assert!(personal_access_token.is_expired(later)); } + + #[test] + fn expiry_at_should_be_refused_given_expiry_past_the_end_of_the_clock() { + let now = IggyTimestamp::from(1_000); + let last = IggyExpiry::ExpireDuration(IggyDuration::from(u64::MAX - 1_000)); + let past = IggyExpiry::ExpireDuration(IggyDuration::from(u64::MAX - 999)); + + assert_eq!( + PersonalAccessToken::calculate_expiry_at(now, last), + Ok(Some(IggyTimestamp::from(u64::MAX))) + ); + assert_eq!( + PersonalAccessToken::calculate_expiry_at(now, past), + Err(IggyError::InvalidPersonalAccessTokenExpiry) + ); + } }
diff --git a/core/common/src/types/segment.rs b/core/common/src/types/segment.rs index 492000e..c7c70da 100644 --- a/core/common/src/types/segment.rs +++ b/core/common/src/types/segment.rs
@@ -73,7 +73,7 @@ IggyExpiry::NeverExpire => false, IggyExpiry::ServerDefault => false, IggyExpiry::ExpireDuration(duration) => { - self.end_timestamp + duration.as_micros() <= now.as_micros() + self.end_timestamp.saturating_add(duration.as_micros()) <= now.as_micros() } } } @@ -104,4 +104,16 @@ "BUG: segment with end_timestamp=0 appears instantly expired" ); } + + #[test] + fn segment_with_expiry_past_the_end_of_the_clock_should_not_appear_expired() { + let now = IggyTimestamp::now(); + let mut seg = Segment::new(5000, IggyByteSize::from(128 * 1024 * 1024u64)); + seg.sealed = true; + seg.end_timestamp = now.as_micros(); + + let expiry = IggyExpiry::ExpireDuration(IggyDuration::from(u64::MAX - 1)); + + assert!(!seg.is_expired(now, expiry)); + } }
diff --git a/core/common/src/types/stream/mod.rs b/core/common/src/types/stream/mod.rs index 9a48f1a..0ebcdf1 100644 --- a/core/common/src/types/stream/mod.rs +++ b/core/common/src/types/stream/mod.rs
@@ -15,7 +15,10 @@ // specific language governing permissions and limitations // under the License. -use crate::{IggyByteSize, IggyTimestamp, ResourceOptions, Topic}; +use crate::{ + HeaderKey, IggyByteSize, IggyTimestamp, OptionKey, ResourceOptions, StreamEncryptionSource, + Topic, +}; use serde::{Deserialize, Serialize}; /// `Stream` represents the highest level of logical separation of data. @@ -40,7 +43,8 @@ pub messages_count: u64, /// The total number of topics in the stream. pub topics_count: u32, - /// Creation options, all client-explicit (streams have no derived keys). + /// Options by provenance, plus the read-only encryption key source and + /// fingerprints as derived entries. #[serde(default, with = "crate::resource_options_json")] pub options: ResourceOptions, } @@ -70,7 +74,70 @@ pub topics_count: u32, /// The collection of topics in the stream. pub topics: Vec<Topic>, - /// Creation options, all client-explicit (streams have no derived keys). + /// Options by provenance, plus the read-only encryption key source and + /// fingerprints as derived entries. #[serde(default, with = "crate::resource_options_json")] pub options: ResourceOptions, } + +impl Stream { + /// Who chose the stream's at-rest key, `None` when the server reported no + /// source. + #[must_use] + pub fn encryption_key_source(&self) -> Option<StreamEncryptionSource> { + encryption_key_source(&self.options) + } + + /// Fingerprint of the stream's key, absent for a plaintext stream. + #[must_use] + pub fn encryption_key_fingerprint(&self) -> Option<String> { + text_option(&self.options, OptionKey::EncryptionKeyFingerprint) + } + + /// Fingerprints of the server keys that wrap the stream's key: one + /// outside a key rotation, empty for a plaintext stream. + #[must_use] + pub fn encryption_kek_fingerprints(&self) -> Vec<String> { + kek_fingerprints(&self.options) + } +} + +impl StreamDetails { + /// Who chose the stream's at-rest key, `None` when the server reported no + /// source. + #[must_use] + pub fn encryption_key_source(&self) -> Option<StreamEncryptionSource> { + encryption_key_source(&self.options) + } + + /// Fingerprint of the stream's key, absent for a plaintext stream. + #[must_use] + pub fn encryption_key_fingerprint(&self) -> Option<String> { + text_option(&self.options, OptionKey::EncryptionKeyFingerprint) + } + + /// Fingerprints of the server keys that wrap the stream's key: one + /// outside a key rotation, empty for a plaintext stream. + #[must_use] + pub fn encryption_kek_fingerprints(&self) -> Vec<String> { + kek_fingerprints(&self.options) + } +} + +fn encryption_key_source(options: &ResourceOptions) -> Option<StreamEncryptionSource> { + options + .get(&HeaderKey::from(OptionKey::EncryptionKeySource)) + .and_then(|option| option.value.as_str().ok()?.parse().ok()) +} + +fn text_option(options: &ResourceOptions, key: OptionKey) -> Option<String> { + options + .get(&HeaderKey::from(key)) + .and_then(|option| option.value.as_str().ok().map(str::to_owned)) +} + +fn kek_fingerprints(options: &ResourceOptions) -> Vec<String> { + text_option(options, OptionKey::EncryptionKekFingerprint).map_or_else(Vec::new, |text| { + text.split(',').map(str::to_owned).collect() + }) +}
diff --git a/core/common/src/types/topic/mod.rs b/core/common/src/types/topic/mod.rs index 9379c84..1efd89d 100644 --- a/core/common/src/types/topic/mod.rs +++ b/core/common/src/types/topic/mod.rs
@@ -15,13 +15,11 @@ // specific language governing permissions and limitations // under the License. -use crate::CompressionAlgorithm; use crate::Partition; use crate::ResourceOptions; +use crate::TopicResolvedOptions; use crate::utils::byte_size::IggyByteSize; -use crate::utils::expiry::IggyExpiry; use crate::utils::timestamp::IggyTimestamp; -use crate::utils::topic_size::MaxTopicSize; use serde::{Deserialize, Serialize}; /// `Topic` represents the medium level of logical separation of data as it's a part of the stream. @@ -30,8 +28,6 @@ /// - `created_at`: the timestamp when the topic was created. /// - `name`: the unique name of the topic. /// - `size`: the total size of the topic in bytes. -/// - `message_expiry`: the expiry of the messages in the topic. -/// - `max_topic_size`: the maximum size of the topic. /// - `messages_count`: the total number of messages in the topic. /// - `partitions_count`: the total number of partitions in the topic. #[derive(Debug, Serialize, Deserialize)] @@ -44,19 +40,14 @@ pub name: String, /// The total size of the topic in bytes. pub size: IggyByteSize, - /// The expiry of the messages in the topic. - pub message_expiry: IggyExpiry, - /// Compression algorithm for the topic. - pub compression_algorithm: CompressionAlgorithm, - /// The optional maximum size of the topic. - /// Can't be lower than segment size in the config. - pub max_topic_size: MaxTopicSize, /// The total number of messages in the topic. pub messages_count: u64, /// The total number of partitions in the topic. pub partitions_count: u32, - /// Creation options: client-explicit keys plus admission-derived - /// defaults, distinguished by each entry's `explicit` flag. + /// Settings: client-explicit keys plus admission-derived defaults, + /// distinguished by each entry's `explicit` flag. Message expiry, + /// compression and the size cap live here, read them typed through + /// `resolved_options`. #[serde(default, with = "crate::resource_options_json")] pub options: ResourceOptions, } @@ -67,8 +58,6 @@ /// - `created_at`: the timestamp when the topic was created. /// - `name`: the unique name of the topic. /// - `size`: the total size of the topic. -/// - `message_expiry`: the expiry of the messages in the topic. -/// - `max_topic_size`: the maximum size of the topic. /// - `messages_count`: the total number of messages in the topic. /// - `partitions_count`: the total number of partitions in the topic. /// - `partitions`: the collection of partitions in the topic. @@ -82,21 +71,34 @@ pub name: String, /// The total size of the topic. pub size: IggyByteSize, - /// The expiry of the messages in the topic. - pub message_expiry: IggyExpiry, - /// Compression algorithm for the topic. - pub compression_algorithm: CompressionAlgorithm, - /// The optional maximum size of the topic. - /// Can't be lower than segment size in the config. - pub max_topic_size: MaxTopicSize, /// The total number of messages in the topic. pub messages_count: u64, /// The total number of partitions in the topic. pub partitions_count: u32, /// The collection of partitions in the topic. pub partitions: Vec<Partition>, - /// Creation options: client-explicit keys plus admission-derived - /// defaults, distinguished by each entry's `explicit` flag. + /// Settings: client-explicit keys plus admission-derived defaults, + /// distinguished by each entry's `explicit` flag. Message expiry, + /// compression and the size cap live here, read them typed through + /// `resolved_options`. #[serde(default, with = "crate::resource_options_json")] pub options: ResourceOptions, } + +impl Topic { + /// Every setting of the topic, typed and resolved against the catalog + /// defaults. + #[must_use] + pub fn resolved_options(&self) -> TopicResolvedOptions { + TopicResolvedOptions::from(&self.options) + } +} + +impl TopicDetails { + /// Every setting of the topic, typed and resolved against the catalog + /// defaults. + #[must_use] + pub fn resolved_options(&self) -> TopicResolvedOptions { + TopicResolvedOptions::from(&self.options) + } +}
diff --git a/core/common/src/utils/crypto.rs b/core/common/src/utils/crypto.rs index 5440851..051362e 100644 --- a/core/common/src/utils/crypto.rs +++ b/core/common/src/utils/crypto.rs
@@ -19,22 +19,32 @@ use crate::text; use aes_gcm::aead::{Aead, Generate}; use aes_gcm::{Aes256Gcm, KeyInit, Nonce}; +use aes_gcm_siv::aead::AeadInOut; +use aes_gcm_siv::{Aes256GcmSiv, Nonce as SivNonce, Tag as SivTag}; use std::fmt::Debug; +const AES_GCM_NONCE_SIZE: usize = 12; +const AES_GCM_TAG_SIZE: usize = 16; +const AES_GCM_SIV_NONCE_SIZE: usize = 12; +const AES_GCM_SIV_TAG_SIZE: usize = 16; + #[derive(Debug, Clone)] pub enum EncryptorKind { Aes256Gcm(Aes256GcmEncryptor), + Aes256GcmSiv(Aes256GcmSivEncryptor), } impl EncryptorKind { pub fn encrypt(&self, data: &[u8]) -> Result<Vec<u8>, IggyError> { match self { EncryptorKind::Aes256Gcm(e) => e.encrypt(data), + EncryptorKind::Aes256GcmSiv(e) => e.encrypt(data), } } pub fn decrypt(&self, data: &[u8]) -> Result<Vec<u8>, IggyError> { match self { EncryptorKind::Aes256Gcm(e) => e.decrypt(data), + EncryptorKind::Aes256GcmSiv(e) => e.decrypt(data), } } } @@ -79,14 +89,65 @@ } fn decrypt(&self, data: &[u8]) -> Result<Vec<u8>, IggyError> { - let nonce = (&data[0..12]) - .try_into() - .map_err(|_| IggyError::CannotDecryptData)?; - let payload = self.cipher.decrypt(nonce, &data[12..]); - if payload.is_err() { + if data.len() < AES_GCM_NONCE_SIZE + AES_GCM_TAG_SIZE { return Err(IggyError::CannotDecryptData); } - Ok(payload.unwrap()) + let (nonce, ciphertext) = data.split_at(AES_GCM_NONCE_SIZE); + let nonce = nonce.try_into().map_err(|_| IggyError::CannotDecryptData)?; + self.cipher + .decrypt(nonce, ciphertext) + .map_err(|_| IggyError::CannotDecryptData) + } +} + +#[derive(Clone)] +pub struct Aes256GcmSivEncryptor { + cipher: Aes256GcmSiv, +} + +impl Debug for Aes256GcmSivEncryptor { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Aes256GcmSivEncryptor").finish() + } +} + +impl Aes256GcmSivEncryptor { + pub fn new(key: &[u8]) -> Result<Self, IggyError> { + Ok(Self { + cipher: Aes256GcmSiv::new_from_slice(key) + .map_err(|_| IggyError::InvalidEncryptionKey)?, + }) + } +} + +impl Encryptor for Aes256GcmSivEncryptor { + fn encrypt(&self, data: &[u8]) -> Result<Vec<u8>, IggyError> { + let nonce = SivNonce::generate(); + let mut output = + Vec::with_capacity(AES_GCM_SIV_NONCE_SIZE + data.len() + AES_GCM_SIV_TAG_SIZE); + output.extend_from_slice(&nonce); + output.extend_from_slice(data); + let tag = self + .cipher + .encrypt_inout_detached(&nonce, b"", (&mut output[AES_GCM_SIV_NONCE_SIZE..]).into()) + .map_err(|_| IggyError::CannotEncryptData)?; + output.extend_from_slice(&tag); + Ok(output) + } + + fn decrypt(&self, data: &[u8]) -> Result<Vec<u8>, IggyError> { + if data.len() < AES_GCM_SIV_NONCE_SIZE + AES_GCM_SIV_TAG_SIZE { + return Err(IggyError::CannotDecryptData); + } + let (nonce, body) = data.split_at(AES_GCM_SIV_NONCE_SIZE); + let (ciphertext, tag) = body.split_at(body.len() - AES_GCM_SIV_TAG_SIZE); + let nonce = SivNonce::try_from(nonce).map_err(|_| IggyError::CannotDecryptData)?; + let tag = SivTag::try_from(tag).map_err(|_| IggyError::CannotDecryptData)?; + let mut plaintext = ciphertext.to_vec(); + self.cipher + .decrypt_inout_detached(&nonce, b"", plaintext.as_mut_slice().into(), &tag) + .map_err(|_| IggyError::CannotDecryptData)?; + Ok(plaintext) } } @@ -96,6 +157,80 @@ use crate::{HeaderKey, HeaderValue, IggyMessage}; #[test] + fn given_siv_ciphertext_when_decrypting_should_authenticate_every_byte() { + let encryptor = Aes256GcmSivEncryptor::new(&[7u8; 32]).expect("valid key"); + let other = Aes256GcmSivEncryptor::new(&[8u8; 32]).expect("valid key"); + let first = encryptor.encrypt(b"stream payload").expect("encrypts"); + let second = encryptor + .encrypt(b"stream payload") + .expect("encrypts again"); + assert_ne!(first, second); + assert_eq!( + encryptor.decrypt(&first).expect("decrypts"), + b"stream payload" + ); + assert_eq!(other.decrypt(&first), Err(IggyError::CannotDecryptData)); + + let mut tampered_ciphertext = first.clone(); + tampered_ciphertext[AES_GCM_SIV_NONCE_SIZE] ^= 1; + assert_eq!( + encryptor.decrypt(&tampered_ciphertext), + Err(IggyError::CannotDecryptData) + ); + let mut tampered_tag = first.clone(); + *tampered_tag.last_mut().expect("tag exists") ^= 1; + assert_eq!( + encryptor.decrypt(&tampered_tag), + Err(IggyError::CannotDecryptData) + ); + } + + #[test] + fn given_short_or_empty_siv_payload_when_decrypting_should_handle_lengths() { + let encryptor = Aes256GcmSivEncryptor::new(&[7u8; 32]).expect("valid key"); + for length in 0..AES_GCM_SIV_NONCE_SIZE + AES_GCM_SIV_TAG_SIZE { + assert_eq!( + encryptor.decrypt(&vec![0x5a; length]), + Err(IggyError::CannotDecryptData), + "length {length}" + ); + } + let encrypted = encryptor.encrypt(&[]).expect("empty plaintext encrypts"); + assert_eq!( + encrypted.len(), + AES_GCM_SIV_NONCE_SIZE + AES_GCM_SIV_TAG_SIZE + ); + assert_eq!( + encryptor + .decrypt(&encrypted) + .expect("empty plaintext decrypts"), + b"" + ); + } + + #[test] + fn given_short_ciphertext_when_decrypting_should_return_cannot_decrypt_data() { + let encryptor = Aes256GcmEncryptor::new(&[3; 32]).expect("valid key"); + for length in 0..28 { + let ciphertext = vec![0xA5; length]; + assert_eq!( + encryptor + .decrypt(&ciphertext) + .expect_err("ciphertext shorter than nonce and tag must fail") + .as_code(), + IggyError::CannotDecryptData.as_code(), + "length {length}" + ); + } + let empty = encryptor.encrypt(&[]).expect("encrypt empty payload"); + assert_eq!(empty.len(), 28); + assert_eq!( + encryptor.decrypt(&empty).expect("decrypt empty payload"), + Vec::<u8>::new() + ); + } + + #[test] fn given_the_same_key_data_should_be_encrypted_and_decrypted_correctly() { let key = [1; 32]; let encryptor = Aes256GcmEncryptor::new(&key).unwrap();
diff --git a/core/common/src/utils/mod.rs b/core/common/src/utils/mod.rs index 4ac92d8..f6bf807 100644 --- a/core/common/src/utils/mod.rs +++ b/core/common/src/utils/mod.rs
@@ -25,6 +25,7 @@ pub(crate) mod personal_access_token_expiry; pub mod random_id; pub mod serde_secret; +pub mod stream_key; pub mod text; pub(crate) mod timestamp; pub(crate) mod topic_size;
diff --git a/core/common/src/utils/stream_key.rs b/core/common/src/utils/stream_key.rs new file mode 100644 index 0000000..f75e57f --- /dev/null +++ b/core/common/src/utils/stream_key.rs
@@ -0,0 +1,1436 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::IggyError; +use crate::utils::crypto::{Aes256GcmSivEncryptor, EncryptorKind}; +use crate::utils::text; +use aes_gcm::aead::{Aead, Generate, Payload}; +use aes_gcm::{Aes256Gcm, KeyInit, Nonce}; +use base64::Engine; +use base64::engine::general_purpose::STANDARD; +use ring::rand::{SecureRandom, SystemRandom}; +use ring::{digest, hkdf}; +use secrecy::zeroize::Zeroizing; +use secrecy::{ExposeSecret, ExposeSecretMut, SecretBox}; +use std::fmt::{Debug, Formatter}; +use std::str::FromStr; +use std::sync::Arc; +use thiserror::Error; + +pub use iggy_binary_protocol::requests::streams::{ + MAX_WRAP_SET_ENVELOPES, WRAPPED_STREAM_KEY_LENGTH, +}; + +pub const STREAM_KEY_LENGTH: usize = 32; +/// Most retired config keys a node keeps for unwrapping during a rotation. +pub const MAX_PREVIOUS_KEYS: usize = MAX_WRAP_SET_ENVELOPES - 1; + +const WRAPPED_STREAM_KEY_VERSION: u8 = 1; +const FINGERPRINT_LENGTH: usize = 8; +const NONCE_LENGTH: usize = 12; +const WRAPPED_KEY_CIPHERTEXT_LENGTH: usize = STREAM_KEY_LENGTH + 16; +const ORIGIN_OFFSET: usize = 1; +const KEK_FINGERPRINT_OFFSET: usize = ORIGIN_OFFSET + 1; +const NONCE_OFFSET: usize = KEK_FINGERPRINT_OFFSET + FINGERPRINT_LENGTH; +const CIPHERTEXT_OFFSET: usize = NONCE_OFFSET + NONCE_LENGTH; +const KEY_FINGERPRINT_OFFSET: usize = CIPHERTEXT_OFFSET + WRAPPED_KEY_CIPHERTEXT_LENGTH; +const STREAM_KEY_WRAP_INFO: &[u8] = b"iggy/stream-key-wrap/v1"; +const STREAM_KEY_AAD_LABEL: &[u8] = b"iggy/stream-key/v1"; +const STREAM_KEY_FINGERPRINT_LABEL: &[u8] = b"iggy/stream-key-fingerprint/v1"; +const KEK_FINGERPRINT_LABEL: &[u8] = b"iggy/stream-kek-fingerprint/v1"; + +/// A stream's own data encryption key: 32 uniformly random bytes. +/// +/// Never printed and never serialized. The bytes leave this type only to be +/// wrapped, encoded into a create request, or turned into an encryptor. A key +/// derived from a password turns its public fingerprint into an offline +/// guessing oracle, so callers must draw it from a secure random generator. +pub struct StreamEncryptionKey(SecretBox<[u8; STREAM_KEY_LENGTH]>); + +/// Who chose a stream's key, recorded and authenticated in its envelope. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +pub enum StreamKeyOrigin { + /// The client sent the key with the create. + Client = 1, + /// The server generated the key because the client sent none. + Server = 2, +} + +/// A stream key wrapped under one key-encryption key. +/// +/// Layout: `[version:1][origin:1][kek_fingerprint:8][nonce:12][ciphertext:32][tag:16][key_fingerprint:8]`. +/// The version, the origin and both fingerprints are authenticated as +/// associated data. +#[derive(Clone, PartialEq, Eq)] +pub struct WrappedStreamKey([u8; WRAPPED_STREAM_KEY_LENGTH]); + +/// One stream key wrapped under one or more key-encryption keys: one envelope +/// outside a rotation, one per configured key during it. All envelopes hold +/// the same key from the same origin, sorted by KEK fingerprint, none twice. +#[derive(Clone, PartialEq, Eq)] +pub struct WrapSet { + envelopes: Vec<WrappedStreamKey>, + bytes: Vec<u8>, +} + +/// The key-encryption keys derived from the server config: the current one, +/// the previous ones a rotation still unwraps with, and the fingerprints of +/// retired ones whose wraps a rewrap drops. +#[derive(Clone)] +pub struct ServerKeyContext { + current: Option<KeyEncryptionKey>, + previous: Vec<KeyEncryptionKey>, + retired: Vec<KekFingerprint>, + generates: bool, +} + +/// Why a node cannot plan the rewrap a stored wrap set calls for. +#[derive(Debug, Error)] +pub enum RewrapPlanError { + /// The set holds the most wraps, none under the current key and none + /// under a retired one, so a wrap under the current key does not fit. + #[error( + "wrap set already holds {MAX_WRAP_SET_ENVELOPES} wraps and none is retired, retire a key-encryption key fingerprint" + )] + WrapSetFull, + #[error(transparent)] + Key(#[from] IggyError), +} + +#[derive(Clone)] +struct KeyEncryptionKey { + cipher: Aes256Gcm, + fingerprint: [u8; FINGERPRINT_LENGTH], +} + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +struct KekFingerprint([u8; FINGERPRINT_LENGTH]); + +struct DerivedKeyLength; + +impl StreamEncryptionKey { + /// A new random key from the operating system's secure generator. + /// + /// # Errors + /// + /// `CannotEncryptData` when the generator fails. + pub fn generate() -> Result<Self, IggyError> { + let mut key = SecretBox::new(Box::new([0u8; STREAM_KEY_LENGTH])); + SystemRandom::new() + .fill(key.expose_secret_mut()) + .map_err(|_| IggyError::CannotEncryptData)?; + Ok(Self(key)) + } + + /// 16 lowercase hex characters identifying this key without revealing it, + /// as long as the key is uniformly random. + #[must_use] + pub fn fingerprint(&self) -> String { + hex(&self.fingerprint_bytes()) + } + + /// The raw key, for the request encoder and the stream encryptor. + #[must_use] + pub fn as_bytes(&self) -> &[u8; STREAM_KEY_LENGTH] { + self.0.expose_secret() + } + + /// The encryptor for this stream's message payloads and user headers. + /// + /// # Errors + /// + /// `InvalidEncryptionKey` when the cipher refuses the key. + pub fn encryptor(&self) -> Result<Arc<EncryptorKind>, IggyError> { + Ok(Arc::new(EncryptorKind::Aes256GcmSiv( + Aes256GcmSivEncryptor::new(self.as_bytes())?, + ))) + } + + fn fingerprint_bytes(&self) -> [u8; FINGERPRINT_LENGTH] { + labelled_fingerprint(STREAM_KEY_FINGERPRINT_LABEL, self.as_bytes()) + } +} + +impl WrappedStreamKey { + /// Who chose the key inside this envelope. + #[must_use] + pub fn origin(&self) -> StreamKeyOrigin { + // Construction refuses any other origin byte. + if self.0[ORIGIN_OFFSET] == StreamKeyOrigin::Server as u8 { + StreamKeyOrigin::Server + } else { + StreamKeyOrigin::Client + } + } + + /// Fingerprint of the stream key inside this envelope. + #[must_use] + pub fn key_fingerprint(&self) -> String { + hex(self.key_fingerprint_bytes()) + } + + /// Fingerprint of the key-encryption key that wrapped this envelope. + #[must_use] + pub fn kek_fingerprint(&self) -> String { + hex(self.kek_fingerprint_bytes()) + } + + fn key_fingerprint_bytes(&self) -> &[u8] { + &self.0[KEY_FINGERPRINT_OFFSET..] + } + + fn kek_fingerprint_bytes(&self) -> &[u8] { + &self.0[KEK_FINGERPRINT_OFFSET..NONCE_OFFSET] + } + + fn associated_data(&self) -> Vec<u8> { + associated_data( + self.0[ORIGIN_OFFSET], + self.kek_fingerprint_bytes(), + self.key_fingerprint_bytes(), + ) + } +} + +impl WrapSet { + /// A set of `envelopes` in canonical order. + /// + /// # Errors + /// + /// `UnsupportedCriticalOption` for an empty set, more than three + /// envelopes, a KEK fingerprint twice, or envelopes of different keys or + /// origins. + pub fn new(mut envelopes: Vec<WrappedStreamKey>) -> Result<Self, IggyError> { + envelopes.sort_by(|left, right| { + left.kek_fingerprint_bytes() + .cmp(right.kek_fingerprint_bytes()) + }); + let Some(first) = envelopes.first() else { + return Err(unsupported_wrapped_key()); + }; + let consistent = envelopes.len() <= MAX_WRAP_SET_ENVELOPES + && envelopes.iter().all(|envelope| { + envelope.origin() == first.origin() + && envelope.key_fingerprint_bytes() == first.key_fingerprint_bytes() + }) + && envelopes + .windows(2) + .all(|pair| pair[0].kek_fingerprint_bytes() < pair[1].kek_fingerprint_bytes()); + if !consistent { + return Err(unsupported_wrapped_key()); + } + let bytes = envelopes.iter().flat_map(|envelope| envelope.0).collect(); + Ok(Self { envelopes, bytes }) + } + + #[must_use] + pub fn envelopes(&self) -> &[WrappedStreamKey] { + &self.envelopes + } + + /// Who chose the key every envelope holds. + #[must_use] + pub fn origin(&self) -> StreamKeyOrigin { + self.envelopes[0].origin() + } + + /// Fingerprint of the key every envelope holds. + #[must_use] + pub fn key_fingerprint(&self) -> String { + self.envelopes[0].key_fingerprint() + } + + /// The raw fingerprint of the key every envelope holds. + #[must_use] + pub fn key_fingerprint_bytes(&self) -> &[u8] { + self.envelopes[0].key_fingerprint_bytes() + } + + /// Fingerprints of the key-encryption keys, in canonical order. + #[must_use] + pub fn kek_fingerprints(&self) -> Vec<String> { + self.envelopes + .iter() + .map(WrappedStreamKey::kek_fingerprint) + .collect() + } +} + +impl ServerKeyContext { + /// No config key: streams stay plaintext and stream keys are refused. + #[must_use] + pub const fn disabled() -> Self { + Self { + current: None, + previous: Vec::new(), + retired: Vec::new(), + generates: false, + } + } + + /// Derive the key-encryption keys from the base64 config keys. + /// + /// An empty `key` is valid only when encryption is disabled and no + /// previous key is set, and then nothing is derived. A supplied `key` + /// always derives the key-encryption key, so a client may bring its own + /// stream key while keyless streams stay plaintext. `enabled` decides + /// whether a keyless stream gets a generated key. + /// + /// # Errors + /// + /// `InvalidEncryptionKey` when a key is not 32 bytes of base64, when + /// `key` is empty while `enabled` is true or previous keys are set, for + /// more than two previous keys, or for a key configured twice. + pub fn from_config( + key: &str, + previous_keys: &[String], + enabled: bool, + ) -> Result<Self, IggyError> { + if key.is_empty() { + return if enabled || !previous_keys.is_empty() { + Err(IggyError::InvalidEncryptionKey) + } else { + Ok(Self::disabled()) + }; + } + if previous_keys.len() > MAX_PREVIOUS_KEYS { + return Err(IggyError::InvalidEncryptionKey); + } + let current = KeyEncryptionKey::from_config_key(key)?; + let previous = previous_keys + .iter() + .map(|previous| KeyEncryptionKey::from_config_key(previous)) + .collect::<Result<Vec<_>, _>>()?; + let mut fingerprints: Vec<_> = previous.iter().map(|kek| kek.fingerprint).collect(); + fingerprints.push(current.fingerprint); + fingerprints.sort_unstable(); + if fingerprints.windows(2).any(|pair| pair[0] == pair[1]) { + return Err(IggyError::InvalidEncryptionKey); + } + Ok(Self { + current: Some(current), + previous, + retired: Vec::new(), + generates: enabled, + }) + } + + /// This context with the key-encryption key fingerprints whose wraps a + /// rewrap drops, as reads print them. + /// + /// # Errors + /// + /// `InvalidEncryptionKey` without a config key, for an entry that is not + /// 16 lowercase hex characters, one naming a configured key, or one given + /// twice. + pub fn with_retired_kek_fingerprints( + mut self, + fingerprints: &[String], + ) -> Result<Self, IggyError> { + if fingerprints.is_empty() { + return Ok(self); + } + if self.current.is_none() { + return Err(IggyError::InvalidEncryptionKey); + } + let mut retired = fingerprints + .iter() + .map(|fingerprint| fingerprint.parse::<KekFingerprint>()) + .collect::<Result<Vec<_>, _>>()?; + retired.sort_unstable(); + let repeated = retired.windows(2).any(|pair| pair[0] == pair[1]); + let configured = retired + .iter() + .any(|retired| self.keks().any(|kek| kek.fingerprint == retired.0)); + if repeated || configured { + return Err(IggyError::InvalidEncryptionKey); + } + self.retired = retired; + Ok(self) + } + + /// Whether a stream may bring its own key on this node. + #[must_use] + pub const fn accepts_stream_keys(&self) -> bool { + self.current.is_some() + } + + /// Whether this node gives a keyless stream a generated key. + #[must_use] + pub const fn generates_stream_keys(&self) -> bool { + self.current.is_some() && self.generates + } + + /// Fingerprint of the key-encryption key that wraps, when a key is set. + #[must_use] + pub fn kek_fingerprint(&self) -> Option<String> { + self.current.as_ref().map(|kek| hex(&kek.fingerprint)) + } + + /// Whether this node's keys can call for a rewrap: a previous key, whose + /// streams need a wrap under the current key, or a retired key to drop. + /// Without either no stored set can differ from its target. + #[must_use] + pub const fn rotating(&self) -> bool { + self.current.is_some() && !(self.previous.is_empty() && self.retired.is_empty()) + } + + /// Fingerprints of every configured key-encryption key, current first. + #[must_use] + pub fn kek_fingerprints(&self) -> Vec<String> { + self.keks().map(|kek| hex(&kek.fingerprint)).collect() + } + + /// Wrap a stream key under the current key-encryption key. + /// + /// # Errors + /// + /// `StreamEncryptionUnavailable` without a config key, `CannotEncryptData` + /// when the cipher fails. + pub fn wrap( + &self, + key: &StreamEncryptionKey, + origin: StreamKeyOrigin, + ) -> Result<WrappedStreamKey, IggyError> { + self.current + .as_ref() + .ok_or(IggyError::StreamEncryptionUnavailable)? + .wrap(key, origin) + } + + /// The wrap set a new stream stores: its key wrapped under every + /// configured key-encryption key, so it also opens on nodes that still + /// hold only a previous key while a rotation rolls out. + /// + /// # Errors + /// + /// As [`Self::wrap`]. + pub fn wrap_set( + &self, + key: &StreamEncryptionKey, + origin: StreamKeyOrigin, + ) -> Result<WrapSet, IggyError> { + if self.current.is_none() { + return Err(IggyError::StreamEncryptionUnavailable); + } + let envelopes = self + .keks() + .map(|kek| kek.wrap(key, origin)) + .collect::<Result<Vec<_>, _>>()?; + WrapSet::new(envelopes) + } + + /// Authenticate and unwrap one envelope with the configured key-encryption + /// key it names. + /// + /// # Errors + /// + /// `StreamEncryptionUnavailable` without a config key, `CannotDecryptData` + /// when no configured key matches, the envelope does not authenticate, or + /// its key fingerprint does not match the key inside. + pub fn unwrap(&self, wrapped: &WrappedStreamKey) -> Result<StreamEncryptionKey, IggyError> { + if self.current.is_none() { + return Err(IggyError::StreamEncryptionUnavailable); + } + let kek = self + .keks() + .find(|kek| kek.fingerprint == wrapped.kek_fingerprint_bytes()) + .ok_or(IggyError::CannotDecryptData)?; + let nonce = Nonce::try_from(&wrapped.0[NONCE_OFFSET..CIPHERTEXT_OFFSET]) + .map_err(|_| IggyError::CannotDecryptData)?; + let plaintext = Zeroizing::new( + kek.cipher + .decrypt( + &nonce, + Payload { + msg: &wrapped.0[CIPHERTEXT_OFFSET..KEY_FINGERPRINT_OFFSET], + aad: &wrapped.associated_data(), + }, + ) + .map_err(|_| IggyError::CannotDecryptData)?, + ); + let key = StreamEncryptionKey::try_from(plaintext.as_slice()) + .map_err(|_| IggyError::CannotDecryptData)?; + if key.fingerprint_bytes() != wrapped.key_fingerprint_bytes() { + return Err(IggyError::CannotDecryptData); + } + Ok(key) + } + + /// The stream key of a wrap set, from the first envelope that opens. + /// + /// # Errors + /// + /// As [`Self::unwrap`], from the last envelope tried. + pub fn open(&self, set: &WrapSet) -> Result<StreamEncryptionKey, IggyError> { + let mut last = IggyError::CannotDecryptData; + for envelope in set.envelopes() { + match self.unwrap(envelope) { + Ok(key) => return Ok(key), + Err(error) => last = error, + } + } + Err(last) + } + + /// Whether a stored wrap set already is what a rewrap on this node would + /// store: one envelope under the current key-encryption key and none + /// under a retired one. Reads only the fingerprints in the raw stored + /// value, so a sweep over every stream allocates nothing. Without a + /// config key nothing can change, so every set is settled. + #[must_use] + pub fn is_settled(&self, stored: &[u8]) -> bool { + let Some(current) = &self.current else { + return true; + }; + let (envelopes, rest) = stored.as_chunks::<WRAPPED_STREAM_KEY_LENGTH>(); + rest.is_empty() + && envelopes.iter().any(|envelope| { + envelope[KEK_FINGERPRINT_OFFSET..NONCE_OFFSET] == current.fingerprint + }) + && !envelopes + .iter() + .any(|envelope| self.retires(&envelope[KEK_FINGERPRINT_OFFSET..NONCE_OFFSET])) + } + + /// Whether a stored wrap set holds an envelope under a configured + /// key-encryption key. Reads only the fingerprints, like + /// [`Self::is_settled`], so a scan over every stream allocates nothing. + #[must_use] + pub fn holds_wrap_for(&self, stored: &[u8]) -> bool { + let (envelopes, rest) = stored.as_chunks::<WRAPPED_STREAM_KEY_LENGTH>(); + rest.is_empty() + && envelopes.iter().any(|envelope| { + self.keks() + .any(|kek| envelope[KEK_FINGERPRINT_OFFSET..NONCE_OFFSET] == kek.fingerprint) + }) + } + + /// The set a rewrap should store instead of `stored`: every envelope not + /// under a retired key-encryption key, plus one under the current key + /// when missing. Envelopes under keys this node does not hold stay, + /// because a node still being rolled out may need them. `None` when + /// nothing changes. + /// + /// # Errors + /// + /// When the set must change but this node cannot do it, and the set is + /// left alone: `WrapSetFull` when the set already holds the most + /// envelopes and none is retired, `CannotDecryptData` when no envelope + /// opens here, `CannotEncryptData` when wrapping fails. + pub fn target_set(&self, stored: &WrapSet) -> Result<Option<WrapSet>, RewrapPlanError> { + let Some(current) = &self.current else { + return Ok(None); + }; + if self.is_settled(stored.as_ref()) { + return Ok(None); + } + let key = self.open(stored)?; + let mut envelopes: Vec<WrappedStreamKey> = stored + .envelopes() + .iter() + .filter(|envelope| !self.retires(envelope.kek_fingerprint_bytes())) + .cloned() + .collect(); + if !envelopes + .iter() + .any(|envelope| envelope.kek_fingerprint_bytes() == current.fingerprint) + { + if envelopes.len() == MAX_WRAP_SET_ENVELOPES { + return Err(RewrapPlanError::WrapSetFull); + } + envelopes.push(current.wrap(&key, stored.origin())?); + } + Ok(Some(WrapSet::new(envelopes)?)) + } + + fn keks(&self) -> impl Iterator<Item = &KeyEncryptionKey> { + self.current.iter().chain(&self.previous) + } + + fn retires(&self, kek_fingerprint: &[u8]) -> bool { + self.retired + .iter() + .any(|retired| retired.0 == kek_fingerprint) + } +} + +impl Default for ServerKeyContext { + fn default() -> Self { + Self::disabled() + } +} + +impl KeyEncryptionKey { + fn from_config_key(config_key: &str) -> Result<Self, IggyError> { + let config_key = Zeroizing::new( + text::from_base64_as_bytes(config_key).map_err(|_| IggyError::InvalidEncryptionKey)?, + ); + if config_key.len() != STREAM_KEY_LENGTH { + return Err(IggyError::InvalidEncryptionKey); + } + let prk = hkdf::Salt::new(hkdf::HKDF_SHA256, &[]).extract(config_key.as_slice()); + let wrapping_key = derive_key(&prk, STREAM_KEY_WRAP_INFO)?; + Ok(Self { + cipher: Aes256Gcm::new_from_slice(wrapping_key.expose_secret()) + .map_err(|_| IggyError::InvalidEncryptionKey)?, + fingerprint: labelled_fingerprint(KEK_FINGERPRINT_LABEL, wrapping_key.expose_secret()), + }) + } + + fn wrap( + &self, + key: &StreamEncryptionKey, + origin: StreamKeyOrigin, + ) -> Result<WrappedStreamKey, IggyError> { + let key_fingerprint = key.fingerprint_bytes(); + let aad = associated_data(origin as u8, &self.fingerprint, &key_fingerprint); + let nonce = Nonce::generate(); + let sealed = self + .cipher + .encrypt( + &nonce, + Payload { + msg: key.as_bytes(), + aad: &aad, + }, + ) + .map_err(|_| IggyError::CannotEncryptData)?; + if sealed.len() != WRAPPED_KEY_CIPHERTEXT_LENGTH { + return Err(IggyError::CannotEncryptData); + } + let mut envelope = [0u8; WRAPPED_STREAM_KEY_LENGTH]; + envelope[0] = WRAPPED_STREAM_KEY_VERSION; + envelope[ORIGIN_OFFSET] = origin as u8; + envelope[KEK_FINGERPRINT_OFFSET..NONCE_OFFSET].copy_from_slice(&self.fingerprint); + envelope[NONCE_OFFSET..CIPHERTEXT_OFFSET].copy_from_slice(&nonce); + envelope[CIPHERTEXT_OFFSET..KEY_FINGERPRINT_OFFSET].copy_from_slice(&sealed); + envelope[KEY_FINGERPRINT_OFFSET..].copy_from_slice(&key_fingerprint); + Ok(WrappedStreamKey(envelope)) + } +} + +impl FromStr for KekFingerprint { + type Err = IggyError; + + /// Exactly 16 lowercase hex characters, the form reads print. + fn from_str(text: &str) -> Result<Self, Self::Err> { + let lowercase_hex = text + .bytes() + .all(|digit| matches!(digit, b'0'..=b'9' | b'a'..=b'f')); + if text.len() != 2 * FINGERPRINT_LENGTH || !lowercase_hex { + return Err(IggyError::InvalidEncryptionKey); + } + let mut fingerprint = [0u8; FINGERPRINT_LENGTH]; + for (index, byte) in fingerprint.iter_mut().enumerate() { + *byte = u8::from_str_radix(&text[2 * index..2 * index + 2], 16) + .map_err(|_| IggyError::InvalidEncryptionKey)?; + } + Ok(Self(fingerprint)) + } +} + +impl TryFrom<&[u8]> for StreamEncryptionKey { + type Error = IggyError; + + fn try_from(bytes: &[u8]) -> Result<Self, Self::Error> { + if bytes.len() != STREAM_KEY_LENGTH { + return Err(IggyError::InvalidEncryptionKey); + } + // Filled in place so no copy of the key outlives this call on the stack. + let mut key = SecretBox::new(Box::new([0u8; STREAM_KEY_LENGTH])); + key.expose_secret_mut().copy_from_slice(bytes); + Ok(Self(key)) + } +} + +impl FromStr for StreamEncryptionKey { + type Err = IggyError; + + /// Canonical standard padded base64 of exactly 32 random bytes. Decoding and + /// re-encoding must reproduce the input, which refuses whitespace, the + /// URL-safe alphabet, missing padding and noncanonical trailing bits. + fn from_str(text: &str) -> Result<Self, Self::Err> { + let bytes = Zeroizing::new( + STANDARD + .decode(text) + .map_err(|_| IggyError::InvalidEncryptionKey)?, + ); + if *Zeroizing::new(STANDARD.encode(bytes.as_slice())) != text { + return Err(IggyError::InvalidEncryptionKey); + } + Self::try_from(bytes.as_slice()) + } +} + +impl Clone for StreamEncryptionKey { + fn clone(&self) -> Self { + let mut key = SecretBox::new(Box::new([0u8; STREAM_KEY_LENGTH])); + key.expose_secret_mut().copy_from_slice(self.as_bytes()); + Self(key) + } +} + +impl PartialEq for StreamEncryptionKey { + fn eq(&self, other: &Self) -> bool { + self.as_bytes() == other.as_bytes() + } +} + +impl Eq for StreamEncryptionKey {} + +impl Debug for StreamEncryptionKey { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("StreamEncryptionKey") + .field("fingerprint", &self.fingerprint()) + .finish() + } +} + +impl TryFrom<u8> for StreamKeyOrigin { + type Error = IggyError; + + fn try_from(code: u8) -> Result<Self, Self::Error> { + match code { + 1 => Ok(Self::Client), + 2 => Ok(Self::Server), + _ => Err(unsupported_wrapped_key()), + } + } +} + +impl TryFrom<&[u8]> for WrappedStreamKey { + type Error = IggyError; + + /// # Errors + /// + /// `UnsupportedCriticalOption` when the envelope has the wrong length, a + /// version this build does not understand, or an unknown origin. + fn try_from(bytes: &[u8]) -> Result<Self, Self::Error> { + let envelope: [u8; WRAPPED_STREAM_KEY_LENGTH] = + bytes.try_into().map_err(|_| unsupported_wrapped_key())?; + if envelope[0] != WRAPPED_STREAM_KEY_VERSION { + return Err(unsupported_wrapped_key()); + } + StreamKeyOrigin::try_from(envelope[ORIGIN_OFFSET])?; + Ok(Self(envelope)) + } +} + +impl AsRef<[u8]> for WrappedStreamKey { + fn as_ref(&self) -> &[u8] { + &self.0 + } +} + +impl TryFrom<&[u8]> for WrapSet { + type Error = IggyError; + + /// # Errors + /// + /// `UnsupportedCriticalOption` when the bytes are not whole envelopes in + /// canonical order, see [`WrapSet::new`]. + fn try_from(bytes: &[u8]) -> Result<Self, Self::Error> { + if bytes.is_empty() || !bytes.len().is_multiple_of(WRAPPED_STREAM_KEY_LENGTH) { + return Err(unsupported_wrapped_key()); + } + let (chunks, _) = bytes.as_chunks::<WRAPPED_STREAM_KEY_LENGTH>(); + let envelopes = chunks + .iter() + .map(|chunk| WrappedStreamKey::try_from(chunk.as_slice())) + .collect::<Result<Vec<_>, _>>()?; + let set = Self::new(envelopes)?; + if set.bytes != bytes { + return Err(unsupported_wrapped_key()); + } + Ok(set) + } +} + +impl From<WrappedStreamKey> for WrapSet { + fn from(envelope: WrappedStreamKey) -> Self { + Self { + bytes: envelope.0.to_vec(), + envelopes: vec![envelope], + } + } +} + +impl AsRef<[u8]> for WrapSet { + fn as_ref(&self) -> &[u8] { + &self.bytes + } +} + +impl Debug for WrappedStreamKey { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("WrappedStreamKey") + .field("origin", &self.origin()) + .field("kek_fingerprint", &self.kek_fingerprint()) + .field("key_fingerprint", &self.key_fingerprint()) + .finish() + } +} + +impl Debug for WrapSet { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("WrapSet") + .field("origin", &self.origin()) + .field("key_fingerprint", &self.key_fingerprint()) + .field("kek_fingerprints", &self.kek_fingerprints()) + .finish() + } +} + +impl Debug for ServerKeyContext { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("ServerKeyContext") + .field("kek_fingerprints", &self.kek_fingerprints()) + .field( + "retired_kek_fingerprints", + &self + .retired + .iter() + .map(|retired| hex(&retired.0)) + .collect::<Vec<_>>(), + ) + .field("generates_stream_keys", &self.generates_stream_keys()) + .finish() + } +} + +impl hkdf::KeyType for DerivedKeyLength { + fn len(&self) -> usize { + STREAM_KEY_LENGTH + } +} + +/// Fingerprint of a stream key given as 32 raw bytes, without the server. +/// It hides only a uniformly random key, never one derived from a password. +/// +/// # Errors +/// +/// `InvalidEncryptionKey` when `key` is not exactly 32 bytes. +pub fn stream_key_fingerprint(key: &[u8]) -> Result<String, IggyError> { + StreamEncryptionKey::try_from(key).map(|key| key.fingerprint()) +} + +fn derive_key( + prk: &hkdf::Prk, + info: &[u8], +) -> Result<SecretBox<[u8; STREAM_KEY_LENGTH]>, IggyError> { + let mut key = SecretBox::new(Box::new([0u8; STREAM_KEY_LENGTH])); + let info = [info]; + prk.expand(&info, DerivedKeyLength) + .and_then(|okm| okm.fill(key.expose_secret_mut())) + .map_err(|_| IggyError::InvalidEncryptionKey)?; + Ok(key) +} + +fn labelled_fingerprint(label: &[u8], key: &[u8]) -> [u8; FINGERPRINT_LENGTH] { + let mut context = digest::Context::new(&digest::SHA256); + context.update(label); + context.update(key); + let digest = context.finish(); + let mut fingerprint = [0u8; FINGERPRINT_LENGTH]; + fingerprint.copy_from_slice(&digest.as_ref()[..FINGERPRINT_LENGTH]); + fingerprint +} + +fn associated_data(origin: u8, kek_fingerprint: &[u8], key_fingerprint: &[u8]) -> Vec<u8> { + let mut aad = Vec::with_capacity(STREAM_KEY_AAD_LABEL.len() + 2 + 2 * FINGERPRINT_LENGTH); + aad.extend_from_slice(STREAM_KEY_AAD_LABEL); + aad.push(WRAPPED_STREAM_KEY_VERSION); + aad.push(origin); + aad.extend_from_slice(kek_fingerprint); + aad.extend_from_slice(key_fingerprint); + aad +} + +fn unsupported_wrapped_key() -> IggyError { + IggyError::UnsupportedCriticalOption("encryption_key_wrapped".to_owned()) +} + +fn hex(bytes: &[u8]) -> String { + const DIGITS: &[u8; 16] = b"0123456789abcdef"; + let mut text = String::with_capacity(2 * bytes.len()); + for byte in bytes { + text.push(char::from(DIGITS[usize::from(byte >> 4)])); + text.push(char::from(DIGITS[usize::from(byte & 0x0f)])); + } + text +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::utils::crypto::{Aes256GcmEncryptor, Encryptor}; + + const CONFIG_KEY: &str = "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE="; + const OTHER_CONFIG_KEY: &str = "AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA="; + const THIRD_CONFIG_KEY: &str = "ICEiIyQlJicoKSorLC0uLzAxMjM0NTY3ODk6Ozw9Pj8="; + const FOURTH_CONFIG_KEY: &str = "QEFCQ0RFRkdISUpLTE1OT1BRUlNUVVZXWFlaW1xdXl8="; + + #[test] + fn given_stream_key_when_encrypting_should_use_a_distinct_cipher_from_client_payloads() { + let key = StreamEncryptionKey::try_from([7u8; 32].as_slice()).expect("valid key"); + let server = key.encryptor().expect("server encryptor"); + let client = Aes256GcmEncryptor::new(key.as_bytes()).expect("client encryptor"); + let encrypted = server + .encrypt(b"stream payload") + .expect("encrypted payload"); + assert_eq!( + server.decrypt(&encrypted).expect("server decrypts"), + b"stream payload" + ); + assert!(client.decrypt(&encrypted).is_err()); + } + + #[test] + fn given_config_key_when_deriving_should_produce_only_the_pinned_kek() { + let config = text::from_base64_as_bytes(CONFIG_KEY).expect("valid base64"); + let prk = hkdf::Salt::new(hkdf::HKDF_SHA256, &[]).extract(&config); + let wrapping_key = derive_key(&prk, STREAM_KEY_WRAP_INFO).expect("derive kek"); + assert_ne!(wrapping_key.expose_secret().as_slice(), config.as_slice()); + // Pinned against an independent HKDF-SHA256 computation (RFC 5869, + // empty salt, 32-byte output) of the same config key and label. + assert_eq!(hex(wrapping_key.expose_secret()), WRAPPING_KEY_VECTOR); + + let enabled = context(CONFIG_KEY, &[], true); + assert_eq!( + enabled.kek_fingerprint().as_deref(), + Some(KEK_FINGERPRINT_VECTOR) + ); + assert!(enabled.accepts_stream_keys()); + assert!(enabled.generates_stream_keys()); + let disabled = context(CONFIG_KEY, &[], false); + assert!(disabled.accepts_stream_keys()); + assert!(!disabled.generates_stream_keys()); + let empty = context("", &[], false); + assert!(!empty.accepts_stream_keys()); + assert!(!empty.generates_stream_keys()); + assert!(ServerKeyContext::from_config("", &[], true).is_err()); + assert!(ServerKeyContext::from_config("c2hvcnQ=", &[], false).is_err()); + assert_eq!( + context(OTHER_CONFIG_KEY, &[], true) + .kek_fingerprint() + .as_deref(), + Some(OTHER_KEK_FINGERPRINT_VECTOR) + ); + } + + #[test] + fn given_wrapped_key_when_tampering_should_reject_every_envelope_component() { + let context = context(CONFIG_KEY, &[], false); + let key = fixed_key(); + let wrapped = context.wrap(&key, StreamKeyOrigin::Client).expect("wrap"); + assert_eq!(wrapped.as_ref().len(), WRAPPED_STREAM_KEY_LENGTH); + assert_eq!(context.unwrap(&wrapped).expect("unwrap"), key); + assert_eq!(wrapped.key_fingerprint(), key.fingerprint()); + assert_eq!(wrapped.origin(), StreamKeyOrigin::Client); + + for (component, position) in [ + ("version", 0), + ("origin", ORIGIN_OFFSET), + ("kek fingerprint", KEK_FINGERPRINT_OFFSET), + ("nonce", NONCE_OFFSET), + ("ciphertext", CIPHERTEXT_OFFSET), + ("tag", KEY_FINGERPRINT_OFFSET - 1), + ("key fingerprint", KEY_FINGERPRINT_OFFSET), + ] { + let mut tampered = wrapped.0; + tampered[position] ^= 0x03; + let outcome = WrappedStreamKey::try_from(tampered.as_slice()) + .and_then(|tampered| context.unwrap(&tampered)); + assert!(outcome.is_err(), "tampered {component} must not unwrap"); + } + // An origin flipped to the other valid value parses, and still fails + // to authenticate. + let mut relabelled = wrapped.0; + relabelled[ORIGIN_OFFSET] = StreamKeyOrigin::Server as u8; + let relabelled = WrappedStreamKey::try_from(relabelled.as_slice()).expect("valid origin"); + assert!(context.unwrap(&relabelled).is_err()); + } + + #[test] + fn given_same_key_when_wrapping_twice_should_use_a_fresh_nonce_each_time() { + let context = context(CONFIG_KEY, &[], true); + let key = fixed_key(); + let first = context.wrap(&key, StreamKeyOrigin::Client).expect("wrap"); + let second = context.wrap(&key, StreamKeyOrigin::Client).expect("wrap"); + assert_ne!( + first.0[NONCE_OFFSET..CIPHERTEXT_OFFSET], + second.0[NONCE_OFFSET..CIPHERTEXT_OFFSET] + ); + assert_ne!( + first.0[CIPHERTEXT_OFFSET..KEY_FINGERPRINT_OFFSET], + second.0[CIPHERTEXT_OFFSET..KEY_FINGERPRINT_OFFSET] + ); + assert_eq!(context.unwrap(&first).expect("unwrap"), key); + assert_eq!(context.unwrap(&second).expect("unwrap"), key); + } + + #[test] + fn given_wrong_config_key_when_unwrapping_should_return_error() { + let context = context(CONFIG_KEY, &[], true); + let other = self::context(OTHER_CONFIG_KEY, &[], true); + let wrapped = context + .wrap(&fixed_key(), StreamKeyOrigin::Client) + .expect("wrap"); + assert_eq!( + other.unwrap(&wrapped).expect_err("wrong kek").as_code(), + IggyError::CannotDecryptData.as_code() + ); + assert_ne!(other.kek_fingerprint(), Some(wrapped.kek_fingerprint())); + assert_eq!(context.kek_fingerprint(), Some(wrapped.kek_fingerprint())); + assert_eq!( + ServerKeyContext::disabled() + .wrap(&fixed_key(), StreamKeyOrigin::Server) + .expect_err("no kek") + .as_code(), + IggyError::StreamEncryptionUnavailable.as_code() + ); + } + + #[test] + fn given_generated_keys_when_wrapping_should_differ_and_record_server_origin() { + let context = context(CONFIG_KEY, &[], true); + let first = StreamEncryptionKey::generate().expect("random key"); + let second = StreamEncryptionKey::generate().expect("random key"); + assert_ne!(first, second); + let wrapped = context.wrap(&first, StreamKeyOrigin::Server).expect("wrap"); + let set = WrapSet::from(wrapped); + assert_eq!(set.origin(), StreamKeyOrigin::Server); + assert_eq!(set.key_fingerprint(), first.fingerprint()); + assert_eq!(context.open(&set).expect("opens"), first); + assert_eq!( + WrapSet::try_from(set.as_ref()).expect("canonical bytes"), + set + ); + } + + #[test] + fn given_wrap_set_bytes_when_parsing_should_enforce_canonical_form() { + let key = fixed_key(); + let wraps: Vec<WrappedStreamKey> = [CONFIG_KEY, OTHER_CONFIG_KEY, THIRD_CONFIG_KEY] + .iter() + .map(|config_key| { + context(config_key, &[], true) + .wrap(&key, StreamKeyOrigin::Client) + .expect("wrap") + }) + .collect(); + let set = WrapSet::new(wraps.clone()).expect("three wraps"); + let mut sorted = wraps.clone(); + sorted.sort_by_key(WrappedStreamKey::kek_fingerprint); + assert_eq!(set.envelopes(), sorted.as_slice()); + + let concat = |envelopes: &[WrappedStreamKey]| -> Vec<u8> { + envelopes.iter().flat_map(|envelope| envelope.0).collect() + }; + let other_key = context(CONFIG_KEY, &[], true) + .wrap( + &StreamEncryptionKey::try_from([9u8; 32].as_slice()).unwrap(), + StreamKeyOrigin::Client, + ) + .expect("wrap"); + let server_origin = context(OTHER_CONFIG_KEY, &[], true) + .wrap(&key, StreamKeyOrigin::Server) + .expect("wrap"); + let mut unsorted = sorted.clone(); + unsorted.reverse(); + for (case, bytes) in [ + ("empty", Vec::new()), + ("partial envelope", sorted[0].0[..40].to_vec()), + ("unsorted", concat(&unsorted)), + ( + "duplicate kek", + concat(&[sorted[0].clone(), sorted[0].clone()]), + ), + ( + "mixed keys", + concat(&[sorted[0].clone(), other_key.clone()]), + ), + ( + "mixed origins", + concat(&[wraps[0].clone(), server_origin.clone()]), + ), + ( + "four envelopes", + concat(&[sorted.clone(), vec![other_key]].concat()), + ), + ] { + assert!(WrapSet::try_from(bytes.as_slice()).is_err(), "{case}"); + } + } + + #[test] + fn given_previous_keys_when_opening_wrap_set_should_use_any_configured_kek() { + let key = fixed_key(); + let old = context(CONFIG_KEY, &[], true); + let set = WrapSet::from(old.wrap(&key, StreamKeyOrigin::Server).expect("wrap")); + let rotated = context(OTHER_CONFIG_KEY, &[CONFIG_KEY.to_owned()], true); + assert_eq!(rotated.open(&set).expect("previous key opens"), key); + let new_only = context(OTHER_CONFIG_KEY, &[], true); + assert!(new_only.open(&set).is_err()); + let rewrapped = rotated.wrap(&key, StreamKeyOrigin::Server).expect("wrap"); + assert_eq!(Some(rewrapped.kek_fingerprint()), rotated.kek_fingerprint()); + + for invalid in [ + vec![CONFIG_KEY.to_owned(); 2], + vec![OTHER_CONFIG_KEY.to_owned()], + vec![ + CONFIG_KEY.to_owned(), + THIRD_CONFIG_KEY.to_owned(), + "AAAA".to_owned(), + ], + vec!["c2hvcnQ=".to_owned()], + ] { + assert!( + ServerKeyContext::from_config(OTHER_CONFIG_KEY, &invalid, true).is_err(), + "{} previous keys", + invalid.len() + ); + } + assert!(ServerKeyContext::from_config("", &[CONFIG_KEY.to_owned()], false).is_err()); + } + + #[test] + fn given_rotation_phases_when_planning_should_add_current_and_drop_only_retired_wraps() { + let key = fixed_key(); + let old_only = WrapSet::from( + context(CONFIG_KEY, &[], true) + .wrap(&key, StreamKeyOrigin::Client) + .expect("wrap"), + ); + let phase_one = context(OTHER_CONFIG_KEY, &[CONFIG_KEY.to_owned()], true); + let both = phase_one + .target_set(&old_only) + .expect("plans") + .expect("adds the current wrap"); + assert_eq!( + both.kek_fingerprints(), + sorted(&[KEK_FINGERPRINT_VECTOR, OTHER_KEK_FINGERPRINT_VECTOR]) + ); + assert_eq!(both.origin(), StreamKeyOrigin::Client); + assert_eq!(phase_one.open(&both).expect("opens"), key); + assert_eq!(phase_one.target_set(&both).expect("plans"), None); + + let new_alone = context(OTHER_CONFIG_KEY, &[], true); + assert_eq!(new_alone.target_set(&both).expect("plans"), None); + + let phase_two = new_alone + .clone() + .with_retired_kek_fingerprints(&[KEK_FINGERPRINT_VECTOR.to_owned()]) + .expect("valid retirement"); + let new_only = phase_two + .target_set(&both) + .expect("plans") + .expect("drops the retired wrap"); + assert_eq!( + new_only.kek_fingerprints(), + vec![OTHER_KEK_FINGERPRINT_VECTOR] + ); + assert_eq!(phase_two.open(&new_only).expect("opens"), key); + assert_eq!(phase_two.target_set(&new_only).expect("plans"), None); + assert_eq!( + phase_one.target_set(&new_only).expect("plans"), + None, + "a phase 1 primary must not re-add a wrap phase 2 dropped" + ); + + for (node, stuck) in [ + (context(THIRD_CONFIG_KEY, &[], true), &old_only), + (phase_two, &old_only), + ] { + assert!(matches!( + node.target_set(stuck).expect_err("no wrap opens here"), + RewrapPlanError::Key(IggyError::CannotDecryptData) + )); + } + } + + #[test] + fn given_full_wrap_set_when_planning_should_refuse_until_a_wrap_is_retired() { + let key = fixed_key(); + let full = WrapSet::new( + [CONFIG_KEY, OTHER_CONFIG_KEY, THIRD_CONFIG_KEY] + .iter() + .map(|config_key| { + context(config_key, &[], true) + .wrap(&key, StreamKeyOrigin::Server) + .expect("wrap") + }) + .collect(), + ) + .expect("three wraps"); + let fourth = context(FOURTH_CONFIG_KEY, &[CONFIG_KEY.to_owned()], true); + assert!(matches!( + fourth + .target_set(&full) + .expect_err("a fourth wrap does not fit"), + RewrapPlanError::WrapSetFull + )); + + let retired = fourth + .with_retired_kek_fingerprints(&[OTHER_KEK_FINGERPRINT_VECTOR.to_owned()]) + .expect("valid retirement"); + let replaced = retired + .target_set(&full) + .expect("plans") + .expect("swaps the retired wrap for the current one"); + assert_eq!(replaced.envelopes().len(), 3); + assert!( + !replaced + .kek_fingerprints() + .contains(&OTHER_KEK_FINGERPRINT_VECTOR.to_owned()) + ); + assert_eq!(retired.open(&replaced).expect("opens"), key); + } + + #[test] + fn given_retired_fingerprints_when_configuring_should_accept_only_unconfigured_lowercase_hex() { + let rotated = context(OTHER_CONFIG_KEY, &[THIRD_CONFIG_KEY.to_owned()], true); + let accepted = rotated + .clone() + .with_retired_kek_fingerprints(&[KEK_FINGERPRINT_VECTOR.to_owned()]) + .expect("an unconfigured fingerprint"); + assert!(accepted.rotating()); + let third_fingerprint = context(THIRD_CONFIG_KEY, &[], true) + .kek_fingerprint() + .expect("a key"); + for invalid in [ + vec![KEK_FINGERPRINT_VECTOR.to_uppercase()], + vec![KEK_FINGERPRINT_VECTOR[..15].to_owned()], + vec![format!("{KEK_FINGERPRINT_VECTOR}0")], + vec!["39e4f0f95c3693cg".to_owned()], + vec![OTHER_KEK_FINGERPRINT_VECTOR.to_owned()], + vec![third_fingerprint], + vec![KEK_FINGERPRINT_VECTOR.to_owned(); 2], + ] { + assert!( + rotated + .clone() + .with_retired_kek_fingerprints(&invalid) + .is_err(), + "{invalid:?} must be refused" + ); + } + assert!( + ServerKeyContext::disabled() + .with_retired_kek_fingerprints(&[KEK_FINGERPRINT_VECTOR.to_owned()]) + .is_err() + ); + assert!( + !ServerKeyContext::disabled() + .with_retired_kek_fingerprints(&[]) + .expect("nothing to retire") + .rotating() + ); + + assert!(!context(CONFIG_KEY, &[], true).rotating()); + assert!(!context(CONFIG_KEY, &[], false).rotating()); + assert!(rotated.rotating()); + assert!( + context(OTHER_CONFIG_KEY, &[], true) + .with_retired_kek_fingerprints(&[KEK_FINGERPRINT_VECTOR.to_owned()]) + .expect("valid retirement") + .rotating() + ); + } + + #[test] + fn given_raw_stored_sets_when_checking_settled_should_match_the_planner() { + let key = fixed_key(); + let old_only = WrapSet::from( + context(CONFIG_KEY, &[], true) + .wrap(&key, StreamKeyOrigin::Server) + .expect("wrap"), + ); + let phase_one = context(OTHER_CONFIG_KEY, &[CONFIG_KEY.to_owned()], true); + let both = phase_one + .wrap_set(&key, StreamKeyOrigin::Server) + .expect("wraps"); + let phase_two = context(OTHER_CONFIG_KEY, &[], true) + .with_retired_kek_fingerprints(&[KEK_FINGERPRINT_VECTOR.to_owned()]) + .expect("valid retirement"); + + for (node, set, settled) in [ + (&phase_one, &old_only, false), + (&phase_one, &both, true), + (&phase_two, &both, false), + (&phase_two, &old_only, false), + ] { + assert_eq!(node.is_settled(set.as_ref()), settled, "{set:?}"); + assert_eq!( + node.target_set(set).is_ok_and(|target| target.is_none()), + settled, + "{set:?}" + ); + } + assert!(!phase_one.is_settled(&both.as_ref()[..WRAPPED_STREAM_KEY_LENGTH + 1])); + assert!(!phase_one.is_settled(&[])); + assert!(ServerKeyContext::disabled().is_settled(old_only.as_ref())); + } + + #[test] + fn given_previous_keys_when_wrapping_new_key_should_wrap_under_every_configured_kek() { + let key = fixed_key(); + let phase_one = context( + OTHER_CONFIG_KEY, + &[CONFIG_KEY.to_owned(), THIRD_CONFIG_KEY.to_owned()], + true, + ); + let set = phase_one + .wrap_set(&key, StreamKeyOrigin::Client) + .expect("wraps"); + assert_eq!(set.envelopes().len(), 3); + assert_eq!(set.origin(), StreamKeyOrigin::Client); + for config_key in [CONFIG_KEY, OTHER_CONFIG_KEY, THIRD_CONFIG_KEY] { + assert_eq!( + context(config_key, &[], true) + .open(&set) + .expect("opens under each key alone"), + key + ); + } + let single = context(CONFIG_KEY, &[], true) + .wrap_set(&key, StreamKeyOrigin::Client) + .expect("wraps"); + assert_eq!(single.kek_fingerprints(), vec![KEK_FINGERPRINT_VECTOR]); + assert_eq!( + ServerKeyContext::disabled() + .wrap_set(&key, StreamKeyOrigin::Client) + .expect_err("no kek") + .as_code(), + IggyError::StreamEncryptionUnavailable.as_code() + ); + } + + #[test] + fn given_wraps_under_unknown_keks_when_planning_should_keep_them() { + let key = fixed_key(); + let phase_one = context(OTHER_CONFIG_KEY, &[CONFIG_KEY.to_owned()], true); + let both = WrapSet::new(vec![ + phase_one.wrap(&key, StreamKeyOrigin::Client).expect("wrap"), + context(CONFIG_KEY, &[], true) + .wrap(&key, StreamKeyOrigin::Client) + .expect("wrap"), + ]) + .expect("two wraps"); + + for (config, previous) in [(CONFIG_KEY, &[][..]), (OTHER_CONFIG_KEY, &[][..])] { + assert_eq!( + context(config, previous, true) + .target_set(&both) + .expect("plans"), + None, + "a node holding one of the keys must keep the other wrap" + ); + } + } + + #[test] + fn given_fixed_key_when_fingerprinting_should_match_shared_vector() { + let key = fixed_key(); + assert_eq!(key.fingerprint(), FINGERPRINT_VECTOR); + assert_eq!( + stream_key_fingerprint(key.as_bytes()).expect("32 bytes"), + FINGERPRINT_VECTOR + ); + assert!(stream_key_fingerprint(&[0u8; 31]).is_err()); + assert!(stream_key_fingerprint(&[0u8; 33]).is_err()); + } + + #[test] + fn given_secret_values_when_debugging_should_redact_raw_and_base64_forms() { + let key = fixed_key(); + let base64 = STANDARD.encode(key.as_bytes()); + let context = context(CONFIG_KEY, &[OTHER_CONFIG_KEY.to_owned()], true); + let wrapped = context.wrap(&key, StreamKeyOrigin::Client).expect("wrap"); + let set = WrapSet::from(wrapped.clone()); + for rendered in [ + format!("{key:?}"), + format!("{wrapped:?}"), + format!("{set:?}"), + format!("{context:?}"), + ] { + assert!(!rendered.contains(&base64), "{rendered}"); + assert!(!rendered.contains(CONFIG_KEY), "{rendered}"); + assert!(!rendered.contains(OTHER_CONFIG_KEY), "{rendered}"); + assert!( + !rendered.contains(&format!("{:?}", key.as_bytes())), + "{rendered}" + ); + } + } + + #[test] + fn given_noncanonical_base64_when_parsing_key_should_reject() { + let canonical = STANDARD.encode(fixed_key().as_bytes()); + assert_eq!( + canonical.parse::<StreamEncryptionKey>().expect("canonical"), + fixed_key() + ); + for invalid in [ + format!(" {canonical}"), + canonical.trim_end_matches('=').to_owned(), + STANDARD + .encode([0xFB; 32]) + .replace('+', "-") + .replace('/', "_"), + STANDARD.encode([7u8; 31]), + STANDARD.encode([7u8; 33]), + // Same 32 bytes, noncanonical trailing bits in the last symbol. + format!("{}B=", &canonical[..canonical.len() - 2]), + ] { + assert!( + invalid.parse::<StreamEncryptionKey>().is_err(), + "{invalid} must be refused" + ); + } + } + + fn context(key: &str, previous_keys: &[String], enabled: bool) -> ServerKeyContext { + ServerKeyContext::from_config(key, previous_keys, enabled).expect("valid config") + } + + fn sorted(fingerprints: &[&str]) -> Vec<String> { + let mut sorted: Vec<String> = fingerprints + .iter() + .map(|fingerprint| (*fingerprint).to_owned()) + .collect(); + sorted.sort(); + sorted + } + + fn fixed_key() -> StreamEncryptionKey { + let bytes: Vec<u8> = (0u8..32).collect(); + StreamEncryptionKey::try_from(bytes.as_slice()).expect("32 bytes") + } + + const FINGERPRINT_VECTOR: &str = "8441e38538cbfa57"; + const WRAPPING_KEY_VECTOR: &str = + "8433bbdfaceaf9e7540529aad95a49cd8680ce3d30a1ef1103d5c6d122a1ba3e"; + const KEK_FINGERPRINT_VECTOR: &str = "39e4f0f95c3693c7"; + const OTHER_KEK_FINGERPRINT_VECTOR: &str = "d55d479829bc2936"; +}
diff --git a/core/common/src/wire_conversions.rs b/core/common/src/wire_conversions.rs index c5d0f93..1932229 100644 --- a/core/common/src/wire_conversions.rs +++ b/core/common/src/wire_conversions.rs
@@ -23,13 +23,14 @@ use crate::{ CacheMetrics, CacheMetricsKey, ClientInfo, ClientInfoDetails, ClusterMetadata, ClusterNode, - ClusterNodeRole, ClusterNodeStatus, CompressionAlgorithm, Consumer, ConsumerGroup, - ConsumerGroupDetails, ConsumerGroupInfo, ConsumerGroupMember, ConsumerOffsetInfo, - GlobalPermissions, HeaderKey, HeaderKind, HeaderValue, IdKind, Identifier, IdentityInfo, - IggyByteSize, IggyError, IggyExpiry, MaxTopicSize, OptionSpec, OptionValue, OptionsProvenance, - Partition, Permissions, PersonalAccessTokenInfo, RawPersonalAccessToken, ResourceOptions, - Stats, Stream, StreamDetails, StreamPermissions, Topic, TopicDetails, TopicPermissions, - TransportEndpoints, UserInfo, UserInfoDetails, UserStatus, + ClusterNodeRole, ClusterNodeStatus, Consumer, ConsumerGroup, ConsumerGroupDetails, + ConsumerGroupInfo, ConsumerGroupMember, ConsumerOffsetInfo, GlobalPermissions, HeaderKey, + HeaderKind, HeaderValue, IdKind, Identifier, IdentityInfo, IggyByteSize, IggyError, + OPTION_FLAG_CRITICAL, OPTION_FLAG_READ_ONLY, OPTION_FLAG_RESETTABLE, OPTION_FLAG_SECRET, + OPTION_FLAG_UPDATABLE, OptionSpec, OptionValue, OptionsProvenance, OptionsScope, Partition, + Permissions, PersonalAccessTokenInfo, RawPersonalAccessToken, ResourceOptions, Stats, Stream, + StreamDetails, StreamPermissions, Topic, TopicDetails, TopicPermissions, TransportEndpoints, + UserInfo, UserInfoDetails, UserStatus, descriptors_for, }; use bytes::{BufMut, BytesMut}; use iggy_binary_protocol::primitives::options::{MAX_OPTIONS, MAX_OPTIONS_BYTES, WireOptions}; @@ -84,7 +85,7 @@ size: IggyByteSize::from(w.size_bytes), messages_count: w.messages_count, topics_count: w.topics_count, - options: resource_options_from_wire(&w.options, true)?, + options: resource_options_from_wire_split(&w.options, &w.derived_options)?, }) } } @@ -107,7 +108,10 @@ messages_count: w.stream.messages_count, topics_count: w.stream.topics_count, topics, - options: resource_options_from_wire(&w.stream.options, true)?, + options: resource_options_from_wire_split( + &w.stream.options, + &w.stream.derived_options, + )?, }) } } @@ -130,11 +134,6 @@ type Error = IggyError; fn try_from(w: TopicHeader) -> Result<Self, Self::Error> { - let message_expiry = match w.message_expiry { - 0 => IggyExpiry::NeverExpire, - v => v.into(), - }; - let max_topic_size: MaxTopicSize = w.max_topic_size.into(); let options = resource_options_from_wire_split(&w.options, &w.derived_options)?; Ok(Self { id: w.id, @@ -143,9 +142,6 @@ partitions_count: w.partitions_count, size: IggyByteSize::from(w.size_bytes), messages_count: w.messages_count, - message_expiry, - compression_algorithm: CompressionAlgorithm::from_code(w.compression_algorithm)?, - max_topic_size, options, }) } @@ -179,9 +175,6 @@ name: topic.name, size: topic.size, messages_count: topic.messages_count, - message_expiry: topic.message_expiry, - compression_algorithm: topic.compression_algorithm, - max_topic_size: topic.max_topic_size, partitions_count: topic.partitions_count, partitions, options: topic.options, @@ -212,7 +205,7 @@ created_at: w.created_at.into(), status: UserStatus::from_code(w.status)?, username: w.username.to_string(), - options: resource_options_from_wire(&w.options, true)?, + options: resource_options_from_wire_split(&w.options, &w.derived_options)?, }) } } @@ -876,6 +869,36 @@ // -- Options conversions -- +/// The `DescribeOptions` response for `scope`: every catalog entry a client +/// may see, with its kind, flags, default and description. Internal keys stay +/// out. +/// +/// # Errors +/// +/// Propagates a catalog key or default that does not fit the wire, which the +/// catalog tests rule out. +pub fn describe_options_response( + scope: OptionsScope, +) -> Result<iggy_binary_protocol::responses::system::DescribeOptionsResponse, IggyError> { + let entries = descriptors_for(scope) + .filter(|descriptor| descriptor.is_described()) + .map(|descriptor| { + Ok(iggy_binary_protocol::responses::system::OptionDescriptor { + key: iggy_binary_protocol::WireName::new(descriptor.key.to_string()) + .map_err(|_| IggyError::InvalidFormat)?, + kind: descriptor.kind.as_code(), + flags: descriptor.flags(), + default_value: descriptor + .default_value()? + .map(|value| bytes::Bytes::copy_from_slice(value.as_bytes())) + .unwrap_or_default(), + description: (descriptor.description)(), + }) + }) + .collect::<Result<Vec<_>, IggyError>>()?; + Ok(iggy_binary_protocol::responses::system::DescribeOptionsResponse { entries }) +} + /// Decode a `DescribeOptions` response into domain option specs. /// /// # Errors @@ -891,6 +914,11 @@ Ok(OptionSpec { key: entry.key.to_string(), kind: HeaderKind::from_code(entry.kind)?, + updatable: entry.flags & OPTION_FLAG_UPDATABLE != 0, + resettable: entry.flags & OPTION_FLAG_RESETTABLE != 0, + secret: entry.flags & OPTION_FLAG_SECRET != 0, + read_only: entry.flags & OPTION_FLAG_READ_ONLY != 0, + critical: entry.flags & OPTION_FLAG_CRITICAL != 0, default_value: entry.default_value.to_vec(), description: entry.description, }) @@ -983,23 +1011,7 @@ if entries.is_empty() { return Ok(WireOptions::empty()); } - if entries.len() > MAX_OPTIONS as usize { - return Err(IggyError::OptionsBlockTooLarge(format!( - "{} entries, maximum {MAX_OPTIONS}", - entries.len() - ))); - } - let size: usize = entries - .iter() - .map(|(key, option)| { - OPTION_ENTRY_OVERHEAD + key.as_bytes().len() + option.value.as_bytes().len() - }) - .sum(); - if size > MAX_OPTIONS_BYTES { - return Err(IggyError::OptionsBlockTooLarge(format!( - "{size} bytes, maximum {MAX_OPTIONS_BYTES}" - ))); - } + let size = options_block_size(entries.iter().copied())?; let mut buf = BytesMut::with_capacity(size); for (key, option) in entries { // A non-string key would encode a block the receiving peer refuses, @@ -1027,6 +1039,35 @@ Ok(WireOptions::from_validated(buf.freeze())) } +/// The encoded size of an options block holding `entries`, without encoding +/// it. +/// +/// # Errors +/// +/// Returns `IggyError::OptionsBlockTooLarge` when the entries exceed +/// `MAX_OPTIONS` or `MAX_OPTIONS_BYTES`. +pub(crate) fn options_block_size<'a>( + entries: impl Iterator<Item = (&'a HeaderKey, &'a OptionValue)>, +) -> Result<usize, IggyError> { + let (count, size) = entries.fold((0usize, 0usize), |(count, size), (key, option)| { + ( + count + 1, + size + OPTION_ENTRY_OVERHEAD + key.as_bytes().len() + option.value.as_bytes().len(), + ) + }); + if count > MAX_OPTIONS as usize { + return Err(IggyError::OptionsBlockTooLarge(format!( + "{count} entries, maximum {MAX_OPTIONS}" + ))); + } + if size > MAX_OPTIONS_BYTES { + return Err(IggyError::OptionsBlockTooLarge(format!( + "{size} bytes, maximum {MAX_OPTIONS_BYTES}" + ))); + } + Ok(size) +} + #[cfg(test)] mod tests { use super::*;
diff --git a/core/common/tests/resource_options_fixtures.rs b/core/common/tests/resource_options_fixtures.rs new file mode 100644 index 0000000..dbf5d0c --- /dev/null +++ b/core/common/tests/resource_options_fixtures.rs
@@ -0,0 +1,776 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#[path = "../examples/resource_options_fixtures/schema.rs"] +mod schema; + +use base64::Engine; +use base64::engine::general_purpose::STANDARD; +use iggy_binary_protocol::requests::streams::CreateStreamRequest; +use iggy_binary_protocol::requests::system::{DescribeOptionsRequest, UpdateResourceRequest}; +use iggy_binary_protocol::requests::topics::CreateTopicRequest; +use iggy_binary_protocol::requests::users::CreateUserRequest; +use iggy_binary_protocol::responses::streams::{GetStreamResponse, StreamResponse, TopicHeader}; +use iggy_binary_protocol::responses::system::DescribeOptionsResponse; +use iggy_binary_protocol::responses::topics::GetTopicResponse; +use iggy_binary_protocol::responses::users::{UserDetailsResponse, UserResponse}; +use iggy_binary_protocol::version::pack_protocol_version; +use iggy_binary_protocol::{Operation, WireDecode, WireEncode, WireName, lookup_command}; +use iggy_common::create_stream::CreateStream; +use iggy_common::create_topic::CreateTopic; +use iggy_common::wire_conversions::{resource_options_from_wire, resource_options_to_wire_split}; +use iggy_common::{ + CATALOG, HeaderKey, OPTION_FLAG_CRITICAL, OPTION_FLAG_READ_ONLY, OPTION_FLAG_RESETTABLE, + OPTION_FLAG_SECRET, OPTION_FLAG_UPDATABLE, OptionColumns, OptionKey, OptionsScope, + ServerKeyContext, StreamDetails, StreamEncryptionSource, TopicDetails, UserInfoDetails, + public_options, stream_key_fingerprint, +}; +use ring::digest::{SHA256, digest}; +use schema::{ + CANONICAL_PATH, CatalogRow, CommandCode, DescribedOption, ErrorExpectation, GENERATOR, + HttpOutcome, KeyInput, LANGUAGE_COPIES, ResourceOptionsFixture, SCHEMA_VERSION, SCOPES, + StreamProjection, TopicProjection, UserProjection, WireConstants, admit_create, admit_update, + contains_bytes, describe_options_response, evaluate_http_update, update_resource_request, +}; +use std::collections::BTreeSet; +use std::fs; +use std::path::Path; + +const KNOWN_FINGERPRINT: &str = "8441e38538cbfa57"; +const ABOVE_2_POW_53: u64 = (1 << 53) + 1; + +#[test] +fn given_live_wire_constants_when_comparing_fixture_should_match_codes_and_versions() { + let fixture = fixture(); + assert_eq!(fixture.schema_version, SCHEMA_VERSION); + assert_eq!(fixture.generated_by, GENERATOR); + let constants = &fixture.constants; + assert_eq!( + *constants, + WireConstants::live().expect("live constants build") + ); + assert_eq!( + constants.protocol_version.packed, + pack_protocol_version(0, 12, 0) + ); + assert_eq!(constants.protocol_version.text, "0.12.0"); + assert_eq!( + constants.command_codes["update_resource"], + CommandCode { + code: 16, + operation: Some(150), + } + ); + for (name, code) in &constants.retired_command_codes { + assert!( + lookup_command(*code).is_none(), + "retired command {name} ({code}) is still dispatched" + ); + } + for (name, value) in &constants.retired_operations { + assert!( + !Operation::is_known_code(*value), + "retired operation {name} ({value}) is still known" + ); + } + let update_resource = &constants.operations["update_resource"]; + assert!(!update_resource.internal); + for name in [ + "update_stream_resolved", + "update_topic_resolved", + "update_user_resolved", + ] { + let resolved = &constants.operations[name]; + assert!(resolved.internal, "{name} must be internal"); + assert_eq!(resolved.request_operation, update_resource.value, "{name}"); + } + assert!(constants.operations["rewrap_stream_keys"].internal); + let scope_codes: Vec<(&str, u8)> = constants + .scope_codes + .iter() + .map(|(name, code)| (name.as_str(), *code)) + .collect(); + assert_eq!(scope_codes, [("stream", 2), ("topic", 1), ("user", 3)]); + assert_eq!( + constants.never_returned_keys, + [ + OptionKey::EncryptionKey, + OptionKey::EncryptionKeyWrapped, + OptionKey::CriticalKeys, + ] + ); +} + +#[test] +fn given_live_catalog_when_comparing_fixture_should_match_keys_kinds_defaults_and_flags() { + let fixture = fixture(); + let live = CATALOG + .iter() + .map(CatalogRow::try_from) + .collect::<Result<Vec<_>, _>>() + .expect("live catalog rows build"); + assert_eq!(fixture.catalog, live); + + let row = |key: OptionKey| { + fixture + .catalog + .iter() + .find(|row| row.key == key) + .expect("catalog row exists") + }; + assert_eq!(row(OptionKey::Name).flags, OPTION_FLAG_UPDATABLE); + assert_eq!( + row(OptionKey::CompressionAlgorithm).flags, + 0, + "compression_algorithm is create-only" + ); + assert_eq!( + row(OptionKey::CompressionAlgorithm).default.as_deref(), + Some("none") + ); + assert_eq!( + row(OptionKey::MessageExpiry).flags, + OPTION_FLAG_UPDATABLE | OPTION_FLAG_RESETTABLE + ); + assert_eq!( + row(OptionKey::MaxTopicSize).default.as_deref(), + Some("18446744073709551615") + ); + assert_eq!( + row(OptionKey::EncryptionKey).flags, + OPTION_FLAG_SECRET | OPTION_FLAG_CRITICAL + ); + assert_eq!( + row(OptionKey::EncryptionKekFingerprint).flags, + OPTION_FLAG_READ_ONLY + ); + assert!(!row(OptionKey::EncryptionKeyWrapped).described); + assert!(!row(OptionKey::CriticalKeys).described); +} + +#[test] +fn given_describe_options_vectors_when_decoding_should_match_catalog_flags() { + let fixture = fixture(); + let scopes: Vec<OptionsScope> = fixture + .describe_options + .iter() + .map(|vector| vector.scope) + .collect(); + assert_eq!(scopes, SCOPES); + for vector in &fixture.describe_options { + let request = DescribeOptionsRequest { + scope: vector.scope.as_code(), + }; + assert_eq!(hex::encode(request.to_bytes()), vector.request_hex); + + let bytes = hex::decode(&vector.response_hex).expect("response is hex"); + let (response, consumed) = + DescribeOptionsResponse::decode(&bytes).expect("describe response decodes"); + assert_eq!(consumed, bytes.len(), "{}", vector.scope); + let entries = response + .entries + .iter() + .map(DescribedOption::try_from) + .collect::<Result<Vec<_>, _>>() + .expect("entries decode"); + assert_eq!(entries, vector.entries, "{}", vector.scope); + + let rebuilt = describe_options_response(vector.scope).expect("server response builds"); + assert_eq!( + rebuilt.to_bytes().as_ref(), + bytes.as_slice(), + "{}", + vector.scope + ); + + let from_catalog: Vec<DescribedOption> = fixture + .catalog + .iter() + .filter(|row| row.described && row.scopes.contains(&vector.scope)) + .map(|row| DescribedOption { + key: row.key.to_string(), + kind: row.kind, + flags: row.flags, + default: row.default.clone(), + }) + .collect(); + assert_eq!(entries, from_catalog, "{}", vector.scope); + } +} + +#[test] +fn given_fixed_key_when_comparing_fixture_should_match_fingerprint() { + let vectors = fixture().stream_key; + let key = hex::decode(&vectors.key_hex).expect("key is hex"); + assert_eq!(key, (0u8..32).collect::<Vec<_>>()); + assert_eq!(STANDARD.encode(&key), vectors.key_base64); + assert_eq!(vectors.fingerprint, KNOWN_FINGERPRINT); + assert_eq!( + stream_key_fingerprint(&key).expect("32-byte key"), + vectors.fingerprint + ); + assert_eq!( + KeyInput::Base64(vectors.key_base64.clone()) + .parse() + .expect("canonical base64 key") + .fingerprint(), + vectors.fingerprint + ); + let mut labelled = vectors.fingerprint_label.as_bytes().to_vec(); + labelled.extend_from_slice(&key); + assert_eq!( + hex::encode(&digest(&SHA256, &labelled).as_ref()[..8]), + vectors.fingerprint + ); + + let cases: BTreeSet<&str> = vectors + .invalid_keys + .iter() + .map(|vector| vector.case.as_str()) + .collect(); + assert_eq!( + cases, + BTreeSet::from([ + "base64_missing_padding", + "base64_noncanonical_trailing_bits", + "base64_with_whitespace", + "raw_31_bytes", + "raw_33_bytes", + ]) + ); + for vector in &vectors.invalid_keys { + let refused = vector + .input + .parse() + .err() + .map(|error| ErrorExpectation::from(&error)); + assert_eq!(refused.as_ref(), Some(&vector.error), "{}", vector.case); + assert_eq!( + vector.error.name, "invalid_encryption_key", + "{}", + vector.case + ); + } +} + +#[test] +fn given_update_resource_vectors_when_decoding_should_match_all_scopes() { + let fixture = fixture(); + let mut scopes = Vec::new(); + for vector in &fixture.update_resource { + let bytes = hex::decode(&vector.body_hex).expect("body is hex"); + let decoded = UpdateResourceRequest::decode_from(&bytes).expect("vector decodes"); + let typed = update_resource_request(vector.scope, &vector.target, &vector.input) + .expect("typed input encodes"); + assert_eq!(decoded, typed, "{}", vector.case); + assert_eq!( + typed.to_bytes().as_ref(), + bytes.as_slice(), + "{}", + vector.case + ); + assert_eq!( + resource_options_from_wire(&decoded.options, true).expect("set block decodes"), + vector.wire_set, + "{}", + vector.case + ); + let reset: Vec<&str> = decoded.reset.iter().map(WireName::as_str).collect(); + assert_eq!(reset, vector.wire_reset, "{}", vector.case); + assert_eq!(admit_update(&decoded), Ok(()), "{}", vector.case); + scopes.push(vector.scope); + } + for scope in SCOPES { + assert!(scopes.contains(&scope), "no update vector for {scope}"); + } + let keeps_u64_precision = fixture + .update_resource + .iter() + .filter_map(|vector| { + vector + .wire_set + .get(&HeaderKey::from(OptionKey::MaxTopicSize)) + }) + .any(|option| option.value.as_uint64() == Ok(ABOVE_2_POW_53)); + assert!(keeps_u64_precision, "no u64 value above 2^53"); +} + +#[test] +fn given_invalid_update_inputs_when_encoding_should_refuse_before_sending() { + let fixture = fixture(); + for vector in &fixture.update_resource_errors { + let refused = update_resource_request(vector.scope, &vector.target, &vector.input) + .err() + .map(|error| ErrorExpectation::from(&error)); + assert_eq!(refused.as_ref(), Some(&vector.error), "{}", vector.case); + } + let names: BTreeSet<&str> = fixture + .update_resource_errors + .iter() + .map(|vector| vector.error.name.as_str()) + .collect(); + assert_eq!( + names, + BTreeSet::from([ + "invalid_command", + "invalid_stream_name", + "invalid_topic_name", + "invalid_username", + "option_not_resettable", + "option_not_updatable", + "read_only_option" + ]) + ); +} + +#[test] +fn given_create_vectors_when_decoding_should_match_typed_requests() { + let fixture = fixture(); + for vector in &fixture.create_stream { + let bytes = hex::decode(&vector.body_hex).expect("body is hex"); + let decoded = CreateStreamRequest::decode_from(&bytes).expect("vector decodes"); + let typed = vector.input.request().expect("typed input encodes"); + assert_eq!(decoded, typed, "{}", vector.case); + assert_eq!( + typed.to_bytes().as_ref(), + bytes.as_slice(), + "{}", + vector.case + ); + assert_eq!( + resource_options_from_wire(&decoded.options, true).expect("options decode"), + vector.wire_options, + "{}", + vector.case + ); + let columns = OptionColumns { + name: Some(vector.input.name.clone()), + ..OptionColumns::default() + }; + let admission = admit_create(OptionsScope::Stream, columns.clone(), &decoded.options) + .err() + .map(|error| ErrorExpectation::from(&error)); + assert_eq!(admission, vector.admission_error, "{}", vector.case); + assert_eq!( + vector.input.http_body().expect("HTTP body encodes"), + vector.http_body, + "{}", + vector.case + ); + let posted: CreateStream = + serde_json::from_str(&vector.http_body).expect("HTTP body parses"); + let http_admission = admit_create( + OptionsScope::Stream, + columns, + &posted.options.to_wire().expect("HTTP options encode"), + ) + .err() + .map(|error| ErrorExpectation::from(&error)); + assert_eq!(http_admission, vector.admission_error, "{}", vector.case); + } + let keyed = fixture + .create_stream + .iter() + .filter_map(|vector| { + vector + .wire_options + .get(&HeaderKey::from(OptionKey::EncryptionKey)) + }) + .count(); + assert_eq!(keyed, 1, "exactly one create carries a stream key"); + + for vector in &fixture.create_topic { + let bytes = hex::decode(&vector.body_hex).expect("body is hex"); + let decoded = CreateTopicRequest::decode_from(&bytes).expect("vector decodes"); + let typed = vector.input.request().expect("typed input encodes"); + assert_eq!(decoded, typed, "{}", vector.case); + assert_eq!( + typed.to_bytes().as_ref(), + bytes.as_slice(), + "{}", + vector.case + ); + assert_eq!( + resource_options_from_wire(&decoded.options, true).expect("options decode"), + vector.wire_options, + "{}", + vector.case + ); + let columns = OptionColumns { + name: Some(vector.input.name.clone()), + ..OptionColumns::default() + }; + let admission = admit_create(OptionsScope::Topic, columns.clone(), &decoded.options) + .err() + .map(|error| ErrorExpectation::from(&error)); + assert_eq!(admission, vector.admission_error, "{}", vector.case); + assert_eq!( + vector.input.http_body().expect("HTTP body encodes"), + vector.http_body, + "{}", + vector.case + ); + let posted: CreateTopic = + serde_json::from_str(&vector.http_body).expect("HTTP body parses"); + let http_admission = admit_create( + OptionsScope::Topic, + columns, + &posted.options.to_wire().expect("HTTP options encode"), + ) + .err() + .map(|error| ErrorExpectation::from(&error)); + assert_eq!(http_admission, vector.admission_error, "{}", vector.case); + } + + for vector in &fixture.create_user { + let bytes = hex::decode(&vector.body_hex).expect("body is hex"); + let decoded = CreateUserRequest::decode_from(&bytes).expect("vector decodes"); + let typed = vector.input.request().expect("typed input encodes"); + assert_eq!(decoded, typed, "{}", vector.case); + assert_eq!( + typed.to_bytes().as_ref(), + bytes.as_slice(), + "{}", + vector.case + ); + assert!(vector.wire_options.is_empty(), "{}", vector.case); + let columns = OptionColumns { + username: Some(vector.input.username.clone()), + status: Some(vector.input.status), + ..OptionColumns::default() + }; + let admission = admit_create(OptionsScope::User, columns, &decoded.options) + .err() + .map(|error| ErrorExpectation::from(&error)); + assert_eq!(admission, vector.admission_error, "{}", vector.case); + assert_eq!( + vector.input.http_body().expect("HTTP body encodes"), + vector.http_body, + "{}", + vector.case + ); + } +} + +#[test] +fn given_http_update_vectors_when_parsing_should_match_update_resource() { + let fixture = fixture(); + let mut outcomes = Vec::new(); + for vector in &fixture.http_update { + let evaluation = evaluate_http_update(vector.scope, &vector.target, &vector.body) + .expect("body converts"); + assert_eq!( + evaluation.update_resource.as_deref().map(hex::encode), + vector.update_resource_hex, + "{}", + vector.case + ); + assert_eq!(evaluation.outcome, vector.outcome, "{}", vector.case); + assert_eq!(evaluation.error, vector.error, "{}", vector.case); + outcomes.push(vector.outcome); + } + for outcome in [ + HttpOutcome::Accepted, + HttpOutcome::JsonRefused, + HttpOutcome::DecodeRefused, + HttpOutcome::AdmissionRefused, + ] { + assert!(outcomes.contains(&outcome), "no vector for {outcome:?}"); + } + + let null_ignored = fixture + .http_update + .iter() + .find(|vector| vector.case == "topic_null_ignored") + .and_then(|vector| vector.update_resource_hex.as_deref()) + .expect("null vector has a request"); + let request = + UpdateResourceRequest::decode_from(&hex::decode(null_ignored).expect("request is hex")) + .expect("null vector decodes"); + let set_keys: Vec<&[u8]> = (&request.options) + .into_iter() + .map(|entry| entry.key) + .collect(); + assert_eq!(set_keys, [b"message_expiry".as_slice()]); + assert!(request.reset.is_empty()); +} + +#[test] +fn given_response_vectors_when_decoding_should_match_typed_projection() { + let fixture = fixture(); + let never_returned: Vec<HeaderKey> = fixture + .constants + .never_returned_keys + .iter() + .map(|key| HeaderKey::from(*key)) + .collect(); + + let mut sources = Vec::new(); + for vector in &fixture.stream_responses { + let bytes = hex::decode(&vector.response_hex).expect("response is hex"); + let (response, consumed) = + GetStreamResponse::decode(&bytes).expect("stream response decodes"); + assert_eq!(consumed, bytes.len(), "{}", vector.case); + let details = StreamDetails::try_from(response).expect("stream converts"); + assert_eq!( + StreamProjection::from(&details), + vector.expected, + "{}", + vector.case + ); + + let stored = vector.state.stored_options().expect("stored state builds"); + let public = public_options(OptionsScope::Stream, &stored).expect("stream projects"); + let (options, derived_options) = + resource_options_to_wire_split(&public).expect("blocks encode"); + let rebuilt = GetStreamResponse { + stream: StreamResponse { + id: vector.expected.id, + created_at: vector.expected.created_at, + topics_count: vector.expected.topics_count, + size_bytes: vector.expected.size_bytes, + messages_count: vector.expected.messages_count, + name: WireName::new(vector.expected.name.as_str()).expect("stream name"), + options, + derived_options, + }, + topics: Vec::new(), + }; + assert_eq!( + rebuilt.to_bytes().as_ref(), + bytes.as_slice(), + "{}", + vector.case + ); + + for key in &never_returned { + assert!( + !vector.expected.options.contains_key(key), + "{} returns a secret or internal key", + vector.case + ); + } + if let Some(wrapped) = stored.get(&HeaderKey::from(OptionKey::EncryptionKeyWrapped)) { + assert!( + !contains_bytes(&bytes, wrapped.value.as_bytes()), + "{} carries the wrap set", + vector.case + ); + } + match &vector.state.data_key { + Some(data_key) => { + let key = data_key.parse().expect("state key parses"); + assert!(!contains_bytes(&bytes, key.as_bytes()), "{}", vector.case); + assert!( + !contains_bytes(&bytes, STANDARD.encode(key.as_bytes()).as_bytes()), + "{}", + vector.case + ); + assert_eq!( + vector.expected.encryption_key_fingerprint, + Some(key.fingerprint()), + "{}", + vector.case + ); + } + None => assert_eq!( + vector.expected.encryption_key_fingerprint, None, + "{}", + vector.case + ), + } + let mut kek_fingerprints: Vec<String> = vector + .state + .kek_config_keys + .iter() + .map(|config_key| { + ServerKeyContext::from_config(config_key, &[], true) + .expect("config key is valid") + .kek_fingerprint() + .expect("config key derives a KEK") + }) + .collect(); + kek_fingerprints.sort(); + assert_eq!( + vector.expected.encryption_kek_fingerprints, kek_fingerprints, + "{}", + vector.case + ); + assert_eq!( + vector.expected.encryption_key_source, + Some(vector.state.encryption_key_source), + "{}", + vector.case + ); + sources.push(vector.state.encryption_key_source); + } + for source in [ + StreamEncryptionSource::Stream, + StreamEncryptionSource::Server, + StreamEncryptionSource::None, + ] { + assert!(sources.contains(&source), "no stream vector for {source}"); + } + assert!( + fixture.stream_responses.iter().any(|vector| vector + .expected + .encryption_kek_fingerprints + .len() + == 2), + "no stream wrapped under two KEKs" + ); + + for vector in &fixture.topic_responses { + let bytes = hex::decode(&vector.response_hex).expect("response is hex"); + let (response, consumed) = + GetTopicResponse::decode(&bytes).expect("topic response decodes"); + assert_eq!(consumed, bytes.len(), "{}", vector.case); + let details = TopicDetails::try_from(response).expect("topic converts"); + assert_eq!( + TopicProjection::from(&details), + vector.expected, + "{}", + vector.case + ); + assert_eq!( + public_options(OptionsScope::Topic, &vector.expected.options).expect("topic projects"), + vector.expected.options, + "{}", + vector.case + ); + let (options, derived_options) = + resource_options_to_wire_split(&vector.expected.options).expect("blocks encode"); + assert!( + !options.is_empty() && !derived_options.is_empty(), + "{}", + vector.case + ); + let rebuilt = GetTopicResponse { + topic: TopicHeader { + id: vector.expected.id, + created_at: vector.expected.created_at, + partitions_count: vector.expected.partitions_count, + size_bytes: vector.expected.size_bytes, + messages_count: vector.expected.messages_count, + name: WireName::new(vector.expected.name.as_str()).expect("topic name"), + options, + derived_options, + }, + partitions: Vec::new(), + }; + assert_eq!( + rebuilt.to_bytes().as_ref(), + bytes.as_slice(), + "{}", + vector.case + ); + for key in [ + OptionKey::CompressionAlgorithm, + OptionKey::MessageExpiry, + OptionKey::MaxTopicSize, + ] { + assert!( + vector.expected.options.contains_key(&HeaderKey::from(key)), + "{} lacks {key}", + vector.case + ); + } + } + + for vector in &fixture.user_responses { + let bytes = hex::decode(&vector.response_hex).expect("response is hex"); + let (response, consumed) = + UserDetailsResponse::decode(&bytes).expect("user response decodes"); + assert_eq!(consumed, bytes.len(), "{}", vector.case); + let details = UserInfoDetails::try_from(response).expect("user converts"); + assert_eq!( + UserProjection::from(&details), + vector.expected, + "{}", + vector.case + ); + assert_eq!( + public_options(OptionsScope::User, &vector.expected.options).expect("user projects"), + vector.expected.options, + "{}", + vector.case + ); + for key in &never_returned { + assert!( + !vector.expected.options.contains_key(key), + "{}", + vector.case + ); + } + let (options, derived_options) = + resource_options_to_wire_split(&vector.expected.options).expect("blocks encode"); + let rebuilt = UserDetailsResponse { + user: UserResponse { + id: vector.expected.id, + created_at: vector.expected.created_at, + status: vector.expected.status.as_code(), + username: WireName::new(vector.expected.username.as_str()).expect("username"), + options, + derived_options, + }, + permissions: None, + }; + assert_eq!( + rebuilt.to_bytes().as_ref(), + bytes.as_slice(), + "{}", + vector.case + ); + } + let user_blocks: BTreeSet<(bool, bool)> = fixture + .user_responses + .iter() + .map(|vector| { + let options = &vector.expected.options; + ( + options.values().any(|option| option.explicit), + options.values().any(|option| !option.explicit), + ) + }) + .collect(); + assert_eq!(user_blocks, BTreeSet::from([(false, false), (true, true)])); +} + +#[test] +fn given_language_copies_when_comparing_should_match_canonical_bytes() { + let canonical = read_manifest_file(CANONICAL_PATH); + for copy in LANGUAGE_COPIES { + assert!( + read_manifest_file(copy) == canonical, + "{copy} differs from {CANONICAL_PATH}, rerun the resource_options_fixtures example" + ); + } +} + +fn fixture() -> ResourceOptionsFixture { + serde_json::from_slice(&read_manifest_file(CANONICAL_PATH)).expect("fixture matches the schema") +} + +fn read_manifest_file(relative: &str) -> Vec<u8> { + let path = Path::new(env!("CARGO_MANIFEST_DIR")).join(relative); + fs::read(&path).unwrap_or_else(|error| { + panic!( + "cannot read {}: {error}. Generate it with: cargo run -p iggy_common --example resource_options_fixtures", + path.display() + ) + }) +}
diff --git a/core/common/tests/stream_key_erasure.rs b/core/common/tests/stream_key_erasure.rs new file mode 100644 index 0000000..a7c9bb9 --- /dev/null +++ b/core/common/tests/stream_key_erasure.rs
@@ -0,0 +1,277 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::cell::Cell; +use std::collections::{BTreeMap, BTreeSet}; + +use bytes::Bytes; +use iggy_binary_protocol::WireOptions; +use iggy_common::{ + IggyError, OptionAdmissionContext, OptionColumns, OptionInputMap, OptionInputValue, OptionKey, + OptionOperation, OptionsScope, OptionsUpdate, ServerKeyContext, StreamCreateOptions, + StreamEncryptionKey, StreamKeyOrigin, admit, erase_buffer, erase_options, +}; + +const KEY: [u8; 32] = *b"erasure-probe-key-32-bytes-long!"; +const KEY_BASE64: &str = "ZXJhc3VyZS1wcm9iZS1rZXktMzItYnl0ZXMtbG9uZyE="; +const CONFIG_KEY: &str = "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE="; +const PREVIOUS_CONFIG_KEY: &str = "AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA="; + +#[global_allocator] +static ALLOCATOR: FreedKeyProbe = FreedKeyProbe; + +/// Counts the blocks freed on a watching thread that still hold the probe key, +/// raw or in base64. Every block starts zeroed, so a scan reads only bytes +/// that were written. +struct FreedKeyProbe; + +thread_local! { + static WATCHING: Cell<bool> = const { Cell::new(false) }; + static FREED_COPIES: Cell<usize> = const { Cell::new(0) }; +} + +#[test] +fn given_plain_drop_of_a_key_copy_when_probing_should_count_it() { + let copy = Bytes::copy_from_slice(&KEY); + let ((), freed) = freed_copies_during(|| drop(copy)); + assert_eq!(freed, 1, "the probe must see a key freed without zeroing"); +} + +#[test] +fn given_stream_key_when_encoding_create_options_should_free_no_copy_of_it() { + let typed = StreamCreateOptions { + encryption_key: Some(StreamEncryptionKey::try_from(KEY.as_slice()).expect("32 bytes")), + raw: BTreeMap::new(), + }; + let raw = StreamCreateOptions { + encryption_key: None, + raw: BTreeMap::from([(OptionKey::EncryptionKey.to_string(), KEY_BASE64.to_owned())]), + }; + for options in [typed, raw] { + let (wire, freed) = freed_copies_during(|| options.to_wire()); + let wire = wire.expect("encodes"); + assert!(holds_key(wire.as_bytes()), "the block carries the key"); + assert_eq!(freed, 0, "to_wire, {options:?}"); + + let (input, freed) = freed_copies_during(|| options.to_input_map()); + assert!(input.is_ok(), "encodes for HTTP"); + assert_eq!(freed, 0, "to_input_map, {options:?}"); + } +} + +#[test] +fn given_stream_key_in_either_form_when_admitting_create_should_free_no_copy_of_it() { + let context = OptionAdmissionContext { + current: None, + create_columns: OptionColumns { + name: Some("orders".to_owned()), + ..OptionColumns::default() + }, + }; + for block in [raw_key_block(), text_key_block()] { + let (admitted, freed) = freed_copies_during(|| { + admit( + OptionsScope::Stream, + OptionOperation::Create, + &block, + &[], + &context, + ) + }); + let admitted = admitted.expect("admits"); + assert_eq!(freed, 0, "admission frees a copy of the key"); + + let ((), freed) = freed_copies_during(|| erase_options(admitted.explicit)); + assert_eq!(freed, 0, "erasing the admitted options"); + } +} + +#[test] +fn given_refused_raw_entry_after_the_key_when_encoding_should_free_no_copy_of_the_key() { + let options = StreamCreateOptions { + encryption_key: None, + raw: BTreeMap::from([ + (OptionKey::EncryptionKey.to_string(), KEY_BASE64.to_owned()), + ("zz_empty".to_owned(), String::new()), + ]), + }; + let (wire, freed) = freed_copies_during(|| options.to_wire()); + assert!(wire.is_err(), "an empty raw value is refused"); + assert_eq!(freed, 0); +} + +#[test] +fn given_stream_key_in_http_create_body_when_encoding_should_free_no_copy_of_it() { + let body = OptionInputMap(BTreeMap::from([( + OptionKey::EncryptionKey.to_string(), + OptionInputValue::String(KEY_BASE64.to_owned().into()), + )])); + let (wire, freed) = freed_copies_during(|| body.to_wire()); + let wire = wire.expect("encodes"); + assert!(holds_key(wire.as_bytes()), "the block carries the key"); + assert_eq!(freed, 0); +} + +#[test] +fn given_refused_http_key_after_the_stream_key_when_encoding_should_free_no_copy_of_the_key() { + let body = OptionInputMap(BTreeMap::from([ + ( + OptionKey::EncryptionKey.to_string(), + OptionInputValue::String(KEY_BASE64.to_owned().into()), + ), + ("z".repeat(256), OptionInputValue::Bool(true)), + ])); + let (wire, freed) = freed_copies_during(|| body.to_wire()); + assert!( + matches!(wire, Err(IggyError::UnsupportedOptionKey(_))), + "a name over 255 bytes is refused" + ); + assert_eq!(freed, 0); +} + +#[test] +fn given_stream_key_in_http_update_body_when_splitting_should_free_no_copy_of_it() { + let set = BTreeMap::from([( + OptionKey::EncryptionKey.to_string(), + OptionInputValue::String(KEY_BASE64.to_owned().into()), + )]); + let accepted = OptionsUpdate { + set: set.clone(), + reset: BTreeSet::from(["message_expiry".to_owned()]), + }; + let refused = OptionsUpdate { + set, + reset: BTreeSet::from([String::new()]), + }; + let (patch, freed) = freed_copies_during(|| accepted.to_patch()); + assert!(patch.is_ok(), "splits"); + assert_eq!(freed, 0, "accepted update"); + + let (patch, freed) = freed_copies_during(|| refused.to_patch()); + assert!( + matches!(patch, Err(IggyError::UnsupportedOptionKey(_))), + "an empty reset name is refused" + ); + assert_eq!(freed, 0, "refused reset name"); +} + +#[test] +fn given_shared_buffer_when_erasing_should_leave_it_to_its_other_owner() { + let buffer = Bytes::copy_from_slice(&KEY); + let other = buffer.clone(); + let ((), freed) = freed_copies_during(|| erase_buffer(buffer)); + assert_eq!(freed, 0, "a shared buffer is not freed"); + assert_eq!( + other.as_ref(), + KEY.as_slice(), + "the other owner still reads it" + ); + + let ((), freed) = freed_copies_during(|| erase_buffer(other)); + assert_eq!(freed, 0, "the last handle is zeroed before it is freed"); +} + +#[test] +fn given_rotating_context_when_wrapping_and_opening_should_free_no_copy_of_the_key() { + let context = + ServerKeyContext::from_config(CONFIG_KEY, &[PREVIOUS_CONFIG_KEY.to_owned()], true) + .expect("valid keys"); + let key = StreamEncryptionKey::try_from(KEY.as_slice()).expect("32 bytes"); + + let (set, freed) = freed_copies_during(|| context.wrap_set(&key, StreamKeyOrigin::Client)); + let set = set.expect("wraps"); + assert_eq!(freed, 0, "wrap_set"); + + let (opened, freed) = freed_copies_during(|| context.open(&set).map(|opened| opened == key)); + assert!(opened.expect("opens"), "opens to the same key"); + assert_eq!(freed, 0, "open"); +} + +#[test] +fn given_text_stream_key_when_parsing_should_free_no_copy_of_it() { + let (parsed, freed) = freed_copies_during(|| { + KEY_BASE64 + .parse::<StreamEncryptionKey>() + .map(|key| key.as_bytes() == &KEY) + }); + assert!(parsed.expect("parses"), "parses to the key"); + assert_eq!(freed, 0); +} + +unsafe impl GlobalAlloc for FreedKeyProbe { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + // SAFETY: forwarded with the caller's layout. + unsafe { System.alloc_zeroed(layout) } + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + // SAFETY: forwarded with the caller's layout. + unsafe { System.alloc_zeroed(layout) } + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + if WATCHING.try_with(Cell::get).unwrap_or(false) { + // SAFETY: the block is live until the call below and was zeroed at + // allocation, so all of its bytes are initialized. + let block = unsafe { std::slice::from_raw_parts(ptr, layout.size()) }; + if holds_key(block) { + let _ = FREED_COPIES.try_with(|freed| freed.set(freed.get() + 1)); + } + } + // SAFETY: the block came from `alloc` above with this layout. + unsafe { System.dealloc(ptr, layout) } + } +} + +/// Run `work` with the probe watching this thread. Returns its output, freed +/// after the probe stops, and how many freed blocks held the key. +fn freed_copies_during<T>(work: impl FnOnce() -> T) -> (T, usize) { + FREED_COPIES.with(|freed| freed.set(0)); + WATCHING.with(|watching| watching.set(true)); + let output = work(); + WATCHING.with(|watching| watching.set(false)); + (output, FREED_COPIES.with(Cell::get)) +} + +fn holds_key(block: &[u8]) -> bool { + [KEY.as_slice(), KEY_BASE64.as_bytes()] + .iter() + .any(|pattern| { + block + .windows(pattern.len()) + .any(|window| window == *pattern) + }) +} + +fn raw_key_block() -> WireOptions { + StreamCreateOptions { + encryption_key: Some(StreamEncryptionKey::try_from(KEY.as_slice()).expect("32 bytes")), + raw: BTreeMap::new(), + } + .to_wire() + .expect("encodes") +} + +fn text_key_block() -> WireOptions { + StreamCreateOptions { + encryption_key: None, + raw: BTreeMap::from([(OptionKey::EncryptionKey.to_string(), KEY_BASE64.to_owned())]), + } + .to_wire() + .expect("encodes") +}
diff --git a/core/configs/src/common/defaults.rs b/core/configs/src/common/defaults.rs index 6d44232..a96ea24 100644 --- a/core/configs/src/common/defaults.rs +++ b/core/configs/src/common/defaults.rs
@@ -236,6 +236,8 @@ EncryptionConfig { enabled: SERVER_CONFIG.encryption.enabled, key: SERVER_CONFIG.encryption.key.parse().unwrap(), + previous_keys: Vec::new(), + retired_kek_fingerprints: Vec::new(), } } }
diff --git a/core/configs/src/common/displays.rs b/core/configs/src/common/displays.rs index 8fdf94c..af37e48 100644 --- a/core/configs/src/common/displays.rs +++ b/core/configs/src/common/displays.rs
@@ -23,7 +23,7 @@ http::{HttpConfig, HttpCorsConfig, HttpJwtConfig, HttpMetricsConfig, HttpTlsConfig}, system::{EncryptionConfig, LoggingConfig}, }; -use std::fmt::{Display, Formatter}; +use std::fmt::{Debug, Display, Formatter}; impl Display for HttpConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { @@ -126,7 +126,24 @@ impl Display for EncryptionConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{{ enabled: {} }}", self.enabled) + write!( + f, + "{{ enabled: {}, previous_keys: {}, retired_kek_fingerprints: {:?} }}", + self.enabled, + self.previous_keys.len(), + self.retired_kek_fingerprints + ) + } +} + +impl Debug for EncryptionConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("EncryptionConfig") + .field("enabled", &self.enabled) + .field("key_set", &!self.key.is_empty()) + .field("previous_keys", &self.previous_keys.len()) + .field("retired_kek_fingerprints", &self.retired_kek_fingerprints) + .finish() } }
diff --git a/core/configs/src/common/system.rs b/core/configs/src/common/system.rs index 640a68b..31ffb98 100644 --- a/core/configs/src/common/system.rs +++ b/core/configs/src/common/system.rs
@@ -21,7 +21,8 @@ use iggy_common::IggyDuration; use serde::{Deserialize, Serialize}; use serde_with::DisplayFromStr; -use serde_with::serde_as; +use serde_with::formats::PreferMany; +use serde_with::{OneOrMany, serde_as}; use server_common::log::LoggingSettings; #[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] @@ -65,7 +66,8 @@ } } -#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] +#[serde_as] +#[derive(Deserialize, Serialize, Clone, ConfigEnv)] pub struct EncryptionConfig { pub enabled: bool, // skip_serializing keeps the key out of the runtime current_config.toml (and @@ -74,11 +76,23 @@ #[serde(default, skip_serializing)] #[config_env(secret)] pub key: String, + /// Retired keys a node still unwraps stream keys with during a rotation. + // A bare string is one key, so a type error can never echo the key into the log. + #[serde_as(as = "OneOrMany<_, PreferMany>")] + #[serde(default, skip_serializing)] + #[config_env(secret)] + pub previous_keys: Vec<String>, + /// Fingerprints of retired keys whose stream key wraps the primary drops. + #[serde_as(as = "OneOrMany<_, PreferMany>")] + #[serde(default)] + pub retired_kek_fingerprints: Vec<String>, } #[cfg(test)] mod tests { use super::*; + use figment::Figment; + use figment::providers::{Format, Toml}; #[test] fn encryption_key_is_never_serialized() { @@ -89,11 +103,57 @@ let config = EncryptionConfig { enabled: true, key: "encryption-key-MUST-NOT-be-persisted".to_owned(), + previous_keys: vec!["previous-key-MUST-NOT-be-persisted".to_owned()], + retired_kek_fingerprints: Vec::new(), }; let serialized = serde_json::to_string(&config).expect("serialize encryption config"); assert!( !serialized.contains("MUST-NOT-be-persisted"), "encryption key leaked into serialized config: {serialized}" ); + let debugged = format!("{config:?}"); + assert!( + !debugged.contains("MUST-NOT-be-persisted"), + "encryption key leaked into debug output: {debugged}" + ); + } + + #[test] + fn given_bare_previous_key_string_when_extracting_should_read_one_key_without_echoing_it() { + let previous_key = "previous-key-MUST-NOT-be-logged"; + let extracted = Figment::from(Toml::string(&format!( + "enabled = true\nprevious_keys = \"{previous_key}\"" + ))) + .extract::<EncryptionConfig>(); + + match extracted { + Ok(config) => assert_eq!(config.previous_keys, vec![previous_key.to_owned()]), + Err(error) => panic!( + "bare previous key string was refused, echoed: {}", + error.to_string().contains(previous_key) + ), + } + } + + #[test] + fn given_retired_kek_fingerprints_when_extracting_should_read_one_or_many_and_serialize_them() { + let fingerprint = "39e4f0f95c3693c7"; + for value in [format!("\"{fingerprint}\""), format!("[\"{fingerprint}\"]")] { + let config = Figment::from(Toml::string(&format!( + "enabled = true\nretired_kek_fingerprints = {value}" + ))) + .extract::<EncryptionConfig>() + .expect("extracts"); + assert_eq!( + config.retired_kek_fingerprints, + vec![fingerprint.to_owned()] + ); + let serialized = serde_json::to_string(&config).expect("serialize encryption config"); + assert!(serialized.contains(fingerprint), "{serialized}"); + } + let defaulted = Figment::from(Toml::string("enabled = true")) + .extract::<EncryptionConfig>() + .expect("extracts"); + assert!(defaulted.retired_kek_fingerprints.is_empty()); } }
diff --git a/core/configs/src/common/validators.rs b/core/configs/src/common/validators.rs index d649a77..657ef71 100644 --- a/core/configs/src/common/validators.rs +++ b/core/configs/src/common/validators.rs
@@ -18,11 +18,11 @@ use super::COMPONENT; use super::server::{DataMaintenanceConfig, MessagesMaintenanceConfig, TelemetryConfig}; use super::server::{MemoryPoolConfig, PersonalAccessTokenConfig}; -use super::system::LoggingConfig; +use super::system::{EncryptionConfig, LoggingConfig}; use crate::ConfigurationError; use cpu_allocation::{CpuAllocation, allowed_cpus}; use err_trail::ErrContext; -use iggy_common::Validatable; +use iggy_common::{ServerKeyContext, Validatable}; use server_common::log::LogFilter; use std::thread::available_parallelism; @@ -54,6 +54,27 @@ } } +impl Validatable<ConfigurationError> for EncryptionConfig { + /// A supplied key must be 32 bytes of base64, and `enabled` needs one. + /// The key values are never printed. + fn validate(&self) -> Result<(), ConfigurationError> { + ServerKeyContext::from_config(&self.key, &self.previous_keys, self.enabled) + .and_then(|context| { + context.with_retired_kek_fingerprints(&self.retired_kek_fingerprints) + }) + .map(|_| ()) + .map_err(|_| { + eprintln!( + "encryption.key must be 32 bytes of standard base64, set when encryption is \ + enabled, encryption.previous_keys may hold at most two other such keys, and \ + encryption.retired_kek_fingerprints needs a key and holds 16 lowercase hex \ + characters per entry, none naming a configured key" + ); + ConfigurationError::InvalidConfigurationValue + }) + } +} + impl Validatable<ConfigurationError> for DataMaintenanceConfig { fn validate(&self) -> Result<(), ConfigurationError> { self.messages.validate().error(|e: &ConfigurationError| { @@ -388,3 +409,94 @@ assert!(unlimited.validate().is_ok()); } } + +#[cfg(test)] +mod tests { + use super::*; + + const KEY: &str = "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE="; + const OTHER_KEY: &str = "AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA="; + const THIRD_KEY: &str = "ICEiIyQlJicoKSorLC0uLzAxMjM0NTY3ODk6Ozw9Pj8="; + + #[test] + fn given_encryption_config_when_validating_should_require_valid_key_only_when_supplied_or_enabled() + { + for (enabled, key, valid) in [ + (false, "", true), + (false, KEY, true), + (true, KEY, true), + (true, "", false), + (false, "c2hvcnQ=", false), + (true, "not base64", false), + ] { + assert_eq!( + config(enabled, key, &[]).validate().is_ok(), + valid, + "enabled={enabled}, key length {}", + key.len() + ); + } + } + + #[test] + fn given_previous_keys_when_validating_should_require_valid_current_key_and_at_most_two() { + for (key, previous, valid) in [ + (KEY, vec![], true), + (KEY, vec![OTHER_KEY], true), + (KEY, vec![OTHER_KEY, THIRD_KEY], true), + (KEY, vec![OTHER_KEY, THIRD_KEY, "AAAA"], false), + (KEY, vec![KEY], false), + (KEY, vec![OTHER_KEY, OTHER_KEY], false), + (KEY, vec!["c2hvcnQ="], false), + ("", vec![OTHER_KEY], false), + ] { + assert_eq!( + config(true, key, &previous).validate().is_ok(), + valid, + "{} previous keys", + previous.len() + ); + } + } + + #[test] + fn given_retired_kek_fingerprints_when_validating_should_require_a_key_and_unconfigured_hex() { + let configured = ServerKeyContext::from_config(KEY, &[], true) + .expect("valid key") + .kek_fingerprint() + .expect("a fingerprint"); + let unconfigured = ServerKeyContext::from_config(THIRD_KEY, &[], true) + .expect("valid key") + .kek_fingerprint() + .expect("a fingerprint"); + for (key, previous, retired, valid) in [ + (KEY, vec![], vec![unconfigured.clone()], true), + (KEY, vec![], vec![], true), + ("", vec![], vec![], true), + ("", vec![], vec![unconfigured.clone()], false), + (KEY, vec![], vec![configured], false), + (KEY, vec![THIRD_KEY], vec![unconfigured.clone()], false), + (KEY, vec![], vec![unconfigured.to_uppercase()], false), + (KEY, vec![], vec![unconfigured[..15].to_owned()], false), + ( + KEY, + vec![], + vec![unconfigured.clone(), unconfigured.clone()], + false, + ), + ] { + let mut config = config(!key.is_empty(), key, &previous); + config.retired_kek_fingerprints = retired.clone(); + assert_eq!(config.validate().is_ok(), valid, "{retired:?}"); + } + } + + fn config(enabled: bool, key: &str, previous_keys: &[&str]) -> EncryptionConfig { + EncryptionConfig { + enabled, + key: key.to_owned(), + previous_keys: previous_keys.iter().map(|key| (*key).to_owned()).collect(), + retired_kek_fingerprints: Vec::new(), + } + } +}
diff --git a/core/configs/src/server_config/validators.rs b/core/configs/src/server_config/validators.rs index ce7caf8..e37a025 100644 --- a/core/configs/src/server_config/validators.rs +++ b/core/configs/src/server_config/validators.rs
@@ -89,6 +89,9 @@ self.logging.validate().error(|e: &ConfigurationError| { format!("{COMPONENT} (error: {e}) - failed to validate logging config") })?; + self.encryption.validate().error(|e: &ConfigurationError| { + format!("{COMPONENT} (error: {e}) - failed to validate encryption config") + })?; if self.http.enabled && let IggyExpiry::ServerDefault = self.http.jwt.access_token_expiry
diff --git a/core/connectors/runtime/Cargo.toml b/core/connectors/runtime/Cargo.toml index 03cc82b..2e8644a 100644 --- a/core/connectors/runtime/Cargo.toml +++ b/core/connectors/runtime/Cargo.toml
@@ -17,7 +17,7 @@ [package] name = "iggy-connectors" -version = "0.5.1-edge.1" +version = "0.5.1-edge.2" description = "Connectors runtime for Iggy message streaming platform" edition = "2024" license = "Apache-2.0"
diff --git a/core/connectors/runtime/src/source.rs b/core/connectors/runtime/src/source.rs index f6ab81a..6f2c528 100644 --- a/core/connectors/runtime/src/source.rs +++ b/core/connectors/runtime/src/source.rs
@@ -18,6 +18,7 @@ use dashmap::DashMap; use dlopen2::wrapper::Container; use flume::{Receiver, Sender}; +use iggy::prelude::StreamCreateOptions; use iggy::prelude::{ Client, DirectConfig, HeaderKey, HeaderValue, Identifier, IggyClient, IggyDuration, IggyError, IggyMessage, StreamClient, TopicClient, TopicCreateOptions, @@ -516,7 +517,9 @@ ) -> Result<(), RuntimeError> { let stream_id = Identifier::named(stream_name)?; if client.get_stream(&stream_id).await?.is_none() { - client.create_stream(stream_name).await?; + client + .create_stream(stream_name, &StreamCreateOptions::default()) + .await?; } let topic_id = Identifier::named(topic_name)?;
diff --git a/core/connectors/sources/mqtt_source/Cargo.toml b/core/connectors/sources/mqtt_source/Cargo.toml index 4ffc480..c10b39d 100644 --- a/core/connectors/sources/mqtt_source/Cargo.toml +++ b/core/connectors/sources/mqtt_source/Cargo.toml
@@ -42,7 +42,8 @@ humantime = { workspace = true } iggy_common = { workspace = true } iggy_connector_sdk = { workspace = true } -rumqttc = { workspace = true } +rumqttc-v4 = { workspace = true } +rumqttc-v5 = { workspace = true } rustls-native-certs = { workspace = true } rustls-pemfile = { workspace = true } secrecy = { workspace = true }
diff --git a/core/connectors/sources/mqtt_source/README.md b/core/connectors/sources/mqtt_source/README.md index 677fb2e..41b8c36 100644 --- a/core/connectors/sources/mqtt_source/README.md +++ b/core/connectors/sources/mqtt_source/README.md
@@ -1,6 +1,6 @@ # Apache Iggy MQTT Source Connector (`iggy_connector_mqtt_source`) -The **MQTT Source Connector** is a dynamically loaded shared library plugin (`.so` / `.dylib` / `.dll`) for Apache Iggy. Built using the `iggy_connector_sdk::source_connector!` C-FFI ABI and the [`rumqttc`](https://crates.io/crates/rumqttc) asynchronous client, it ingests telemetry and event streams from external MQTT brokers (such as **EMQX** or Mosquitto) directly into persistent **Apache Iggy** streams, topics, and partitions. +The **MQTT Source Connector** is a dynamically loaded shared library plugin (`.so` / `.dylib` / `.dll`) for Apache Iggy. Built using the `iggy_connector_sdk::source_connector!` C-FFI ABI and the `rumqttc-v4-next` and `rumqttc-v5-next` asynchronous clients, it ingests telemetry and event streams from external MQTT brokers (such as **EMQX** or Mosquitto) directly into persistent **Apache Iggy** streams, topics, and partitions. --- @@ -88,7 +88,7 @@ | `client_id` | String | Conditional | `iggy-mqtt-source-{id}` when `clean_start = true` | MQTT client identifier. Required and non-empty when `clean_start = false` so a restart resumes the same broker session. | | `username` | String | No | None | Broker authentication username (must be supplied together with `password`). | | `password` | String | No | None | Broker authentication password (wrapped as a secret, never logged). | -| `clean_start` | Boolean | No | `false` | MQTT clean start flag (or clean session for MQTT 3.1.1). | +| `clean_start` | Boolean | No | `false` | MQTT clean start flag (or clean session for MQTT 3.1.1). When an MQTT 5 broker resumes the session, the source keeps the subscriptions stored in that session and does not subscribe again, so a changed `subscriptions` list takes effect once the broker starts a fresh session. | | `session_expiry_interval` | Integer | No | `3600` | MQTT 5 session expiry interval in seconds. Set to `0` only when immediate session discard is intentional. | | `keep_alive` | Duration | No | `"30s"` | Ping interval string (minimum `1s` for MQTT 3.1.1, minimum `5s` for MQTT 5). | | `poll_timeout` | Duration | No | `"1s"` | Maximum duration spent waiting on network events per driver poll tick. |
diff --git a/core/connectors/sources/mqtt_source/src/driver.rs b/core/connectors/sources/mqtt_source/src/driver.rs index 4cc8c24..199c1ee 100644 --- a/core/connectors/sources/mqtt_source/src/driver.rs +++ b/core/connectors/sources/mqtt_source/src/driver.rs
@@ -18,19 +18,21 @@ use super::{MqttProtocol, MqttSourceConfig, Qos, qos_for_subscription}; use base64::Engine; use iggy_common::{HeaderKey, HeaderValue}; -use rumqttc::tokio_rustls::rustls::{ +use rumqttc_v4::tokio_rustls::rustls::{ self, ClientConfig, RootCertStore, pki_types::{CertificateDer, PrivateKeyDer}, }; -use rumqttc::v5::{ - AsyncClient as Mqtt5Client, Event as Mqtt5Event, EventLoop as Mqtt5EventLoop, - Incoming as Mqtt5Incoming, MqttOptions as Mqtt5Options, - mqttbytes::{QoS as Mqtt5Qos, v5::Publish as Mqtt5Publish}, +use rumqttc_v4::{ + AckMode as Mqtt311AckMode, AsyncClient as Mqtt311Client, Event as Mqtt311Event, + EventLoop as Mqtt311EventLoop, Incoming as Mqtt311Incoming, MqttOptions as Mqtt311Options, + TlsConfiguration, Transport as Mqtt311Transport, + mqttbytes::{QoS as MqttQos, v4::Publish as Mqtt311Publish}, }; -use rumqttc::{ - AsyncClient as Mqtt311Client, Event as Mqtt311Event, EventLoop as Mqtt311EventLoop, - Incoming as Mqtt311Incoming, MqttOptions as Mqtt311Options, TlsConfiguration, Transport, - mqttbytes::{QoS as Mqtt311Qos, v4::Publish as Mqtt311Publish}, +use rumqttc_v5::{ + AckMode as Mqtt5AckMode, AsyncClient as Mqtt5Client, BrokerSessionResumePolicy, + Event as Mqtt5Event, EventLoop as Mqtt5EventLoop, Incoming as Mqtt5Incoming, + MqttOptions as Mqtt5Options, Transport as Mqtt5Transport, + mqttbytes::v5::Publish as Mqtt5Publish, }; use rustls_native_certs::load_native_certs; use rustls_pemfile::{certs, private_key}; @@ -48,6 +50,7 @@ const MAX_HEADER_VALUE_LENGTH: usize = 255; const RECONNECT_RETRY_DELAY: Duration = Duration::from_millis(10); +const MQTTS_DEFAULT_PORT: u16 = 8883; const MQTT_PROTOCOL_HEADER: &str = "mqtt.protocol"; const MQTT_TOPIC_HEADER: &str = "mqtt.topic"; @@ -119,8 +122,8 @@ #[derive(Debug)] pub(crate) enum AckToken { - Mqtt311 { qos: Mqtt311Qos, packet_id: u16 }, - Mqtt5 { qos: Mqtt5Qos, packet_id: u16 }, + Mqtt311 { qos: MqttQos, packet_id: u16 }, + Mqtt5 { qos: MqttQos, packet_id: u16 }, } enum MqttConnection { @@ -145,6 +148,11 @@ None, } +struct BrokerUrl { + url: String, + tls: bool, +} + /// Protocol-specific MQTT client and event loop behind one common driver API. pub(crate) struct MqttDriver { // Messages received while flushing acknowledgements are retained here rather @@ -170,11 +178,17 @@ max_buffered_messages: usize, ) -> Result<Self, iggy_connector_sdk::Error> { install_rustls_provider(); + let keep_alive = u16::try_from(keep_alive.as_secs()).map_err(|_| { + iggy_connector_sdk::Error::InvalidConfigValue(format!( + "keep_alive must be at most {}s", + u16::MAX + )) + })?; let broker_url = broker_url_with_client_id(config, id)?; let subscriptions = configured_subscriptions(config, qos)?; match config.protocol { MqttProtocol::Mqtt311 => { - let mut options = Mqtt311Options::parse_url(&broker_url).map_err(|error| { + let mut options = Mqtt311Options::parse_url(&broker_url.url).map_err(|error| { iggy_connector_sdk::Error::InvalidConfigValue(format!("broker_url: {error}")) })?; options @@ -184,13 +198,19 @@ .set_request_channel_capacity(request_capacity) // Manual acknowledgements let Iggy persistence happen before // PUBACK/PUBREC is sent to the broker. - .set_manual_acks(true); + .set_ack_mode(Mqtt311AckMode::Manual); set_mqtt311_credentials(&mut options, config); - if let Some(transport) = tls_transport(config)? { - options.set_transport(transport); + if broker_url.tls { + options.set_transport(Mqtt311Transport::tls_with_config(tls_configuration( + config, + )?)); } - let (client, mut event_loop) = Mqtt311Client::new(options, request_capacity); + let (client, mut event_loop) = Mqtt311Client::builder(options) + .try_build() + .map_err(|error| { + iggy_connector_sdk::Error::InvalidConfigValue(error.to_string()) + })?; subscribe_mqtt311(&client, &subscriptions).await?; let buffered_messages = poll_mqtt311( &mut event_loop, @@ -213,7 +233,7 @@ }) } MqttProtocol::Mqtt5 => { - let mut options = Mqtt5Options::parse_url(&broker_url).map_err(|error| { + let mut options = Mqtt5Options::parse_url(&broker_url.url).map_err(|error| { iggy_connector_sdk::Error::InvalidConfigValue(format!("broker_url: {error}")) })?; options @@ -223,23 +243,37 @@ .set_request_channel_capacity(request_capacity) // MQTT 5 uses the same deferred-acknowledgement lifecycle; // rumqttc maps the token to PUBREC/PUBREL/PUBCOMP internally. - .set_manual_acks(true) + .set_ack_mode(Mqtt5AckMode::Manual) + // The source keeps no local session store, so after a restart the + // broker's session is what redelivers unacknowledged publishes. + .set_broker_session_resume_policy(BrokerSessionResumePolicy::AllowBrokerOnly) .set_session_expiry_interval(config.session_expiry_interval); set_mqtt5_credentials(&mut options, config); - if let Some(transport) = tls_transport(config)? { - options.set_transport(transport); + if broker_url.tls { + options + .set_transport(Mqtt5Transport::tls_with_config(tls_configuration(config)?)); } - let (client, mut event_loop) = Mqtt5Client::new(options, request_capacity); - subscribe_mqtt5(&client, &subscriptions).await?; - let buffered_messages = poll_mqtt5( - &mut event_loop, - subscriptions.len(), - poll_timeout, - config.include_metadata, - max_buffered_messages, - ) - .await?; + let (client, mut event_loop) = + Mqtt5Client::builder(options).try_build().map_err(|error| { + iggy_connector_sdk::Error::InvalidConfigValue(error.to_string()) + })?; + // A resumed broker session keeps its subscriptions, and broker-only + // resume refuses new SUBSCRIBE packets, so only a fresh session subscribes. + let buffered_messages = + if mqtt5_session_present(&mut event_loop, poll_timeout).await? { + VecDeque::new() + } else { + subscribe_mqtt5(&client, &subscriptions).await?; + poll_mqtt5( + &mut event_loop, + subscriptions.len(), + poll_timeout, + config.include_metadata, + max_buffered_messages, + ) + .await? + }; Ok(Self { connection: MqttConnection::Mqtt5 { @@ -544,21 +578,21 @@ let _ = rustls::crypto::ring::default_provider().install_default(); } -fn tls_transport( +fn tls_configuration( config: &MqttSourceConfig, -) -> Result<Option<Transport>, iggy_connector_sdk::Error> { - // No TLS table means plaintext MQTT. When present, build a complete rustls - // transport from either the supplied CA or the system trust store. - let Some(tls) = &config.tls else { - return Ok(None); +) -> Result<TlsConfiguration, iggy_connector_sdk::Error> { + // A secure broker URL always gets TLS. The optional tls table supplies a + // custom CA and client certificate, otherwise the system trust store is used. + let (ca_file, client_cert_file, client_key_file) = match &config.tls { + Some(tls) => (&tls.ca_file, &tls.client_cert_file, &tls.client_key_file), + None => (&None, &None, &None), }; - let ca = tls - .ca_file + let ca = ca_file .as_deref() .map(|path| read_certificate_file(path, "tls.ca_file")) .transpose()?; - let client_auth = match (&tls.client_cert_file, &tls.client_key_file) { + let client_auth = match (client_cert_file, client_key_file) { (Some(cert_path), Some(key_path)) => Some(( read_certificate_file(cert_path, "tls.client_cert_file")?, read_private_key_file(key_path, "tls.client_key_file")?, @@ -567,9 +601,10 @@ _ => unreachable!("TLS client certificate and key validation occurs before connect"), }; - let tls_config = TlsConfiguration::Rustls(Arc::new(build_tls_config(ca, client_auth)?)); - - Ok(Some(Transport::tls_with_config(tls_config))) + Ok(TlsConfiguration::Rustls(Arc::new(build_tls_config( + ca, + client_auth, + )?))) } fn read_file(path: &str, field: &str) -> Result<Vec<u8>, iggy_connector_sdk::Error> { @@ -681,13 +716,28 @@ fn broker_url_with_client_id( config: &MqttSourceConfig, id: u32, -) -> Result<String, iggy_connector_sdk::Error> { +) -> Result<BrokerUrl, iggy_connector_sdk::Error> { // rumqttc requires client_id to be present in the URL before parse_url can // create MqttOptions. set_client_id below still makes the precedence // explicit for both protocol versions. let mut url = Url::parse(&config.broker_url).map_err(|error| { iggy_connector_sdk::Error::InvalidConfigValue(format!("broker_url: {error}")) })?; + // parse_url refuses secure schemes, so a secure URL is parsed as mqtt:// on + // its secure port and the caller must set the TLS transport itself. + let tls = matches!(url.scheme(), "mqtts" | "ssl"); + if tls { + if url.port().is_none() { + url.set_port(Some(MQTTS_DEFAULT_PORT)).map_err(|()| { + iggy_connector_sdk::Error::InvalidConfigValue("broker_url has no host".to_string()) + })?; + } + url.set_scheme("mqtt").map_err(|()| { + iggy_connector_sdk::Error::InvalidConfigValue( + "broker_url scheme cannot be normalized".to_string(), + ) + })?; + } let existing_query = url .query_pairs() .filter(|(key, _)| key != "client_id") @@ -699,7 +749,10 @@ } query.append_pair("client_id", &client_id(config, id)); url.set_query(Some(&query.finish())); - Ok(url.into()) + Ok(BrokerUrl { + url: url.into(), + tls, + }) } fn set_mqtt311_credentials(options: &mut Mqtt311Options, config: &MqttSourceConfig) { @@ -816,7 +869,9 @@ .await?, )) } - Mqtt5Event::Outgoing(_) | Mqtt5Event::Incoming(_) => Ok(PollResult::None), + Mqtt5Event::Outgoing(_) | Mqtt5Event::Incoming(_) | Mqtt5Event::Auth(_) => { + Ok(PollResult::None) + } } } @@ -844,7 +899,7 @@ if let Some(reason) = suback.return_codes.iter().find(|reason| { !matches!( reason, - rumqttc::mqttbytes::v4::SubscribeReasonCode::Success(_) + rumqttc_v4::mqttbytes::v4::SubscribeReasonCode::Success(_) ) }) { return Err(iggy_connector_sdk::Error::Connection(format!( @@ -870,6 +925,25 @@ Ok(buffered_messages) } +async fn mqtt5_session_present( + event_loop: &mut Mqtt5EventLoop, + poll_timeout: Duration, +) -> Result<bool, iggy_connector_sdk::Error> { + loop { + let event = timeout(poll_timeout, event_loop.poll()) + .await + .map_err(|_| { + iggy_connector_sdk::Error::Connection( + "timed out waiting for the MQTT 5 connection acknowledgement".to_string(), + ) + })? + .map_err(|error| iggy_connector_sdk::Error::Connection(error.to_string()))?; + if let Mqtt5Event::Incoming(Mqtt5Incoming::ConnAck(connack)) = event { + return Ok(connack.session_present); + } + } +} + async fn poll_mqtt5( event_loop: &mut Mqtt5EventLoop, subscription_count: usize, @@ -895,7 +969,7 @@ if let Some(reason) = suback.return_codes.iter().find(|reason| { !matches!( reason, - rumqttc::v5::mqttbytes::v5::SubscribeReasonCode::Success(_) + rumqttc_v5::mqttbytes::v5::SubscribeReasonCode::Success(_) ) }) { return Err(iggy_connector_sdk::Error::Connection(format!( @@ -914,7 +988,7 @@ } buffered_messages.push_back(normalize_mqtt5(publish, include_metadata)?); } - Mqtt5Event::Outgoing(_) | Mqtt5Event::Incoming(_) => {} + Mqtt5Event::Outgoing(_) | Mqtt5Event::Incoming(_) | Mqtt5Event::Auth(_) => {} } } @@ -933,16 +1007,17 @@ dup: publish.dup, retain: publish.retain, }; - let headers = metadata_headers("mqtt311", &publish.topic, metadata)?; + let topic = topic_string(&publish.topic)?; + let headers = metadata_headers("mqtt311", &topic, metadata)?; let message = MqttMessage { payload: publish.payload.to_vec(), - topic: publish.topic.clone(), + topic, headers, metadata, mqtt5_properties: None, }; let ack_token = match publish.qos { - Mqtt311Qos::AtMostOnce => None, + MqttQos::AtMostOnce => None, qos => Some(AckToken::Mqtt311 { qos, packet_id: publish.pkid, @@ -981,7 +1056,7 @@ mqtt5_properties, }; let ack_token = match publish.qos { - Mqtt5Qos::AtMostOnce => None, + MqttQos::AtMostOnce => None, qos => Some(AckToken::Mqtt5 { qos, packet_id: publish.pkid, @@ -990,8 +1065,8 @@ Ok(ReceivedMessage { message, ack_token }) } -impl From<&rumqttc::v5::mqttbytes::v5::PublishProperties> for Mqtt5EnvelopeProperties { - fn from(properties: &rumqttc::v5::mqttbytes::v5::PublishProperties) -> Self { +impl From<&rumqttc_v5::mqttbytes::v5::PublishProperties> for Mqtt5EnvelopeProperties { + fn from(properties: &rumqttc_v5::mqttbytes::v5::PublishProperties) -> Self { Self { payload_format_indicator: properties.payload_format_indicator, message_expiry_interval: properties.message_expiry_interval, @@ -1088,7 +1163,7 @@ String::from_utf8(topic.to_vec()).map_err(|_| iggy_connector_sdk::Error::InvalidTextPayload) } -impl From<Qos> for Mqtt311Qos { +impl From<Qos> for MqttQos { fn from(qos: Qos) -> Self { match qos { Qos::Zero => Self::AtMostOnce, @@ -1098,32 +1173,12 @@ } } -impl From<Qos> for Mqtt5Qos { - fn from(qos: Qos) -> Self { +impl From<MqttQos> for Qos { + fn from(qos: MqttQos) -> Self { match qos { - Qos::Zero => Self::AtMostOnce, - Qos::One => Self::AtLeastOnce, - Qos::Two => Self::ExactlyOnce, - } - } -} - -impl From<Mqtt311Qos> for Qos { - fn from(qos: Mqtt311Qos) -> Self { - match qos { - Mqtt311Qos::AtMostOnce => Self::Zero, - Mqtt311Qos::AtLeastOnce => Self::One, - Mqtt311Qos::ExactlyOnce => Self::Two, - } - } -} - -impl From<Mqtt5Qos> for Qos { - fn from(qos: Mqtt5Qos) -> Self { - match qos { - Mqtt5Qos::AtMostOnce => Self::Zero, - Mqtt5Qos::AtLeastOnce => Self::One, - Mqtt5Qos::ExactlyOnce => Self::Two, + MqttQos::AtMostOnce => Self::Zero, + MqttQos::AtLeastOnce => Self::One, + MqttQos::ExactlyOnce => Self::Two, } } } @@ -1131,25 +1186,23 @@ #[cfg(test)] mod tests { use super::*; - use rumqttc::v5::mqttbytes::v5::PublishProperties; + use rumqttc_v5::mqttbytes::v5::PublishProperties; + use tokio::io::AsyncReadExt; + use tokio::net::TcpListener; + + // First byte of a TLS record carrying a handshake, as opposed to 0x10 for an MQTT CONNECT. + const TLS_HANDSHAKE_RECORD: u8 = 0x16; #[test] - fn given_qos_should_convert_to_mqtt311_qos() { - assert_eq!(Mqtt311Qos::AtMostOnce, Qos::Zero.into()); - assert_eq!(Mqtt311Qos::AtLeastOnce, Qos::One.into()); - assert_eq!(Mqtt311Qos::ExactlyOnce, Qos::Two.into()); - } - - #[test] - fn given_qos_should_convert_to_mqtt5_qos() { - assert_eq!(Mqtt5Qos::AtMostOnce, Qos::Zero.into()); - assert_eq!(Mqtt5Qos::AtLeastOnce, Qos::One.into()); - assert_eq!(Mqtt5Qos::ExactlyOnce, Qos::Two.into()); + fn given_qos_should_convert_to_mqtt_qos() { + assert_eq!(MqttQos::AtMostOnce, Qos::Zero.into()); + assert_eq!(MqttQos::AtLeastOnce, Qos::One.into()); + assert_eq!(MqttQos::ExactlyOnce, Qos::Two.into()); } #[test] fn given_mqtt311_qos_zero_should_not_create_ack_token() { - let publish = Mqtt311Publish::new("devices/test", Mqtt311Qos::AtMostOnce, b"payload"); + let publish = Mqtt311Publish::new("devices/test", MqttQos::AtMostOnce, b"payload"); let received = normalize_mqtt311(publish).expect("MQTT 3.1.1 publish should normalize"); assert!(received.ack_token.is_none()); @@ -1159,7 +1212,7 @@ #[test] fn given_mqtt311_publish_should_map_metadata_to_headers() { - let mut publish = Mqtt311Publish::new("devices/test", Mqtt311Qos::AtLeastOnce, b"payload"); + let mut publish = Mqtt311Publish::new("devices/test", MqttQos::AtLeastOnce, b"payload"); publish.dup = true; publish.retain = true; @@ -1183,7 +1236,7 @@ #[test] fn given_oversized_topic_should_preserve_message_and_omit_topic_header() { let topic = "t".repeat(MAX_HEADER_VALUE_LENGTH + 1); - let publish = Mqtt311Publish::new(&topic, Mqtt311Qos::AtMostOnce, b"payload"); + let publish = Mqtt311Publish::new(&topic, MqttQos::AtMostOnce, b"payload"); let received = normalize_mqtt311(publish).expect("oversized topic should be tolerated"); @@ -1210,7 +1263,7 @@ }; let publish = Mqtt5Publish::new( "devices/test", - Mqtt5Qos::AtMostOnce, + MqttQos::AtMostOnce, b"payload".as_slice(), Some(properties), ); @@ -1236,7 +1289,7 @@ fn given_mqtt5_qos_two_should_create_deferred_ack_token() { let mut publish = Mqtt5Publish::new( "devices/test", - Mqtt5Qos::ExactlyOnce, + MqttQos::ExactlyOnce, b"payload".as_slice(), None, ); @@ -1246,7 +1299,7 @@ assert!(matches!( received.ack_token, Some(AckToken::Mqtt5 { - qos: Mqtt5Qos::ExactlyOnce, + qos: MqttQos::ExactlyOnce, packet_id: 42, }) )); @@ -1256,14 +1309,14 @@ #[test] fn given_mqtt311_qos_one_should_create_deferred_ack_token() { - let mut publish = Mqtt311Publish::new("devices/test", Mqtt311Qos::AtLeastOnce, b"payload"); + let mut publish = Mqtt311Publish::new("devices/test", MqttQos::AtLeastOnce, b"payload"); publish.pkid = 17; let received = normalize_mqtt311(publish).expect("MQTT 3.1.1 publish should normalize"); assert!(matches!( received.ack_token, Some(AckToken::Mqtt311 { - qos: Mqtt311Qos::AtLeastOnce, + qos: MqttQos::AtLeastOnce, packet_id: 17, }) )); @@ -1273,7 +1326,7 @@ #[test] fn given_mqtt311_qos_two_should_create_deferred_ack_token() { - let publish = Mqtt311Publish::new("devices/test", Mqtt311Qos::ExactlyOnce, b"payload"); + let publish = Mqtt311Publish::new("devices/test", MqttQos::ExactlyOnce, b"payload"); let received = normalize_mqtt311(publish).expect("MQTT 3.1.1 publish should normalize"); assert!(received.ack_token.is_some()); @@ -1284,14 +1337,14 @@ #[test] fn given_unsent_acknowledgements_should_retain_them_in_order() { let mut first_publish = - Mqtt311Publish::new("devices/first", Mqtt311Qos::AtLeastOnce, b"first"); + Mqtt311Publish::new("devices/first", MqttQos::AtLeastOnce, b"first"); first_publish.pkid = 1; let first = normalize_mqtt311(first_publish) .expect("first MQTT publish should normalize") .ack_token .expect("QoS 1 publish should have an acknowledgement token"); let mut second_publish = - Mqtt311Publish::new("devices/second", Mqtt311Qos::AtLeastOnce, b"second"); + Mqtt311Publish::new("devices/second", MqttQos::AtLeastOnce, b"second"); second_publish.pkid = 2; let second = normalize_mqtt311(second_publish) .expect("second MQTT publish should normalize") @@ -1318,7 +1371,7 @@ fn given_mqtt5_qos_zero_should_not_create_ack_token() { let publish = Mqtt5Publish::new( "devices/test", - Mqtt5Qos::AtMostOnce, + MqttQos::AtMostOnce, b"payload".as_slice(), None, ); @@ -1333,7 +1386,7 @@ fn given_mqtt5_qos_one_should_create_deferred_ack_token() { let publish = Mqtt5Publish::new( "devices/test", - Mqtt5Qos::AtLeastOnce, + MqttQos::AtLeastOnce, b"payload".as_slice(), None, ); @@ -1358,7 +1411,7 @@ }; let publish = Mqtt5Publish::new( "devices/test", - Mqtt5Qos::AtLeastOnce, + MqttQos::AtLeastOnce, b"payload".as_slice(), Some(properties), ); @@ -1395,7 +1448,7 @@ }; let publish = Mqtt5Publish::new( "devices/test", - Mqtt5Qos::AtLeastOnce, + MqttQos::AtLeastOnce, b"payload".as_slice(), Some(properties), ); @@ -1494,6 +1547,72 @@ } #[test] + fn given_secure_broker_url_without_tls_table_should_open_with_a_tls_handshake() { + let runtime = tokio::runtime::Runtime::new().expect("runtime should start"); + runtime.block_on(async { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("listener should bind"); + let port = listener + .local_addr() + .expect("listener has an address") + .port(); + for (protocol, scheme) in [ + (MqttProtocol::Mqtt5, "mqtts"), + (MqttProtocol::Mqtt311, "ssl"), + ] { + let config = MqttSourceConfig { + broker_url: format!("{scheme}://127.0.0.1:{port}"), + subscriptions: vec!["devices/test".to_string()], + subscription_qos: BTreeMap::new(), + protocol, + include_metadata: false, + qos: 1, + tls: None, + client_id: Some("test-source".to_string()), + username: None, + password: None, + clean_start: false, + session_expiry_interval: None, + keep_alive: Some("5s".to_string()), + poll_timeout: Some("100ms".to_string()), + request_capacity: Some(4), + batch_size: Some(3), + batch_timeout: Some("10ms".to_string()), + max_retries: None, + verbose_logging: None, + }; + let connecting = tokio::spawn(async move { + MqttDriver::connect( + 7, + &config, + Qos::One, + Duration::from_secs(5), + Duration::from_millis(100), + 4, + 3, + ) + .await + }); + let (mut stream, _) = timeout(Duration::from_secs(5), listener.accept()) + .await + .expect("the source should dial the broker") + .expect("the connection should be accepted"); + let mut first_byte = [0u8; 1]; + stream + .read_exact(&mut first_byte) + .await + .expect("the source should send its first byte"); + connecting.abort(); + assert_eq!( + first_byte[0], TLS_HANDSHAKE_RECORD, + "{protocol:?} over {scheme}" + ); + } + }); + } + + #[test] fn given_broker_url_without_client_id_should_add_configured_client_id() { let config = MqttSourceConfig { broker_url: "mqtt://127.0.0.1:1883?keep_alive_secs=30".to_string(), @@ -1518,10 +1637,51 @@ }; let broker_url = broker_url_with_client_id(&config, 7).expect("URL should be valid"); - let parsed = Url::parse(&broker_url).expect("URL should parse"); + let parsed = Url::parse(&broker_url.url).expect("URL should parse"); let query = parsed.query_pairs().collect::<Vec<_>>(); + assert!(!broker_url.tls); assert!(query.contains(&("keep_alive_secs".into(), "30".into()))); assert!(query.contains(&("client_id".into(), "configured-client".into()))); } + + #[test] + fn given_secure_broker_url_should_parse_as_mqtt_on_its_secure_port_with_tls() { + for (broker_url, expected_port) in [ + ("mqtts://broker.example", 8883), + ("ssl://broker.example:9883", 9883), + ] { + let config = MqttSourceConfig { + broker_url: broker_url.to_string(), + subscriptions: vec!["devices/test".to_string()], + subscription_qos: BTreeMap::new(), + protocol: MqttProtocol::Mqtt5, + include_metadata: false, + qos: 1, + tls: None, + client_id: Some("configured-client".to_string()), + username: None, + password: None, + clean_start: false, + session_expiry_interval: None, + keep_alive: Some("30s".to_string()), + poll_timeout: Some("1s".to_string()), + request_capacity: Some(4), + batch_size: Some(3), + batch_timeout: Some("10ms".to_string()), + max_retries: None, + verbose_logging: None, + }; + + let normalized = broker_url_with_client_id(&config, 7).expect("URL should be valid"); + let parsed = Url::parse(&normalized.url).expect("URL should parse"); + + assert!(normalized.tls, "{broker_url}"); + assert_eq!( + (parsed.scheme(), parsed.host_str(), parsed.port()), + ("mqtt", Some("broker.example"), Some(expected_port)), + "{broker_url}" + ); + } + } }
diff --git a/core/connectors/sources/mqtt_source/src/lib.rs b/core/connectors/sources/mqtt_source/src/lib.rs index ca13b85..7edaa92 100644 --- a/core/connectors/sources/mqtt_source/src/lib.rs +++ b/core/connectors/sources/mqtt_source/src/lib.rs
@@ -711,6 +711,12 @@ minimum_keep_alive, protocol ))); } + if keep_alive.as_secs() > u64::from(u16::MAX) { + return Err(Error::InvalidConfigValue(format!( + "{field} must be at most {}s, the MQTT keep alive limit", + u16::MAX + ))); + } Ok(()) } @@ -829,9 +835,9 @@ let source = MqttSource::new(7, test_config(), None); let runtime = tokio::runtime::Runtime::new().expect("runtime should start"); runtime.block_on(async { - let publish = rumqttc::v5::mqttbytes::v5::Publish::new( + let publish = rumqttc_v5::mqttbytes::v5::Publish::new( "devices/test", - rumqttc::v5::mqttbytes::QoS::AtLeastOnce, + rumqttc_v5::mqttbytes::QoS::AtLeastOnce, b"payload".as_slice(), None, ); @@ -955,13 +961,13 @@ #[test] fn given_mqtt5_metadata_enabled_should_serialize_payload_and_properties() { - let properties = rumqttc::v5::mqttbytes::v5::PublishProperties { + let properties = rumqttc_v5::mqttbytes::v5::PublishProperties { correlation_data: Some(vec![0, 255].into()), ..Default::default() }; - let publish = rumqttc::v5::mqttbytes::v5::Publish::new( + let publish = rumqttc_v5::mqttbytes::v5::Publish::new( "devices/test", - rumqttc::v5::mqttbytes::QoS::AtMostOnce, + rumqttc_v5::mqttbytes::QoS::AtMostOnce, vec![0, 255], Some(properties), ); @@ -983,9 +989,9 @@ #[test] fn given_oversized_topic_should_be_preserved_by_mqtt5_envelope() { let topic = format!("devices/{}/telemetry", "x".repeat(280)); - let publish = rumqttc::v5::mqttbytes::v5::Publish::new( + let publish = rumqttc_v5::mqttbytes::v5::Publish::new( topic.clone(), - rumqttc::v5::mqttbytes::QoS::AtMostOnce, + rumqttc_v5::mqttbytes::QoS::AtMostOnce, b"payload".to_vec(), None, ); @@ -1320,6 +1326,36 @@ } #[test] + fn given_keep_alive_above_the_mqtt_maximum_should_reject_configuration() { + for protocol in [MqttProtocol::Mqtt311, MqttProtocol::Mqtt5] { + let mut config = test_config(); + config.protocol = protocol; + config.keep_alive = Some("65536s".to_string()); + let source = MqttSource::new(7, config, None); + + assert!(source.validate_config().is_err(), "{protocol:?}"); + } + } + + #[test] + fn given_keep_alive_at_the_mqtt_maximum_should_accept_configuration() { + let mut config = test_config(); + config.keep_alive = Some("65535s".to_string()); + let source = MqttSource::new(7, config, None); + + assert!(source.validate_config().is_ok()); + } + + #[test] + fn given_url_keep_alive_above_the_mqtt_maximum_should_reject_configuration() { + let mut config = test_config(); + config.broker_url = "mqtt://localhost:1883?keep_alive_secs=65536".to_string(); + let source = MqttSource::new(7, config, None); + + assert!(source.validate_config().is_err()); + } + + #[test] fn given_mqtt5_url_keep_alive_below_minimum_should_reject_configuration() { let mut config = test_config(); config.broker_url = "mqtt://localhost:1883?keep_alive_secs=4".to_string();
diff --git a/core/consensus/src/client_table.rs b/core/consensus/src/client_table.rs index 39a569a..7fb462b 100644 --- a/core/consensus/src/client_table.rs +++ b/core/consensus/src/client_table.rs
@@ -1857,7 +1857,9 @@ || (ended == self.latest_commit && !matches!( header.operation, - Operation::Logout | Operation::UpdateUser | Operation::DeleteUser + Operation::Logout + | Operation::UpdateUserResolved + | Operation::DeleteUser )) }) { @@ -2245,7 +2247,7 @@ let secret = [0x5a; BIND_SECRET_BYTES]; for self_revocation in [ None, - Some(Operation::UpdateUser), + Some(Operation::UpdateUserResolved), Some(Operation::DeleteUser), ] { let mut table = ClientTable::new(1); @@ -2296,6 +2298,73 @@ } } + #[test] + fn given_self_deactivation_through_generic_patch_when_validating_protection_should_accept_resolved_operation() + { + const CLIENT: u128 = 7; + const EPOCH: u64 = 10; + const END: u64 = 40; + let secret = [0x5a; BIND_SECRET_BYTES]; + for (operation, accepted) in [ + (Operation::UpdateUserResolved, true), + (Operation::UpdateStreamResolved, false), + (Operation::UpdateTopicResolved, false), + ] { + let mut table = ClientTable::new(1); + table + .commit_register( + CLIENT, + TEST_USER_ID, + bind_verifier(CLIENT, TEST_USER_ID, &secret), + make_register_reply(CLIENT, EPOCH), + ) + .unwrap(); + table.commit_reply(CLIENT, TEST_USER_ID, make_reply_for(CLIENT, 5, END - 1)); + assert_eq!(table.end_user_sessions(TEST_USER_ID, END).len(), 1); + let receipt = make_reply_for(CLIENT, 6, END).transmute_header(|old, header| { + *header = old; + header.operation = operation; + }); + table.commit_reply(CLIENT, TEST_USER_ID, receipt); + assert_eq!( + ClientTable::from_snapshot(table.to_snapshot()).is_ok(), + accepted, + "{operation:?}" + ); + } + } + + #[test] + fn given_resolved_update_receipt_when_retrying_client_operation_should_return_cached_reply() { + for resolved in [ + Operation::UpdateStreamResolved, + Operation::UpdateTopicResolved, + Operation::UpdateUserResolved, + ] { + let (mut table, epoch) = table_with_client(); + let reply = make_reply_for(1, 1, 11).transmute_header(|old, header| { + *header = old; + header.operation = resolved; + }); + table.commit_reply(1, TEST_USER_ID, reply.clone()); + for restored in [ + ClientTable::from_snapshot(table.to_snapshot()).unwrap(), + ClientTable::decode(&table.encode()).unwrap(), + ] { + let RequestStatus::Duplicate(cached) = + restored.check_request(1, epoch, 1, Operation::UpdateResource) + else { + panic!("lost the receipt for {resolved:?}"); + }; + assert_eq!(cached.into_message().as_slice(), reply.as_slice()); + assert!(matches!( + restored.check_request(1, epoch, 1, Operation::DeleteStream), + RequestStatus::OperationMismatch { .. } + )); + } + } + } + /// A request admitted before its user's revocation committed commits /// right behind it. Its session already ended, so the op stays uncertain. #[test] @@ -2308,7 +2377,7 @@ let secret = [0x5a; BIND_SECRET_BYTES]; for self_revocation in [ None, - Some(Operation::UpdateUser), + Some(Operation::UpdateUserResolved), Some(Operation::DeleteUser), ] { let mut table = ClientTable::new(2); @@ -2729,7 +2798,7 @@ }; assert_eq!(cached.into_message().as_slice(), reply.as_slice()); assert!(matches!( - restored.check_request(1, epoch, 1, Operation::UpdateStream), + restored.check_request(1, epoch, 1, Operation::UpdateResource), RequestStatus::OperationMismatch { .. } )); }
diff --git a/core/consensus/src/observability.rs b/core/consensus/src/observability.rs index a2b1f9e..c4508b6 100644 --- a/core/consensus/src/observability.rs +++ b/core/consensus/src/observability.rs
@@ -708,14 +708,17 @@ Operation::FinalizeSession => "finalize_session", Operation::RetireSession => "retire_session", Operation::CompleteLifecycle => "complete_lifecycle", + Operation::RewrapStreamKeys => "rewrap_stream_keys", Operation::TransitionPartitionHistory => "transition_partition_history", Operation::RetireConsumerGroupOwners => "retire_consumer_group_owners", Operation::InstallConsumerGroupOwner => "install_consumer_group_owner", + Operation::UpdateStreamResolved => "update_stream_resolved", + Operation::UpdateTopicResolved => "update_topic_resolved", + Operation::UpdateUserResolved => "update_user_resolved", + Operation::UpdateResource => "update_resource", Operation::CreateStream => "create_stream", - Operation::UpdateStream => "update_stream", Operation::DeleteStream => "delete_stream", Operation::CreateTopic => "create_topic", - Operation::UpdateTopic => "update_topic", Operation::DeleteTopic => "delete_topic", Operation::CreatePartitions => "create_partitions", Operation::DeletePartitions => "delete_partitions", @@ -725,7 +728,6 @@ Operation::JoinConsumerGroup => "join_consumer_group", Operation::LeaveConsumerGroup => "leave_consumer_group", Operation::CreateUser => "create_user", - Operation::UpdateUser => "update_user", Operation::DeleteUser => "delete_user", Operation::ChangePassword => "change_password", Operation::UpdatePermissions => "update_permissions",
diff --git a/core/integration/Cargo.toml b/core/integration/Cargo.toml index b8a7978..22a7787 100644 --- a/core/integration/Cargo.toml +++ b/core/integration/Cargo.toml
@@ -94,7 +94,8 @@ "transport-streamable-http-client", "transport-streamable-http-client-reqwest", ] } -rumqttc = { workspace = true } +rumqttc-v4 = { workspace = true } +rumqttc-v5 = { workspace = true } rust-s3 = { workspace = true } secrecy = { workspace = true } serde = { workspace = true }
diff --git a/core/integration/src/harness/helpers.rs b/core/integration/src/harness/helpers.rs index 47bb1ba..eb60fb7 100644 --- a/core/integration/src/harness/helpers.rs +++ b/core/integration/src/harness/helpers.rs
@@ -16,6 +16,7 @@ // under the License. use crate::harness::{TestBinary, TestHarness}; +use iggy::prelude::UserCreateOptions; use iggy::prelude::{ ConsumerGroupClient, ConsumerGroupDetails, DEFAULT_ROOT_PASSWORD, DEFAULT_ROOT_USERNAME, GlobalPermissions, Identifier, IdentityInfo, Permissions, StreamClient, TopicClient, @@ -173,6 +174,7 @@ USER_PASSWORD, UserStatus::Active, Some(full_permissions()), + &UserCreateOptions::default(), ) .await .expect("Failed to create user");
diff --git a/core/integration/src/harness/mod.rs b/core/integration/src/harness/mod.rs index 53e72f1..12a3cc2 100644 --- a/core/integration/src/harness/mod.rs +++ b/core/integration/src/harness/mod.rs
@@ -34,7 +34,7 @@ //! //! // Client is already logged in as root //! let client = harness.client(); -//! // client.create_stream(...).await.unwrap(); +//! // client.create_stream(..., &StreamCreateOptions::default()).await.unwrap(); //! //! harness.stop().await.unwrap(); //! }
diff --git a/core/integration/src/harness/seeds.rs b/core/integration/src/harness/seeds.rs index b0f54f6..be99260 100644 --- a/core/integration/src/harness/seeds.rs +++ b/core/integration/src/harness/seeds.rs
@@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +use iggy::prelude::StreamCreateOptions; +use iggy::prelude::UserCreateOptions; use iggy::prelude::{IggyClient, StreamClient, TopicClient, TopicCreateOptions}; use iggy_common::{ Consumer, Durability, Identifier, IggyExpiry, IggyMessage, MaxTopicSize, Partitioning, @@ -44,13 +46,17 @@ /// Creates a single stream named "test". pub async fn stream_only(client: &IggyClient) -> Result<(), SeedError> { - client.create_stream("test").await?; + client + .create_stream("test", &StreamCreateOptions::default()) + .await?; Ok(()) } /// Creates a stream with a topic. pub async fn stream_with_topic(client: &IggyClient) -> Result<(), SeedError> { - client.create_stream("test_stream").await?; + client + .create_stream("test_stream", &StreamCreateOptions::default()) + .await?; client .create_topic( &"test_stream".try_into()?, @@ -70,7 +76,9 @@ pub async fn connector_stream(client: &IggyClient) -> Result<(), SeedError> { let stream_id: Identifier = names::STREAM.try_into()?; - client.create_stream(names::STREAM).await?; + client + .create_stream(names::STREAM, &StreamCreateOptions::default()) + .await?; client .create_topic( @@ -94,7 +102,9 @@ pub async fn connector_multi_topic_stream(client: &IggyClient) -> Result<(), SeedError> { let stream_id: Identifier = names::STREAM.try_into()?; - client.create_stream(names::STREAM).await?; + client + .create_stream(names::STREAM, &StreamCreateOptions::default()) + .await?; client .create_topic( @@ -130,7 +140,9 @@ let stream_id: Identifier = names::STREAM.try_into()?; let topic_id: Identifier = names::TOPIC.try_into()?; - client.create_stream(names::STREAM).await?; + client + .create_stream(names::STREAM, &StreamCreateOptions::default()) + .await?; client .create_topic( @@ -168,7 +180,13 @@ .await?; client - .create_user(names::USER, names::USER_PASSWORD, UserStatus::Active, None) + .create_user( + names::USER, + names::USER_PASSWORD, + UserStatus::Active, + None, + &UserCreateOptions::default(), + ) .await?; client
diff --git a/core/integration/tests/cli/common/mod.rs b/core/integration/tests/cli/common/mod.rs index 8e1e77c..5c419da 100644 --- a/core/integration/tests/cli/common/mod.rs +++ b/core/integration/tests/cli/common/mod.rs
@@ -145,6 +145,18 @@ } } + pub(crate) fn with_server(config: TestServerConfig) -> Self { + let harness = TestHarnessBuilder::default() + .server(config) + .build() + .unwrap(); + + Self { + harness, + start_server: true, + } + } + pub(crate) fn help_message() -> Self { Self::new(false) }
diff --git a/core/integration/tests/cli/consumer_group/test_consumer_group_create_command.rs b/core/integration/tests/cli/consumer_group/test_consumer_group_create_command.rs index 97c18f7..e3ac54d 100644 --- a/core/integration/tests/cli/consumer_group/test_consumer_group_create_command.rs +++ b/core/integration/tests/cli/consumer_group/test_consumer_group_create_command.rs
@@ -21,6 +21,7 @@ }; use assert_cmd::assert::Assert; use async_trait::async_trait; +use iggy::prelude::StreamCreateOptions; use iggy::prelude::{Client, IggyExpiry, TopicCreateOptions}; use predicates::str::diff; use serial_test::parallel; @@ -76,7 +77,9 @@ #[async_trait] impl IggyCmdTestCase for TestConsumerGroupCreateCmd { async fn prepare_server_state(&mut self, client: &dyn Client) { - let stream = client.create_stream(&self.stream_name).await; + let stream = client + .create_stream(&self.stream_name, &StreamCreateOptions::default()) + .await; assert!(stream.is_ok()); let stream_details = stream.unwrap(); self.stream_id = stream_details.id;
diff --git a/core/integration/tests/cli/consumer_group/test_consumer_group_delete_command.rs b/core/integration/tests/cli/consumer_group/test_consumer_group_delete_command.rs index 21fbdb1..2167705 100644 --- a/core/integration/tests/cli/consumer_group/test_consumer_group_delete_command.rs +++ b/core/integration/tests/cli/consumer_group/test_consumer_group_delete_command.rs
@@ -23,6 +23,7 @@ use async_trait::async_trait; use iggy::prelude::Client; use iggy::prelude::IggyExpiry; +use iggy::prelude::StreamCreateOptions; use iggy::prelude::TopicCreateOptions; use predicates::str::diff; use serial_test::parallel; @@ -88,7 +89,9 @@ #[async_trait] impl IggyCmdTestCase for TestConsumerGroupDeleteCmd { async fn prepare_server_state(&mut self, client: &dyn Client) { - let stream = client.create_stream(&self.stream_name).await; + let stream = client + .create_stream(&self.stream_name, &StreamCreateOptions::default()) + .await; assert!(stream.is_ok()); let stream_details = stream.unwrap(); self.stream_id = stream_details.id;
diff --git a/core/integration/tests/cli/consumer_group/test_consumer_group_get_command.rs b/core/integration/tests/cli/consumer_group/test_consumer_group_get_command.rs index 6ceb3c9..76b0974 100644 --- a/core/integration/tests/cli/consumer_group/test_consumer_group_get_command.rs +++ b/core/integration/tests/cli/consumer_group/test_consumer_group_get_command.rs
@@ -23,6 +23,7 @@ use async_trait::async_trait; use iggy::prelude::Client; use iggy::prelude::IggyExpiry; +use iggy::prelude::StreamCreateOptions; use iggy::prelude::TopicCreateOptions; use predicates::str::{contains, starts_with}; use serial_test::parallel; @@ -88,7 +89,9 @@ #[async_trait] impl IggyCmdTestCase for TestConsumerGroupGetCmd { async fn prepare_server_state(&mut self, client: &dyn Client) { - let stream = client.create_stream(&self.stream_name).await; + let stream = client + .create_stream(&self.stream_name, &StreamCreateOptions::default()) + .await; assert!(stream.is_ok()); let stream_details = stream.unwrap(); self.stream_id = stream_details.id;
diff --git a/core/integration/tests/cli/consumer_group/test_consumer_group_list_command.rs b/core/integration/tests/cli/consumer_group/test_consumer_group_list_command.rs index 8cf6e2f..9bd1110 100644 --- a/core/integration/tests/cli/consumer_group/test_consumer_group_list_command.rs +++ b/core/integration/tests/cli/consumer_group/test_consumer_group_list_command.rs
@@ -23,6 +23,7 @@ use async_trait::async_trait; use iggy::prelude::Client; use iggy::prelude::IggyExpiry; +use iggy::prelude::StreamCreateOptions; use iggy::prelude::TopicCreateOptions; use predicates::str::{contains, starts_with}; use serial_test::parallel; @@ -84,7 +85,7 @@ async fn prepare_server_state(&mut self, client: &dyn Client) { // Create stream and capture its actual ID let stream = client - .create_stream(&self.stream_name) + .create_stream(&self.stream_name, &StreamCreateOptions::default()) .await .expect("Failed to create stream"); self.actual_stream_id = Some(stream.id);
diff --git a/core/integration/tests/cli/consumer_offset/test_consumer_offset_external_group_command.rs b/core/integration/tests/cli/consumer_offset/test_consumer_offset_external_group_command.rs index 669e5b3..5713baa 100644 --- a/core/integration/tests/cli/consumer_offset/test_consumer_offset_external_group_command.rs +++ b/core/integration/tests/cli/consumer_offset/test_consumer_offset_external_group_command.rs
@@ -55,7 +55,10 @@ if matches!(self.step, Step::Get) { return; } - client.create_stream(STREAM_NAME).await.unwrap(); + client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); client .create_topic( &stream_id(),
diff --git a/core/integration/tests/cli/consumer_offset/test_consumer_offset_get_command.rs b/core/integration/tests/cli/consumer_offset/test_consumer_offset_get_command.rs index 0928f8d..885780f 100644 --- a/core/integration/tests/cli/consumer_offset/test_consumer_offset_get_command.rs +++ b/core/integration/tests/cli/consumer_offset/test_consumer_offset_get_command.rs
@@ -94,7 +94,7 @@ async fn prepare_server_state(&mut self, client: &dyn Client) { // Create stream and capture its actual ID let stream = client - .create_stream(&self.stream_name) + .create_stream(&self.stream_name, &StreamCreateOptions::default()) .await .expect("Failed to create stream"); self.actual_stream_id = Some(stream.id);
diff --git a/core/integration/tests/cli/consumer_offset/test_consumer_offset_set_command.rs b/core/integration/tests/cli/consumer_offset/test_consumer_offset_set_command.rs index d545a5d..fe27d2e 100644 --- a/core/integration/tests/cli/consumer_offset/test_consumer_offset_set_command.rs +++ b/core/integration/tests/cli/consumer_offset/test_consumer_offset_set_command.rs
@@ -96,7 +96,9 @@ #[async_trait] impl IggyCmdTestCase for TestConsumerOffsetSetCmd { async fn prepare_server_state(&mut self, client: &dyn Client) { - let stream = client.create_stream(&self.stream_name).await; + let stream = client + .create_stream(&self.stream_name, &StreamCreateOptions::default()) + .await; assert!(stream.is_ok()); let stream_details = stream.unwrap(); self.stream_id = stream_details.id;
diff --git a/core/integration/tests/cli/general/test_help_command.rs b/core/integration/tests/cli/general/test_help_command.rs index 24ca261..110245c 100644 --- a/core/integration/tests/cli/general/test_help_command.rs +++ b/core/integration/tests/cli/general/test_help_command.rs
@@ -38,7 +38,7 @@ segment segments operations [alias: seg] ping ping iggy server me get current client info - options list the options a resource's create command accepts + options list the option catalog of a resource scope stats get iggy server statistics snapshot collect iggy server troubleshooting data pat personal access token operations
diff --git a/core/integration/tests/cli/general/test_overview_command.rs b/core/integration/tests/cli/general/test_overview_command.rs index f8f447e..20c63fc 100644 --- a/core/integration/tests/cli/general/test_overview_command.rs +++ b/core/integration/tests/cli/general/test_overview_command.rs
@@ -49,7 +49,7 @@ segment segments operations [alias: seg] ping ping iggy server me get current client info - options list the options a resource's create command accepts + options list the option catalog of a resource scope stats get iggy server statistics snapshot collect iggy server troubleshooting data pat personal access token operations
diff --git a/core/integration/tests/cli/message/test_message_poll_command.rs b/core/integration/tests/cli/message/test_message_poll_command.rs index cc91fec..cfbad07 100644 --- a/core/integration/tests/cli/message/test_message_poll_command.rs +++ b/core/integration/tests/cli/message/test_message_poll_command.rs
@@ -115,7 +115,9 @@ #[async_trait] impl IggyCmdTestCase for TestMessagePollCmd { async fn prepare_server_state(&mut self, client: &dyn Client) { - let stream = client.create_stream(&self.stream_name).await; + let stream = client + .create_stream(&self.stream_name, &StreamCreateOptions::default()) + .await; assert!(stream.is_ok()); let stream = stream.unwrap(); self.actual_stream_id = Some(stream.id);
diff --git a/core/integration/tests/cli/message/test_message_poll_to_file_command.rs b/core/integration/tests/cli/message/test_message_poll_to_file_command.rs index a433191..1be13e0 100644 --- a/core/integration/tests/cli/message/test_message_poll_to_file_command.rs +++ b/core/integration/tests/cli/message/test_message_poll_to_file_command.rs
@@ -108,7 +108,9 @@ #[async_trait] impl IggyCmdTestCase for TestMessagePollToFileCmd<'_> { async fn prepare_server_state(&mut self, client: &dyn Client) { - let stream = client.create_stream(&self.stream_name).await; + let stream = client + .create_stream(&self.stream_name, &StreamCreateOptions::default()) + .await; assert!(stream.is_ok()); let stream = stream.unwrap(); self.actual_stream_id = Some(stream.id);
diff --git a/core/integration/tests/cli/message/test_message_send_command.rs b/core/integration/tests/cli/message/test_message_send_command.rs index 718d08c..ed708b6 100644 --- a/core/integration/tests/cli/message/test_message_send_command.rs +++ b/core/integration/tests/cli/message/test_message_send_command.rs
@@ -159,7 +159,7 @@ async fn prepare_server_state(&mut self, client: &dyn Client) { // Create stream and capture its actual ID let stream = client - .create_stream(&self.stream_name) + .create_stream(&self.stream_name, &StreamCreateOptions::default()) .await .expect("Failed to create stream"); self.actual_stream_id = Some(stream.id);
diff --git a/core/integration/tests/cli/message/test_message_send_from_file_command.rs b/core/integration/tests/cli/message/test_message_send_from_file_command.rs index cc67a0d..9a025b0 100644 --- a/core/integration/tests/cli/message/test_message_send_from_file_command.rs +++ b/core/integration/tests/cli/message/test_message_send_from_file_command.rs
@@ -95,7 +95,7 @@ async fn prepare_server_state(&mut self, client: &dyn Client) { // Create stream and capture its actual ID let stream = client - .create_stream(&self.stream_name) + .create_stream(&self.stream_name, &StreamCreateOptions::default()) .await .expect("Failed to create stream"); self.actual_stream_id = Some(stream.id);
diff --git a/core/integration/tests/cli/partition/test_partition_create_command.rs b/core/integration/tests/cli/partition/test_partition_create_command.rs index ff243f0..7bee9a2 100644 --- a/core/integration/tests/cli/partition/test_partition_create_command.rs +++ b/core/integration/tests/cli/partition/test_partition_create_command.rs
@@ -23,6 +23,7 @@ use iggy::prelude::Client; use iggy::prelude::CompressionAlgorithm; use iggy::prelude::IggyExpiry; +use iggy::prelude::StreamCreateOptions; use iggy::prelude::TopicCreateOptions; use predicates::str::diff; use serial_test::parallel; @@ -83,7 +84,9 @@ #[async_trait] impl IggyCmdTestCase for TestPartitionCreateCmd { async fn prepare_server_state(&mut self, client: &dyn Client) { - let stream = client.create_stream(&self.stream_name).await; + let stream = client + .create_stream(&self.stream_name, &StreamCreateOptions::default()) + .await; assert!(stream.is_ok()); self.actual_stream_id = Some(stream.unwrap().id);
diff --git a/core/integration/tests/cli/partition/test_partition_delete_command.rs b/core/integration/tests/cli/partition/test_partition_delete_command.rs index d74424b..53fe123 100644 --- a/core/integration/tests/cli/partition/test_partition_delete_command.rs +++ b/core/integration/tests/cli/partition/test_partition_delete_command.rs
@@ -22,6 +22,7 @@ use async_trait::async_trait; use iggy::prelude::Client; use iggy::prelude::IggyExpiry; +use iggy::prelude::StreamCreateOptions; use iggy::prelude::TopicCreateOptions; use predicates::str::diff; use serial_test::parallel; @@ -71,7 +72,9 @@ #[async_trait] impl IggyCmdTestCase for TestPartitionDeleteCmd { async fn prepare_server_state(&mut self, client: &dyn Client) { - let stream = client.create_stream(&self.stream_name).await; + let stream = client + .create_stream(&self.stream_name, &StreamCreateOptions::default()) + .await; assert!(stream.is_ok()); self.actual_stream_id = Some(stream.unwrap().id);
diff --git a/core/integration/tests/cli/stream/test_stream_create_command.rs b/core/integration/tests/cli/stream/test_stream_create_command.rs index 19d6f74..4a915d9 100644 --- a/core/integration/tests/cli/stream/test_stream_create_command.rs +++ b/core/integration/tests/cli/stream/test_stream_create_command.rs
@@ -15,12 +15,25 @@ // specific language governing permissions and limitations // under the License. -use crate::cli::common::{IggyCmdCommand, IggyCmdTest, IggyCmdTestCase, TestHelpCmd, USAGE_PREFIX}; +use crate::cli::common::{ + CLAP_INDENT, IggyCmdCommand, IggyCmdTest, IggyCmdTestCase, TestHelpCmd, USAGE_PREFIX, +}; use assert_cmd::assert::Assert; use async_trait::async_trait; -use iggy::prelude::Client; -use predicates::str::diff; +use base64::Engine; +use base64::engine::general_purpose::STANDARD; +use iggy::prelude::{Client, StreamEncryptionKey, StreamEncryptionSource}; +use iggy_common::ServerKeyContext; +use integration::harness::{EncryptionConfig, TestServerConfig}; +use predicates::prelude::PredicateBooleanExt; +use predicates::str::{contains, diff, starts_with}; use serial_test::parallel; +use std::fs; +use std::path::PathBuf; +use tempfile::TempDir; + +pub(super) const SERVER_KEY: &str = "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE="; +const STREAM_KEY_ENV: &str = "IGGY_STREAM_ENCRYPTION_KEY"; struct TestStreamCreateCmd { stream_id: Option<u32>, @@ -87,6 +100,179 @@ } } +struct TestStreamCreateWithKeyCmd { + stream_name: String, + _key_directory: TempDir, + key_file: Option<PathBuf>, + file_key: Option<String>, + environment_key: Option<String>, +} + +/// What the server must end up with for one create. +enum ExpectedKey { + Stream(StreamEncryptionKey), + Generated, + Refused, +} + +impl TestStreamCreateWithKeyCmd { + fn new(stream_name: &str, file_key: Option<String>, environment_key: Option<String>) -> Self { + let key_directory = TempDir::new().expect("key directory"); + let key_file = file_key.as_ref().map(|key| { + let key_file = key_directory.path().join("stream.key"); + fs::write(&key_file, format!("{key}\n")).expect("key file"); + key_file + }); + Self { + stream_name: stream_name.to_owned(), + _key_directory: key_directory, + key_file, + file_key, + environment_key, + } + } + + /// Only the file supplies a key, the environment never does. + fn expected_key(&self) -> ExpectedKey { + match &self.file_key { + Some(text) => text + .parse() + .map_or(ExpectedKey::Refused, ExpectedKey::Stream), + None => ExpectedKey::Generated, + } + } + + fn secrets(&self) -> Vec<String> { + self.file_key + .iter() + .chain(self.environment_key.iter()) + .cloned() + .collect() + } +} + +#[async_trait] +impl IggyCmdTestCase for TestStreamCreateWithKeyCmd { + async fn prepare_server_state(&mut self, _client: &dyn Client) {} + + fn get_command(&self) -> IggyCmdCommand { + let mut command = IggyCmdCommand::new().arg("stream").arg("create"); + if let Some(key_file) = &self.key_file { + command = command + .arg("--stream-key-file") + .arg(key_file.to_str().expect("UTF-8 path")); + } + let command = command.arg(&self.stream_name).with_env_credentials(); + match &self.environment_key { + Some(key) => command.env(STREAM_KEY_ENV, key), + None => command, + } + } + + fn verify_command(&self, command_state: Assert) { + let name = &self.stream_name; + let created = format!( + "Executing create stream with name: {name} and ID auto incremented\n\ + Stream with name: {name} and ID auto incremented created\n" + ); + let mut command_state = match self.expected_key() { + ExpectedKey::Stream(key) => { + let kek_fingerprint = ServerKeyContext::from_config(SERVER_KEY, &[], true) + .expect("server key") + .kek_fingerprint() + .expect("KEK fingerprint"); + command_state.success().stdout(diff(format!( + "{created}Stream encryption key source: stream, fingerprint: {}, KEK fingerprints: {kek_fingerprint}\n", + key.fingerprint() + ))) + } + ExpectedKey::Generated => command_state.success().stdout(starts_with(format!( + "{created}Stream encryption key source: server, fingerprint: " + ))), + ExpectedKey::Refused => command_state.failure().stderr(contains( + "Stream encryption key must be standard base64 of exactly 32 bytes", + )), + }; + for secret in self.secrets() { + command_state = command_state + .stdout(contains(secret.as_str()).not()) + .stderr(contains(secret).not()); + } + } + + async fn verify_server_state(&self, client: &dyn Client) { + let stream_id = self.stream_name.clone().try_into().unwrap(); + let stream = client.get_stream(&stream_id).await.expect("get stream"); + let (source, fingerprint) = match self.expected_key() { + ExpectedKey::Refused => { + assert!(stream.is_none(), "a refused key must not create the stream"); + return; + } + ExpectedKey::Stream(key) => (StreamEncryptionSource::Stream, Some(key.fingerprint())), + ExpectedKey::Generated => (StreamEncryptionSource::Server, None), + }; + let stream = stream.expect("stream exists"); + assert_eq!(stream.encryption_key_source(), Some(source)); + if let Some(fingerprint) = fingerprint { + assert_eq!(stream.encryption_key_fingerprint(), Some(fingerprint)); + } + let environment_fingerprint = self + .environment_key + .as_ref() + .and_then(|text| text.parse::<StreamEncryptionKey>().ok()) + .map(|key| key.fingerprint()); + if environment_fingerprint.is_some() { + assert_ne!(stream.encryption_key_fingerprint(), environment_fingerprint); + } + client.delete_stream(&stream_id).await.expect("delete"); + } +} + +#[tokio::test] +#[parallel] +pub async fn given_key_file_when_creating_stream_should_show_only_projections() { + let mut iggy_cmd_test = IggyCmdTest::with_server(encrypted_server()); + + iggy_cmd_test.setup().await; + iggy_cmd_test + .execute_test(TestStreamCreateWithKeyCmd::new( + "secured", + Some(STANDARD.encode([7; 32])), + None, + )) + .await; +} + +#[tokio::test] +#[parallel] +pub async fn given_environment_key_when_creating_stream_should_ignore_it() { + let mut iggy_cmd_test = IggyCmdTest::with_server(encrypted_server()); + + iggy_cmd_test.setup().await; + iggy_cmd_test + .execute_test(TestStreamCreateWithKeyCmd::new( + "from-environment", + None, + Some(STANDARD.encode([9; 32])), + )) + .await; +} + +#[tokio::test] +#[parallel] +pub async fn given_malformed_key_file_when_creating_stream_should_refuse_without_echoing_it() { + let mut iggy_cmd_test = IggyCmdTest::with_server(encrypted_server()); + + iggy_cmd_test.setup().await; + iggy_cmd_test + .execute_test(TestStreamCreateWithKeyCmd::new( + "short-key-file", + Some(STANDARD.encode([5; 31])), + None, + )) + .await; +} + #[tokio::test] #[parallel] pub async fn should_be_successful() { @@ -119,6 +305,7 @@ Examples: iggy stream create prod iggy stream create -s 1 test + iggy stream create --stream-key-file stream.key payments {USAGE_PREFIX} stream create [OPTIONS] <NAME> @@ -130,6 +317,15 @@ -s, --stream-id <STREAM_ID> Legacy stream ID flag (ignored) + --stream-key-file <PATH> + File holding the stream's own at-rest encryption key +{CLAP_INDENT} + Standard base64 of exactly 32 random bytes from a secure source, such + as "head -c 32 /dev/urandom | base64". Use a new key for every stream. + Without this flag the server encryption setting applies. The server + stores the key wrapped and reads report only its fingerprint. Unrelated + to --encryption-key, which encrypts message payloads on the client. + -h, --help Print help (see a summary with '-h') "#, @@ -155,10 +351,17 @@ <NAME> Name of the stream Options: - -s, --stream-id <STREAM_ID> Legacy stream ID flag (ignored) - -h, --help Print help (see more with '--help') + -s, --stream-id <STREAM_ID> Legacy stream ID flag (ignored) + --stream-key-file <PATH> File holding the stream's own at-rest encryption key + -h, --help Print help (see more with '--help') "#, ), )) .await; } + +pub(super) fn encrypted_server() -> TestServerConfig { + TestServerConfig::builder() + .encryption(EncryptionConfig::new(SERVER_KEY)) + .build() +}
diff --git a/core/integration/tests/cli/stream/test_stream_delete_command.rs b/core/integration/tests/cli/stream/test_stream_delete_command.rs index b57b73c..30d8d54 100644 --- a/core/integration/tests/cli/stream/test_stream_delete_command.rs +++ b/core/integration/tests/cli/stream/test_stream_delete_command.rs
@@ -22,6 +22,7 @@ use assert_cmd::assert::Assert; use async_trait::async_trait; use iggy::prelude::Client; +use iggy::prelude::StreamCreateOptions; use predicates::str::diff; use serial_test::parallel; @@ -51,7 +52,9 @@ #[async_trait] impl IggyCmdTestCase for TestStreamDeleteCmd { async fn prepare_server_state(&mut self, client: &dyn Client) { - let stream = client.create_stream(&self.stream_name).await; + let stream = client + .create_stream(&self.stream_name, &StreamCreateOptions::default()) + .await; assert!(stream.is_ok()); }
diff --git a/core/integration/tests/cli/stream/test_stream_get_command.rs b/core/integration/tests/cli/stream/test_stream_get_command.rs index 445c2b8..158fef8 100644 --- a/core/integration/tests/cli/stream/test_stream_get_command.rs +++ b/core/integration/tests/cli/stream/test_stream_get_command.rs
@@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use super::test_stream_create_command::{SERVER_KEY, encrypted_server}; use crate::cli::common::{ CLAP_INDENT, IggyCmdCommand, IggyCmdTest, IggyCmdTestCase, TestHelpCmd, TestStreamId, USAGE_PREFIX, @@ -22,6 +23,8 @@ use assert_cmd::assert::Assert; use async_trait::async_trait; use iggy::prelude::Client; +use iggy::prelude::{StreamCreateOptions, StreamEncryptionKey}; +use iggy_common::ServerKeyContext; use predicates::str::{contains, starts_with}; use serial_test::parallel; @@ -51,7 +54,9 @@ #[async_trait] impl IggyCmdTestCase for TestStreamGetCmd { async fn prepare_server_state(&mut self, client: &dyn Client) { - let stream = client.create_stream(&self.stream_name).await; + let stream = client + .create_stream(&self.stream_name, &StreamCreateOptions::default()) + .await; assert!(stream.is_ok()); } @@ -81,12 +86,80 @@ ))) .stdout(contains("Stream size | 0")) .stdout(contains("Stream message count | 0")) - .stdout(contains("Stream topics count | 0")); + .stdout(contains("Stream topics count | 0")) + .stdout(contains("Encryption source | none")); } async fn verify_server_state(&self, _client: &dyn Client) {} } +struct TestStreamGetEncryptedCmd { + stream_name: String, + key: StreamEncryptionKey, +} + +#[async_trait] +impl IggyCmdTestCase for TestStreamGetEncryptedCmd { + async fn prepare_server_state(&mut self, client: &dyn Client) { + client + .create_stream( + &self.stream_name, + &StreamCreateOptions { + encryption_key: Some(self.key.clone()), + ..StreamCreateOptions::default() + }, + ) + .await + .expect("create encrypted stream"); + } + + fn get_command(&self) -> IggyCmdCommand { + IggyCmdCommand::new() + .arg("stream") + .arg("get") + .arg(&self.stream_name) + .with_env_credentials() + } + + fn verify_command(&self, command_state: Assert) { + let kek_fingerprint = ServerKeyContext::from_config(SERVER_KEY, &[], true) + .expect("server key") + .kek_fingerprint() + .expect("KEK fingerprint"); + command_state + .success() + .stdout(contains("Encryption source | stream")) + .stdout(contains(format!( + "Key fingerprint | {}", + self.key.fingerprint() + ))) + .stdout(contains(format!( + "KEK fingerprints | {kek_fingerprint}" + ))); + } + + async fn verify_server_state(&self, client: &dyn Client) { + client + .delete_stream(&self.stream_name.clone().try_into().unwrap()) + .await + .expect("delete"); + } +} + +#[tokio::test] +#[parallel] +pub async fn given_encrypted_stream_when_getting_should_show_source_and_fingerprints() { + let mut iggy_cmd_test = IggyCmdTest::with_server(encrypted_server()); + + iggy_cmd_test.setup().await; + iggy_cmd_test + .execute_test(TestStreamGetEncryptedCmd { + stream_name: String::from("secured"), + key: StreamEncryptionKey::try_from([3u8; 32].as_slice()).expect("valid key"), + }) + .await; +} + #[tokio::test] #[parallel] pub async fn should_be_successful() {
diff --git a/core/integration/tests/cli/stream/test_stream_help_command.rs b/core/integration/tests/cli/stream/test_stream_help_command.rs index 3fbc0b3..7984beb 100644 --- a/core/integration/tests/cli/stream/test_stream_help_command.rs +++ b/core/integration/tests/cli/stream/test_stream_help_command.rs
@@ -34,7 +34,7 @@ Commands: create Create stream with given name [alias: c] delete Delete stream with given ID [alias: d] - update Update stream name for given stream ID [alias: u] + update Update stream name or options for given stream ID [alias: u] get Get details of a single stream with given ID [alias: g] list List all streams [alias: l] help Print this message or the help of the given subcommand(s)
diff --git a/core/integration/tests/cli/stream/test_stream_list_command.rs b/core/integration/tests/cli/stream/test_stream_list_command.rs index 0a6f614..23129fa 100644 --- a/core/integration/tests/cli/stream/test_stream_list_command.rs +++ b/core/integration/tests/cli/stream/test_stream_list_command.rs
@@ -22,6 +22,7 @@ use assert_cmd::assert::Assert; use async_trait::async_trait; use iggy::prelude::Client; +use iggy::prelude::StreamCreateOptions; use predicates::str::{contains, starts_with}; use serial_test::parallel; @@ -43,7 +44,9 @@ #[async_trait] impl IggyCmdTestCase for TestStreamListCmd { async fn prepare_server_state(&mut self, client: &dyn Client) { - let stream = client.create_stream(&self.name).await; + let stream = client + .create_stream(&self.name, &StreamCreateOptions::default()) + .await; assert!(stream.is_ok()); } @@ -62,7 +65,11 @@ "Executing list streams in {} mode", self.output ))) - .stdout(contains(self.name.clone())); + .stdout(contains(self.name.clone())) + .stdout(contains(match self.output { + OutputFormat::List => format!("{}|0 B|0|0|none\n", self.name), + OutputFormat::Default | OutputFormat::Table => "| Encryption |".to_owned(), + })); } async fn verify_server_state(&self, _client: &dyn Client) {}
diff --git a/core/integration/tests/cli/stream/test_stream_update_command.rs b/core/integration/tests/cli/stream/test_stream_update_command.rs index 1e47ac7..5af59d9 100644 --- a/core/integration/tests/cli/stream/test_stream_update_command.rs +++ b/core/integration/tests/cli/stream/test_stream_update_command.rs
@@ -22,6 +22,7 @@ use assert_cmd::assert::Assert; use async_trait::async_trait; use iggy::prelude::Client; +use iggy::prelude::StreamCreateOptions; use predicates::str::diff; use serial_test::parallel; @@ -44,10 +45,16 @@ fn to_args(&self) -> Vec<String> { match self.using_identifier { - TestStreamId::Named => vec![self.stream_name.clone(), self.new_name.clone()], - TestStreamId::Numeric => { - vec![format!("{}", self.stream_id), self.new_name.clone()] - } + TestStreamId::Named => vec![ + self.stream_name.clone(), + "--name".to_owned(), + self.new_name.clone(), + ], + TestStreamId::Numeric => vec![ + format!("{}", self.stream_id), + "--name".to_owned(), + self.new_name.clone(), + ], } } } @@ -55,7 +62,9 @@ #[async_trait] impl IggyCmdTestCase for TestStreamUpdateCmd { async fn prepare_server_state(&mut self, client: &dyn Client) { - let stream = client.create_stream(&self.stream_name).await; + let stream = client + .create_stream(&self.stream_name, &StreamCreateOptions::default()) + .await; assert!(stream.is_ok()); } @@ -70,11 +79,11 @@ fn verify_command(&self, command_state: Assert) { let message = match self.using_identifier { TestStreamId::Named => format!( - "Executing update stream with ID: {} and name: {}\nStream with ID: {} updated name: {}\n", + "Executing update stream with ID: {} (name: {})\nStream with ID: {} updated (name: {})\n", self.stream_name, self.new_name, self.stream_name, self.new_name ), TestStreamId::Numeric => format!( - "Executing update stream with ID: {} and name: {}\nStream with ID: {} updated name: {}\n", + "Executing update stream with ID: {} (name: {})\nStream with ID: {} updated (name: {})\n", self.stream_id, self.new_name, self.stream_id, self.new_name ), }; @@ -130,15 +139,16 @@ .execute_test_for_help_command(TestHelpCmd::new( vec!["stream", "update", "--help"], format!( - r#"Update stream name for given stream ID + r#"Update stream name or options for given stream ID Stream ID can be specified as a stream name or ID +Only the settings passed change, every other setting keeps its value. Examples: - iggy stream update 1 production - iggy stream update test development + iggy stream update 1 --name production + iggy stream update test --name development -{USAGE_PREFIX} stream update <STREAM_ID> <NAME> +{USAGE_PREFIX} stream update [OPTIONS] <STREAM_ID> Arguments: <STREAM_ID> @@ -146,10 +156,21 @@ {CLAP_INDENT} Stream ID can be specified as a stream name or ID - <NAME> +Options: + -n, --name <NAME> New name for the stream -Options: + --set <KEY=VALUE> + Updatable stream option as key=value, repeatable +{CLAP_INDENT} + Values are sent as strings and parsed server-side. + Run "iggy options stream" to list the keys and which ones can change. + + -r, --reset <RESET> + Setting to return to its default, repeatable +{CLAP_INDENT} + The server refuses a key that cannot be reset. + -h, --help Print help (see a summary with '-h') "#, @@ -167,16 +188,18 @@ .execute_test_for_help_command(TestHelpCmd::new( vec!["stream", "update", "-h"], format!( - r#"Update stream name for given stream ID + r#"Update stream name or options for given stream ID -{USAGE_PREFIX} stream update <STREAM_ID> <NAME> +{USAGE_PREFIX} stream update [OPTIONS] <STREAM_ID> Arguments: <STREAM_ID> Stream ID to update - <NAME> New name for the stream Options: - -h, --help Print help (see more with '--help') + -n, --name <NAME> New name for the stream + --set <KEY=VALUE> Updatable stream option as key=value, repeatable + -r, --reset <RESET> Setting to return to its default, repeatable + -h, --help Print help (see more with '--help') "#, ), ))
diff --git a/core/integration/tests/cli/system/test_stats_command.rs b/core/integration/tests/cli/system/test_stats_command.rs index 03918f5..527d317 100644 --- a/core/integration/tests/cli/system/test_stats_command.rs +++ b/core/integration/tests/cli/system/test_stats_command.rs
@@ -25,6 +25,7 @@ use iggy::prelude::IggyExpiry; use iggy::prelude::IggyMessage; use iggy::prelude::Partitioning; +use iggy::prelude::StreamCreateOptions; use iggy::prelude::TopicCreateOptions; use iggy_cli::commands::binary_system::stats::GetStatsOutput; use iggy_common::Stats; @@ -67,7 +68,9 @@ impl IggyCmdTestCase for TestStatsCmd { async fn prepare_server_state(&mut self, client: &dyn Client) { let stream_id = Identifier::from_str_value("logs").unwrap(); - let stream = client.create_stream(&stream_id.as_string()).await; + let stream = client + .create_stream(&stream_id.as_string(), &StreamCreateOptions::default()) + .await; assert!(stream.is_ok()); let topic = client @@ -183,7 +186,9 @@ #[async_trait] impl IggyCmdTestCase for TestStatsCmdWithMessages { async fn prepare_server_state(&mut self, client: &dyn Client) { - let stream = client.create_stream("size-test").await; + let stream = client + .create_stream("size-test", &StreamCreateOptions::default()) + .await; assert!(stream.is_ok()); let stream_details = stream.unwrap(); self.stream_id = stream_details.id;
diff --git a/core/integration/tests/cli/topic/test_topic_create_command.rs b/core/integration/tests/cli/topic/test_topic_create_command.rs index 89a7be9..da3d0b4 100644 --- a/core/integration/tests/cli/topic/test_topic_create_command.rs +++ b/core/integration/tests/cli/topic/test_topic_create_command.rs
@@ -27,6 +27,7 @@ use iggy::prelude::IggyByteSize; use iggy::prelude::IggyExpiry; use iggy::prelude::MaxTopicSize; +use iggy::prelude::StreamCreateOptions; use predicates::str::diff; use serial_test::parallel; use std::str::FromStr; @@ -89,7 +90,9 @@ #[async_trait] impl IggyCmdTestCase for TestTopicCreateCmd { async fn prepare_server_state(&mut self, client: &dyn Client) { - let stream = client.create_stream(&self.stream_name).await; + let stream = client + .create_stream(&self.stream_name, &StreamCreateOptions::default()) + .await; assert!(stream.is_ok()); } @@ -150,7 +153,7 @@ .parse::<HumanDuration>() .unwrap(); assert_eq!( - topic_details.message_expiry, + topic_details.resolved_options().message_expiry, IggyExpiry::ExpireDuration(duration.into()) ); }
diff --git a/core/integration/tests/cli/topic/test_topic_delete_command.rs b/core/integration/tests/cli/topic/test_topic_delete_command.rs index 24b572c..d9c4ad3 100644 --- a/core/integration/tests/cli/topic/test_topic_delete_command.rs +++ b/core/integration/tests/cli/topic/test_topic_delete_command.rs
@@ -23,6 +23,7 @@ use async_trait::async_trait; use iggy::prelude::Client; use iggy::prelude::IggyExpiry; +use iggy::prelude::StreamCreateOptions; use iggy::prelude::TopicCreateOptions; use predicates::str::diff; use serial_test::parallel; @@ -73,7 +74,9 @@ #[async_trait] impl IggyCmdTestCase for TestTopicDeleteCmd { async fn prepare_server_state(&mut self, client: &dyn Client) { - let stream = client.create_stream(&self.stream_name).await; + let stream = client + .create_stream(&self.stream_name, &StreamCreateOptions::default()) + .await; assert!(stream.is_ok()); let topic = client
diff --git a/core/integration/tests/cli/topic/test_topic_get_command.rs b/core/integration/tests/cli/topic/test_topic_get_command.rs index ddaecb8..48a6a6d 100644 --- a/core/integration/tests/cli/topic/test_topic_get_command.rs +++ b/core/integration/tests/cli/topic/test_topic_get_command.rs
@@ -26,7 +26,9 @@ use iggy::prelude::Durability; use iggy::prelude::IggyExpiry; use iggy::prelude::defaults::{DEFAULT_ROOT_PASSWORD, DEFAULT_ROOT_USERNAME}; -use iggy::prelude::{IggyMessage, Partitioning, StreamClient, TopicClient, TopicCreateOptions}; +use iggy::prelude::{ + IggyMessage, Partitioning, StreamClient, StreamCreateOptions, TopicClient, TopicCreateOptions, +}; use integration::harness::{TestHarnessBuilder, TestServerConfig, wait_for_topic_messages_count}; use predicates::prelude::PredicateBooleanExt; use predicates::str::{contains, starts_with}; @@ -115,7 +117,7 @@ impl IggyCmdTestCase for TestTopicGetCmd { async fn prepare_server_state(&mut self, client: &dyn Client) { let stream = client - .create_stream(&self.stream_name) + .create_stream(&self.stream_name, &StreamCreateOptions::default()) .await .expect("stream should be created"); self.stream_id = stream.id; @@ -366,7 +368,10 @@ #[async_trait] impl IggyCmdTestCase for TestMissingTopicGetCmd { async fn prepare_server_state(&mut self, client: &dyn Client) { - client.create_stream("missing-topic-stream").await.unwrap(); + client + .create_stream("missing-topic-stream", &StreamCreateOptions::default()) + .await + .unwrap(); } fn get_command(&self) -> IggyCmdCommand { @@ -482,7 +487,10 @@ .unwrap(); harness.start().await.expect("test server should start"); let client = harness.tcp_root_client().await.unwrap(); - let stream = client.create_stream("cli-persisted-stream").await.unwrap(); + let stream = client + .create_stream("cli-persisted-stream", &StreamCreateOptions::default()) + .await + .unwrap(); let address = harness.server().raw_tcp_addr().unwrap(); let socket_address = harness.server().tcp_addr().unwrap(); clear_session_entry(socket_address);
diff --git a/core/integration/tests/cli/topic/test_topic_help_command.rs b/core/integration/tests/cli/topic/test_topic_help_command.rs index d21e47a..14be102 100644 --- a/core/integration/tests/cli/topic/test_topic_help_command.rs +++ b/core/integration/tests/cli/topic/test_topic_help_command.rs
@@ -34,7 +34,7 @@ Commands: create Create topic with given name, number of partitions, compression algorithm and expiry time for given stream ID [alias: c] delete Delete topic with given ID in given stream ID [alias: d] - update Update topic name, compression algorithm and message expiry time for given topic ID in given stream ID [alias: u] + update Update topic name, message expiry or max topic size for given topic ID in given stream ID [alias: u] get Get topic detail for given topic ID and stream ID [alias: g] list List all topics in given stream ID [alias: l] help Print this message or the help of the given subcommand(s)
diff --git a/core/integration/tests/cli/topic/test_topic_list_command.rs b/core/integration/tests/cli/topic/test_topic_list_command.rs index 08a277e..4c1c4d0 100644 --- a/core/integration/tests/cli/topic/test_topic_list_command.rs +++ b/core/integration/tests/cli/topic/test_topic_list_command.rs
@@ -23,6 +23,7 @@ use async_trait::async_trait; use iggy::prelude::Client; use iggy::prelude::IggyExpiry; +use iggy::prelude::StreamCreateOptions; use iggy::prelude::TopicCreateOptions; use predicates::str::{contains, starts_with}; use serial_test::parallel; @@ -67,7 +68,9 @@ #[async_trait] impl IggyCmdTestCase for TestTopicListCmd { async fn prepare_server_state(&mut self, client: &dyn Client) { - let stream = client.create_stream(&self.stream_name).await; + let stream = client + .create_stream(&self.stream_name, &StreamCreateOptions::default()) + .await; assert!(stream.is_ok()); let topic = client
diff --git a/core/integration/tests/cli/topic/test_topic_update_command.rs b/core/integration/tests/cli/topic/test_topic_update_command.rs index d284ab2..3a94526 100644 --- a/core/integration/tests/cli/topic/test_topic_update_command.rs +++ b/core/integration/tests/cli/topic/test_topic_update_command.rs
@@ -22,12 +22,10 @@ use assert_cmd::assert::Assert; use async_trait::async_trait; use humantime::Duration as HumanDuration; -use iggy::prelude::Client; -use iggy::prelude::CompressionAlgorithm; -use iggy::prelude::IggyByteSize; -use iggy::prelude::IggyExpiry; -use iggy::prelude::MaxTopicSize; -use iggy::prelude::TopicCreateOptions; +use iggy::prelude::{ + Client, HeaderKey, IggyByteSize, IggyExpiry, MaxTopicSize, OptionKey, StreamCreateOptions, + TopicCreateOptions, +}; use predicates::str::diff; use serial_test::parallel; use std::str::FromStr; @@ -38,101 +36,104 @@ stream_name: String, topic_id: u32, topic_name: String, - compression_algorithm: CompressionAlgorithm, message_expiry: Option<Vec<String>>, max_topic_size: MaxTopicSize, - topic_new_name: String, - topic_new_compression_algorithm: CompressionAlgorithm, - topic_new_message_expiry: Option<Vec<String>>, - topic_new_max_size: MaxTopicSize, + new_name: Option<String>, + new_message_expiry: Option<Vec<String>>, + new_max_topic_size: Option<MaxTopicSize>, + reset: Vec<OptionKey>, using_stream_id: TestStreamId, using_topic_id: TestTopicId, } impl TestTopicUpdateCmd { - #[allow(clippy::too_many_arguments)] - fn new( - stream_id: u32, - stream_name: String, - topic_id: u32, - topic_name: String, - compression_algorithm: CompressionAlgorithm, - message_expiry: Option<Vec<String>>, - max_topic_size: MaxTopicSize, - topic_new_name: String, - topic_new_compression_algorithm: CompressionAlgorithm, - topic_new_message_expiry: Option<Vec<String>>, - topic_new_max_size: MaxTopicSize, - using_stream_id: TestStreamId, - using_topic_id: TestTopicId, - ) -> Self { + fn new(stream_id: u32, stream_name: &str, topic_id: u32, topic_name: &str) -> Self { Self { stream_id, - stream_name, + stream_name: stream_name.to_owned(), topic_id, - topic_name, - compression_algorithm, - message_expiry, - max_topic_size, - topic_new_name, - topic_new_compression_algorithm, - topic_new_message_expiry, - topic_new_max_size, - using_stream_id, - using_topic_id, + topic_name: topic_name.to_owned(), + message_expiry: None, + max_topic_size: MaxTopicSize::ServerDefault, + new_name: None, + new_message_expiry: None, + new_max_topic_size: None, + reset: Vec::new(), + using_stream_id: TestStreamId::Named, + using_topic_id: TestTopicId::Named, } } + fn final_name(&self) -> &str { + self.new_name.as_deref().unwrap_or(&self.topic_name) + } + fn to_args(&self) -> Vec<String> { let mut command = match self.using_stream_id { TestStreamId::Numeric => vec![format!("{}", self.stream_id)], TestStreamId::Named => vec![self.stream_name.clone()], }; - command.push(match self.using_topic_id { TestTopicId::Numeric => format!("{}", self.topic_id), TestTopicId::Named => self.topic_name.clone(), }); - - command.push(self.topic_new_name.clone()); - command.push(self.topic_new_compression_algorithm.to_string()); - command.push(format!("--max-topic-size={}", self.topic_new_max_size)); - - if let Some(message_expiry) = &self.topic_new_message_expiry { + if let Some(name) = &self.new_name { + command.push(format!("--name={name}")); + } + if let Some(max_topic_size) = &self.new_max_topic_size { + command.push(format!( + "--max-topic-size={}", + max_topic_size.as_bytes_u64() + )); + } + if let Some(message_expiry) = &self.new_message_expiry { + command.push("--message-expiry".to_owned()); command.extend(message_expiry.clone()); } - + for key in &self.reset { + command.push(format!("--reset={key}")); + } command } + + fn changes(&self) -> String { + let mut changes = Vec::new(); + if let Some(name) = &self.new_name { + changes.push(format!("name: {name}")); + } + if let Some(message_expiry) = &self.new_message_expiry { + changes.push(format!("message expiry: {}", expiry(message_expiry))); + } + if let Some(max_topic_size) = &self.new_max_topic_size { + changes.push(format!("max topic size: {max_topic_size}")); + } + for key in &self.reset { + changes.push(format!("reset: {key}")); + } + changes.join(", ") + } +} + +fn expiry(parts: &[String]) -> IggyExpiry { + let duration: Duration = *parts.join(" ").parse::<HumanDuration>().unwrap(); + IggyExpiry::ExpireDuration(duration.into()) } #[async_trait] impl IggyCmdTestCase for TestTopicUpdateCmd { async fn prepare_server_state(&mut self, client: &dyn Client) { - let stream = client.create_stream(&self.stream_name).await; + let stream = client + .create_stream(&self.stream_name, &StreamCreateOptions::default()) + .await; assert!(stream.is_ok()); - let message_expiry = match &self.message_expiry { - None => IggyExpiry::ServerDefault, - Some(message_expiry) => { - let duration: Duration = - *message_expiry.join(" ").parse::<HumanDuration>().unwrap(); - - IggyExpiry::ExpireDuration(duration.into()) - } - }; - let topic = client .create_topic( &self.stream_name.clone().try_into().unwrap(), &self.topic_name, &TopicCreateOptions { partitions_count: Some(1), - compression_algorithm: (self.compression_algorithm - != CompressionAlgorithm::default()) - .then_some(self.compression_algorithm), - message_expiry: (message_expiry != IggyExpiry::ServerDefault) - .then_some(message_expiry), + message_expiry: self.message_expiry.as_deref().map(expiry), max_topic_size: (self.max_topic_size != MaxTopicSize::ServerDefault) .then_some(self.max_topic_size), ..TopicCreateOptions::default() @@ -155,30 +156,14 @@ TestStreamId::Numeric => format!("{}", self.stream_id), TestStreamId::Named => self.stream_name.clone(), }; - let topic_id = match self.using_topic_id { TestTopicId::Numeric => format!("{}", self.topic_id), TestTopicId::Named => self.topic_name.clone(), }; - - let compression_algorithm = &self.topic_new_compression_algorithm; - - let message_expiry = (match &self.topic_new_message_expiry { - Some(value) => value.join(" "), - None => IggyExpiry::ServerDefault.to_string(), - }) - .to_string(); - - let new_topic_name = &self.topic_new_name; - let new_max_topic_size = self.topic_new_max_size.to_string(); - + let changes = self.changes(); let expected_message = format!( - "Executing update topic with ID: {topic_id}, name: {new_topic_name}, \ - message expiry: {message_expiry}, compression algorithm: {compression_algorithm}, max topic size: \ - {new_max_topic_size}, in stream with ID: {stream_id}\n\ - Topic with ID: {topic_id} updated name: {new_topic_name}, updated message expiry: {message_expiry}, \ - updated compression algorithm: {compression_algorithm}, \ - updated max topic size: {new_max_topic_size} in stream with ID: {stream_id}\n" + "Executing update topic with ID: {topic_id} ({changes}) in stream with ID: {stream_id}\n\ + Topic with ID: {topic_id} updated ({changes}) in stream with ID: {stream_id}\n" ); command_state.success().stdout(diff(expected_message)); @@ -188,32 +173,39 @@ let topic = client .get_topic( &self.stream_name.clone().try_into().unwrap(), - &self.topic_new_name.clone().try_into().unwrap(), + &self.final_name().try_into().unwrap(), ) .await; assert!(topic.is_ok()); let topic_details = topic.unwrap().expect("Failed to get topic"); - assert_eq!(topic_details.name, self.topic_new_name); - assert_eq!(topic_details.messages_count, 0); + assert_eq!(topic_details.name, self.final_name()); + let resolved = topic_details.resolved_options(); - if self.topic_new_message_expiry.is_some() { - let duration: Duration = *self - .topic_new_message_expiry - .clone() - .unwrap() - .join(" ") - .parse::<HumanDuration>() - .unwrap(); - assert_eq!( - topic_details.message_expiry, - IggyExpiry::ExpireDuration(duration.into()) - ); + let expected_expiry = self + .new_message_expiry + .as_deref() + .or(self.message_expiry.as_deref()) + .filter(|_| !self.reset.contains(&OptionKey::MessageExpiry)) + .map_or(IggyExpiry::NeverExpire, expiry); + assert_eq!(resolved.message_expiry, expected_expiry); + let expected_max_topic_size = self + .new_max_topic_size + .or((self.max_topic_size != MaxTopicSize::ServerDefault).then_some(self.max_topic_size)) + .filter(|_| !self.reset.contains(&OptionKey::MaxTopicSize)) + .unwrap_or(MaxTopicSize::Unlimited); + assert_eq!(resolved.max_topic_size, expected_max_topic_size); + for key in &self.reset { + let option = topic_details + .options + .get(&HeaderKey::from(*key)) + .expect("a reset key holds its default"); + assert!(!option.explicit, "{key} must read back as derived"); } let topic = client .delete_topic( &self.stream_name.clone().try_into().unwrap(), - &self.topic_new_name.clone().try_into().unwrap(), + &self.final_name().try_into().unwrap(), ) .await; assert!(topic.is_ok()); @@ -231,44 +223,36 @@ let mut iggy_cmd_test = IggyCmdTest::default(); iggy_cmd_test.setup().await; iggy_cmd_test - .execute_test(TestTopicUpdateCmd::new( - 0, - String::from("main"), - 0, - String::from("sync"), - Default::default(), - None, - MaxTopicSize::ServerDefault, - String::from("new_name"), - CompressionAlgorithm::Gzip, - None, - MaxTopicSize::Custom(IggyByteSize::from_str("2GiB").unwrap()), - TestStreamId::Named, - TestTopicId::Named, - )) + .execute_test(TestTopicUpdateCmd { + new_name: Some("new_name".to_owned()), + new_max_topic_size: Some(MaxTopicSize::Custom( + IggyByteSize::from_str("2GiB").unwrap(), + )), + ..TestTopicUpdateCmd::new(0, "main", 0, "sync") + }) .await; iggy_cmd_test - .execute_test(TestTopicUpdateCmd::new( - 1, - String::from("production"), - 0, - String::from("topic"), - Default::default(), - None, - MaxTopicSize::ServerDefault, - String::from("testing"), - CompressionAlgorithm::Gzip, - None, - MaxTopicSize::Unlimited, - TestStreamId::Named, - TestTopicId::Named, - )) + .execute_test(TestTopicUpdateCmd { + new_name: Some("testing".to_owned()), + new_message_expiry: Some(vec!["1day".to_owned(), "1hour".to_owned()]), + ..TestTopicUpdateCmd::new(1, "production", 0, "topic") + }) .await; - // There used to be 3 more test cases, but they don't fit anymore the test scenario - // Previously we allowed to update topic name to an already existing one, which is now forbidden. +} - // Imagine a scenario where user tries to use any of our APIs with the `Identifier::named` variant for topic identifier - // How to distinguish between topics with same name. +#[tokio::test] +#[parallel] +pub async fn given_topic_reset_when_using_cli_should_send_only_requested_keys() { + let mut iggy_cmd_test = IggyCmdTest::default(); + iggy_cmd_test.setup().await; + iggy_cmd_test + .execute_test(TestTopicUpdateCmd { + message_expiry: Some(vec!["10m".to_owned()]), + max_topic_size: MaxTopicSize::Custom(IggyByteSize::from_str("5GiB").unwrap()), + reset: vec![OptionKey::MaxTopicSize], + ..TestTopicUpdateCmd::new(2, "reset-stream", 0, "reset-topic") + }) + .await; } #[tokio::test] @@ -280,19 +264,21 @@ .execute_test_for_help_command(TestHelpCmd::new( vec!["topic", "update", "--help"], format!( - r#"Update topic name, compression algorithm and message expiry time for given topic ID in given stream ID + r#"Update topic name, message expiry or max topic size for given topic ID in given stream ID Stream ID can be specified as a stream name or ID Topic ID can be specified as a topic name or ID +Only the settings passed change, every other setting keeps its value. +Compression, segment size, durability and the other create-time settings cannot change. Examples - iggy topic update 1 1 sensor3 none - iggy topic update prod sensor3 old-sensor none - iggy topic update test debugs ready gzip 15days - iggy topic update 1 1 new-name gzip - iggy topic update 1 2 new-name none 1day 1hour 1min 1sec + iggy topic update 1 1 --name sensor3 + iggy topic update prod sensor3 --name old-sensor --message-expiry 15days + iggy topic update test debugs --max-topic-size 15GB + iggy topic update 1 2 --message-expiry 1day 1hour 1min 1sec + iggy topic update 1 2 --reset message_expiry --reset max_topic_size -{USAGE_PREFIX} topic update [OPTIONS] <STREAM_ID> <TOPIC_ID> <NAME> <COMPRESSION_ALGORITHM> [MESSAGE_EXPIRY]... +{USAGE_PREFIX} topic update [OPTIONS] <STREAM_ID> <TOPIC_ID> Arguments: <STREAM_ID> @@ -305,27 +291,29 @@ {CLAP_INDENT} Topic ID can be specified as a topic name or ID - <NAME> +Options: + -n, --name <NAME> New name for the topic - <COMPRESSION_ALGORITHM> - Compression metadata (none or gzip). Payload compression is not implemented - - [MESSAGE_EXPIRY]... - New message expiry time in human-readable format like "unlimited" or "15days 2min 2s" -{CLAP_INDENT} - Skipping this parameter or using "server_default" preserves the current message expiry. -{CLAP_INDENT} - [default: server_default] - -Options: -m, --max-topic-size <MAX_TOPIC_SIZE> New max topic size in human-readable format like "unlimited" or "15GB" {CLAP_INDENT} - Skipping this parameter or using "server_default" preserves the current max topic size. A finite size cannot be lower than the topic segment size. + + -e, --message-expiry <MESSAGE_EXPIRY>... + New message expiry time in human-readable format like "unlimited" or "15days 2min 2s" + + --set <KEY=VALUE> + Updatable topic option as key=value, repeatable {CLAP_INDENT} - [default: server_default] + Values are sent as strings and parsed server-side. + Run "iggy options topic" to list the keys and which ones can change. + + -r, --reset <RESET> + Setting to return to its default, like message_expiry or max_topic_size +{CLAP_INDENT} + Repeat the flag to reset several settings. + The server refuses a key that cannot be reset. -h, --help Print help (see a summary with '-h') @@ -344,20 +332,21 @@ .execute_test_for_help_command(TestHelpCmd::new( vec!["topic", "update", "-h"], format!( - r#"Update topic name, compression algorithm and message expiry time for given topic ID in given stream ID + r#"Update topic name, message expiry or max topic size for given topic ID in given stream ID -{USAGE_PREFIX} topic update [OPTIONS] <STREAM_ID> <TOPIC_ID> <NAME> <COMPRESSION_ALGORITHM> [MESSAGE_EXPIRY]... +{USAGE_PREFIX} topic update [OPTIONS] <STREAM_ID> <TOPIC_ID> Arguments: - <STREAM_ID> Stream ID to update topic - <TOPIC_ID> Topic ID to update - <NAME> New name for the topic - <COMPRESSION_ALGORITHM> Compression metadata (none or gzip). Payload compression is not implemented - [MESSAGE_EXPIRY]... New message expiry time in human-readable format like "unlimited" or "15days 2min 2s" [default: server_default] + <STREAM_ID> Stream ID to update topic + <TOPIC_ID> Topic ID to update Options: - -m, --max-topic-size <MAX_TOPIC_SIZE> New max topic size in human-readable format like "unlimited" or "15GB" [default: server_default] - -h, --help Print help (see more with '--help') + -n, --name <NAME> New name for the topic + -m, --max-topic-size <MAX_TOPIC_SIZE> New max topic size in human-readable format like "unlimited" or "15GB" + -e, --message-expiry <MESSAGE_EXPIRY>... New message expiry time in human-readable format like "unlimited" or "15days 2min 2s" + --set <KEY=VALUE> Updatable topic option as key=value, repeatable + -r, --reset <RESET> Setting to return to its default, like message_expiry or max_topic_size + -h, --help Print help (see more with '--help') "#, ), ))
diff --git a/core/integration/tests/cli/user/common.rs b/core/integration/tests/cli/user/common.rs index 758810d..e147576 100644 --- a/core/integration/tests/cli/user/common.rs +++ b/core/integration/tests/cli/user/common.rs
@@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use iggy::prelude::StreamCreateOptions; use iggy::prelude::{Client, Identifier, Permissions, TopicCreateOptions}; use std::collections::BTreeMap; @@ -30,7 +31,10 @@ let mut created = 0; while client.get_stream(&stream).await.unwrap().is_none() { client - .create_stream(&format!("grant-target-{stream_id}-{created}")) + .create_stream( + &format!("grant-target-{stream_id}-{created}"), + &StreamCreateOptions::default(), + ) .await .unwrap(); created += 1;
diff --git a/core/integration/tests/cli/user/test_user_delete_command.rs b/core/integration/tests/cli/user/test_user_delete_command.rs index a909f02..c7df37b 100644 --- a/core/integration/tests/cli/user/test_user_delete_command.rs +++ b/core/integration/tests/cli/user/test_user_delete_command.rs
@@ -22,6 +22,7 @@ use assert_cmd::assert::Assert; use async_trait::async_trait; use iggy::prelude::Client; +use iggy::prelude::UserCreateOptions; use iggy::prelude::UserStatus; use predicates::str::diff; use serial_test::parallel; @@ -63,7 +64,13 @@ impl IggyCmdTestCase for TestUserDeleteCmd { async fn prepare_server_state(&mut self, client: &dyn Client) { let user = client - .create_user(&self.username, &self.password, self.status, None) + .create_user( + &self.username, + &self.password, + self.status, + None, + &UserCreateOptions::default(), + ) .await; assert!(user.is_ok()); }
diff --git a/core/integration/tests/cli/user/test_user_get_command.rs b/core/integration/tests/cli/user/test_user_get_command.rs index da320f6..88e6583 100644 --- a/core/integration/tests/cli/user/test_user_get_command.rs +++ b/core/integration/tests/cli/user/test_user_get_command.rs
@@ -22,6 +22,7 @@ use crate::cli::user::common::create_grant_targets; use assert_cmd::assert::Assert; use async_trait::async_trait; +use iggy::prelude::UserCreateOptions; use iggy::prelude::{Client, Permissions, StreamPermissions, TopicPermissions, UserId, UserStatus}; use predicates::str::{is_match, starts_with}; use serial_test::parallel; @@ -113,6 +114,7 @@ "secret", self.status, self.get_permissions(), + &UserCreateOptions::default(), ) .await; assert!(create_user.is_ok());
diff --git a/core/integration/tests/cli/user/test_user_list_command.rs b/core/integration/tests/cli/user/test_user_list_command.rs index 0559d0b..fbf8360 100644 --- a/core/integration/tests/cli/user/test_user_list_command.rs +++ b/core/integration/tests/cli/user/test_user_list_command.rs
@@ -22,6 +22,7 @@ use assert_cmd::assert::Assert; use async_trait::async_trait; use iggy::prelude::Client; +use iggy::prelude::UserCreateOptions; use iggy::prelude::UserStatus; use predicates::str::{contains, starts_with}; use serial_test::parallel; @@ -50,7 +51,13 @@ impl IggyCmdTestCase for TestUserListCmd { async fn prepare_server_state(&mut self, client: &dyn Client) { let stream = client - .create_user(&self.username, "secret", self.status, None) + .create_user( + &self.username, + "secret", + self.status, + None, + &UserCreateOptions::default(), + ) .await; assert!(stream.is_ok()); }
diff --git a/core/integration/tests/cli/user/test_user_name_command.rs b/core/integration/tests/cli/user/test_user_name_command.rs index 894d8d7..6f238fb 100644 --- a/core/integration/tests/cli/user/test_user_name_command.rs +++ b/core/integration/tests/cli/user/test_user_name_command.rs
@@ -22,6 +22,7 @@ use assert_cmd::assert::Assert; use async_trait::async_trait; use iggy::prelude::Client; +use iggy::prelude::UserCreateOptions; use iggy::prelude::UserId; use iggy::prelude::UserStatus; use predicates::str::diff; @@ -61,7 +62,13 @@ impl IggyCmdTestCase for TestUserNameCmd { async fn prepare_server_state(&mut self, client: &dyn Client) { let create_user = client - .create_user(&self.username, "secret", UserStatus::Active, None) + .create_user( + &self.username, + "secret", + UserStatus::Active, + None, + &UserCreateOptions::default(), + ) .await; assert!(create_user.is_ok()); let user = client
diff --git a/core/integration/tests/cli/user/test_user_password_command.rs b/core/integration/tests/cli/user/test_user_password_command.rs index b9299d9..263a4a0 100644 --- a/core/integration/tests/cli/user/test_user_password_command.rs +++ b/core/integration/tests/cli/user/test_user_password_command.rs
@@ -22,6 +22,7 @@ use assert_cmd::assert::Assert; use async_trait::async_trait; use iggy::prelude::Client; +use iggy::prelude::UserCreateOptions; use iggy::prelude::UserId; use iggy::prelude::UserStatus; use predicates::str::diff; @@ -74,7 +75,13 @@ impl IggyCmdTestCase for TestUserPasswordCmd { async fn prepare_server_state(&mut self, client: &dyn Client) { let create_user = client - .create_user(&self.username, &self.password, UserStatus::Active, None) + .create_user( + &self.username, + &self.password, + UserStatus::Active, + None, + &UserCreateOptions::default(), + ) .await; assert!(create_user.is_ok()); let user = client
diff --git a/core/integration/tests/cli/user/test_user_permissions_command.rs b/core/integration/tests/cli/user/test_user_permissions_command.rs index 21163d1..4c0642e 100644 --- a/core/integration/tests/cli/user/test_user_permissions_command.rs +++ b/core/integration/tests/cli/user/test_user_permissions_command.rs
@@ -24,6 +24,7 @@ use async_trait::async_trait; use iggy::prelude::Client; use iggy::prelude::Permissions; +use iggy::prelude::UserCreateOptions; use iggy::prelude::UserId; use iggy::prelude::UserStatus; use iggy::prelude::{GlobalPermissions, StreamPermissions, TopicPermissions}; @@ -71,7 +72,13 @@ async fn prepare_server_state(&mut self, client: &dyn Client) { create_grant_targets(client, self.permissions.expected_permissions.as_ref()).await; let create_user = client - .create_user(&self.username, "secret", UserStatus::Active, None) + .create_user( + &self.username, + "secret", + UserStatus::Active, + None, + &UserCreateOptions::default(), + ) .await; assert!(create_user.is_ok()); let user = client
diff --git a/core/integration/tests/cli/user/test_user_status_command.rs b/core/integration/tests/cli/user/test_user_status_command.rs index 8978f1d..9101b0d 100644 --- a/core/integration/tests/cli/user/test_user_status_command.rs +++ b/core/integration/tests/cli/user/test_user_status_command.rs
@@ -22,6 +22,7 @@ use assert_cmd::assert::Assert; use async_trait::async_trait; use iggy::prelude::Client; +use iggy::prelude::UserCreateOptions; use iggy::prelude::UserId; use iggy::prelude::UserStatus; use predicates::str::diff; @@ -71,7 +72,13 @@ impl IggyCmdTestCase for TestUserStatusCmd { async fn prepare_server_state(&mut self, client: &dyn Client) { let create_user = client - .create_user(&self.username, "secret", self.status, None) + .create_user( + &self.username, + "secret", + self.status, + None, + &UserCreateOptions::default(), + ) .await; assert!(create_user.is_ok()); let user = client
diff --git a/core/integration/tests/cluster/chaos_monkey.rs b/core/integration/tests/cluster/chaos_monkey.rs index 476b0ce..7f9950e 100644 --- a/core/integration/tests/cluster/chaos_monkey.rs +++ b/core/integration/tests/cluster/chaos_monkey.rs
@@ -1285,7 +1285,7 @@ let target = Target::new(); let client = harness.tcp_root_client().await.expect("root client"); client - .create_stream(STREAM_NAME) + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) .await .expect("create the stream"); let options = TopicCreateOptions {
diff --git a/core/integration/tests/cluster/client_table_adversarial.rs b/core/integration/tests/cluster/client_table_adversarial.rs index 603076f..ea83221 100644 --- a/core/integration/tests/cluster/client_table_adversarial.rs +++ b/core/integration/tests/cluster/client_table_adversarial.rs
@@ -195,7 +195,7 @@ // so the raw session below spends ids only on what this spec is about. let setup = harness.tcp_root_client().await.unwrap(); setup - .create_stream(GAP_STREAM) + .create_stream(GAP_STREAM, &StreamCreateOptions::default()) .await .expect("create stream"); setup
diff --git a/core/integration/tests/cluster/client_table_restart.rs b/core/integration/tests/cluster/client_table_restart.rs index b243ac6..009e58e 100644 --- a/core/integration/tests/cluster/client_table_restart.rs +++ b/core/integration/tests/cluster/client_table_restart.rs
@@ -186,8 +186,10 @@ Exchange::Reply { status, .. } if status == IggyError::Unauthenticated.as_code())); let payload = create_stream_payload("lost-login-shared-binding"); - let original_result = commit_request(&mut bound, session, 1, &payload).await; - let replay = commit_request(&mut retried, session, 1, &payload).await; + let header = + RequestHeader::for_request(CREATE_STREAM_CODE, CLIENT_ID, 1, session, &payload).unwrap(); + let original_result = commit_request_header(&mut bound, &header, &payload).await; + let replay = commit_request_header(&mut retried, &header, &payload).await; assert_eq!(replay.header, original_result.header); assert_eq!(replay.payload, original_result.payload); } @@ -622,6 +624,13 @@ let mut last_failure = "the listener never came back".to_string(); let mut attempt = 0usize; let mut authenticated_attempts = Vec::new(); + let mut header = *header; + if let Some(original) = expect_replay_of { + let receipt = + bytemuck::checked::try_pod_read_unaligned::<ReplyHeader>(original.header.as_ref()) + .unwrap(); + header.request_checksum = receipt.request_checksum; + } while Instant::now() < deadline { let addr = addrs[attempt % addrs.len()]; attempt += 1; @@ -644,7 +653,7 @@ continue; } }; - match exchange(&mut stream, header, body).await.verdict() { + match exchange(&mut stream, &header, body).await.verdict() { Verdict::Success(reply) => { if let Some(original) = expect_replay_of { assert_replayed_from_cache(original, &reply, request); @@ -1313,7 +1322,10 @@ ) { let observer = harness.root_client_for_node(0).await.unwrap(); let partitions_count = u32::try_from(harness.cluster_size()).unwrap(); - observer.create_stream(LIVENESS_STREAM).await.unwrap(); + observer + .create_stream(LIVENESS_STREAM, &StreamCreateOptions::default()) + .await + .unwrap(); observer .create_topic( &Identifier::named(LIVENESS_STREAM).unwrap(), @@ -1463,7 +1475,10 @@ } async fn create_liveness_group(observer: &IggyClient) { - observer.create_stream(LIVENESS_STREAM).await.unwrap(); + observer + .create_stream(LIVENESS_STREAM, &StreamCreateOptions::default()) + .await + .unwrap(); let stream = Identifier::named(LIVENESS_STREAM).unwrap(); let topic = Identifier::named(LIVENESS_TOPIC).unwrap(); observer
diff --git a/core/integration/tests/cluster/consumer_offset_quota.rs b/core/integration/tests/cluster/consumer_offset_quota.rs index bb47b51..831efe3 100644 --- a/core/integration/tests/cluster/consumer_offset_quota.rs +++ b/core/integration/tests/cluster/consumer_offset_quota.rs
@@ -51,7 +51,7 @@ let stream = Identifier::named(STREAM_NAME).expect("stream identifier"); let topic = Identifier::named(TOPIC_NAME).expect("topic identifier"); let stream_details = client - .create_stream(STREAM_NAME) + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) .await .expect("create stream"); let topic_details = client
diff --git a/core/integration/tests/cluster/crash_durability.rs b/core/integration/tests/cluster/crash_durability.rs index a9d13bf..7b9d527 100644 --- a/core/integration/tests/cluster/crash_durability.rs +++ b/core/integration/tests/cluster/crash_durability.rs
@@ -62,7 +62,7 @@ async fn create_stream_and_topic(client: &IggyClient, eager_flush: bool) { client - .create_stream(STREAM_NAME) + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) .await .expect("create stream"); // `eager_flush` persists and fsyncs every committed batch on every @@ -548,7 +548,10 @@ async fn verify_persisted_restart(harness: &mut TestHarness, durability: Durability) { let client = harness.tcp_root_client().await.unwrap(); - client.create_stream(STREAM_NAME).await.unwrap(); + client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); let stream = Identifier::named(STREAM_NAME).unwrap(); client .create_topic( @@ -609,7 +612,10 @@ harness: &mut TestHarness, ) { let client = harness.tcp_root_client().await.unwrap(); - client.create_stream(STREAM_NAME).await.unwrap(); + client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); client .create_topic( &Identifier::named(STREAM_NAME).unwrap(), @@ -643,7 +649,10 @@ harness: &mut TestHarness, ) { let client = harness.tcp_root_client().await.unwrap(); - let stream_details = client.create_stream(STREAM_NAME).await.unwrap(); + let stream_details = client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); let stream = Identifier::numeric(stream_details.id).unwrap(); let topic_details = client .create_topic(
diff --git a/core/integration/tests/cluster/crash_offset_reuse.rs b/core/integration/tests/cluster/crash_offset_reuse.rs index 92e4ae8..b5c7610 100644 --- a/core/integration/tests/cluster/crash_offset_reuse.rs +++ b/core/integration/tests/cluster/crash_offset_reuse.rs
@@ -100,7 +100,7 @@ /// Create the stream and its single-partition topic. async fn create_topic(client: &IggyClient, messages_required_to_save: Option<u32>) { client - .create_stream(STREAM_NAME) + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) .await .expect("create stream"); client
diff --git a/core/integration/tests/cluster/crash_recovery_corruption.rs b/core/integration/tests/cluster/crash_recovery_corruption.rs index 9d36ef3..339b03a 100644 --- a/core/integration/tests/cluster/crash_recovery_corruption.rs +++ b/core/integration/tests/cluster/crash_recovery_corruption.rs
@@ -98,7 +98,7 @@ async fn create_stream_and_topic(client: &IggyClient, durability: Durability) { client - .create_stream(STREAM_NAME) + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) .await .expect("create stream"); // Eager flush makes committed batches visible to the on-disk oracles. @@ -465,7 +465,7 @@ ) { let client = harness.tcp_root_client().await.unwrap(); client - .create_stream(STREAM_NAME) + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) .await .expect("create stream"); // Not `create_stream_and_topic`: its eager flush indexes every batch at @@ -964,7 +964,10 @@ // provably precedes many complete entries. for index in 0..WAL_FODDER_STREAMS { client - .create_stream(&format!("wal-fodder-{index:02}")) + .create_stream( + &format!("wal-fodder-{index:02}"), + &StreamCreateOptions::default(), + ) .await .expect("create fodder stream"); }
diff --git a/core/integration/tests/cluster/failover_client_continuity.rs b/core/integration/tests/cluster/failover_client_continuity.rs index 1e0f3ac..9cd1f9c 100644 --- a/core/integration/tests/cluster/failover_client_continuity.rs +++ b/core/integration/tests/cluster/failover_client_continuity.rs
@@ -63,7 +63,7 @@ ) { let setup_client = harness.tcp_root_client().await.unwrap(); setup_client - .create_stream(STREAM_NAME) + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) .await .expect("create stream"); // Eagerly flushed so every pre-kill ack is durable on the survivors and
diff --git a/core/integration/tests/cluster/fast_primary_rejoin.rs b/core/integration/tests/cluster/fast_primary_rejoin.rs index 3074444..a362df1 100644 --- a/core/integration/tests/cluster/fast_primary_rejoin.rs +++ b/core/integration/tests/cluster/fast_primary_rejoin.rs
@@ -78,7 +78,7 @@ async fn create_stream_and_topic(harness: &TestHarness) { let setup_client = harness.tcp_root_client().await.unwrap(); setup_client - .create_stream(STREAM_NAME) + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) .await .expect("create stream"); let options = TopicCreateOptions {
diff --git a/core/integration/tests/cluster/metadata_checkpoint_restart.rs b/core/integration/tests/cluster/metadata_checkpoint_restart.rs index efcf6ff..df27335 100644 --- a/core/integration/tests/cluster/metadata_checkpoint_restart.rs +++ b/core/integration/tests/cluster/metadata_checkpoint_restart.rs
@@ -350,7 +350,7 @@ let client = connect(harness).await; for index in 0..STREAMS { client - .create_stream(&stream_name(index)) + .create_stream(&stream_name(index), &StreamCreateOptions::default()) .await .unwrap_or_else(|e| panic!("create stream {index}: {e}")); }
diff --git a/core/integration/tests/cluster/mod.rs b/core/integration/tests/cluster/mod.rs index 76ae990..546e1d3 100644 --- a/core/integration/tests/cluster/mod.rs +++ b/core/integration/tests/cluster/mod.rs
@@ -35,3 +35,5 @@ mod replica_read_batching; mod root_credentials; mod staggered_bootstrap; +mod stream_encryption; +mod stream_key_rotation;
diff --git a/core/integration/tests/cluster/multi_shard_partition_convergence.rs b/core/integration/tests/cluster/multi_shard_partition_convergence.rs index ceb72bc..56f9fc8 100644 --- a/core/integration/tests/cluster/multi_shard_partition_convergence.rs +++ b/core/integration/tests/cluster/multi_shard_partition_convergence.rs
@@ -149,7 +149,10 @@ .await .unwrap(); let stream = Identifier::named(STREAM).unwrap(); - client.create_stream(STREAM).await.unwrap(); + client + .create_stream(STREAM, &StreamCreateOptions::default()) + .await + .unwrap(); for index in 0..TOPICS { create_topic(&client, &stream, &topic_name(index)).await; @@ -193,7 +196,10 @@ .await .unwrap(); let stream = Identifier::named(STREAM).unwrap(); - client.create_stream(STREAM).await.unwrap(); + client + .create_stream(STREAM, &StreamCreateOptions::default()) + .await + .unwrap(); for index in 0..TOPICS { let name = topic_name(index);
diff --git a/core/integration/tests/cluster/parked_frame_redispatch.rs b/core/integration/tests/cluster/parked_frame_redispatch.rs index 572ced2..0901a86 100644 --- a/core/integration/tests/cluster/parked_frame_redispatch.rs +++ b/core/integration/tests/cluster/parked_frame_redispatch.rs
@@ -124,7 +124,10 @@ .root_client_for_node(leader) .await .expect("root client on the metadata leader"); - setup.create_stream(STREAM).await.expect("create stream"); + setup + .create_stream(STREAM, &StreamCreateOptions::default()) + .await + .expect("create stream"); let stream = Identifier::named(STREAM).expect("stream identifier"); // Connected and logged in before the first `create_topic`, so the burst
diff --git a/core/integration/tests/cluster/partition_dedup.rs b/core/integration/tests/cluster/partition_dedup.rs index f1fb57c..482edd1 100644 --- a/core/integration/tests/cluster/partition_dedup.rs +++ b/core/integration/tests/cluster/partition_dedup.rs
@@ -354,6 +354,7 @@ PASSWORD, UserStatus::Active, Some(permissions(true)), + &UserCreateOptions::default(), ) .await .expect("create the producer user"); @@ -1056,7 +1057,7 @@ async fn seed_topic(client: &IggyClient) -> PartitionContext { client - .create_stream(STREAM_NAME) + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) .await .expect("create stream"); let stream_id = Identifier::named(STREAM_NAME).expect("stream identifier");
diff --git a/core/integration/tests/cluster/partition_primary_routing.rs b/core/integration/tests/cluster/partition_primary_routing.rs index 408c4b7..7dd60a1 100644 --- a/core/integration/tests/cluster/partition_primary_routing.rs +++ b/core/integration/tests/cluster/partition_primary_routing.rs
@@ -250,7 +250,10 @@ .login_user(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD) .await .unwrap(); - first.create_stream(STREAM_NAME).await.unwrap(); + first + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); let stream = Identifier::named(STREAM_NAME).unwrap(); let options = TopicCreateOptions { partitions_count: Some(INITIAL_PARTITIONS), @@ -399,7 +402,10 @@ .await .unwrap(); let identity = client.session_identity().await.unwrap(); - client.create_stream(STREAM_NAME).await.unwrap(); + client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); let stream = Identifier::named(STREAM_NAME).unwrap(); let options = TopicCreateOptions { partitions_count: Some(1), @@ -616,7 +622,10 @@ .await .unwrap(); let identity = client.session_identity().await.unwrap(); - client.create_stream(STREAM_NAME).await.unwrap(); + client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); let stream = Identifier::named(STREAM_NAME).unwrap(); let options = TopicCreateOptions { partitions_count: Some(1), @@ -874,7 +883,10 @@ .login_user(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD) .await .unwrap(); - client.create_stream(STREAM_NAME).await.unwrap(); + client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); let stream = Identifier::named(STREAM_NAME).unwrap(); let topic = Identifier::named(TOPIC_NAME).unwrap(); let partitions = u32::try_from(harness.cluster_size()).unwrap(); @@ -1103,7 +1115,7 @@ .await .expect("root client on the metadata leader"); setup - .create_stream(STREAM_NAME) + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) .await .expect("create stream"); let stream_id = Identifier::named(STREAM_NAME).expect("stream identifier"); @@ -1183,7 +1195,9 @@ root.login_user(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD) .await .unwrap(); - root.create_stream(STREAM_NAME).await.unwrap(); + root.create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); let stream = Identifier::named(STREAM_NAME).unwrap(); root.create_topic( &stream, @@ -1222,6 +1236,7 @@ global, streams: None, }), + &UserCreateOptions::default(), ) .await .unwrap(); @@ -2306,7 +2321,10 @@ .unwrap(); let stream = Identifier::named(STREAM_NAME).unwrap(); let topic = Identifier::named(TOPIC_NAME).unwrap(); - producer.create_stream(STREAM_NAME).await.unwrap(); + producer + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); producer .create_topic( &stream, @@ -2345,7 +2363,10 @@ // preserves a real durable state instead of fabricating its other fields. for index in 0..METADATA_CHECKPOINT_REQUESTS { producer - .create_stream(&format!("{STREAM_NAME}-{index}")) + .create_stream( + &format!("{STREAM_NAME}-{index}"), + &StreamCreateOptions::default(), + ) .await .expect("fill the metadata journal to trigger its checkpoint"); } @@ -2706,7 +2727,13 @@ ..Default::default() }; member - .create_user(OTHER_USER, OTHER_PASSWORD, UserStatus::Active, None) + .create_user( + OTHER_USER, + OTHER_PASSWORD, + UserStatus::Active, + None, + &UserCreateOptions::default(), + ) .await .unwrap(); let other_user = Identifier::named(OTHER_USER).unwrap(); @@ -3277,6 +3304,7 @@ RELAY_FORGER_PASSWORD, UserStatus::Active, None, + &UserCreateOptions::default(), ) .await .expect("create a user without permissions"); @@ -3486,7 +3514,10 @@ .login_user(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD) .await .unwrap(); - client.create_stream(STREAM_NAME).await.unwrap(); + client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); let stream = Identifier::named(STREAM_NAME).unwrap(); let topic = Identifier::named(TOPIC_NAME).unwrap(); let options = TopicCreateOptions {
diff --git a/core/integration/tests/cluster/partition_state_transfer.rs b/core/integration/tests/cluster/partition_state_transfer.rs index ef16db5..36716ff 100644 --- a/core/integration/tests/cluster/partition_state_transfer.rs +++ b/core/integration/tests/cluster/partition_state_transfer.rs
@@ -413,7 +413,7 @@ /// is what marches `repair_retained_from` forward. async fn seed_topic(client: &IggyClient, segment_size: Option<u64>) { client - .create_stream(STREAM_NAME) + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) .await .expect("create stream"); client
diff --git a/core/integration/tests/cluster/register_forwarding.rs b/core/integration/tests/cluster/register_forwarding.rs index 10d4deb..5614ac0 100644 --- a/core/integration/tests/cluster/register_forwarding.rs +++ b/core/integration/tests/cluster/register_forwarding.rs
@@ -333,7 +333,7 @@ .expect("a backup-dialed login must succeed and settle on the leader"); client - .create_stream(STREAM_NAME) + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) .await .expect("create stream after a backup-dialed login"); let stream_id = Identifier::named(STREAM_NAME).expect("stream identifier");
diff --git a/core/integration/tests/cluster/replica_read_batching.rs b/core/integration/tests/cluster/replica_read_batching.rs index 1f212ca..7744761 100644 --- a/core/integration/tests/cluster/replica_read_batching.rs +++ b/core/integration/tests/cluster/replica_read_batching.rs
@@ -91,7 +91,10 @@ .unwrap(); let stream = Identifier::named(STREAM).unwrap(); let topic = Identifier::named(TOPIC).unwrap(); - client.create_stream(STREAM).await.unwrap(); + client + .create_stream(STREAM, &StreamCreateOptions::default()) + .await + .unwrap(); client .create_topic( &stream,
diff --git a/core/integration/tests/cluster/staggered_bootstrap.rs b/core/integration/tests/cluster/staggered_bootstrap.rs index b3598df..bfedaf7 100644 --- a/core/integration/tests/cluster/staggered_bootstrap.rs +++ b/core/integration/tests/cluster/staggered_bootstrap.rs
@@ -102,7 +102,7 @@ .await .expect("root client on the metadata leader"); let created_stream = setup - .create_stream(STREAM_NAME) + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) .await .expect("create stream"); let stream_id = Identifier::named(STREAM_NAME).expect("stream identifier");
diff --git a/core/integration/tests/cluster/stream_encryption.rs b/core/integration/tests/cluster/stream_encryption.rs new file mode 100644 index 0000000..00e4d03 --- /dev/null +++ b/core/integration/tests/cluster/stream_encryption.rs
@@ -0,0 +1,451 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::time::{Duration, Instant}; + +use iggy::prelude::*; +use iggy_common::ServerKeyContext; +use integration::harness::{TestHarness, disk}; +use integration::iggy_harness; +use tokio::time::sleep; + +const CONFIG_KEY: &str = "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE="; +const OTHER_CONFIG_KEY: &str = "AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA="; +const TOPIC_NAME: &str = "encrypted-cluster-topic"; +const STREAMS: [(&str, Option<u8>); 2] = + [("cluster-own-key", Some(9)), ("cluster-generated", None)]; +const CONVERGE_TIMEOUT: Duration = Duration::from_secs(60); +const POLL_INTERVAL: Duration = Duration::from_millis(250); +const UNBUILT_MARKER: &str = "cannot be built, its key does not open on this node"; + +#[iggy_harness( + cluster_nodes = 3, + server(encryption = "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=") +)] +async fn given_encrypted_streams_when_primary_fails_should_read_after_failover( + harness: &mut TestHarness, +) { + let client = harness.root_client_for_node(0).await.unwrap(); + seed_streams(&client, "before-failover").await; + drop(client); + + let leader = disk::leader_node_index(harness).await; + harness.stop_node(leader).expect("stop the primary"); + let survivors: Vec<usize> = (0..harness.cluster_size()) + .filter(|node| *node != leader) + .collect(); + + let deadline = Instant::now() + CONVERGE_TIMEOUT; + loop { + if let Some(client) = connect_any(harness, &survivors).await + && all_streams_poll(&client, "before-failover").await + { + return; + } + assert!( + Instant::now() < deadline, + "the surviving nodes never served the encrypted streams after the primary stopped" + ); + sleep(POLL_INTERVAL).await; + } +} + +#[iggy_harness( + cluster_nodes = 3, + server(encryption = "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=") +)] +async fn given_empty_replacement_node_when_state_transferring_should_unwrap_stream_keys_before_serving( + harness: &mut TestHarness, +) { + let client = harness.root_client_for_node(0).await.unwrap(); + seed_streams(&client, "before-rejoin").await; + let _seed_client = client; + + harness + .restart_node_from_clean_slate(2) + .expect("clean-slate restart of node 2"); + // With node 1 down, quorum needs the rejoined node, so a send commits only + // once node 2 has opened each stream's key and built its partitions. + harness.stop_node(1).expect("stop node 1"); + let deadline = Instant::now() + CONVERGE_TIMEOUT; + loop { + if let Some(client) = connect_any(harness, &[2, 0]).await + && send_to_all_streams(&client, "after-rejoin").await + && all_streams_poll(&client, "after-rejoin").await + { + break; + } + assert!( + Instant::now() < deadline, + "the rejoined node never served the encrypted streams" + ); + sleep(POLL_INTERVAL).await; + } + assert!(!harness.node(2).stdout_contains(UNBUILT_MARKER)); + + // A replacement holding another key never serves those streams. + harness.restart_node(1).expect("bring node 1 back"); + harness + .node_mut(2) + .add_env("IGGY_ENCRYPTION_KEY", OTHER_CONFIG_KEY); + harness + .restart_node_from_clean_slate(2) + .expect("clean-slate restart of node 2 under another key"); + let deadline = Instant::now() + CONVERGE_TIMEOUT; + while !harness.node(2).stdout_contains(UNBUILT_MARKER) { + assert!( + Instant::now() < deadline, + "a node under another key must refuse to build the encrypted partitions" + ); + sleep(POLL_INTERVAL).await; + } + for stream_id in 0..STREAMS.len() { + let partition_dir = harness + .node(2) + .data_path() + .join(format!("streams/{stream_id}/topics/0/partitions/0")); + assert!( + !partition_dir.exists(), + "a node that cannot open the key must not build partition {stream_id}" + ); + } +} + +#[iggy_harness( + cluster_nodes = 3, + server(encryption = "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=") +)] +async fn given_cluster_when_reading_encrypted_streams_should_report_same_projection_on_every_node( + harness: &mut TestHarness, +) { + let client = harness.root_client_for_node(0).await.unwrap(); + let mut created = Vec::new(); + for (name, seed) in STREAMS { + let details = client + .create_stream( + name, + &StreamCreateOptions { + encryption_key: seed.map(stream_key), + ..StreamCreateOptions::default() + }, + ) + .await + .unwrap(); + created.push(projection(&details.options)); + } + let kek = kek_fingerprint(CONFIG_KEY); + for node in 0..harness.cluster_size() { + let client = harness.root_client_for_node(node).await.unwrap(); + for ((name, _), created) in STREAMS.iter().zip(&created) { + let deadline = Instant::now() + CONVERGE_TIMEOUT; + let details = loop { + if let Ok(Some(details)) = + client.get_stream(&Identifier::named(name).unwrap()).await + { + break details; + } + assert!(Instant::now() < deadline, "node {node} never listed {name}"); + sleep(POLL_INTERVAL).await; + }; + assert_eq!( + &projection(&details.options), + created, + "node {node}, {name}" + ); + assert_eq!(details.encryption_kek_fingerprints(), vec![kek.clone()]); + } + } +} + +#[iggy_harness( + cluster_nodes = 3, + server(encryption = "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=") +)] +async fn given_replica_that_missed_a_rotation_when_booting_with_the_new_key_should_serve_after_catching_up( + harness: &mut TestHarness, +) { + let client = harness.root_client_for_node(0).await.unwrap(); + seed_streams(&client, "before-rotation").await; + drop(client); + harness.stop_node(2).expect("stop node 2"); + for node in [0, 1] { + harness + .node_mut(node) + .add_env("IGGY_ENCRYPTION_KEY", OTHER_CONFIG_KEY); + harness.node_mut(node).add_env( + "IGGY_ENCRYPTION_PREVIOUS_KEYS", + format!("[\"{CONFIG_KEY}\"]"), + ); + harness.restart_node(node).expect("phase one restart"); + } + let mut rotated = [ + kek_fingerprint(CONFIG_KEY), + kek_fingerprint(OTHER_CONFIG_KEY), + ]; + rotated.sort(); + let deadline = Instant::now() + CONVERGE_TIMEOUT; + loop { + if let Some(client) = connect_any(harness, &[0, 1]).await + && streams_report_kek_fingerprints(&client, &rotated).await + { + break; + } + assert!( + Instant::now() < deadline, + "the rotated nodes never added the new wraps" + ); + sleep(POLL_INTERVAL).await; + } + + // Node 2's own metadata still holds only the old wraps, which its new key + // cannot open until catch-up applies the rewrap. + harness + .node_mut(2) + .add_env("IGGY_ENCRYPTION_KEY", OTHER_CONFIG_KEY); + harness + .restart_node(2) + .expect("a replica behind the rotation boots and catches up"); + // With node 1 down, quorum needs node 2, so a send commits only once node 2 + // opened each stream's key under the new wraps and built its partitions. + harness.stop_node(1).expect("stop node 1"); + let deadline = Instant::now() + CONVERGE_TIMEOUT; + loop { + if let Some(client) = connect_any(harness, &[2, 0]).await + && send_to_all_streams(&client, "after-rotation").await + && all_streams_poll(&client, "after-rotation").await + { + return; + } + assert!( + Instant::now() < deadline, + "the replica behind the rotation never served the encrypted streams" + ); + sleep(POLL_INTERVAL).await; + } +} + +#[iggy_harness( + cluster_nodes = 3, + server(encryption = "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=") +)] +async fn given_encrypted_streams_when_replica_restarts_without_a_key_should_refuse_boot( + harness: &mut TestHarness, +) { + let client = harness.root_client_for_node(0).await.unwrap(); + seed_streams(&client, "before-keyless-restart").await; + drop(client); + let deadline = Instant::now() + CONVERGE_TIMEOUT; + loop { + if let Some(client) = connect_any(harness, &[2]).await + && streams_report_kek_fingerprints(&client, &[kek_fingerprint(CONFIG_KEY)]).await + { + break; + } + assert!( + Instant::now() < deadline, + "node 2 never applied the encrypted streams" + ); + sleep(POLL_INTERVAL).await; + } + + // Catch-up never commits a wrap that opens without a key, so a cluster + // replica refuses like a standalone server. + harness + .node_mut(2) + .add_env("IGGY_ENCRYPTION_ENABLED", "false"); + harness.node_mut(2).add_env("IGGY_ENCRYPTION_KEY", ""); + let error = harness + .restart_node(2) + .expect_err("a replica without a key must refuse boot over encrypted streams"); + if stderr_is_captured() { + let diagnostics = error.to_string(); + let refusal = format!("[{}], this node holds []", kek_fingerprint(CONFIG_KEY)); + assert!( + diagnostics.contains(&refusal), + "the refusal must name the stored and the empty node fingerprints, got: {diagnostics}" + ); + } + + harness + .node_mut(2) + .add_env("IGGY_ENCRYPTION_ENABLED", "true"); + harness + .node_mut(2) + .add_env("IGGY_ENCRYPTION_KEY", CONFIG_KEY); + harness + .restart_node(2) + .expect("node 2 boots again under its key"); + // With node 1 down, quorum needs node 2, so a send commits only once node 2 + // serves the encrypted streams again. + harness.stop_node(1).expect("stop node 1"); + let deadline = Instant::now() + CONVERGE_TIMEOUT; + loop { + if let Some(client) = connect_any(harness, &[2, 0]).await + && send_to_all_streams(&client, "after-keyless-restart").await + && all_streams_poll(&client, "after-keyless-restart").await + { + return; + } + assert!( + Instant::now() < deadline, + "node 2 never served the encrypted streams under its key" + ); + sleep(POLL_INTERVAL).await; + } +} + +async fn seed_streams(client: &IggyClient, payload_tag: &str) { + for (name, seed) in STREAMS { + client + .create_stream( + name, + &StreamCreateOptions { + encryption_key: seed.map(stream_key), + ..StreamCreateOptions::default() + }, + ) + .await + .unwrap(); + client + .create_topic( + &Identifier::named(name).unwrap(), + TOPIC_NAME, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + messages_required_to_save: Some(1), + ..TopicCreateOptions::default() + }, + ) + .await + .unwrap(); + } + assert!(send_to_all_streams(client, payload_tag).await); +} + +async fn send_to_all_streams(client: &IggyClient, payload_tag: &str) -> bool { + for (name, _) in STREAMS { + let mut messages = vec![ + IggyMessage::builder() + .payload(payload(name, payload_tag).into()) + .build() + .unwrap(), + ]; + if client + .send_messages( + &Identifier::named(name).unwrap(), + &Identifier::named(TOPIC_NAME).unwrap(), + &Partitioning::partition_id(0), + &mut messages, + ) + .await + .is_err() + { + return false; + } + } + true +} + +/// Whether every stream's messages poll back and include one tagged `payload_tag`. +async fn all_streams_poll(client: &IggyClient, payload_tag: &str) -> bool { + for (name, _) in STREAMS { + let Ok(polled) = client + .poll_messages( + &Identifier::named(name).unwrap(), + &Identifier::named(TOPIC_NAME).unwrap(), + Some(0), + &Consumer::default(), + &PollingStrategy::offset(0), + 100, + false, + ) + .await + else { + return false; + }; + let expected = payload(name, payload_tag); + if !polled + .messages + .iter() + .any(|message| message.payload.as_ref() == expected.as_bytes()) + { + return false; + } + } + true +} + +async fn streams_report_kek_fingerprints(client: &IggyClient, expected: &[String]) -> bool { + for (name, _) in STREAMS { + let Ok(Some(details)) = client.get_stream(&Identifier::named(name).unwrap()).await else { + return false; + }; + if details.encryption_kek_fingerprints() != expected { + return false; + } + } + true +} + +async fn connect_any(harness: &TestHarness, nodes: &[usize]) -> Option<IggyClient> { + for &node in nodes { + if let Ok(builder) = harness.node(node).tcp_client() + && let Ok(client) = builder.with_root_login().connect().await + { + return Some(client); + } + } + None +} + +fn projection(options: &ResourceOptions) -> Vec<(String, Option<String>)> { + [ + OptionKey::EncryptionKeySource, + OptionKey::EncryptionKeyFingerprint, + OptionKey::EncryptionKekFingerprint, + ] + .into_iter() + .map(|key| { + ( + key.to_string(), + options + .get(&HeaderKey::from(key)) + .and_then(|option| option.value.as_str().ok().map(str::to_owned)), + ) + }) + .collect() +} + +fn kek_fingerprint(key: &str) -> String { + ServerKeyContext::from_config(key, &[], true) + .unwrap() + .kek_fingerprint() + .unwrap() +} + +fn stream_key(seed: u8) -> StreamEncryptionKey { + StreamEncryptionKey::try_from([seed; 32].as_slice()).unwrap() +} + +fn payload(stream: &str, tag: &str) -> String { + format!("{tag}-{stream}") +} + +fn stderr_is_captured() -> bool { + std::env::var("IGGY_TEST_VERBOSE").is_err() +}
diff --git a/core/integration/tests/cluster/stream_key_rotation.rs b/core/integration/tests/cluster/stream_key_rotation.rs new file mode 100644 index 0000000..02ce357 --- /dev/null +++ b/core/integration/tests/cluster/stream_key_rotation.rs
@@ -0,0 +1,485 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::BTreeMap; +use std::time::{Duration, Instant}; + +use iggy::prelude::*; +use iggy_common::ServerKeyContext; +use integration::harness::{TestHarness, disk}; +use integration::iggy_harness; +use tokio::time::sleep; + +const OLD_KEY: &str = "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE="; +const NEW_KEY: &str = "AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA="; +const OLD_KEK_FINGERPRINT: &str = "39e4f0f95c3693c7"; +const TOPIC_NAME: &str = "rotated-topic"; +const STREAMS: [(&str, Option<u8>); 2] = + [("rotated-own-key", Some(4)), ("rotated-generated", None)]; +const MID_ROLLOUT_STREAM: &str = "created-mid-rollout"; +const DEFERRED_STREAM_MARKER: &str = + "is not served on this node until metadata catch-up commits a wrap its keys open"; +const CONVERGE_TIMEOUT: Duration = Duration::from_secs(90); +// Several rewrapper ticks, long enough for a primary to strip a wrap if it would. +const HOLD_WINDOW: Duration = Duration::from_secs(20); +const POLL_INTERVAL: Duration = Duration::from_millis(250); +const OLD_ONLY: KeyConfig = KeyConfig { + key: OLD_KEY, + previous_keys: &[], + retired_kek_fingerprints: &[], +}; +const PHASE_ONE: KeyConfig = KeyConfig { + key: NEW_KEY, + previous_keys: &[OLD_KEY], + retired_kek_fingerprints: &[], +}; +const PHASE_TWO: KeyConfig = KeyConfig { + key: NEW_KEY, + previous_keys: &[], + retired_kek_fingerprints: &[OLD_KEK_FINGERPRINT], +}; +const NEW_ONLY: KeyConfig = KeyConfig { + key: NEW_KEY, + previous_keys: &[], + retired_kek_fingerprints: &[], +}; + +/// One node's `[encryption]` keys, applied through the environment. +struct KeyConfig { + key: &'static str, + previous_keys: &'static [&'static str], + retired_kek_fingerprints: &'static [&'static str], +} + +#[iggy_harness( + cluster_nodes = 3, + server(encryption = "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=") +)] +async fn given_phase_one_rollout_when_rotating_should_add_new_wraps_and_keep_serving( + harness: &mut TestHarness, +) { + seed(harness).await; + let segments_before = segment_files(harness, 0); + + roll_out(harness, &PHASE_ONE).await; + wait_for_kek_fingerprints(harness, &STREAMS.map(|(name, _)| name), &[OLD_KEY, NEW_KEY]).await; + + assert_eq!( + segment_files(harness, 0), + segments_before, + "a rewrap must not touch message data" + ); +} + +#[iggy_harness( + cluster_nodes = 3, + server(encryption = "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=") +)] +async fn given_phase_two_rollout_when_rotating_should_drop_old_wraps(harness: &mut TestHarness) { + let streams = STREAMS.map(|(name, _)| name); + seed(harness).await; + roll_out(harness, &PHASE_ONE).await; + wait_for_kek_fingerprints(harness, &streams, &[OLD_KEY, NEW_KEY]).await; + + roll_out(harness, &PHASE_TWO).await; + wait_for_kek_fingerprints(harness, &streams, &[NEW_KEY]).await; + + roll_out(harness, &NEW_ONLY).await; + let client = serving_client(harness).await; + for name in streams { + let polled = poll_all(&client, name) + .await + .expect("the new key reads old messages"); + assert_eq!(polled[0].payload.as_ref(), payload(name).as_bytes()); + } +} + +#[iggy_harness( + cluster_nodes = 3, + server(encryption = "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=") +)] +async fn given_stream_created_on_rotated_primary_when_old_nodes_serve_and_restart_should_open_it( + harness: &mut TestHarness, +) { + seed(harness).await; + let rotated = (leader(harness).await + 1) % harness.cluster_size(); + configure(harness, rotated, &PHASE_ONE); + harness + .restart_node(rotated) + .expect("restart a backup on phase 1"); + make_leader(harness, rotated).await; + + // The primary wraps a binary create, and only one of three nodes holds the + // new key, so the send commits only once an old node opens the stream. + create_with_message(&serving_client(harness).await, MID_ROLLOUT_STREAM, None).await; + wait_for_kek_fingerprints(harness, &[MID_ROLLOUT_STREAM], &[OLD_KEY, NEW_KEY]).await; + + let old = (rotated + 1) % harness.cluster_size(); + harness.restart_node(old).expect("restart an old node"); + let polled = poll_all(&node_client(harness, old).await, MID_ROLLOUT_STREAM) + .await + .expect("serves the stream created mid-rollout"); + assert_eq!( + polled[0].payload.as_ref(), + payload(MID_ROLLOUT_STREAM).as_bytes() + ); + assert!( + !harness.node(old).stdout_contains(DEFERRED_STREAM_MARKER), + "a node on the old key must open every stream at boot" + ); +} + +#[iggy_harness( + cluster_nodes = 3, + server(encryption = "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=") +)] +async fn given_old_config_primary_when_sets_hold_new_wraps_should_keep_them( + harness: &mut TestHarness, +) { + let streams = STREAMS.map(|(name, _)| name); + seed(harness).await; + roll_out(harness, &PHASE_ONE).await; + wait_for_kek_fingerprints(harness, &streams, &[OLD_KEY, NEW_KEY]).await; + + roll_out(harness, &OLD_ONLY).await; + + hold_kek_fingerprints(harness, &streams, &[OLD_KEY, NEW_KEY]).await; +} + +#[iggy_harness( + cluster_nodes = 3, + server(encryption = "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=") +)] +async fn given_new_key_only_before_phase_one_when_booting_in_a_cluster_should_serve_no_old_stream( + harness: &mut TestHarness, +) { + seed(harness).await; + harness.stop_node(2).expect("stop node 2"); + configure(harness, 2, &NEW_ONLY); + harness + .restart_node(2) + .expect("a cluster node defers the streams it cannot open"); + + let deadline = Instant::now() + CONVERGE_TIMEOUT; + while harness.node(2).stdout_occurrences(DEFERRED_STREAM_MARKER) < STREAMS.len() { + assert!( + Instant::now() < deadline, + "node 2 must report every stream it cannot open" + ); + sleep(POLL_INTERVAL).await; + } + let diagnostics = harness.node(2).stdout_plain(); + for key in [OLD_KEY, NEW_KEY] { + let fingerprint = kek_fingerprint(key); + assert!( + diagnostics.contains(&fingerprint), + "the diagnostics must name {fingerprint}" + ); + } + for (name, _) in STREAMS { + let polled = poll_all(&serving_client(harness).await, name) + .await + .expect("the nodes on the old key keep serving"); + assert_eq!(polled[0].payload.as_ref(), payload(name).as_bytes()); + } +} + +#[iggy_harness( + cluster_nodes = 3, + server(encryption = "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=") +)] +async fn given_rewrap_when_persisting_should_store_identical_metadata_on_every_replica( + harness: &mut TestHarness, +) { + seed(harness).await; + roll_out(harness, &PHASE_ONE).await; + wait_for_kek_fingerprints(harness, &STREAMS.map(|(name, _)| name), &[OLD_KEY, NEW_KEY]).await; + + let data_paths: Vec<_> = harness + .all_servers() + .iter() + .map(|server| server.data_path()) + .collect(); + disk::wait_for_log_convergence(&data_paths).await; + harness.stop().await.unwrap(); + disk::assert_replica_data_identical(&data_paths, true); +} + +async fn seed(harness: &TestHarness) { + let client = harness.root_client_for_node(0).await.unwrap(); + assert_eq!(kek_fingerprint(OLD_KEY), OLD_KEK_FINGERPRINT); + for (name, seed) in STREAMS { + create_with_message(&client, name, seed).await; + } + let data_paths: Vec<_> = harness + .all_servers() + .iter() + .map(|server| server.data_path()) + .collect(); + disk::wait_for_log_convergence(&data_paths).await; +} + +async fn create_with_message(client: &IggyClient, name: &str, seed: Option<u8>) { + client + .create_stream( + name, + &StreamCreateOptions { + encryption_key: seed + .map(|seed| StreamEncryptionKey::try_from([seed; 32].as_slice()).unwrap()), + ..StreamCreateOptions::default() + }, + ) + .await + .unwrap(); + client + .create_topic( + &Identifier::named(name).unwrap(), + TOPIC_NAME, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + messages_required_to_save: Some(1), + ..TopicCreateOptions::default() + }, + ) + .await + .unwrap(); + let mut messages = vec![ + IggyMessage::builder() + .payload(payload(name).into()) + .build() + .unwrap(), + ]; + client + .send_messages( + &Identifier::named(name).unwrap(), + &Identifier::named(TOPIC_NAME).unwrap(), + &Partitioning::partition_id(0), + &mut messages, + ) + .await + .unwrap(); +} + +fn configure(harness: &mut TestHarness, node: usize, config: &KeyConfig) { + let list = |values: &[&str]| { + let quoted: Vec<String> = values.iter().map(|value| format!("\"{value}\"")).collect(); + format!("[{}]", quoted.join(",")) + }; + let server = harness.node_mut(node); + server.add_env("IGGY_ENCRYPTION_KEY", config.key); + server.add_env("IGGY_ENCRYPTION_PREVIOUS_KEYS", list(config.previous_keys)); + server.add_env( + "IGGY_ENCRYPTION_RETIRED_KEK_FINGERPRINTS", + list(config.retired_kek_fingerprints), + ); +} + +/// Restart every node in turn under `config`, checking after each restart +/// that the encrypted streams still serve. +async fn roll_out(harness: &mut TestHarness, config: &KeyConfig) { + for node in 0..harness.cluster_size() { + configure(harness, node, config); + harness.restart_node(node).expect("rolling restart"); + let client = serving_client(harness).await; + for (name, _) in STREAMS { + let polled = poll_all(&client, name) + .await + .expect("serves during the rollout"); + assert_eq!(polled[0].payload.as_ref(), payload(name).as_bytes()); + } + } +} + +async fn wait_for_kek_fingerprints(harness: &TestHarness, streams: &[&str], keys: &[&str]) { + let expected = expected_fingerprints(keys); + let deadline = Instant::now() + CONVERGE_TIMEOUT; + while reported_fingerprints(harness, streams).await + != vec![Some(expected.clone()); streams.len()] + { + assert!( + Instant::now() < deadline, + "streams never reported KEK fingerprints {expected:?}" + ); + sleep(POLL_INTERVAL).await; + } +} + +async fn hold_kek_fingerprints(harness: &TestHarness, streams: &[&str], keys: &[&str]) { + let expected = expected_fingerprints(keys); + let deadline = Instant::now() + HOLD_WINDOW; + let mut reads = 0; + while Instant::now() < deadline { + for fingerprints in reported_fingerprints(harness, streams) + .await + .into_iter() + .flatten() + { + assert_eq!( + fingerprints, expected, + "a primary dropped a wrap no retired fingerprint names" + ); + reads += 1; + } + sleep(POLL_INTERVAL).await; + } + assert!(reads > 0, "no stream was readable during the hold window"); +} + +async fn reported_fingerprints( + harness: &TestHarness, + streams: &[&str], +) -> Vec<Option<Vec<String>>> { + let client = serving_client(harness).await; + let mut reported = Vec::with_capacity(streams.len()); + for name in streams { + reported.push( + client + .get_stream(&Identifier::named(name).unwrap()) + .await + .ok() + .flatten() + .map(|details| details.encryption_kek_fingerprints()), + ); + } + reported +} + +fn expected_fingerprints(keys: &[&str]) -> Vec<String> { + let mut expected: Vec<String> = keys.iter().map(|key| kek_fingerprint(key)).collect(); + expected.sort(); + expected +} + +async fn serving_client(harness: &TestHarness) -> IggyClient { + let deadline = Instant::now() + CONVERGE_TIMEOUT; + loop { + for node in 0..harness.cluster_size() { + if let Some(client) = try_node_client(harness, node).await { + return client; + } + } + assert!( + Instant::now() < deadline, + "no node served the encrypted streams" + ); + sleep(POLL_INTERVAL).await; + } +} + +/// The index of the metadata leader, once a serving node names one. +async fn leader(harness: &TestHarness) -> usize { + let deadline = Instant::now() + CONVERGE_TIMEOUT; + loop { + let roster = serving_client(harness).await.get_cluster_metadata().await; + let leader_port = roster.ok().and_then(|roster| { + roster + .nodes + .iter() + .find(|node| node.role == ClusterNodeRole::Leader) + .map(|node| node.endpoints.tcp) + }); + if let Some(index) = (0..harness.cluster_size()).find(|index| { + leader_port.is_some_and(|port| { + harness + .node(*index) + .tcp_addr() + .is_some_and(|address| address.port() == port) + }) + }) { + return index; + } + assert!(Instant::now() < deadline, "no metadata leader was elected"); + sleep(POLL_INTERVAL).await; + } +} + +/// Restart whichever node leads until `target` does. A view change hands +/// leadership to the next replica in order, so this takes at most one +/// restart per other node. +async fn make_leader(harness: &mut TestHarness, target: usize) { + for _ in 0..harness.cluster_size() { + let current = leader(harness).await; + if current == target { + return; + } + harness + .restart_node(current) + .expect("restart the leader to move leadership"); + } + assert_eq!(leader(harness).await, target, "node {target} never led"); +} + +async fn node_client(harness: &TestHarness, node: usize) -> IggyClient { + let deadline = Instant::now() + CONVERGE_TIMEOUT; + loop { + if let Some(client) = try_node_client(harness, node).await { + return client; + } + assert!( + Instant::now() < deadline, + "node {node} never served the encrypted streams" + ); + sleep(POLL_INTERVAL).await; + } +} + +async fn try_node_client(harness: &TestHarness, node: usize) -> Option<IggyClient> { + let client = harness + .node(node) + .tcp_client() + .ok()? + .with_root_login() + .connect() + .await + .ok()?; + poll_all(&client, STREAMS[0].0).await.ok()?; + Some(client) +} + +async fn poll_all(client: &IggyClient, stream: &str) -> Result<Vec<IggyMessage>, IggyError> { + let polled = client + .poll_messages( + &Identifier::named(stream).unwrap(), + &Identifier::named(TOPIC_NAME).unwrap(), + Some(0), + &Consumer::default(), + &PollingStrategy::offset(0), + 10, + false, + ) + .await?; + if polled.messages.is_empty() { + return Err(IggyError::ResourceNotFound(stream.to_owned())); + } + Ok(polled.messages) +} + +fn segment_files(harness: &TestHarness, node: usize) -> BTreeMap<String, Vec<u8>> { + disk::collect_comparable_files(&harness.node(node).data_path(), false) +} + +fn kek_fingerprint(key: &str) -> String { + ServerKeyContext::from_config(key, &[], true) + .unwrap() + .kek_fingerprint() + .unwrap() +} + +fn payload(stream: &str) -> String { + format!("before-rotation-{stream}") +}
diff --git a/core/integration/tests/connectors/fixtures/mqtt/mod.rs b/core/integration/tests/connectors/fixtures/mqtt/mod.rs index 8a6abc0..770fc20 100644 --- a/core/integration/tests/connectors/fixtures/mqtt/mod.rs +++ b/core/integration/tests/connectors/fixtures/mqtt/mod.rs
@@ -17,7 +17,7 @@ use async_trait::async_trait; use integration::harness::{TestBinaryError, TestFixture}; -use rumqttc::v5::mqttbytes::v5::PublishProperties; +use rumqttc_v5::mqttbytes::v5::PublishProperties; use std::collections::HashMap; mod container;
diff --git a/core/integration/tests/connectors/fixtures/mqtt/publisher.rs b/core/integration/tests/connectors/fixtures/mqtt/publisher.rs index e7d2c0c..0f9c7b4 100644 --- a/core/integration/tests/connectors/fixtures/mqtt/publisher.rs +++ b/core/integration/tests/connectors/fixtures/mqtt/publisher.rs
@@ -15,14 +15,15 @@ // specific language governing permissions and limitations // under the License. -use rumqttc::v5::{ - AsyncClient as Mqtt5Client, Event as Mqtt5Event, Incoming as Mqtt5Incoming, - MqttOptions as Mqtt5Options, - mqttbytes::{QoS as Mqtt5Qos, v5::PublishProperties}, +use rumqttc_v4::{ + AsyncClient as Mqtt311Client, Broker as Mqtt311Broker, Event as Mqtt311Event, + Incoming as Mqtt311Incoming, MqttOptions as Mqtt311Options, Outgoing as Mqtt311Outgoing, + PublishOptions as Mqtt311PublishOptions, mqttbytes::QoS as MqttQos, }; -use rumqttc::{ - AsyncClient as Mqtt311Client, Event as Mqtt311Event, Incoming as Mqtt311Incoming, - MqttOptions as Mqtt311Options, Outgoing, mqttbytes::QoS as Mqtt311Qos, +use rumqttc_v5::{ + AsyncClient as Mqtt5Client, Broker as Mqtt5Broker, Event as Mqtt5Event, + Incoming as Mqtt5Incoming, MqttOptions as Mqtt5Options, Outgoing as Mqtt5Outgoing, + PublishOptions as Mqtt5PublishOptions, mqttbytes::v5::PublishProperties, }; use std::time::Duration; use tokio::time::timeout; @@ -73,7 +74,11 @@ let (client, mut event_loop) = connect_mqtt311(broker_url, username, password).await?; for payload in payloads { client - .publish(topic, qos311(qos)?, false, payload.clone()) + .publish( + topic, + payload.clone(), + Mqtt311PublishOptions::new(mqtt_qos(qos)?), + ) .await .map_err(|error| { format!("failed to enqueue MQTT 3.1.1 batch publish: {error}") @@ -85,7 +90,11 @@ let (client, mut event_loop) = connect_mqtt5(broker_url, username, password).await?; for payload in payloads { client - .publish(topic, qos5(qos)?, false, payload.clone()) + .publish( + topic, + payload.clone(), + Mqtt5PublishOptions::new(mqtt_qos(qos)?), + ) .await .map_err(|error| format!("failed to enqueue MQTT 5 batch publish: {error}"))?; } @@ -107,7 +116,11 @@ let (client, mut event_loop) = connect_mqtt311(broker_url, username, password).await?; for (qos, payload) in messages { client - .publish(topic, qos311(*qos)?, false, payload.clone()) + .publish( + topic, + payload.clone(), + Mqtt311PublishOptions::new(mqtt_qos(*qos)?), + ) .await .map_err(|error| { format!("failed to enqueue mixed MQTT 3.1.1 publish: {error}") @@ -119,7 +132,11 @@ let (client, mut event_loop) = connect_mqtt5(broker_url, username, password).await?; for (qos, payload) in messages { client - .publish(topic, qos5(*qos)?, false, payload.clone()) + .publish( + topic, + payload.clone(), + Mqtt5PublishOptions::new(mqtt_qos(*qos)?), + ) .await .map_err(|error| format!("failed to enqueue mixed MQTT 5 publish: {error}"))?; } @@ -138,7 +155,11 @@ ) -> Result<(), String> { let (client, mut event_loop) = connect_mqtt311(broker_url, username, password).await?; client - .publish(topic, qos311(qos)?, false, payload.to_vec()) + .publish( + topic, + payload.to_vec(), + Mqtt311PublishOptions::new(mqtt_qos(qos)?), + ) .await .map_err(|error| format!("failed to enqueue MQTT 3.1.1 publish: {error}"))?; @@ -155,7 +176,11 @@ ) -> Result<(), String> { let (client, mut event_loop) = connect_mqtt5(broker_url, username, password).await?; client - .publish(topic, qos5(qos)?, false, payload.to_vec()) + .publish( + topic, + payload.to_vec(), + Mqtt5PublishOptions::new(mqtt_qos(qos)?), + ) .await .map_err(|error| format!("failed to enqueue MQTT 5 publish: {error}"))?; @@ -173,33 +198,28 @@ ) -> Result<(), String> { let (client, mut event_loop) = connect_mqtt5(broker_url, username, password).await?; client - .publish_with_properties(topic, qos5(qos)?, false, payload.to_vec(), properties) + .publish( + topic, + payload.to_vec(), + Mqtt5PublishOptions::new(mqtt_qos(qos)?).properties(properties), + ) .await .map_err(|error| format!("failed to enqueue MQTT 5 publish: {error}"))?; wait_for_mqtt5_publishes(&mut event_loop, qos, 1).await } -fn qos311(qos: u8) -> Result<Mqtt311Qos, String> { +fn mqtt_qos(qos: u8) -> Result<MqttQos, String> { match qos { - 0 => Ok(Mqtt311Qos::AtMostOnce), - 1 => Ok(Mqtt311Qos::AtLeastOnce), - 2 => Ok(Mqtt311Qos::ExactlyOnce), - qos => Err(format!("unsupported MQTT QoS: {qos}")), - } -} - -fn qos5(qos: u8) -> Result<Mqtt5Qos, String> { - match qos { - 0 => Ok(Mqtt5Qos::AtMostOnce), - 1 => Ok(Mqtt5Qos::AtLeastOnce), - 2 => Ok(Mqtt5Qos::ExactlyOnce), + 0 => Ok(MqttQos::AtMostOnce), + 1 => Ok(MqttQos::AtLeastOnce), + 2 => Ok(MqttQos::ExactlyOnce), qos => Err(format!("unsupported MQTT QoS: {qos}")), } } async fn wait_for_mqtt311_publishes( - event_loop: &mut rumqttc::EventLoop, + event_loop: &mut rumqttc_v4::EventLoop, qos: u8, expected: usize, ) -> Result<(), String> { @@ -211,7 +231,7 @@ .await .map_err(|error| format!("failed to publish MQTT 3.1.1 batch: {error}"))? { - rumqttc::Event::Outgoing(Outgoing::Publish(_)) if qos == 0 => completed += 1, + Mqtt311Event::Outgoing(Mqtt311Outgoing::Publish(_)) if qos == 0 => completed += 1, Mqtt311Event::Incoming(Mqtt311Incoming::PubAck(_)) if qos == 1 => completed += 1, Mqtt311Event::Incoming(Mqtt311Incoming::PubComp(_)) if qos == 2 => completed += 1, _ => {} @@ -224,7 +244,7 @@ } async fn wait_for_mqtt311_mixed_publishes( - event_loop: &mut rumqttc::EventLoop, + event_loop: &mut rumqttc_v4::EventLoop, messages: &[(u8, Vec<u8>)], ) -> Result<(), String> { // A mixed batch has different broker handshakes, so completion is counted @@ -245,7 +265,7 @@ .await .map_err(|error| format!("failed to publish mixed MQTT 3.1.1 batch: {error}"))? { - rumqttc::Event::Outgoing(Outgoing::Publish(_)) + Mqtt311Event::Outgoing(Mqtt311Outgoing::Publish(_)) if completed_qos0 < expected_qos0 => { completed_qos0 += 1 @@ -270,7 +290,7 @@ } async fn wait_for_mqtt5_publishes( - event_loop: &mut rumqttc::v5::EventLoop, + event_loop: &mut rumqttc_v5::EventLoop, qos: u8, expected: usize, ) -> Result<(), String> { @@ -282,7 +302,7 @@ .await .map_err(|error| format!("failed to publish MQTT 5 batch: {error}"))? { - Mqtt5Event::Outgoing(Outgoing::Publish(_)) if qos == 0 => completed += 1, + Mqtt5Event::Outgoing(Mqtt5Outgoing::Publish(_)) if qos == 0 => completed += 1, Mqtt5Event::Incoming(Mqtt5Incoming::PubAck(_)) if qos == 1 => completed += 1, Mqtt5Event::Incoming(Mqtt5Incoming::PubComp(_)) if qos == 2 => completed += 1, _ => {} @@ -295,7 +315,7 @@ } async fn wait_for_mqtt5_mixed_publishes( - event_loop: &mut rumqttc::v5::EventLoop, + event_loop: &mut rumqttc_v5::EventLoop, messages: &[(u8, Vec<u8>)], ) -> Result<(), String> { // QoS 0, 1, and 2 use different completion packets, so keep separate @@ -316,7 +336,9 @@ .await .map_err(|error| format!("failed to publish mixed MQTT 5 batch: {error}"))? { - Mqtt5Event::Outgoing(Outgoing::Publish(_)) if completed_qos0 < expected_qos0 => { + Mqtt5Event::Outgoing(Mqtt5Outgoing::Publish(_)) + if completed_qos0 < expected_qos0 => + { completed_qos0 += 1 } Mqtt5Event::Incoming(Mqtt5Incoming::PubAck(_)) @@ -342,7 +364,7 @@ broker_url: &str, username: &str, password: &str, -) -> Result<(Mqtt311Client, rumqttc::EventLoop), String> { +) -> Result<(Mqtt311Client, rumqttc_v4::EventLoop), String> { let url = url::Url::parse(broker_url).map_err(|error| format!("invalid MQTT broker URL: {error}"))?; let host = url @@ -353,12 +375,14 @@ .ok_or_else(|| "MQTT broker URL has no port".to_string())?; let mut options = Mqtt311Options::new( format!("iggy-integration-publisher-{}", Uuid::new_v4().simple()), - host, - port, + Mqtt311Broker::tcp(host, port), ); - options.set_credentials(username, password); - options.set_keep_alive(Duration::from_secs(5)); - let (client, mut event_loop) = Mqtt311Client::new(options, 10); + options.set_credentials(username, password.to_owned()); + options.set_keep_alive(5); + let (client, mut event_loop) = Mqtt311Client::builder(options) + .capacity(10) + .try_build() + .map_err(|error| format!("failed to build MQTT client: {error}"))?; timeout(Duration::from_secs(10), async { loop { @@ -381,7 +405,7 @@ broker_url: &str, username: &str, password: &str, -) -> Result<(Mqtt5Client, rumqttc::v5::EventLoop), String> { +) -> Result<(Mqtt5Client, rumqttc_v5::EventLoop), String> { let url = url::Url::parse(broker_url).map_err(|error| format!("invalid MQTT broker URL: {error}"))?; let host = url @@ -392,12 +416,14 @@ .ok_or_else(|| "MQTT broker URL has no port".to_string())?; let mut options = Mqtt5Options::new( format!("iggy-integration-publisher-{}", Uuid::new_v4().simple()), - host, - port, + Mqtt5Broker::tcp(host, port), ); - options.set_credentials(username, password); - options.set_keep_alive(Duration::from_secs(5)); - let (client, mut event_loop) = Mqtt5Client::new(options, 10); + options.set_credentials(username, password.to_owned()); + options.set_keep_alive(5); + let (client, mut event_loop) = Mqtt5Client::builder(options) + .capacity(10) + .try_build() + .map_err(|error| format!("failed to build MQTT client: {error}"))?; timeout(Duration::from_secs(10), async { loop {
diff --git a/core/integration/tests/connectors/mqtt/mqtt_source.rs b/core/integration/tests/connectors/mqtt/mqtt_source.rs index d6fda98..fb57286 100644 --- a/core/integration/tests/connectors/mqtt/mqtt_source.rs +++ b/core/integration/tests/connectors/mqtt/mqtt_source.rs
@@ -25,7 +25,7 @@ use integration::harness::{TestBinary, TestHarness, seeds}; use integration::iggy_harness; use reqwest::Client; -use rumqttc::v5::mqttbytes::v5::PublishProperties; +use rumqttc_v5::mqttbytes::v5::PublishProperties; use serde_json::Value; use std::time::Duration; use std::{collections::HashSet, future::Future};
diff --git a/core/integration/tests/data_integrity/storage_compat.rs b/core/integration/tests/data_integrity/storage_compat.rs index ba9339c..a9f7017 100644 --- a/core/integration/tests/data_integrity/storage_compat.rs +++ b/core/integration/tests/data_integrity/storage_compat.rs
@@ -321,7 +321,10 @@ // 1. The topic carrying the segment chain. The flush thresholds are load // bearing: without them committed messages stay in the journal and the // partition directory looks empty until shutdown. - let data_stream_details = client.create_stream(DATA_STREAM).await.unwrap(); + let data_stream_details = client + .create_stream(DATA_STREAM, &StreamCreateOptions::default()) + .await + .unwrap(); let data_stream = Identifier::numeric(data_stream_details.id).unwrap(); let data_topic_details = create_baseline_topic(&client, &data_stream, DATA_TOPIC, &data_topic_options()) @@ -475,7 +478,10 @@ // 5. A second topic, in its own stream: the only snapshot topic that sends // `compression_algorithm`. - let compression_stream_details = client.create_stream(COMPRESSION_STREAM).await.unwrap(); + let compression_stream_details = client + .create_stream(COMPRESSION_STREAM, &StreamCreateOptions::default()) + .await + .unwrap(); let compression_stream = Identifier::numeric(compression_stream_details.id).unwrap(); create_baseline_topic( &client, @@ -494,6 +500,7 @@ USER_PASSWORD, UserStatus::Active, Some(permissions.clone()), + &UserCreateOptions::default(), ) .await .unwrap(); @@ -516,7 +523,10 @@ // records replayed on top of it. for index in 0..SEED_STREAMS { client - .create_stream(&format!("compat-seed-{index}")) + .create_stream( + &format!("compat-seed-{index}"), + &StreamCreateOptions::default(), + ) .await .unwrap_or_else(|error| panic!("create seed stream {index}: {error}")); } @@ -553,6 +563,7 @@ USER_PASSWORD, UserStatus::Active, Some(tail_permissions.clone()), + &UserCreateOptions::default(), ) .await .unwrap(); @@ -573,7 +584,10 @@ for index in 0..WAL_TAIL_STREAMS { client - .create_stream(&format!("compat-tail-{index}")) + .create_stream( + &format!("compat-tail-{index}"), + &StreamCreateOptions::default(), + ) .await .unwrap_or_else(|error| panic!("create tail stream {index}: {error}")); } @@ -1267,9 +1281,13 @@ partitions_count: seed.partitions_count.map(|_| topic.partitions_count), compression_algorithm: seed .compression_algorithm - .map(|_| topic.compression_algorithm), - message_expiry: seed.message_expiry.map(|_| topic.message_expiry), - max_topic_size: seed.max_topic_size.map(|_| topic.max_topic_size), + .map(|_| topic.resolved_options().compression_algorithm), + message_expiry: seed + .message_expiry + .map(|_| topic.resolved_options().message_expiry), + max_topic_size: seed + .max_topic_size + .map(|_| topic.resolved_options().max_topic_size), segment_size: seed.segment_size.and(recovered.segment_size), durability: recovered.durability, consumer_offset_durability: recovered.consumer_offset_durability,
diff --git a/core/integration/tests/data_integrity/verify_after_server_restart.rs b/core/integration/tests/data_integrity/verify_after_server_restart.rs index 9ed57b6..df290ea 100644 --- a/core/integration/tests/data_integrity/verify_after_server_restart.rs +++ b/core/integration/tests/data_integrity/verify_after_server_restart.rs
@@ -277,9 +277,18 @@ let client = harness.tcp_root_client().await.unwrap(); // Create 3 streams - let stream_0 = client.create_stream("stream-0").await.unwrap(); - let stream_1 = client.create_stream("stream-1").await.unwrap(); - let stream_2 = client.create_stream("stream-2").await.unwrap(); + let stream_0 = client + .create_stream("stream-0", &StreamCreateOptions::default()) + .await + .unwrap(); + let stream_1 = client + .create_stream("stream-1", &StreamCreateOptions::default()) + .await + .unwrap(); + let stream_2 = client + .create_stream("stream-2", &StreamCreateOptions::default()) + .await + .unwrap(); assert_eq!(stream_0.id, 0); assert_eq!(stream_1.id, 1); @@ -372,7 +381,10 @@ // was used instead of vacant_key() to predict the assigned ID. // After deletion: streams are [0, 2], len=2, but vacant_key=1 // Bug: code used len (2) to create directory, but metadata assigned ID 1 - let stream_reused = client.create_stream("stream-reused").await.unwrap(); + let stream_reused = client + .create_stream("stream-reused", &StreamCreateOptions::default()) + .await + .unwrap(); assert_eq!( stream_reused.id, 1, "New stream should reuse deleted ID 1, got {}",
diff --git a/core/integration/tests/data_integrity/verify_auto_commit_offset_replicates.rs b/core/integration/tests/data_integrity/verify_auto_commit_offset_replicates.rs index a2ec465..3de07cc 100644 --- a/core/integration/tests/data_integrity/verify_auto_commit_offset_replicates.rs +++ b/core/integration/tests/data_integrity/verify_auto_commit_offset_replicates.rs
@@ -60,7 +60,10 @@ // Primary (node 0, the view-0 primary): create the topic, produce, then poll // with auto_commit so the offset is committed server-side. let primary = harness.tcp_root_client().await.unwrap(); - primary.create_stream(STREAM_NAME).await.unwrap(); + primary + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); primary .create_topic( &stream,
diff --git a/core/integration/tests/data_integrity/verify_cluster_replica_data_identical.rs b/core/integration/tests/data_integrity/verify_cluster_replica_data_identical.rs index dc8b991..c68afc1 100644 --- a/core/integration/tests/data_integrity/verify_cluster_replica_data_identical.rs +++ b/core/integration/tests/data_integrity/verify_cluster_replica_data_identical.rs
@@ -73,7 +73,10 @@ #[iggy_harness(cluster_nodes = 3)] async fn should_persist_byte_identical_data_across_cluster_replicas(harness: &mut TestHarness) { let client = harness.tcp_root_client().await.unwrap(); - client.create_stream(STREAM_NAME).await.unwrap(); + client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); client .create_topic( &Identifier::named(STREAM_NAME).unwrap(), @@ -175,6 +178,166 @@ disk::assert_replica_data_identical(&data_paths, true); } +const ENCRYPTED_STREAM_NAME: &str = "di-encrypted-stream"; +const ENCRYPTED_PAYLOAD_PREFIX: &str = "server-encrypted-identity-payload"; + +// Server-side encryption happens once on the primary before replication, so +// every replica must persist the same ciphertext and none may hold plaintext. +#[iggy_harness( + cluster_nodes = 3, + server(encryption = "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=") +)] +async fn given_three_nodes_with_server_encryption_when_persisting_should_store_identical_ciphertext( + harness: &mut TestHarness, +) { + let client = harness.tcp_root_client().await.unwrap(); + client + .create_stream(ENCRYPTED_STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); + client + .create_topic( + &Identifier::named(ENCRYPTED_STREAM_NAME).unwrap(), + TOPIC_NAME, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + messages_required_to_save: Some(1), + ..TopicCreateOptions::default() + }, + ) + .await + .unwrap(); + let mut messages: Vec<IggyMessage> = (0..MESSAGES) + .map(|i| { + IggyMessage::builder() + .id(u128::from(i + 1)) + .payload(format!("{ENCRYPTED_PAYLOAD_PREFIX}-{i:04}").into()) + .build() + .unwrap() + }) + .collect(); + client + .send_messages( + &Identifier::named(ENCRYPTED_STREAM_NAME).unwrap(), + &Identifier::named(TOPIC_NAME).unwrap(), + &Partitioning::partition_id(0), + &mut messages, + ) + .await + .unwrap(); + + let data_paths: Vec<PathBuf> = harness + .all_servers() + .iter() + .map(|server| server.data_path()) + .collect(); + assert_eq!(data_paths.len(), CLUSTER_NODES); + disk::wait_for_log_convergence(&data_paths).await; + harness.stop().await.unwrap(); + + disk::assert_replica_data_identical(&data_paths, true); + for data_path in &data_paths { + let files = disk::collect_comparable_files(data_path, true); + assert!( + files.keys().any(|path| path.ends_with(".log")), + "{} holds no segment log", + data_path.display() + ); + for (path, bytes) in files { + assert!( + !bytes + .windows(ENCRYPTED_PAYLOAD_PREFIX.len()) + .any(|window| window == ENCRYPTED_PAYLOAD_PREFIX.as_bytes()), + "{path} on {} holds a plaintext payload", + data_path.display() + ); + } + } +} + +#[iggy_harness( + cluster_nodes = 3, + server(encryption = "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=") +)] +async fn given_three_nodes_with_custom_keys_when_persisting_should_store_identical_ciphertext( + harness: &mut TestHarness, +) { + let streams = [ + ( + "client-key-replicas", + Some(StreamEncryptionKey::try_from([3u8; 32].as_slice()).unwrap()), + ), + ("generated-key-replicas", None), + ]; + let client = harness.tcp_root_client().await.unwrap(); + for (name, encryption_key) in &streams { + client + .create_stream( + name, + &StreamCreateOptions { + encryption_key: encryption_key.clone(), + ..StreamCreateOptions::default() + }, + ) + .await + .unwrap(); + client + .create_topic( + &Identifier::named(name).unwrap(), + TOPIC_NAME, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + messages_required_to_save: Some(1), + ..TopicCreateOptions::default() + }, + ) + .await + .unwrap(); + for batch in 0..4u32 { + let mut messages: Vec<IggyMessage> = (0..MESSAGES / 4) + .map(|i| { + IggyMessage::builder() + .payload(format!("{ENCRYPTED_PAYLOAD_PREFIX}-{name}-{batch}-{i:04}").into()) + .build() + .unwrap() + }) + .collect(); + client + .send_messages( + &Identifier::named(name).unwrap(), + &Identifier::named(TOPIC_NAME).unwrap(), + &Partitioning::partition_id(0), + &mut messages, + ) + .await + .unwrap(); + } + } + + let data_paths: Vec<PathBuf> = harness + .all_servers() + .iter() + .map(|server| server.data_path()) + .collect(); + disk::wait_for_log_convergence(&data_paths).await; + harness.stop().await.unwrap(); + + disk::assert_replica_data_identical(&data_paths, true); + for data_path in &data_paths { + for (path, bytes) in disk::collect_comparable_files(data_path, true) { + assert!( + !bytes + .windows(ENCRYPTED_PAYLOAD_PREFIX.len()) + .any(|window| window == ENCRYPTED_PAYLOAD_PREFIX.as_bytes()), + "{path} on {} holds a plaintext payload", + data_path.display() + ); + } + } +} + /// Drive a concurrent send burst across independent connections so the primary /// pipelines multiple ops at once and backups journal ops ahead of their commit /// frontier. Each client sends its own batches in a loop; the sends race so that
diff --git a/core/integration/tests/data_integrity/verify_consumer_group_partition_assignment.rs b/core/integration/tests/data_integrity/verify_consumer_group_partition_assignment.rs index 618a5c7..24af691 100644 --- a/core/integration/tests/data_integrity/verify_consumer_group_partition_assignment.rs +++ b/core/integration/tests/data_integrity/verify_consumer_group_partition_assignment.rs
@@ -113,7 +113,10 @@ .unwrap(); // 1. Create stream, topic with 3 partitions, consumer group - root_client.create_stream(STREAM_NAME).await.unwrap(); + root_client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); root_client .create_topic( &Identifier::named(STREAM_NAME).unwrap(), @@ -775,7 +778,10 @@ let root_client = harness.root_client().await.unwrap(); // Start with 6 partitions - root_client.create_stream(STREAM_NAME).await.unwrap(); + root_client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); root_client .create_topic( &Identifier::named(STREAM_NAME).unwrap(), @@ -913,7 +919,10 @@ let root_client = harness.root_client().await.unwrap(); // Start with 6 partitions for cleaner math - root_client.create_stream(STREAM_NAME).await.unwrap(); + root_client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); root_client .create_topic( &Identifier::named(STREAM_NAME).unwrap(), @@ -1316,7 +1325,10 @@ let root_client = harness.root_client().await.unwrap(); // Create with 6 partitions - root_client.create_stream(STREAM_NAME).await.unwrap(); + root_client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); root_client .create_topic( &Identifier::named(STREAM_NAME).unwrap(), @@ -1659,7 +1671,10 @@ let root_client = harness.root_client().await.unwrap(); // Create with 6 partitions - root_client.create_stream(STREAM_NAME).await.unwrap(); + root_client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); root_client .create_topic( &Identifier::named(STREAM_NAME).unwrap(), @@ -1943,7 +1958,10 @@ .unwrap(); // 1. Setup - root_client.create_stream(STREAM_NAME).await.unwrap(); + root_client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); root_client .create_topic( &Identifier::named(STREAM_NAME).unwrap(), @@ -2084,7 +2102,10 @@ .expect("Failed to get root client"); // 1. Create stream, topic with 3 partitions, consumer group - root_client.create_stream(STREAM_NAME).await.unwrap(); + root_client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); root_client .create_topic( &Identifier::named(STREAM_NAME).unwrap(), @@ -2286,7 +2307,10 @@ } async fn setup_stream_topic_cg_with_partitions(client: &IggyClient, partitions: u32) { - client.create_stream(STREAM_NAME).await.unwrap(); + client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); client .create_topic( &Identifier::named(STREAM_NAME).unwrap(),
diff --git a/core/integration/tests/data_integrity/verify_no_plaintext_credentials_on_disk.rs b/core/integration/tests/data_integrity/verify_no_plaintext_credentials_on_disk.rs index 3875924..e214fa7 100644 --- a/core/integration/tests/data_integrity/verify_no_plaintext_credentials_on_disk.rs +++ b/core/integration/tests/data_integrity/verify_no_plaintext_credentials_on_disk.rs
@@ -16,10 +16,14 @@ // under the License. use iggy::prelude::*; +use iggy_common::{ + ServerKeyContext, StreamKeyOrigin, WRAPPED_STREAM_KEY_LENGTH, WrappedStreamKey, text, +}; use integration::iggy_harness; use std::fs; use std::path::{Path, PathBuf}; +const CONFIG_KEY: &str = "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE="; const USERNAME: &str = "plaintext-regression-user"; const PLAINTEXT_PASSWORD: &str = "plaintext-password-regression-2943"; const PAT_NAME: &str = "plaintext-regression-pat"; @@ -28,7 +32,13 @@ async fn should_not_persist_plaintext_password_or_pat_to_disk(harness: &mut TestHarness) { let root_client = harness.root_client().await.unwrap(); root_client - .create_user(USERNAME, PLAINTEXT_PASSWORD, UserStatus::Active, None) + .create_user( + USERNAME, + PLAINTEXT_PASSWORD, + UserStatus::Active, + None, + &UserCreateOptions::default(), + ) .await .unwrap(); @@ -49,8 +59,87 @@ assert_secret_not_persisted(&data_path, raw_pat_token, "raw PAT"); } +#[iggy_harness(server( + encryption.enabled = "true", + encryption.key = "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=" +))] +async fn given_stream_keys_when_scanning_state_should_find_only_wrapped_keys( + harness: &mut TestHarness, +) { + let client_key = StreamEncryptionKey::try_from([77u8; 32].as_slice()).unwrap(); + let refused_key = StreamEncryptionKey::try_from([88u8; 32].as_slice()).unwrap(); + let client = harness.root_client().await.unwrap(); + for (name, key) in [ + ("client-key-stream", Some(client_key.clone())), + ("keyless-stream", None), + ] { + client + .create_stream( + name, + &StreamCreateOptions { + encryption_key: key, + ..StreamCreateOptions::default() + }, + ) + .await + .unwrap(); + } + // A failed create carries a key too: it must not be kept either. + let duplicate = client + .create_stream( + "client-key-stream", + &StreamCreateOptions { + encryption_key: Some(refused_key.clone()), + ..StreamCreateOptions::default() + }, + ) + .await; + assert!(duplicate.is_err()); + let data_path = harness.server().data_path(); + drop(client); + harness.stop().await.unwrap(); + + let context = ServerKeyContext::from_config(CONFIG_KEY, &[], true).unwrap(); + let kek = hex_bytes(&context.kek_fingerprint().unwrap()); + let mut stored_keys = Vec::new(); + for origin in [StreamKeyOrigin::Client, StreamKeyOrigin::Server] { + let mut prefix = vec![1u8, origin as u8]; + prefix.extend_from_slice(&kek); + let envelope = collect_files(&data_path) + .iter() + .find_map(|path| { + let contents = fs::read(path).ok()?; + let start = contents + .windows(prefix.len()) + .position(|window| window == prefix.as_slice())?; + contents + .get(start..start + WRAPPED_STREAM_KEY_LENGTH) + .map(<[u8]>::to_vec) + }) + .unwrap_or_else(|| panic!("a stored {origin:?} envelope")); + let wrapped = WrappedStreamKey::try_from(envelope.as_slice()).unwrap(); + stored_keys.push(context.unwrap(&wrapped).expect("the envelope opens")); + } + assert_eq!(stored_keys[0], client_key); + for (index, key) in stored_keys.iter().chain([&refused_key]).enumerate() { + let name = format!("stream key {index}"); + assert_bytes_not_persisted(&data_path, key.as_bytes(), &name); + assert_secret_not_persisted(&data_path, &text::as_base64(key.as_bytes()), &name); + } +} + +fn hex_bytes(hex: &str) -> Vec<u8> { + (0..hex.len()) + .step_by(2) + .map(|index| u8::from_str_radix(&hex[index..index + 2], 16).unwrap()) + .collect() +} + fn assert_secret_not_persisted(root: &Path, secret: &str, secret_name: &str) { - let secret = secret.as_bytes(); + assert_bytes_not_persisted(root, secret.as_bytes(), secret_name); +} + +fn assert_bytes_not_persisted(root: &Path, secret: &[u8], secret_name: &str) { for path in collect_files(root) { let contents = fs::read(&path).unwrap_or_else(|e| { panic!("Failed to read persisted file {}: {e}", path.display());
diff --git a/core/integration/tests/mcp/mod.rs b/core/integration/tests/mcp/mod.rs index 6f0af20..c351d1d 100644 --- a/core/integration/tests/mcp/mod.rs +++ b/core/integration/tests/mcp/mod.rs
@@ -541,6 +541,28 @@ assert!(!token.token.is_empty()); } +#[iggy_harness(server(mcp))] +async fn should_refuse_personal_access_token_with_unparsable_expiry(harness: &TestHarness) { + let mcp_client = harness.mcp_client().await.expect("MCP client required"); + let name = "test_token"; + let arguments = json!({"name": name, "expiry": "tomorrow"}); + let params = CallToolRequestParams::new("create_personal_access_token".to_owned()) + .with_arguments(arguments.as_object().expect("Tool arguments").clone()); + let result = mcp_client.call_tool(params).await; + + assert!( + matches!(&result, Err(ServiceError::McpError(error)) + if error.message.starts_with("Failed to parse personal access token expiry")), + "Expected an expiry parse error, got {result:?}" + ); + let tokens: Vec<PersonalAccessTokenInfo> = + invoke(&mcp_client, "get_personal_access_tokens", None).await; + assert!( + tokens.iter().all(|token| token.name != name), + "A refused expiry must not create a token: {tokens:?}" + ); +} + #[iggy_harness(server(mcp), seed = seeds::mcp_standard)] async fn should_delete_personal_access_token(harness: &TestHarness) { let mcp_client = harness.mcp_client().await.expect("MCP client required");
diff --git a/core/integration/tests/sdk/consumer_group.rs b/core/integration/tests/sdk/consumer_group.rs index 25dc105..ba802e4 100644 --- a/core/integration/tests/sdk/consumer_group.rs +++ b/core/integration/tests/sdk/consumer_group.rs
@@ -60,7 +60,10 @@ streams: None, }; - root_client.create_stream(STREAM_NAME).await.unwrap(); + root_client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); root_client .create_topic( &stream_id, @@ -83,6 +86,7 @@ CONSUMER_PASSWORD, UserStatus::Active, Some(consumer_permissions.clone()), + &UserCreateOptions::default(), ) .await .unwrap(); @@ -191,7 +195,10 @@ let stream_id = Identifier::named(STREAM_NAME).unwrap(); let topic_id = Identifier::named(TOPIC_NAME).unwrap(); - root_client.create_stream(STREAM_NAME).await.unwrap(); + root_client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); root_client .create_topic( &stream_id,
diff --git a/core/integration/tests/sdk/consumer_group_membership.rs b/core/integration/tests/sdk/consumer_group_membership.rs index dbf32ac..ae1a89e 100644 --- a/core/integration/tests/sdk/consumer_group_membership.rs +++ b/core/integration/tests/sdk/consumer_group_membership.rs
@@ -77,7 +77,10 @@ let admin = clients.remove(0); let consumer_client = clients.remove(0); - admin.create_stream(STREAM_NAME).await.unwrap(); + admin + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); admin .create_topic( &stream_id, @@ -183,7 +186,10 @@ let first_member = clients.remove(0); let second_member = clients.remove(0); - first_member.create_stream(STREAM_NAME).await.unwrap(); + first_member + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); first_member .create_topic( &stream_id, @@ -268,7 +274,10 @@ .login_user(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD) .await .unwrap(); - client.create_stream(STREAM_NAME).await.unwrap(); + client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); client .create_topic( &stream_id, @@ -351,7 +360,10 @@ "join with a missing stream", ); - root_client.create_stream(STREAM_NAME).await.unwrap(); + root_client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); assert_rejected( root_client .join_consumer_group(&stream_id, &missing, &group_id) @@ -434,7 +446,10 @@ .login_user(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD) .await .unwrap(); - client.create_stream(STREAM_NAME).await.unwrap(); + client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); client .create_topic( &stream_id, @@ -533,7 +548,10 @@ let topic_id = Identifier::named(TOPIC_NAME).unwrap(); let group_id = Identifier::named(CONSUMER_GROUP_NAME).unwrap(); let consumer = Consumer::group(group_id.clone()); - predecessor.create_stream(STREAM_NAME).await.unwrap(); + predecessor + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); predecessor .create_topic( &stream_id,
diff --git a/core/integration/tests/sdk/consumer_offset.rs b/core/integration/tests/sdk/consumer_offset.rs index 6b49f63..ecf03b2 100644 --- a/core/integration/tests/sdk/consumer_offset.rs +++ b/core/integration/tests/sdk/consumer_offset.rs
@@ -53,7 +53,10 @@ let topic_id = Identifier::named(TOPIC_NAME).unwrap(); let group_id = Identifier::named(CONSUMER_GROUP_NAME).unwrap(); - client.create_stream(STREAM_NAME).await.unwrap(); + client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); client .create_topic(&stream_id, TOPIC_NAME, &TopicCreateOptions::default()) .await @@ -209,7 +212,10 @@ let stream_id = Identifier::named(STREAM_NAME).unwrap(); let topic_id = Identifier::named(TOPIC_NAME).unwrap(); - client.create_stream(STREAM_NAME).await.unwrap(); + client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); client .create_topic(&stream_id, TOPIC_NAME, &TopicCreateOptions::default()) .await @@ -321,7 +327,10 @@ let stream_id = Identifier::named(STREAM_NAME).unwrap(); let topic_id = Identifier::named(TOPIC_NAME).unwrap(); - client.create_stream(STREAM_NAME).await.unwrap(); + client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); client .create_topic( &stream_id,
diff --git a/core/integration/tests/sdk/consumer_shutdown.rs b/core/integration/tests/sdk/consumer_shutdown.rs index 172a569..478878b 100644 --- a/core/integration/tests/sdk/consumer_shutdown.rs +++ b/core/integration/tests/sdk/consumer_shutdown.rs
@@ -39,7 +39,10 @@ let stream_id = Identifier::named(STREAM_NAME).unwrap(); let topic_id = Identifier::named(TOPIC_NAME).unwrap(); - client.create_stream(STREAM_NAME).await.unwrap(); + client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); client .create_topic( &stream_id,
diff --git a/core/integration/tests/sdk/options.rs b/core/integration/tests/sdk/options.rs index 1095c59..44959e0 100644 --- a/core/integration/tests/sdk/options.rs +++ b/core/integration/tests/sdk/options.rs
@@ -16,28 +16,94 @@ // under the License. use iggy::prelude::*; -use iggy_common::{OptionsScope, TOPIC_OPTION_KEYS, topic_option_keys}; +use iggy_common::{OptionsScope, descriptors_for, topic_option_keys}; use integration::harness::{backup_commit_budget, wait_for_topic}; use integration::iggy_harness; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::str::FromStr; +#[iggy_harness(test_client_transport = [Tcp, Http])] +async fn given_manage_users_caller_when_updating_root_should_return_4108_and_keep_root_active( + harness: &TestHarness, +) { + let root = harness.root_client().await.expect("root client"); + let root_id = Identifier::numeric(0).expect("root id"); + root.create_user( + "manager", + "secret123", + UserStatus::Active, + Some(Permissions { + global: GlobalPermissions { + manage_users: true, + ..GlobalPermissions::default() + }, + streams: None, + }), + &UserCreateOptions::default(), + ) + .await + .expect("create manager"); + let manager = harness.new_client().await.expect("manager client"); + manager + .login_user("manager", "secret123") + .await + .expect("manager login"); + + for patch in [ + UserUpdateOptions { + username: Some("root-renamed".to_owned()), + ..UserUpdateOptions::default() + }, + UserUpdateOptions { + status: Some(UserStatus::Inactive), + ..UserUpdateOptions::default() + }, + ] { + let error = manager + .update_user(&root_id, &patch) + .await + .expect_err("manager cannot rename or deactivate root"); + assert_policy_error(error, IggyError::CannotModifyRootUser); + } + + let unchanged = root + .get_user(&root_id) + .await + .expect("read root") + .expect("root exists"); + assert_eq!(unchanged.username, "iggy"); + assert_eq!(unchanged.status, UserStatus::Active); +} + #[iggy_harness] async fn given_topic_scope_when_describing_options_should_list_the_catalog(harness: &TestHarness) { let client = harness.root_client().await.unwrap(); let specs = client.describe_options(OptionsScope::Topic).await.unwrap(); - let mut served: Vec<&str> = specs.iter().map(|spec| spec.key.as_str()).collect(); - served.sort_unstable(); - let mut accepted: Vec<&str> = TOPIC_OPTION_KEYS.to_vec(); - accepted.sort_unstable(); + let served: BTreeSet<&str> = specs.iter().map(|spec| spec.key.as_str()).collect(); + let catalog: BTreeSet<String> = descriptors_for(OptionsScope::Topic) + .filter(|descriptor| descriptor.is_described()) + .map(|descriptor| descriptor.key.to_string()) + .collect(); // Exact equality, not per-key contains: the catalog is the only way a - // client discovers a key, and create rejects anything outside it, so a key - // accepted but unlisted is undiscoverable and one listed but unaccepted is - // a lie. - assert_eq!(served, accepted, "catalog must match the accepted key set"); - // `partitions_count` is deliberately absent: it is a fixed field of the - // CreateTopic command, not a topic setting. - assert!(!served.contains(&"partitions_count")); + // client discovers a key, so a key accepted but unlisted is undiscoverable + // and one listed but unaccepted is a lie. + assert_eq!( + served, + catalog.iter().map(String::as_str).collect::<BTreeSet<_>>() + ); + // `partitions_count` is a fixed field of the CreateTopic command, not a + // topic setting, and internal bookkeeping keys are never described. + assert!(!served.contains("partitions_count")); + assert!(!served.contains("critical_keys")); + let flags = |key: &str| { + let spec = specs.iter().find(|spec| spec.key == key).unwrap(); + (spec.updatable, spec.resettable, spec.secret, spec.read_only) + }; + assert_eq!(flags("name"), (true, false, false, false)); + assert_eq!(flags("message_expiry"), (true, true, false, false)); + assert_eq!(flags("max_topic_size"), (true, true, false, false)); + assert_eq!(flags("compression_algorithm"), (false, false, false, false)); + assert_eq!(flags("segment_size"), (false, false, false, false)); } #[iggy_harness] @@ -45,7 +111,10 @@ harness: &TestHarness, ) { let client = harness.root_client().await.unwrap(); - client.create_stream("knob-stream").await.unwrap(); + client + .create_stream("knob-stream", &StreamCreateOptions::default()) + .await + .unwrap(); let stream = Identifier::named("knob-stream").unwrap(); // Zero is rejected: a flush threshold of zero never trips. @@ -133,16 +202,40 @@ } #[iggy_harness] -async fn given_stream_scope_when_describing_options_should_be_empty(harness: &TestHarness) { +async fn given_stream_and_user_scopes_when_describing_options_should_report_mutability( + harness: &TestHarness, +) { let client = harness.root_client().await.unwrap(); - let specs = client.describe_options(OptionsScope::Stream).await.unwrap(); - assert!(specs.is_empty(), "streams have no catalog keys yet"); + for scope in [OptionsScope::Stream, OptionsScope::User] { + let specs = client.describe_options(scope).await.unwrap(); + let served: BTreeSet<String> = specs.iter().map(|spec| spec.key.clone()).collect(); + let catalog: BTreeSet<String> = descriptors_for(scope) + .filter(|descriptor| descriptor.is_described()) + .map(|descriptor| descriptor.key.to_string()) + .collect(); + assert_eq!(served, catalog, "{scope}"); + } + let stream_specs = client.describe_options(OptionsScope::Stream).await.unwrap(); + let flags = |key: &str| { + let spec = stream_specs.iter().find(|spec| spec.key == key).unwrap(); + (spec.updatable, spec.resettable, spec.secret, spec.read_only) + }; + assert_eq!(flags("name"), (true, false, false, false)); + assert_eq!(flags("encryption_key"), (false, false, true, false)); + assert_eq!(flags("encryption_key_source"), (false, false, false, true)); + assert_eq!( + flags("encryption_key_fingerprint"), + (false, false, false, true) + ); } #[iggy_harness] async fn given_explicit_segment_size_when_creating_topic_should_roll_at_it(harness: &TestHarness) { let client = harness.root_client().await.unwrap(); - client.create_stream("seg-stream").await.unwrap(); + client + .create_stream("seg-stream", &StreamCreateOptions::default()) + .await + .unwrap(); let stream = Identifier::named("seg-stream").unwrap(); // Below the floor: denied with the key name before consensus. @@ -228,7 +321,10 @@ #[iggy_harness] async fn given_explicit_expiry_when_creating_topic_should_echo_provenance(harness: &TestHarness) { let client = harness.root_client().await.unwrap(); - client.create_stream("options-stream").await.unwrap(); + client + .create_stream("options-stream", &StreamCreateOptions::default()) + .await + .unwrap(); client .create_topic( &Identifier::named("options-stream").unwrap(), @@ -275,7 +371,10 @@ #[iggy_harness] async fn given_update_options_when_updating_topic_should_patch_not_replace(harness: &TestHarness) { let client = harness.root_client().await.unwrap(); - client.create_stream("update-stream").await.unwrap(); + client + .create_stream("update-stream", &StreamCreateOptions::default()) + .await + .unwrap(); let stream = Identifier::named("update-stream").unwrap(); let topic = Identifier::named("update-topic").unwrap(); @@ -299,8 +398,8 @@ .update_topic( &stream, &topic, - "update-topic", &TopicUpdateOptions { + name: Some("update-topic".to_owned()), raw: BTreeMap::from([( topic_option_keys::SEGMENT_SIZE.to_string(), "2MiB".to_string(), @@ -315,8 +414,10 @@ .update_topic( &stream, &topic, - "update-topic", - &TopicUpdateOptions::default(), + &TopicUpdateOptions { + name: Some("update-topic".to_owned()), + ..TopicUpdateOptions::default() + }, ) .await .unwrap(); @@ -352,17 +453,20 @@ #[iggy_harness] async fn given_unknown_key_when_updating_stream_or_user_should_reject(harness: &TestHarness) { let client = harness.root_client().await.unwrap(); - client.create_stream("opt-stream").await.unwrap(); + client + .create_stream("opt-stream", &StreamCreateOptions::default()) + .await + .unwrap(); - // Streams and users have no catalog keys yet, so the update blocks exist - // as the extension point and reject every key by name rather than storing - // one nothing reads. + // A key outside the catalog is refused by name rather than stored where + // nothing reads it. let stream_rejected = client .update_stream( &Identifier::named("opt-stream").unwrap(), - "opt-stream", &StreamUpdateOptions { + name: Some("opt-stream".to_owned()), raw: BTreeMap::from([("future_key".to_string(), "1".to_string())]), + ..StreamUpdateOptions::default() }, ) .await; @@ -372,24 +476,32 @@ client .update_stream( &Identifier::named("opt-stream").unwrap(), - "opt-stream-renamed", - &StreamUpdateOptions::default(), + &StreamUpdateOptions { + name: Some("opt-stream-renamed".to_owned()), + ..StreamUpdateOptions::default() + }, ) .await .unwrap(); client - .create_user("opt-user", "secret123", UserStatus::Active, None) + .create_user( + "opt-user", + "secret123", + UserStatus::Active, + None, + &UserCreateOptions::default(), + ) .await .unwrap(); let user = Identifier::named("opt-user").unwrap(); let user_rejected = client .update_user( &user, - Some("opt-user"), - None, &UserUpdateOptions { + username: Some("opt-user".to_owned()), raw: BTreeMap::from([("future_key".to_string(), "1".to_string())]), + ..UserUpdateOptions::default() }, ) .await; @@ -398,9 +510,10 @@ client .update_user( &user, - Some("opt-user-renamed"), - None, - &UserUpdateOptions::default(), + &UserUpdateOptions { + username: Some("opt-user-renamed".to_owned()), + ..UserUpdateOptions::default() + }, ) .await .unwrap(); @@ -411,7 +524,10 @@ harness: &TestHarness, ) { let client = harness.root_client().await.unwrap(); - client.create_stream("sentinel-stream").await.unwrap(); + client + .create_stream("sentinel-stream", &StreamCreateOptions::default()) + .await + .unwrap(); let stream = Identifier::named("sentinel-stream").unwrap(); // 0 means "resolve the server default", not "expire immediately" / "no @@ -463,7 +579,10 @@ #[iggy_harness] async fn given_rename_only_when_updating_topic_should_leave_settings_alone(harness: &TestHarness) { let client = harness.root_client().await.unwrap(); - client.create_stream("patch-stream").await.unwrap(); + client + .create_stream("patch-stream", &StreamCreateOptions::default()) + .await + .unwrap(); let stream = Identifier::named("patch-stream").unwrap(); client @@ -489,8 +608,10 @@ .update_topic( &stream, &topic, - "patch-renamed", - &TopicUpdateOptions::default(), + &TopicUpdateOptions { + name: Some("patch-renamed".to_owned()), + ..TopicUpdateOptions::default() + }, ) .await .unwrap(); @@ -501,16 +622,22 @@ .unwrap() .expect("topic exists"); assert_eq!(details.name, "patch-renamed"); - assert_eq!(details.compression_algorithm, CompressionAlgorithm::Gzip); - assert_eq!(details.message_expiry, IggyExpiry::from(5_000_000u64)); + assert_eq!( + details.resolved_options().compression_algorithm, + CompressionAlgorithm::Gzip + ); + assert_eq!( + details.resolved_options().message_expiry, + IggyExpiry::from(5_000_000u64) + ); // Sending one key changes that key and leaves the other alone. client .update_topic( &stream, &Identifier::named("patch-renamed").unwrap(), - "patch-renamed", &TopicUpdateOptions { + name: Some("patch-renamed".to_owned()), message_expiry: Some(IggyExpiry::from(9_000_000u64)), ..TopicUpdateOptions::default() }, @@ -523,9 +650,12 @@ .await .unwrap() .expect("topic exists"); - assert_eq!(details.message_expiry, IggyExpiry::from(9_000_000u64)); assert_eq!( - details.compression_algorithm, + details.resolved_options().message_expiry, + IggyExpiry::from(9_000_000u64) + ); + assert_eq!( + details.resolved_options().compression_algorithm, CompressionAlgorithm::Gzip, "a key the update did not carry keeps its value" ); @@ -536,7 +666,10 @@ harness: &TestHarness, ) { let client = harness.root_client().await.unwrap(); - client.create_stream("resize-stream").await.unwrap(); + client + .create_stream("resize-stream", &StreamCreateOptions::default()) + .await + .unwrap(); let stream = Identifier::named("resize-stream").unwrap(); let policy_key = HeaderKey::from_str(topic_option_keys::PARTITION_RESIZE_POLICY).unwrap(); @@ -599,8 +732,8 @@ .update_topic( &stream, &topic, - "renamed", &TopicUpdateOptions { + name: Some("renamed".to_owned()), raw: BTreeMap::from([( topic_option_keys::PARTITION_RESIZE_POLICY.to_owned(), replacement.to_owned(), @@ -611,7 +744,7 @@ .await; assert_policy_error( updated.unwrap_err(), - IggyError::UnsupportedOptionKey(String::new()), + IggyError::OptionNotUpdatable(String::new()), ); let details = client.get_topic(&stream, &topic).await.unwrap().unwrap(); assert_eq!(details.name, name); @@ -633,3 +766,155 @@ assert_eq!(error.as_code(), expected.as_code(), "{error:?}"); } } + +#[iggy_harness(test_client_transport = [Tcp, Http, Quic, WebSocket])] +async fn given_topic_patch_when_using_each_transport_should_set_reset_and_preserve_provenance( + harness: &TestHarness, +) { + let client = harness.root_client().await.unwrap(); + client + .create_stream("patch-stream", &StreamCreateOptions::default()) + .await + .unwrap(); + let stream = Identifier::named("patch-stream").unwrap(); + let topic = Identifier::named("patch-topic").unwrap(); + client + .create_topic( + &stream, + "patch-topic", + &TopicCreateOptions { + partitions_count: Some(1), + max_topic_size: Some(MaxTopicSize::from(10 * 1024 * 1024 * 1024u64)), + durability: iggy_common::Durability::Persisted, + ..TopicCreateOptions::default() + }, + ) + .await + .unwrap(); + let before = client + .get_topic(&stream, &topic) + .await + .unwrap() + .expect("topic exists"); + let expiry = IggyExpiry::ExpireDuration(IggyDuration::from(60_000_000u64)); + + client + .update_topic( + &stream, + &topic, + &TopicUpdateOptions { + message_expiry: Some(expiry), + reset: BTreeSet::from([OptionKey::MaxTopicSize]), + ..TopicUpdateOptions::default() + }, + ) + .await + .unwrap(); + + let after = client + .get_topic(&stream, &topic) + .await + .unwrap() + .expect("topic exists"); + assert_eq!(after.name, "patch-topic"); + let resolved = after.resolved_options(); + assert_eq!(resolved.message_expiry, expiry); + assert_eq!( + resolved.max_topic_size, + MaxTopicSize::from(iggy_common::DEFAULT_MAX_TOPIC_SIZE) + ); + let option = |key: &str| { + after + .options + .get(&HeaderKey::from_str(key).unwrap()) + .cloned() + }; + assert!(option(topic_option_keys::MESSAGE_EXPIRY).unwrap().explicit); + assert!(!option(topic_option_keys::MAX_TOPIC_SIZE).unwrap().explicit); + for (key, value) in &before.options { + let name = String::from_utf8_lossy(key.as_bytes()).into_owned(); + if name == topic_option_keys::MESSAGE_EXPIRY || name == topic_option_keys::MAX_TOPIC_SIZE { + continue; + } + assert_eq!( + after.options.get(key), + Some(value), + "{name} must be untouched" + ); + } +} + +#[iggy_harness(test_client_transport = [Tcp, Http, Quic, WebSocket])] +async fn given_stream_or_user_reset_when_updating_should_reject_without_changes( + harness: &TestHarness, +) { + let client = harness.root_client().await.unwrap(); + client + .create_stream("reset-stream", &StreamCreateOptions::default()) + .await + .unwrap(); + client + .create_user( + "reset-user", + "secret123", + UserStatus::Active, + None, + &UserCreateOptions::default(), + ) + .await + .unwrap(); + let stream = Identifier::named("reset-stream").unwrap(); + let user = Identifier::named("reset-user").unwrap(); + + // The typed reset is refused by the client, the raw one by the server. + for patch in [ + StreamUpdateOptions { + reset: BTreeSet::from([OptionKey::Name]), + ..StreamUpdateOptions::default() + }, + StreamUpdateOptions { + raw_reset: BTreeSet::from([OptionKey::Name.to_string()]), + ..StreamUpdateOptions::default() + }, + ] { + let error = client + .update_stream(&stream, &patch) + .await + .expect_err("a stream column is not resettable"); + assert_policy_error( + error, + IggyError::OptionNotResettable(OptionKey::Name.to_string()), + ); + } + for key in [OptionKey::Username, OptionKey::Status] { + for patch in [ + UserUpdateOptions { + reset: BTreeSet::from([key]), + ..UserUpdateOptions::default() + }, + UserUpdateOptions { + raw_reset: BTreeSet::from([key.to_string()]), + ..UserUpdateOptions::default() + }, + ] { + let error = client + .update_user(&user, &patch) + .await + .expect_err("a user column is not resettable"); + assert_policy_error(error, IggyError::OptionNotResettable(key.to_string())); + } + } + + let stream_details = client.get_stream(&stream).await.unwrap(); + assert_eq!( + stream_details.expect("stream kept its name").name, + "reset-stream" + ); + let user_details = client + .get_user(&user) + .await + .unwrap() + .expect("user kept its name"); + assert_eq!(user_details.username, "reset-user"); + assert_eq!(user_details.status, UserStatus::Active); +}
diff --git a/core/integration/tests/sdk/producer/mod.rs b/core/integration/tests/sdk/producer/mod.rs index 6c40095..c5dd320 100644 --- a/core/integration/tests/sdk/producer/mod.rs +++ b/core/integration/tests/sdk/producer/mod.rs
@@ -83,7 +83,10 @@ async fn init_system(client: &IggyClient) { // 1. Create the stream - client.create_stream(STREAM_NAME).await.unwrap(); + client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); // 2. Create the topic client
diff --git a/core/integration/tests/sdk/send_confirmation.rs b/core/integration/tests/sdk/send_confirmation.rs index 647c3e1..02f4b53 100644 --- a/core/integration/tests/sdk/send_confirmation.rs +++ b/core/integration/tests/sdk/send_confirmation.rs
@@ -50,7 +50,7 @@ /// Returns the numeric ids the server assigned to the created stream and topic. async fn create_stream_and_topic(client: &IggyClient, partitions_count: u32) -> (u32, u32) { let stream = client - .create_stream(STREAM_NAME) + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) .await .expect("create_stream"); let topic = client
diff --git a/core/integration/tests/server/a2a_jwt/jwt_tests.rs b/core/integration/tests/server/a2a_jwt/jwt_tests.rs index 76c8e96..6132f13 100644 --- a/core/integration/tests/server/a2a_jwt/jwt_tests.rs +++ b/core/integration/tests/server/a2a_jwt/jwt_tests.rs
@@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +use iggy::prelude::StreamCreateOptions; +use iggy::prelude::UserCreateOptions; use std::sync::Arc; use iggy::http::http_client::HttpClient; @@ -45,6 +47,7 @@ A2A_PASSWORD, UserStatus::Active, Some(streams_permissions(false)), + &UserCreateOptions::default(), ) .await { @@ -569,6 +572,7 @@ A2A_PASSWORD, UserStatus::Active, Some(streams_permissions(false)), + &UserCreateOptions::default(), ) .await .expect("create successor"); @@ -607,15 +611,16 @@ let client = create_client_with_jwt(&http_addr.to_string(), create_valid_jwt(3600)).await; client - .create_stream("a2a-before-rename") + .create_stream("a2a-before-rename", &StreamCreateOptions::default()) .await .expect("A2A token must write as the mapped user"); root.update_user( &mapped, - Some("a2a-renamed"), - None, - &UserUpdateOptions::default(), + &UserUpdateOptions { + username: Some("a2a-renamed".to_owned()), + ..UserUpdateOptions::default() + }, ) .await .expect("rename the mapped user"); @@ -624,6 +629,7 @@ A2A_PASSWORD, UserStatus::Active, Some(streams_permissions(false)), + &UserCreateOptions::default(), ) .await .expect("create a new account under the mapped username"); @@ -633,7 +639,7 @@ "reads must act as the account that holds the username now" ); let error = client - .create_stream("a2a-after-rename") + .create_stream("a2a-after-rename", &StreamCreateOptions::default()) .await .expect_err("writes must not act as the renamed account"); assert_eq!(error.as_code(), IggyError::Unauthorized.as_code());
diff --git a/core/integration/tests/server/cluster_view_durability_vsr.rs b/core/integration/tests/server/cluster_view_durability_vsr.rs index 6ae1024..f88920b 100644 --- a/core/integration/tests/server/cluster_view_durability_vsr.rs +++ b/core/integration/tests/server/cluster_view_durability_vsr.rs
@@ -78,7 +78,7 @@ .await .expect("connect a root client to the node"); client - .create_stream(STREAM_NAME) + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) .await .expect("create stream on the view-0 primary"); assert_eq!(
diff --git a/core/integration/tests/server/consumer_offset_quota_vsr.rs b/core/integration/tests/server/consumer_offset_quota_vsr.rs index 080a632..0970a0c 100644 --- a/core/integration/tests/server/consumer_offset_quota_vsr.rs +++ b/core/integration/tests/server/consumer_offset_quota_vsr.rs
@@ -46,7 +46,7 @@ let stream = Identifier::named(STREAM_NAME).expect("stream identifier"); let topic = Identifier::named(TOPIC_NAME).expect("topic identifier"); let stream_details = client - .create_stream(STREAM_NAME) + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) .await .expect("create stream"); let topic_details = client @@ -100,6 +100,7 @@ }, )])), }), + &UserCreateOptions::default(), ) .await .expect("create a topic-scoped consumer"); @@ -399,7 +400,10 @@ let stream = Identifier::named("consumer-offset-restart-stream").expect("stream identifier"); let topic = Identifier::named("consumer-offset-restart-topic").expect("topic identifier"); let stream_details = client - .create_stream("consumer-offset-restart-stream") + .create_stream( + "consumer-offset-restart-stream", + &StreamCreateOptions::default(), + ) .await .expect("create stream"); let topic_details = client
diff --git a/core/integration/tests/server/descriptor_exhaustion_vsr.rs b/core/integration/tests/server/descriptor_exhaustion_vsr.rs index 697ba73..1e2f9b8 100644 --- a/core/integration/tests/server/descriptor_exhaustion_vsr.rs +++ b/core/integration/tests/server/descriptor_exhaustion_vsr.rs
@@ -103,7 +103,7 @@ let client = harness.tcp_root_client().await.expect("TCP root client"); let stream = client - .create_stream(STREAM_NAME) + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) .await .expect("create stream"); let topic = client
diff --git a/core/integration/tests/server/http_rbac.rs b/core/integration/tests/server/http_rbac.rs index bbed1dc..b2f4407 100644 --- a/core/integration/tests/server/http_rbac.rs +++ b/core/integration/tests/server/http_rbac.rs
@@ -36,9 +36,9 @@ use crate::server::http_client::HttpClient; use iggy::prelude::*; -use iggy_common::IggyMessagesBatch; use iggy_common::create_stream::CreateStream; use iggy_common::create_topic::CreateTopic; +use iggy_common::{IggyMessagesBatch, OptionInputMap}; use integration::iggy_harness; use reqwest::{Response, StatusCode}; use serde_json::{Value, json}; @@ -94,7 +94,11 @@ let response = self .post_json( "/streams", - &serde_json::to_value(CreateStream { name: name.into() }).unwrap(), + &serde_json::to_value(CreateStream { + name: name.into(), + options: OptionInputMap::default(), + }) + .unwrap(), ) .await; assert!( @@ -112,11 +116,8 @@ let command = CreateTopic { stream_id: Identifier::default(), partitions_count: partitions, - compression_algorithm: CompressionAlgorithm::None, - message_expiry: IggyExpiry::NeverExpire, - max_topic_size: MaxTopicSize::ServerDefault, name: topic.to_string(), - options: Default::default(), + options: OptionInputMap::default(), }; let response = self .post_json( @@ -785,7 +786,7 @@ /// Admin-initiated change (target != caller): the current password is checked /// against the TARGET's stored hash, so a wrong one is 400 `InvalidCredentials`; /// a target that does not resolve is left to the replicated apply, which -/// answers 404 `UserNotFound` (the plaintext is still stripped + hashed first). +/// answers 404 `UserNotFound` (both plaintext passwords are stripped first). #[iggy_harness] async fn given_admin_change_password_when_target_current_wrong_or_missing_should_reject( harness: &TestHarness,
diff --git a/core/integration/tests/server/http_read_your_writes.rs b/core/integration/tests/server/http_read_your_writes.rs index dcdd610..1466dca 100644 --- a/core/integration/tests/server/http_read_your_writes.rs +++ b/core/integration/tests/server/http_read_your_writes.rs
@@ -106,7 +106,7 @@ for round in 0..ROUNDS { let stream = format!("read-your-writes-{round}"); seeder - .create_stream(&stream) + .create_stream(&stream, &StreamCreateOptions::default()) .await .expect("the primary must commit the stream this round reads back");
diff --git a/core/integration/tests/server/http_vsr.rs b/core/integration/tests/server/http_vsr.rs index 2079753..23f3493 100644 --- a/core/integration/tests/server/http_vsr.rs +++ b/core/integration/tests/server/http_vsr.rs
@@ -27,7 +27,7 @@ use iggy_common::create_stream::CreateStream; use iggy_common::create_topic::CreateTopic; use iggy_common::store_consumer_offset::StoreConsumerOffset; -use iggy_common::{ConsumerOffsetInfo, IggyMessagesBatch}; +use iggy_common::{ConsumerOffsetInfo, IggyMessagesBatch, OptionInputMap, OptionInputValue}; use integration::iggy_harness; use reqwest::{Response, StatusCode}; use serde_json::json; @@ -110,6 +110,7 @@ .bearer_auth(&self.token) .json(&CreateStream { name: stream.to_string(), + options: OptionInputMap::default(), }) .send() .await @@ -123,17 +124,17 @@ let command = CreateTopic { stream_id: Identifier::default(), partitions_count: partitions, - compression_algorithm: CompressionAlgorithm::None, - message_expiry: IggyExpiry::NeverExpire, - max_topic_size: MaxTopicSize::ServerDefault, name: topic.to_string(), - options: BTreeMap::from([ - ("durability".to_string(), Durability::Persisted.to_string()), + options: OptionInputMap(BTreeMap::from([ + ( + "durability".to_string(), + OptionInputValue::String(Durability::Persisted.to_string().into()), + ), ( "consumer_offset_durability".to_string(), - Durability::Persisted.to_string(), + OptionInputValue::String(Durability::Persisted.to_string().into()), ), - ]), + ])), }; let response = self .client @@ -1207,7 +1208,10 @@ harness: &TestHarness, ) { let client = harness.tcp_root_client().await.unwrap(); - client.create_stream("durability-http").await.unwrap(); + client + .create_stream("durability-http", &StreamCreateOptions::default()) + .await + .unwrap(); let stream = Identifier::named("durability-http").unwrap(); let http = HttpClient::login_root(harness).await; for (index, (message_policy, offset_policy)) in [ @@ -1226,13 +1230,16 @@ .bearer_auth(&http.token) .json(&CreateTopic { name: topic.clone(), - options: BTreeMap::from([ - ("durability".to_string(), message_policy.to_string()), + options: OptionInputMap(BTreeMap::from([ + ( + "durability".to_string(), + OptionInputValue::String(message_policy.to_string().into()), + ), ( "consumer_offset_durability".to_string(), - offset_policy.to_string(), + OptionInputValue::String(offset_policy.to_string().into()), ), - ]), + ])), ..CreateTopic::default() }) .send() @@ -1325,7 +1332,10 @@ .bearer_auth(&http.token) .json(&CreateTopic { name: key.to_string(), - options: BTreeMap::from([(key.to_string(), "persisted".to_string())]), + options: OptionInputMap(BTreeMap::from([( + key.to_string(), + OptionInputValue::String("persisted".to_string().into()), + )])), ..CreateTopic::default() }) .send() @@ -1354,7 +1364,10 @@ let address = harness.server().http_addr().unwrap(); let client = iggy::http::http_client::HttpClient::new(&format!("http://{address}")).unwrap(); client.login_user("iggy", "iggy").await.unwrap(); - client.create_stream("sdk-durability").await.unwrap(); + client + .create_stream("sdk-durability", &StreamCreateOptions::default()) + .await + .unwrap(); let stream = Identifier::named("sdk-durability").unwrap(); client .create_topic( @@ -1411,3 +1424,119 @@ "a follower refused after {elapsed:?}, the replay budget is not for backups" ); } + +/// A PUT body the update shape refuses answers the Iggy error JSON with +/// `InvalidCommand`, the category the binary decoder gives the same patch, +/// naming in `field` the member the shape refuses. A `null` member leaves its +/// key unchanged. +#[iggy_harness] +async fn given_malformed_update_body_when_putting_should_reject_invalid_command( + harness: &TestHarness, +) { + let http = HttpClient::login_root(harness).await; + http.create_stream_and_topic("http-put-shape", "shape", 1) + .await; + let path = "/streams/http-put-shape/topics/shape"; + let stream_path = "/streams/http-put-shape"; + let cases = [ + (path, r#"{"name":"first","name":"second"}"#, Some("name")), + ( + path, + r#"{"message_expiry":{"value":"1 day"}}"#, + Some("message_expiry"), + ), + (path, r#"["name"]"#, None), + (path, r#"{"reset":"message_expiry"}"#, Some("reset")), + ( + path, + r#"{"message_expiry":"1 day","reset":["message_expiry"]}"#, + None, + ), + (path, r#"{}"#, None), + ( + stream_path, + r#"{"name":"renamed","options":{}}"#, + Some("options"), + ), + ]; + for (path, body, field) in cases { + let response = http + .client + .put(http.url(path)) + .bearer_auth(&http.token) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .body(body) + .send() + .await + .expect("put request"); + assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{body}"); + let error: serde_json::Value = response.json().await.expect("decode the error body"); + assert_eq!(error["id"], IggyError::InvalidCommand.as_code(), "{body}"); + assert_eq!( + error["code"], + IggyError::InvalidCommand.as_string(), + "{body}" + ); + assert_eq!(error["field"], json!(field), "{body}"); + } + + let before: serde_json::Value = http.get(path).await.json().await.expect("topic json"); + let response = http + .put_json( + path, + &json!({ "name": "shape-renamed", "message_expiry": null }), + ) + .await; + assert_eq!(response.status(), StatusCode::NO_CONTENT); + let after: serde_json::Value = http + .get("/streams/http-put-shape/topics/shape-renamed") + .await + .json() + .await + .expect("topic json"); + assert_eq!(after["options"], before["options"]); +} + +/// An option placed beside `options` in a POST create body is refused, not +/// dropped, so a stream never commits without the key its client sent. The +/// error names the misplaced member in `field`. +#[iggy_harness] +async fn given_option_beside_options_when_creating_should_reject_invalid_command( + harness: &TestHarness, +) { + let http = HttpClient::login_root(harness).await; + http.create_stream_and_topic("http-misplaced", "kept", 1) + .await; + let cases = [ + ( + "/streams", + json!({ + "name": "misplaced-key", + "encryption_key": "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=" + }), + "encryption_key", + ), + ( + "/streams/http-misplaced/topics", + json!({ "name": "misplaced-expiry", "partitions_count": 1, "message_expiry": "7 days" }), + "message_expiry", + ), + ]; + for (path, body, field) in cases { + let response = http.post_json(path, &body).await; + assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{body}"); + let error: serde_json::Value = response.json().await.expect("decode the error body"); + assert_eq!(error["id"], IggyError::InvalidCommand.as_code(), "{body}"); + assert_eq!(error["field"], field, "{body}"); + } + assert_eq!( + http.get("/streams/misplaced-key").await.status(), + StatusCode::NOT_FOUND + ); + assert_eq!( + http.get("/streams/http-misplaced/topics/misplaced-expiry") + .await + .status(), + StatusCode::NOT_FOUND + ); +}
diff --git a/core/integration/tests/server/partition_view_durability_vsr.rs b/core/integration/tests/server/partition_view_durability_vsr.rs index 4d64b58..bf066de 100644 --- a/core/integration/tests/server/partition_view_durability_vsr.rs +++ b/core/integration/tests/server/partition_view_durability_vsr.rs
@@ -75,7 +75,7 @@ .await .expect("connect a root client to the node"); client - .create_stream(STREAM_NAME) + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) .await .expect("create stream on the view-0 primary"); let stream_id = Identifier::named(STREAM_NAME).expect("stream identifier");
diff --git a/core/integration/tests/server/partitions_limit_vsr.rs b/core/integration/tests/server/partitions_limit_vsr.rs index 414d848..2b88724 100644 --- a/core/integration/tests/server/partitions_limit_vsr.rs +++ b/core/integration/tests/server/partitions_limit_vsr.rs
@@ -64,7 +64,7 @@ async fn given_partitions_cap_when_creating_past_it_should_reject_typed(harness: &TestHarness) { let client = harness.tcp_root_client().await.expect("TCP root client"); client - .create_stream(STREAM_NAME) + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) .await .expect("create stream"); let stream_id = Identifier::named(STREAM_NAME).expect("stream identifier");
diff --git a/core/integration/tests/server/personal_access_token_cleaner_vsr.rs b/core/integration/tests/server/personal_access_token_cleaner_vsr.rs index 914a8b9..fa3a1f0 100644 --- a/core/integration/tests/server/personal_access_token_cleaner_vsr.rs +++ b/core/integration/tests/server/personal_access_token_cleaner_vsr.rs
@@ -64,12 +64,15 @@ // The delete applies before its commit completes, so an empty list alone // does not prove the commit went through. A later write must commit // behind it. - timeout(WRITE_BUDGET, client.create_stream(STREAM_NAME)) - .await - .unwrap_or_else(|_| { - panic!("a write after the cleaner's delete did not commit within {WRITE_BUDGET:?}") - }) - .expect("a write after the cleaner's delete must commit"); + timeout( + WRITE_BUDGET, + client.create_stream(STREAM_NAME, &StreamCreateOptions::default()), + ) + .await + .unwrap_or_else(|_| { + panic!("a write after the cleaner's delete did not commit within {WRITE_BUDGET:?}") + }) + .expect("a write after the cleaner's delete must commit"); } async fn wait_for_no_tokens(client: &IggyClient) {
diff --git a/core/integration/tests/server/pipelined_frames_vsr.rs b/core/integration/tests/server/pipelined_frames_vsr.rs index 05ada2f..f510f32 100644 --- a/core/integration/tests/server/pipelined_frames_vsr.rs +++ b/core/integration/tests/server/pipelined_frames_vsr.rs
@@ -246,7 +246,10 @@ let client = harness.tcp_root_client().await.unwrap(); let stream = Identifier::named(STREAM).unwrap(); let topic = Identifier::named(TOPIC).unwrap(); - client.create_stream(STREAM).await.unwrap(); + client + .create_stream(STREAM, &StreamCreateOptions::default()) + .await + .unwrap(); client .create_topic( &stream,
diff --git a/core/integration/tests/server/poll_semantics_vsr.rs b/core/integration/tests/server/poll_semantics_vsr.rs index 2430397..d031711 100644 --- a/core/integration/tests/server/poll_semantics_vsr.rs +++ b/core/integration/tests/server/poll_semantics_vsr.rs
@@ -36,7 +36,7 @@ ) { let client = harness.tcp_root_client().await.expect("tcp root client"); client - .create_stream("poll-stream") + .create_stream("poll-stream", &StreamCreateOptions::default()) .await .expect("create stream"); let stream_id = Identifier::from_str_value("poll-stream").expect("stream identifier"); @@ -121,7 +121,7 @@ async fn given_missing_topic_when_polling_should_reject_topic_not_found(harness: &TestHarness) { let client = harness.tcp_root_client().await.expect("tcp root client"); client - .create_stream("topicless-stream") + .create_stream("topicless-stream", &StreamCreateOptions::default()) .await .expect("create stream"); let stream_id = Identifier::from_str_value("topicless-stream").expect("stream identifier"); @@ -161,7 +161,7 @@ ) { let client = harness.tcp_root_client().await.expect("tcp root client"); client - .create_stream("offset-stream") + .create_stream("offset-stream", &StreamCreateOptions::default()) .await .expect("create stream"); let stream_id = Identifier::from_str_value("offset-stream").expect("stream identifier"); @@ -238,7 +238,7 @@ ) { let client = harness.tcp_root_client().await.expect("tcp root client"); client - .create_stream("offset-topicless-stream") + .create_stream("offset-topicless-stream", &StreamCreateOptions::default()) .await .expect("create stream"); let stream_id = @@ -264,7 +264,7 @@ async fn given_message_at_polled_timestamp_when_polling_should_include_it(harness: &TestHarness) { let client = harness.tcp_root_client().await.expect("tcp root client"); client - .create_stream("ts-stream") + .create_stream("ts-stream", &StreamCreateOptions::default()) .await .expect("create stream"); let stream_id = Identifier::from_str_value("ts-stream").expect("stream identifier");
diff --git a/core/integration/tests/server/scenarios/authentication_scenario.rs b/core/integration/tests/server/scenarios/authentication_scenario.rs index 609a6cc..9d433a3 100644 --- a/core/integration/tests/server/scenarios/authentication_scenario.rs +++ b/core/integration/tests/server/scenarios/authentication_scenario.rs
@@ -198,15 +198,16 @@ GET_USER_CODE => client.get_user(&ctx.user_id).await.map(|_| ()), GET_USERS_CODE => client.get_users().await.map(|_| ()), CREATE_USER_CODE => client - .create_user("test", "test", UserStatus::Active, None) + .create_user( + "test", + "test", + UserStatus::Active, + None, + &UserCreateOptions::default(), + ) .await .map(|_| ()), DELETE_USER_CODE => client.delete_user(&ctx.user_id).await, - UPDATE_USER_CODE => { - client - .update_user(&ctx.user_id, Some("x"), None, &UserUpdateOptions::default()) - .await - } UPDATE_PERMISSIONS_CODE => client.update_permissions(&ctx.user_id, None).await, CHANGE_PASSWORD_CODE => client.change_password(&ctx.user_id, "old", "new").await, @@ -223,11 +224,20 @@ // Streams GET_STREAM_CODE => client.get_stream(&ctx.stream_id).await.map(|_| ()), GET_STREAMS_CODE => client.get_streams().await.map(|_| ()), - CREATE_STREAM_CODE => client.create_stream("x").await.map(|_| ()), + CREATE_STREAM_CODE => client + .create_stream("x", &StreamCreateOptions::default()) + .await + .map(|_| ()), DELETE_STREAM_CODE => client.delete_stream(&ctx.stream_id).await, - UPDATE_STREAM_CODE => { + UPDATE_RESOURCE_CODE => { client - .update_stream(&ctx.stream_id, "x", &StreamUpdateOptions::default()) + .update_stream( + &ctx.stream_id, + &StreamUpdateOptions { + name: Some("x".to_owned()), + ..StreamUpdateOptions::default() + }, + ) .await } @@ -250,16 +260,6 @@ .await .map(|_| ()), DELETE_TOPIC_CODE => client.delete_topic(&ctx.stream_id, &ctx.topic_id).await, - UPDATE_TOPIC_CODE => { - client - .update_topic( - &ctx.stream_id, - &ctx.topic_id, - "x", - &TopicUpdateOptions::default(), - ) - .await - } // Partitions CREATE_PARTITIONS_CODE => { @@ -429,7 +429,7 @@ async fn setup_test_resources(client: &IggyClient) { client - .create_stream(STREAM_NAME) + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) .await .expect("create stream"); client
diff --git a/core/integration/tests/server/scenarios/concurrent_produce_consume_scenario.rs b/core/integration/tests/server/scenarios/concurrent_produce_consume_scenario.rs index b5adb73..d96d281 100644 --- a/core/integration/tests/server/scenarios/concurrent_produce_consume_scenario.rs +++ b/core/integration/tests/server/scenarios/concurrent_produce_consume_scenario.rs
@@ -55,7 +55,10 @@ let topic_id = Identifier::named(TOPIC_NAME).unwrap(); let setup = harness.tcp_root_client().await.unwrap(); - setup.create_stream(STREAM_NAME).await.unwrap(); + setup + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); setup .create_topic( &stream_id,
diff --git a/core/integration/tests/server/scenarios/concurrent_scenario.rs b/core/integration/tests/server/scenarios/concurrent_scenario.rs index 5103fca..4411a77 100644 --- a/core/integration/tests/server/scenarios/concurrent_scenario.rs +++ b/core/integration/tests/server/scenarios/concurrent_scenario.rs
@@ -74,12 +74,18 @@ // For topic tests, create parent stream first if resource_type == ResourceType::Topic { - root_client.create_stream(TEST_STREAM_NAME).await.unwrap(); + root_client + .create_stream(TEST_STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); } // For partition and consumer group tests, create parent stream and topic first if resource_type == ResourceType::Partition || resource_type == ResourceType::ConsumerGroup { - root_client.create_stream(TEST_STREAM_NAME).await.unwrap(); + root_client + .create_stream(TEST_STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); let stream_id = Identifier::named(TEST_STREAM_NAME).unwrap(); root_client .create_topic( @@ -152,7 +158,13 @@ } let username = format!("race-user-{}", client_id); client - .create_user(&username, USER_PASSWORD, UserStatus::Active, None) + .create_user( + &username, + USER_PASSWORD, + UserStatus::Active, + None, + &UserCreateOptions::default(), + ) .await .map(|_| ()) })); @@ -183,7 +195,13 @@ b.wait().await; } client - .create_user(DUPLICATE_USER, USER_PASSWORD, UserStatus::Active, None) + .create_user( + DUPLICATE_USER, + USER_PASSWORD, + UserStatus::Active, + None, + &UserCreateOptions::default(), + ) .await .map(|_| ()) })); @@ -213,7 +231,10 @@ b.wait().await; } let stream_name = format!("race-stream-{}", client_id); - client.create_stream(&stream_name).await.map(|_| ()) + client + .create_stream(&stream_name, &StreamCreateOptions::default()) + .await + .map(|_| ()) })); } @@ -241,7 +262,10 @@ if let Some(b) = barrier { b.wait().await; } - client.create_stream(DUPLICATE_STREAM).await.map(|_| ()) + client + .create_stream(DUPLICATE_STREAM, &StreamCreateOptions::default()) + .await + .map(|_| ()) })); }
diff --git a/core/integration/tests/server/scenarios/consumer_group_auto_commit_reconnection_scenario.rs b/core/integration/tests/server/scenarios/consumer_group_auto_commit_reconnection_scenario.rs index 3ec00ca..6215146 100644 --- a/core/integration/tests/server/scenarios/consumer_group_auto_commit_reconnection_scenario.rs +++ b/core/integration/tests/server/scenarios/consumer_group_auto_commit_reconnection_scenario.rs
@@ -37,7 +37,10 @@ async fn init_system(client: &IggyClient) { // 1. Create the stream - client.create_stream(STREAM_NAME).await.unwrap(); + client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); // 2. Create the topic client
diff --git a/core/integration/tests/server/scenarios/consumer_group_duplicate_name_create_scenario.rs b/core/integration/tests/server/scenarios/consumer_group_duplicate_name_create_scenario.rs index 6795574..b3a56bc 100644 --- a/core/integration/tests/server/scenarios/consumer_group_duplicate_name_create_scenario.rs +++ b/core/integration/tests/server/scenarios/consumer_group_duplicate_name_create_scenario.rs
@@ -27,6 +27,7 @@ }; use iggy::prelude::Identifier; use iggy::prelude::IggyExpiry; +use iggy::prelude::StreamCreateOptions; use iggy::prelude::{ConsumerGroupClient, StreamClient, TopicClient, TopicCreateOptions}; use integration::harness::{ TestHarness, assert_clean_system, create_user, delete_user, login_user, @@ -39,7 +40,10 @@ .expect("Failed to get root client"); let client1 = create_client(harness).await; - system_client.create_stream(STREAM_NAME).await.unwrap(); + system_client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); system_client .create_topic( &Identifier::named(STREAM_NAME).unwrap(),
diff --git a/core/integration/tests/server/scenarios/consumer_group_join_scenario.rs b/core/integration/tests/server/scenarios/consumer_group_join_scenario.rs index 00845a5..ed43632 100644 --- a/core/integration/tests/server/scenarios/consumer_group_join_scenario.rs +++ b/core/integration/tests/server/scenarios/consumer_group_join_scenario.rs
@@ -24,6 +24,7 @@ use iggy::prelude::ConsumerGroupDetails; use iggy::prelude::Identifier; use iggy::prelude::IggyExpiry; +use iggy::prelude::StreamCreateOptions; use iggy::prelude::{ ConsumerGroupClient, StreamClient, SystemClient, TopicClient, TopicCreateOptions, }; @@ -40,7 +41,10 @@ let client3 = create_client(harness).await; // 1. Create the stream - let stream = system_client.create_stream(STREAM_NAME).await.unwrap(); + let stream = system_client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); let stream_id = stream.id;
diff --git a/core/integration/tests/server/scenarios/consumer_group_new_messages_after_restart_scenario.rs b/core/integration/tests/server/scenarios/consumer_group_new_messages_after_restart_scenario.rs index 4bbceb8..7aaac37 100644 --- a/core/integration/tests/server/scenarios/consumer_group_new_messages_after_restart_scenario.rs +++ b/core/integration/tests/server/scenarios/consumer_group_new_messages_after_restart_scenario.rs
@@ -35,7 +35,10 @@ } async fn init_system(client: &IggyClient) { - client.create_stream(STREAM_NAME).await.unwrap(); + client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); client .create_topic(
diff --git a/core/integration/tests/server/scenarios/consumer_group_offset_cleanup_scenario.rs b/core/integration/tests/server/scenarios/consumer_group_offset_cleanup_scenario.rs index f7d1fb0..b807f58 100644 --- a/core/integration/tests/server/scenarios/consumer_group_offset_cleanup_scenario.rs +++ b/core/integration/tests/server/scenarios/consumer_group_offset_cleanup_scenario.rs
@@ -42,7 +42,10 @@ async fn init_system(client: &IggyClient) { // 1. Create the stream - client.create_stream(STREAM_NAME).await.unwrap(); + client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); // 2. Create the topic with 1 partition client
diff --git a/core/integration/tests/server/scenarios/consumer_group_with_multiple_clients_polling_messages_scenario.rs b/core/integration/tests/server/scenarios/consumer_group_with_multiple_clients_polling_messages_scenario.rs index 152cc21..e0dce7a 100644 --- a/core/integration/tests/server/scenarios/consumer_group_with_multiple_clients_polling_messages_scenario.rs +++ b/core/integration/tests/server/scenarios/consumer_group_with_multiple_clients_polling_messages_scenario.rs
@@ -52,7 +52,10 @@ create_users: bool, ) { // 1. Create the stream - system_client.create_stream(STREAM_NAME).await.unwrap(); + system_client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); // 2. Create the topic system_client
diff --git a/core/integration/tests/server/scenarios/consumer_group_with_single_client_polling_messages_scenario.rs b/core/integration/tests/server/scenarios/consumer_group_with_single_client_polling_messages_scenario.rs index 8e94831..f03da44 100644 --- a/core/integration/tests/server/scenarios/consumer_group_with_single_client_polling_messages_scenario.rs +++ b/core/integration/tests/server/scenarios/consumer_group_with_single_client_polling_messages_scenario.rs
@@ -43,7 +43,10 @@ async fn init_system(client: &IggyClient) { // 1. Create the stream - client.create_stream(STREAM_NAME).await.unwrap(); + client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); // 2. Create the topic client
diff --git a/core/integration/tests/server/scenarios/consumer_timestamp_polling_scenario.rs b/core/integration/tests/server/scenarios/consumer_timestamp_polling_scenario.rs index 68a37a9..5bdefa4 100644 --- a/core/integration/tests/server/scenarios/consumer_timestamp_polling_scenario.rs +++ b/core/integration/tests/server/scenarios/consumer_timestamp_polling_scenario.rs
@@ -129,7 +129,10 @@ } async fn init_system(client: &IggyClient) { - client.create_stream(STREAM_NAME).await.unwrap(); + client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); client .create_topic(
diff --git a/core/integration/tests/server/scenarios/delete_segments_scenario.rs b/core/integration/tests/server/scenarios/delete_segments_scenario.rs index 30d9789..803374f 100644 --- a/core/integration/tests/server/scenarios/delete_segments_scenario.rs +++ b/core/integration/tests/server/scenarios/delete_segments_scenario.rs
@@ -82,7 +82,10 @@ client.connect().await.unwrap(); let data_path = harness.server().data_path().to_path_buf(); - let stream = client.create_stream(STREAM_NAME).await.unwrap(); + let stream = client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); let stream_id = stream.id; let topic = client @@ -366,7 +369,10 @@ client.connect().await.unwrap(); let data_path = harness.server().data_path().to_path_buf(); - let stream = client.create_stream(STREAM_NAME).await.unwrap(); + let stream = client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); let stream_id = stream.id; let topic = client @@ -457,7 +463,10 @@ /// /// No restart_server variant — 25 delete_segments calls would mean 25 restarts. pub async fn run_consumer_group_barrier(client: &IggyClient, data_path: &Path) { - let stream = client.create_stream(STREAM_NAME).await.unwrap(); + let stream = client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); let stream_id = stream.id; let topic = client @@ -598,7 +607,10 @@ client.connect().await.unwrap(); let data_path = harness.server().data_path().to_path_buf(); - let stream = client.create_stream(STREAM_NAME).await.unwrap(); + let stream = client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); let stream_id = stream.id; let topic = client
diff --git a/core/integration/tests/server/scenarios/delete_stats_rollback_scenario.rs b/core/integration/tests/server/scenarios/delete_stats_rollback_scenario.rs index e0fdd61..cbb9745 100644 --- a/core/integration/tests/server/scenarios/delete_stats_rollback_scenario.rs +++ b/core/integration/tests/server/scenarios/delete_stats_rollback_scenario.rs
@@ -85,7 +85,10 @@ /// both the topic and the stream. The retained partition's messages must stay /// counted, so a blanket zeroing fails here as loudly as no rollback at all. async fn delete_partitions_rolls_back_topic_and_stream(client: &IggyClient) { - client.create_stream(STREAM_NAME).await.unwrap(); + client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); create_topic(client, STREAM_NAME, KEPT_TOPIC).await; // One pass per partition, so the delete below removes a known share. @@ -152,7 +155,10 @@ /// Deleting a topic that still holds messages must take its bytes out of the /// stream total, not just remove the topic. async fn delete_topic_rolls_back_the_stream(client: &IggyClient) { - client.create_stream(STREAM_NAME).await.unwrap(); + client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); create_topic(client, STREAM_NAME, KEPT_TOPIC).await; create_topic(client, STREAM_NAME, DELETED_TOPIC).await;
diff --git a/core/integration/tests/server/scenarios/encryption_scenario.rs b/core/integration/tests/server/scenarios/encryption_scenario.rs index 8bab9d5..d02dcaa 100644 --- a/core/integration/tests/server/scenarios/encryption_scenario.rs +++ b/core/integration/tests/server/scenarios/encryption_scenario.rs
@@ -17,14 +17,31 @@ use bytes::Bytes; use iggy::prelude::*; -use iggy_common::TransportProtocol; +use iggy_common::{ + OptionKey, ServerKeyContext, StreamEncryptionKey, TransportProtocol, stream_key_fingerprint, + text, +}; use integration::harness::{TestHarness, TestServerConfig}; use serial_test::parallel; use std::collections::{BTreeMap, HashMap}; use std::path::{Path, PathBuf}; use std::sync::Arc; +use std::time::Duration; use test_case::test_matrix; +const CONFIG_KEY: &str = "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE="; +const SECOND_ENCRYPTION_KEY: &str = "AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA="; +const ALL_TRANSPORTS: [TransportProtocol; 4] = [ + TransportProtocol::Tcp, + TransportProtocol::Quic, + TransportProtocol::WebSocket, + TransportProtocol::Http, +]; +const DISK_WAIT: Duration = Duration::from_secs(10); +// The stored payload is the 12-byte nonce, the ciphertext and the 16-byte tag. +const AES_GCM_OVERHEAD: usize = 28; +const TOPIC_NAME: &str = "encrypted-topic"; + #[test_matrix( [encryption_enabled(), encryption_disabled()] )] @@ -48,7 +65,10 @@ let topic_name = "test-topic-api"; let partition_count = 1; - client.create_stream(stream_name).await.unwrap(); + client + .create_stream(stream_name, &StreamCreateOptions::default()) + .await + .unwrap(); client .create_topic( &Identifier::named(stream_name).unwrap(), @@ -339,7 +359,10 @@ let stream_name = format!("client-enc-{transport}"); let topic_name = "enc-topic"; - setup_client.create_stream(&stream_name).await.unwrap(); + setup_client + .create_stream(&stream_name, &StreamCreateOptions::default()) + .await + .unwrap(); setup_client .create_topic( &Identifier::named(&stream_name).unwrap(), @@ -478,6 +501,525 @@ /// The server flushes on the journal thresholds (there is no flush primitive), /// so force every committed batch straight to disk: the assertions below read /// the segment files directly. Both knobs are topic creation options now. +#[tokio::test] +#[parallel] +async fn given_client_key_and_keyless_streams_when_polling_should_use_their_own_keys() { + let mut harness = single_node(encryption_config(true, CONFIG_KEY)).await; + let tcp = harness.tcp_root_client().await.unwrap(); + let http = harness + .root_client_for(TransportProtocol::Http) + .await + .unwrap(); + let first_key = stream_key(11); + let second_key = stream_key(22); + create_stream_with_topic(&tcp, "own-key-tcp", Some(first_key.clone())).await; + create_stream_with_topic(&http, "own-key-http", Some(second_key.clone())).await; + create_stream_with_topic(&tcp, "generated-key", None).await; + let streams = ["own-key-tcp", "own-key-http", "generated-key"]; + for name in streams { + send_single_message(&tcp, &stream_id(name), &topic_id(), 1, &payload(name)).await; + } + + for transport in ALL_TRANSPORTS { + let client = harness.root_client_for(transport).await.unwrap(); + for name in streams { + let polled = poll_single_partition(&client, &stream_id(name), &topic_id(), 0, 1) + .await + .unwrap_or_else(|error| panic!("{transport} poll of {name}: {error}")); + assert_eq!( + polled.messages[0].payload.as_ref(), + payload(name).as_bytes() + ); + } + } + let kek = kek_fingerprint(CONFIG_KEY); + let projections = [ + ( + "own-key-tcp", + StreamEncryptionSource::Stream, + Some(first_key.fingerprint()), + ), + ( + "own-key-http", + StreamEncryptionSource::Stream, + Some(second_key.fingerprint()), + ), + ("generated-key", StreamEncryptionSource::Server, None), + ]; + for (name, source, fingerprint) in projections { + let details = tcp.get_stream(&stream_id(name)).await.unwrap().unwrap(); + assert_eq!(details.encryption_key_source(), Some(source), "{name}"); + if let Some(fingerprint) = fingerprint { + assert_eq!(details.encryption_key_fingerprint(), Some(fingerprint)); + } + assert_eq!( + details.encryption_kek_fingerprints(), + vec![kek.clone()], + "{name}" + ); + } + for name in streams { + assert_plaintext_on_disk(&harness, &payload(name), false).await; + } + let first_encryptor = first_key.encryptor().unwrap(); + let second_encryptor = second_key.encryptor().unwrap(); + let config_encryptor = + EncryptorKind::Aes256Gcm(Aes256GcmEncryptor::from_base64_key(CONFIG_KEY).unwrap()); + assert_ciphertext_on_disk(&harness, &first_encryptor, &payload("own-key-tcp")).await; + assert_ciphertext_on_disk(&harness, &second_encryptor, &payload("own-key-http")).await; + let foreign_keys = [ + ("own-key-tcp", vec![&*second_encryptor, &config_encryptor]), + ("own-key-http", vec![&*first_encryptor, &config_encryptor]), + ( + "generated-key", + vec![&*first_encryptor, &*second_encryptor, &config_encryptor], + ), + ]; + for (name, encryptors) in foreign_keys { + for encryptor in encryptors { + assert!( + opened_on_disk(&harness, encryptor, payload(name).len()).is_empty(), + "{name} ciphertext opened under a key that is not its own" + ); + } + } + harness.restart_server().await.unwrap(); + let client = harness.tcp_root_client().await.unwrap(); + for name in streams { + let polled = poll_single_partition(&client, &stream_id(name), &topic_id(), 0, 1) + .await + .expect("restart reopens every stream key"); + assert_eq!( + polled.messages[0].payload.as_ref(), + payload(name).as_bytes() + ); + } +} + +#[tokio::test] +#[parallel] +async fn given_keyless_stream_with_encryption_enabled_when_reading_should_report_generated_key() { + let harness = single_node(encryption_config(true, CONFIG_KEY)).await; + let client = harness.tcp_root_client().await.unwrap(); + for name in ["generated-first", "generated-second"] { + create_stream_with_topic(&client, name, None).await; + send_single_message(&client, &stream_id(name), &topic_id(), 1, &payload(name)).await; + } + let first = client + .get_stream(&stream_id("generated-first")) + .await + .unwrap() + .unwrap(); + let second = client + .get_stream(&stream_id("generated-second")) + .await + .unwrap() + .unwrap(); + for details in [&first, &second] { + assert_eq!( + details.encryption_key_source(), + Some(StreamEncryptionSource::Server) + ); + assert_eq!( + details.encryption_kek_fingerprints(), + vec![kek_fingerprint(CONFIG_KEY)] + ); + } + assert_ne!( + first.encryption_key_fingerprint(), + second.encryption_key_fingerprint() + ); + assert!(first.encryption_key_fingerprint().is_some()); + for name in ["generated-first", "generated-second"] { + assert_plaintext_on_disk(&harness, &payload(name), false).await; + let polled = poll_single_partition(&client, &stream_id(name), &topic_id(), 0, 1) + .await + .unwrap(); + assert_eq!( + polled.messages[0].payload.as_ref(), + payload(name).as_bytes() + ); + } +} + +#[tokio::test] +#[parallel] +async fn given_disabled_encryption_with_config_key_when_creating_client_key_stream_should_encrypt_only_that_stream() + { + let harness = single_node(encryption_config(false, CONFIG_KEY)).await; + for transport in [TransportProtocol::Tcp, TransportProtocol::Http] { + let client = harness.root_client_for(transport).await.unwrap(); + let own = format!("disabled-own-{transport}"); + let plain = format!("disabled-plain-{transport}"); + create_stream_with_topic(&client, &own, Some(stream_key(33))).await; + create_stream_with_topic(&client, &plain, None).await; + send_single_message(&client, &stream_id(&own), &topic_id(), 1, &payload(&own)).await; + send_single_message( + &client, + &stream_id(&plain), + &topic_id(), + 1, + &payload(&plain), + ) + .await; + assert_eq!( + source_of(&client, &own).await, + Some(StreamEncryptionSource::Stream) + ); + assert_eq!( + source_of(&client, &plain).await, + Some(StreamEncryptionSource::None) + ); + assert_plaintext_on_disk(&harness, &payload(&own), false).await; + assert_plaintext_on_disk(&harness, &payload(&plain), true).await; + for name in [&own, &plain] { + let polled = poll_single_partition(&client, &stream_id(name), &topic_id(), 0, 1) + .await + .unwrap(); + assert_eq!( + polled.messages[0].payload.as_ref(), + payload(name).as_bytes() + ); + } + } +} + +#[tokio::test] +#[parallel] +async fn given_stream_created_while_disabled_when_enabling_encryption_should_stay_plaintext() { + let mut harness = single_node(encryption_config(false, CONFIG_KEY)).await; + let client = harness.tcp_root_client().await.unwrap(); + create_stream_with_topic(&client, "before-enable", None).await; + + harness + .server_mut() + .add_env("IGGY_ENCRYPTION_ENABLED", "true"); + harness.restart_server().await.unwrap(); + let client = harness.tcp_root_client().await.unwrap(); + send_single_message( + &client, + &stream_id("before-enable"), + &topic_id(), + 1, + &payload("before-enable"), + ) + .await; + create_stream_with_topic(&client, "after-enable", None).await; + + assert_eq!( + source_of(&client, "before-enable").await, + Some(StreamEncryptionSource::None) + ); + assert_eq!( + source_of(&client, "after-enable").await, + Some(StreamEncryptionSource::Server) + ); + assert_plaintext_on_disk(&harness, &payload("before-enable"), true).await; +} + +#[tokio::test] +#[parallel] +async fn given_custom_key_without_config_key_when_creating_should_reject_unavailable() { + let harness = single_node(TestServerConfig::default()).await; + for transport in ALL_TRANSPORTS { + let client = harness.root_client_for(transport).await.unwrap(); + let name = format!("refused-{transport}"); + let error = client + .create_stream( + &name, + &StreamCreateOptions { + encryption_key: Some(stream_key(44)), + ..StreamCreateOptions::default() + }, + ) + .await + .expect_err("a stream key needs a server key"); + assert!( + is_error(&error, &IggyError::StreamEncryptionUnavailable), + "{transport} returned {error}" + ); + assert!( + client + .get_stream(&stream_id(&name)) + .await + .unwrap() + .is_none() + ); + } + let client = harness.tcp_root_client().await.unwrap(); + create_stream_with_topic(&client, "plain-without-key", None).await; + assert_eq!( + source_of(&client, "plain-without-key").await, + Some(StreamEncryptionSource::None) + ); +} + +#[tokio::test] +#[parallel] +async fn given_custom_key_when_reading_every_response_should_return_only_public_projections() { + let harness = single_node(encryption_config(true, CONFIG_KEY)).await; + let key = stream_key(55); + let encoded = text::as_base64(key.as_bytes()); + let hidden = [ + OptionKey::EncryptionKey, + OptionKey::EncryptionKeyWrapped, + OptionKey::CriticalKeys, + ]; + for transport in ALL_TRANSPORTS { + let client = harness.root_client_for(transport).await.unwrap(); + let name = format!("projected-{transport}"); + let created = client + .create_stream( + &name, + &StreamCreateOptions { + encryption_key: Some(key.clone()), + ..StreamCreateOptions::default() + }, + ) + .await + .unwrap(); + let fetched = client.get_stream(&stream_id(&name)).await.unwrap().unwrap(); + let listed = client + .get_streams() + .await + .unwrap() + .into_iter() + .find(|stream| stream.name == name) + .expect("listed"); + for options in [&created.options, &fetched.options, &listed.options] { + for key in hidden { + assert!( + !options.contains_key(&HeaderKey::from(key)), + "{transport}: {key}" + ); + } + let rendered = format!("{options:?}"); + assert!(!rendered.contains(&encoded), "{transport} rendered the key"); + } + for details in [ + created.encryption_key_fingerprint(), + fetched.encryption_key_fingerprint(), + ] { + assert_eq!(details, Some(key.fingerprint()), "{transport}"); + } + assert_eq!( + listed.encryption_key_fingerprint(), + Some(stream_key_fingerprint(key.as_bytes()).unwrap()) + ); + assert_eq!( + fetched.encryption_kek_fingerprints(), + vec![kek_fingerprint(CONFIG_KEY)] + ); + } +} + +#[tokio::test] +#[parallel] +async fn given_changed_config_key_when_restarting_should_refuse_boot_naming_both_fingerprints() { + let mut harness = single_node(encryption_config(true, CONFIG_KEY)).await; + let client = harness.tcp_root_client().await.unwrap(); + create_stream_with_topic(&client, "kept-own", Some(stream_key(66))).await; + create_stream_with_topic(&client, "kept-generated", None).await; + for name in ["kept-own", "kept-generated"] { + send_single_message(&client, &stream_id(name), &topic_id(), 1, &payload(name)).await; + } + + harness + .server_mut() + .add_env("IGGY_ENCRYPTION_KEY", SECOND_ENCRYPTION_KEY); + let error = harness + .restart_server() + .await + .expect_err("a changed config key must refuse boot"); + if stderr_is_captured() { + let diagnostics = error.to_string(); + for fingerprint in [ + kek_fingerprint(CONFIG_KEY), + kek_fingerprint(SECOND_ENCRYPTION_KEY), + ] { + assert!( + diagnostics.contains(&fingerprint), + "the refusal must name {fingerprint}, got: {diagnostics}" + ); + } + } + + harness + .server_mut() + .add_env("IGGY_ENCRYPTION_ENABLED", "false"); + harness.server_mut().add_env("IGGY_ENCRYPTION_KEY", ""); + harness + .restart_server() + .await + .expect_err("no config key must refuse boot over encrypted streams"); + + harness + .server_mut() + .add_env("IGGY_ENCRYPTION_ENABLED", "true"); + harness + .server_mut() + .add_env("IGGY_ENCRYPTION_KEY", CONFIG_KEY); + harness.restart_server().await.unwrap(); + let client = harness.tcp_root_client().await.unwrap(); + for name in ["kept-own", "kept-generated"] { + let polled = poll_single_partition(&client, &stream_id(name), &topic_id(), 0, 1) + .await + .expect("the original key reads the stored messages"); + assert_eq!( + polled.messages[0].payload.as_ref(), + payload(name).as_bytes() + ); + } +} + +async fn single_node(config: TestServerConfig) -> TestHarness { + let mut harness = TestHarness::builder() + .cluster_nodes(1) + .server(config) + .build() + .unwrap(); + harness.start().await.unwrap(); + harness +} + +fn encryption_config(enabled: bool, key: &str) -> TestServerConfig { + TestServerConfig::builder() + .extra_envs(HashMap::from([ + ("IGGY_ENCRYPTION_ENABLED".to_string(), enabled.to_string()), + ("IGGY_ENCRYPTION_KEY".to_string(), key.to_string()), + ])) + .build() +} + +async fn create_stream_with_topic( + client: &IggyClient, + name: &str, + encryption_key: Option<StreamEncryptionKey>, +) { + client + .create_stream( + name, + &StreamCreateOptions { + encryption_key, + ..StreamCreateOptions::default() + }, + ) + .await + .unwrap_or_else(|error| panic!("create stream {name}: {error}")); + client + .create_topic( + &stream_id(name), + TOPIC_NAME, + &TopicCreateOptions { + partitions_count: Some(1), + ..eager_flush_options() + }, + ) + .await + .unwrap(); +} + +async fn source_of(client: &IggyClient, name: &str) -> Option<StreamEncryptionSource> { + client + .get_stream(&stream_id(name)) + .await + .unwrap() + .expect("stream exists") + .encryption_key_source() +} + +/// Wait until the stream data reaches disk, then check whether `payload` +/// appears there as plaintext. +async fn assert_plaintext_on_disk(harness: &TestHarness, payload: &str, expected: bool) { + let deadline = tokio::time::Instant::now() + DISK_WAIT; + loop { + let bytes: Vec<u8> = find_log_files(&harness.server().data_path()) + .iter() + .flat_map(|path| std::fs::read(path).unwrap_or_default()) + .collect(); + let found = bytes + .windows(payload.len()) + .any(|window| window == payload.as_bytes()); + if found == expected && (found || bytes.len() >= payload.len()) { + return; + } + assert!( + tokio::time::Instant::now() < deadline, + "payload {payload} plaintext on disk: expected {expected}, found {found}" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } +} + +/// Wait until `encryptor` opens `payload` from a segment file. +async fn assert_ciphertext_on_disk( + harness: &TestHarness, + encryptor: &EncryptorKind, + payload: &str, +) { + let deadline = tokio::time::Instant::now() + DISK_WAIT; + while !opened_on_disk(harness, encryptor, payload.len()) + .iter() + .any(|plaintext| plaintext == payload.as_bytes()) + { + assert!( + tokio::time::Instant::now() < deadline, + "payload {payload} is not stored under the expected key" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } +} + +/// Every plaintext of `length` bytes that `encryptor` opens from any window of a segment file. +fn opened_on_disk(harness: &TestHarness, encryptor: &EncryptorKind, length: usize) -> Vec<Vec<u8>> { + find_log_files(&harness.server().data_path()) + .iter() + .flat_map(|path| { + std::fs::read(path) + .unwrap_or_default() + .windows(length + AES_GCM_OVERHEAD) + .filter_map(|window| encryptor.decrypt(window).ok()) + .collect::<Vec<_>>() + }) + .collect() +} + +fn is_error(error: &IggyError, expected: &IggyError) -> bool { + match error { + IggyError::HttpResponseError(_, body) => { + body.contains(&format!("\"id\":{}", expected.as_code())) + } + other => other.as_code() == expected.as_code(), + } +} + +fn kek_fingerprint(config_key: &str) -> String { + ServerKeyContext::from_config(config_key, &[], true) + .unwrap() + .kek_fingerprint() + .unwrap() +} + +fn stream_key(seed: u8) -> StreamEncryptionKey { + StreamEncryptionKey::try_from([seed; 32].as_slice()).unwrap() +} + +fn stream_id(name: &str) -> Identifier { + Identifier::named(name).unwrap() +} + +fn topic_id() -> Identifier { + Identifier::named(TOPIC_NAME).unwrap() +} + +fn payload(stream: &str) -> String { + format!("payload-of-{stream}") +} + +fn stderr_is_captured() -> bool { + std::env::var("IGGY_TEST_VERBOSE").is_err() +} + fn eager_flush_options() -> TopicCreateOptions { TopicCreateOptions { durability: iggy_common::Durability::Persisted, @@ -486,15 +1028,57 @@ } } +async fn send_single_message( + client: &IggyClient, + stream_id: &Identifier, + topic_id: &Identifier, + id: u128, + payload: &str, +) { + let mut messages = vec![ + IggyMessage::builder() + .id(id) + .payload(Bytes::copy_from_slice(payload.as_bytes())) + .build() + .unwrap(), + ]; + client + .send_messages( + stream_id, + topic_id, + &Partitioning::partition_id(0), + &mut messages, + ) + .await + .unwrap(); +} + +async fn poll_single_partition( + client: &IggyClient, + stream_id: &Identifier, + topic_id: &Identifier, + offset: u64, + count: u32, +) -> Result<PolledMessages, IggyError> { + client + .poll_messages( + stream_id, + topic_id, + Some(0), + &Consumer::default(), + &PollingStrategy::offset(offset), + count, + false, + ) + .await +} + fn build_server_config(encryption: bool) -> TestServerConfig { let mut extra_envs = HashMap::new(); if encryption { extra_envs.insert("IGGY_ENCRYPTION_ENABLED".to_string(), "true".to_string()); - extra_envs.insert( - "IGGY_ENCRYPTION_KEY".to_string(), - "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=".to_string(), - ); + extra_envs.insert("IGGY_ENCRYPTION_KEY".to_string(), CONFIG_KEY.to_string()); } TestServerConfig::builder().extra_envs(extra_envs).build()
diff --git a/core/integration/tests/server/scenarios/external_group_offsets_scenario.rs b/core/integration/tests/server/scenarios/external_group_offsets_scenario.rs index 29fa84a..1cbf1b9 100644 --- a/core/integration/tests/server/scenarios/external_group_offsets_scenario.rs +++ b/core/integration/tests/server/scenarios/external_group_offsets_scenario.rs
@@ -33,7 +33,11 @@ .expect("Failed to get root client"); let stream = Identifier::named(STREAM_NAME).unwrap(); let topic = Identifier::named(TOPIC_NAME).unwrap(); - let stream_id = client.create_stream(STREAM_NAME).await.unwrap().id; + let stream_id = client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap() + .id; let topic_id = client .create_topic( &stream,
diff --git a/core/integration/tests/server/scenarios/invalid_consumer_offset_scenario.rs b/core/integration/tests/server/scenarios/invalid_consumer_offset_scenario.rs index 731c34c..71d1991 100644 --- a/core/integration/tests/server/scenarios/invalid_consumer_offset_scenario.rs +++ b/core/integration/tests/server/scenarios/invalid_consumer_offset_scenario.rs
@@ -86,7 +86,10 @@ } async fn initialize(client: &IggyClient, stream: &Identifier, topic: &Identifier) -> bool { - client.create_stream(STREAM_NAME).await.unwrap(); + client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); client .create_topic( stream,
diff --git a/core/integration/tests/server/scenarios/log_rotation_scenario.rs b/core/integration/tests/server/scenarios/log_rotation_scenario.rs index ad1c809..0acecca 100644 --- a/core/integration/tests/server/scenarios/log_rotation_scenario.rs +++ b/core/integration/tests/server/scenarios/log_rotation_scenario.rs
@@ -281,7 +281,7 @@ let topic_name = format!("{TOPIC_NAME}-{i}"); client - .create_stream(&stream_name) + .create_stream(&stream_name, &StreamCreateOptions::default()) .await .map_err(|e| format!("Failed to create {stream_name}: {e}"))?;
diff --git a/core/integration/tests/server/scenarios/message_cleanup_scenario.rs b/core/integration/tests/server/scenarios/message_cleanup_scenario.rs index d7fab23..da8a32e 100644 --- a/core/integration/tests/server/scenarios/message_cleanup_scenario.rs +++ b/core/integration/tests/server/scenarios/message_cleanup_scenario.rs
@@ -73,7 +73,10 @@ /// Tests time-based retention: segments are cleaned up after expiry. pub async fn run_expiry_after_rotation(client: &IggyClient, data_path: &Path) { - let stream = client.create_stream(STREAM_NAME).await.unwrap(); + let stream = client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); let stream_id = stream.id; let topic = client @@ -177,7 +180,10 @@ /// Tests that the active segment is never deleted, even if expired. pub async fn run_active_segment_protection(client: &IggyClient, data_path: &Path) { - let stream = client.create_stream(STREAM_NAME).await.unwrap(); + let stream = client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); let stream_id = stream.id; let expiry = Duration::from_secs(1); @@ -240,7 +246,10 @@ /// Tests size-based retention: oldest segments deleted when topic exceeds max_size. pub async fn run_size_based_retention(client: &IggyClient, data_path: &Path) { - let stream = client.create_stream(STREAM_NAME).await.unwrap(); + let stream = client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); let stream_id = stream.id; // 4 MiB max, cleanup at 90% = 3.6 MiB. The cap has to sit above the 1 MiB @@ -337,7 +346,10 @@ /// Tests both retention policies together: time-based AND size-based. pub async fn run_combined_retention(client: &IggyClient, data_path: &Path) { - let stream = client.create_stream(STREAM_NAME).await.unwrap(); + let stream = client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); let stream_id = stream.id; let topic = client @@ -403,7 +415,10 @@ pub async fn run_expiry_with_multiple_partitions(client: &IggyClient, data_path: &Path) { const PARTITIONS_COUNT: u32 = 3; - let stream = client.create_stream(STREAM_NAME).await.unwrap(); + let stream = client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); let stream_id = stream.id; let topic = client @@ -498,7 +513,10 @@ pub async fn run_fair_size_based_cleanup_multipartition(client: &IggyClient, data_path: &Path) { const PARTITIONS_COUNT: u32 = 3; - let stream = client.create_stream(STREAM_NAME).await.unwrap(); + let stream = client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); let stream_id = stream.id; // 12 MiB over 3 partitions = a 4 MiB share each. The SHARE is what has to @@ -626,7 +644,10 @@ const TEST_STREAM: &str = "test_cleaner_barrier_stream"; const TEST_TOPIC: &str = "test_cleaner_barrier_topic"; - let stream = client.create_stream(TEST_STREAM).await.unwrap(); + let stream = client + .create_stream(TEST_STREAM, &StreamCreateOptions::default()) + .await + .unwrap(); let stream_id = stream.id; let topic = client @@ -757,8 +778,8 @@ .update_topic( &Identifier::named(stream).unwrap(), &Identifier::named(topic).unwrap(), - topic, &TopicUpdateOptions { + name: Some(topic.to_owned()), message_expiry: Some(IggyExpiry::ExpireDuration(IggyDuration::from( MESSAGE_EXPIRY, ))),
diff --git a/core/integration/tests/server/scenarios/message_headers_scenario.rs b/core/integration/tests/server/scenarios/message_headers_scenario.rs index 968ec1e..e0c780b 100644 --- a/core/integration/tests/server/scenarios/message_headers_scenario.rs +++ b/core/integration/tests/server/scenarios/message_headers_scenario.rs
@@ -107,7 +107,10 @@ async fn init_system(client: &IggyClient) { // 1. Create the stream - client.create_stream(STREAM_NAME).await.unwrap(); + client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); // 2. Create the topic client
diff --git a/core/integration/tests/server/scenarios/message_size_scenario.rs b/core/integration/tests/server/scenarios/message_size_scenario.rs index 087787d..9a63a07 100644 --- a/core/integration/tests/server/scenarios/message_size_scenario.rs +++ b/core/integration/tests/server/scenarios/message_size_scenario.rs
@@ -105,7 +105,10 @@ async fn init_system(client: &IggyClient) { // 1. Create the stream - client.create_stream(STREAM_NAME).await.unwrap(); + client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); // 2. Create the topic client
diff --git a/core/integration/tests/server/scenarios/offset_scenario.rs b/core/integration/tests/server/scenarios/offset_scenario.rs index 7fdc311..987c8a8 100644 --- a/core/integration/tests/server/scenarios/offset_scenario.rs +++ b/core/integration/tests/server/scenarios/offset_scenario.rs
@@ -127,7 +127,10 @@ topic_name: &str, topic_options: &TopicCreateOptions, ) { - client.create_stream(stream_name).await.unwrap(); + client + .create_stream(stream_name, &StreamCreateOptions::default()) + .await + .unwrap(); client .create_topic( &Identifier::named(stream_name).unwrap(),
diff --git a/core/integration/tests/server/scenarios/permissions_scenario.rs b/core/integration/tests/server/scenarios/permissions_scenario.rs index fb069a9..c643307 100644 --- a/core/integration/tests/server/scenarios/permissions_scenario.rs +++ b/core/integration/tests/server/scenarios/permissions_scenario.rs
@@ -85,12 +85,12 @@ async fn setup_test_resources(root_client: &IggyClient) { root_client - .create_stream(STREAM_1) + .create_stream(STREAM_1, &StreamCreateOptions::default()) .await .expect("create stream 1"); root_client - .create_stream(STREAM_2) + .create_stream(STREAM_2, &StreamCreateOptions::default()) .await .expect("create stream 2"); @@ -135,7 +135,13 @@ .await; root_client - .create_user(username, "password123", UserStatus::Active, permissions) + .create_user( + username, + "password123", + UserStatus::Active, + permissions, + &UserCreateOptions::default(), + ) .await .expect("create test user"); } @@ -182,7 +188,12 @@ assert_unauthorized(client.get_stream(&stream_id).await, "get_stream"); assert_unauthorized(client.get_topics(&stream_id).await, "get_topics"); assert_unauthorized(client.get_topic(&stream_id, &topic_id).await, "get_topic"); - assert_unauthorized(client.create_stream("x").await, "create_stream"); + assert_unauthorized( + client + .create_stream("x", &StreamCreateOptions::default()) + .await, + "create_stream", + ); assert_unauthorized( client .create_topic( @@ -379,6 +390,7 @@ "validpassword123", UserStatus::Active, None, + &UserCreateOptions::default(), ) .await, "read_users: create_user", @@ -415,7 +427,13 @@ // Create a temporary user client - .create_user("temp-user", "temp", UserStatus::Active, None) + .create_user( + "temp-user", + "temp", + UserStatus::Active, + None, + &UserCreateOptions::default(), + ) .await .expect("manage_users: create_user should work"); @@ -423,9 +441,10 @@ client .update_user( &Identifier::named("temp-user").unwrap(), - Some("temp-user-updated"), - None, - &UserUpdateOptions::default(), + &UserUpdateOptions { + username: Some("temp-user-updated".to_owned()), + ..UserUpdateOptions::default() + }, ) .await .expect("manage_users: update_user should work"); @@ -472,12 +491,20 @@ // Cannot create/update/delete streams assert_unauthorized( - client.create_stream("x").await, + client + .create_stream("x", &StreamCreateOptions::default()) + .await, "read_streams: create_stream", ); assert_unauthorized( client - .update_stream(&stream_id, "new-name", &StreamUpdateOptions::default()) + .update_stream( + &stream_id, + &StreamUpdateOptions { + name: Some("new-name".to_owned()), + ..StreamUpdateOptions::default() + }, + ) .await, "read_streams: update_stream", ); @@ -511,15 +538,17 @@ // Create, update, delete stream client - .create_stream("temp-stream") + .create_stream("temp-stream", &StreamCreateOptions::default()) .await .expect("manage_streams: create_stream should work"); client .update_stream( &Identifier::named("temp-stream").unwrap(), - "temp-stream-v2", - &StreamUpdateOptions::default(), + &StreamUpdateOptions { + name: Some("temp-stream-v2".to_owned()), + ..StreamUpdateOptions::default() + }, ) .await .expect("manage_streams: update_stream should work"); @@ -602,8 +631,10 @@ .update_topic( &stream_id, &topic_id, - "new-name", - &TopicUpdateOptions::default(), + &TopicUpdateOptions { + name: Some("new-name".to_owned()), + ..TopicUpdateOptions::default() + }, ) .await, "read_topics: update_topic", @@ -647,8 +678,10 @@ .update_topic( &stream_id, &Identifier::named("temp-topic").unwrap(), - "temp-topic-v2", - &TopicUpdateOptions::default(), + &TopicUpdateOptions { + name: Some("temp-topic-v2".to_owned()), + ..TopicUpdateOptions::default() + }, ) .await .expect("manage_topics: update_topic should work"); @@ -661,7 +694,9 @@ // Cannot create streams assert_unauthorized( - client.create_stream("x").await, + client + .create_stream("x", &StreamCreateOptions::default()) + .await, "manage_topics: create_stream", ); @@ -1152,7 +1187,9 @@ // manage operations NOT implied assert_unauthorized( - client.create_stream("x").await, + client + .create_stream("x", &StreamCreateOptions::default()) + .await, "read_streams does NOT imply manage_streams", ); assert_unauthorized( @@ -1583,8 +1620,10 @@ .update_topic( &stream_id, &Identifier::named("temp-manage-stream-topic").unwrap(), - "temp-manage-stream-topic-v2", - &TopicUpdateOptions::default(), + &TopicUpdateOptions { + name: Some("temp-manage-stream-topic-v2".to_owned()), + ..TopicUpdateOptions::default() + }, ) .await .expect("stream.manage_stream → manage_topics: update_topic"); @@ -2712,7 +2751,7 @@ async fn create_stream_with_topic(root_client: &IggyClient, name: &str) -> StreamDetails { let stream = root_client - .create_stream(name) + .create_stream(name, &StreamCreateOptions::default()) .await .expect("create stream"); root_client @@ -2752,6 +2791,7 @@ global: GlobalPermissions::default(), streams: Some(BTreeMap::from([(MISSING_ID, StreamPermissions::default())])), }), + &UserCreateOptions::default(), ) .await, IggyError::StreamIdNotFound(dummy_id.clone()),
diff --git a/core/integration/tests/server/scenarios/read_during_persistence_scenario.rs b/core/integration/tests/server/scenarios/read_during_persistence_scenario.rs index 8861b83..0964d83 100644 --- a/core/integration/tests/server/scenarios/read_during_persistence_scenario.rs +++ b/core/integration/tests/server/scenarios/read_during_persistence_scenario.rs
@@ -32,7 +32,10 @@ async fn should_read_messages_still_in_the_journal(harness: &integration::harness::TestHarness) { let producer = harness.tcp_root_client().await.unwrap(); - producer.create_stream(STREAM_NAME).await.unwrap(); + producer + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); producer .create_topic( &Identifier::named(STREAM_NAME).unwrap(),
diff --git a/core/integration/tests/server/scenarios/reconnect_after_restart_scenario.rs b/core/integration/tests/server/scenarios/reconnect_after_restart_scenario.rs index 32033f9..6f71817 100644 --- a/core/integration/tests/server/scenarios/reconnect_after_restart_scenario.rs +++ b/core/integration/tests/server/scenarios/reconnect_after_restart_scenario.rs
@@ -41,7 +41,10 @@ pub async fn run_producer(harness: &mut TestHarness) { let client = create_client(harness); Client::connect(&client).await.expect("Failed to connect"); - client.create_stream(STREAM_NAME).await.unwrap(); + client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); client .create_topic( &Identifier::named(STREAM_NAME).unwrap(), @@ -133,7 +136,7 @@ .expect("Failed to create setup client"); setup_client - .create_stream(STREAM_NAME) + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) .await .expect("Failed to create stream"); setup_client @@ -290,7 +293,10 @@ .await .expect("Failed to create client"); - client.create_stream(STREAM).await.unwrap(); + client + .create_stream(STREAM, &StreamCreateOptions::default()) + .await + .unwrap(); client .create_topic( &Identifier::named(STREAM).unwrap(), @@ -405,7 +411,10 @@ .await .expect("Failed to create client"); - let stream = client.create_stream(STREAM).await.unwrap(); + let stream = client + .create_stream(STREAM, &StreamCreateOptions::default()) + .await + .unwrap(); let stream_id = stream.id; let topic = client @@ -541,7 +550,7 @@ .expect("Failed to create setup client"); setup_client - .create_stream(STREAM_NAME) + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) .await .expect("Failed to create stream"); setup_client @@ -645,7 +654,7 @@ .await .expect("Failed to create setup client"); setup_client - .create_stream(STREAM_NAME) + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) .await .expect("Failed to create stream"); setup_client
diff --git a/core/integration/tests/server/scenarios/restart_offset_skip_scenario.rs b/core/integration/tests/server/scenarios/restart_offset_skip_scenario.rs index d4eb19f..e806e10 100644 --- a/core/integration/tests/server/scenarios/restart_offset_skip_scenario.rs +++ b/core/integration/tests/server/scenarios/restart_offset_skip_scenario.rs
@@ -57,7 +57,10 @@ // Step 1-2: Create stream/topic, send pre-restart messages let setup_client = harness.tcp_root_client().await.unwrap(); - setup_client.create_stream(STREAM_NAME).await.unwrap(); + setup_client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); setup_client .create_topic( &stream_id,
diff --git a/core/integration/tests/server/scenarios/segment_rotation_race_scenario.rs b/core/integration/tests/server/scenarios/segment_rotation_race_scenario.rs index f3c7f8a..751b8b7 100644 --- a/core/integration/tests/server/scenarios/segment_rotation_race_scenario.rs +++ b/core/integration/tests/server/scenarios/segment_rotation_race_scenario.rs
@@ -184,7 +184,10 @@ } async fn init_system(client: &IggyClient, total_producers: usize) { - client.create_stream(STREAM_NAME).await.unwrap(); + client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); client .create_topic(
diff --git a/core/integration/tests/server/scenarios/single_message_per_batch_scenario.rs b/core/integration/tests/server/scenarios/single_message_per_batch_scenario.rs index ba22668..9f71daa 100644 --- a/core/integration/tests/server/scenarios/single_message_per_batch_scenario.rs +++ b/core/integration/tests/server/scenarios/single_message_per_batch_scenario.rs
@@ -45,7 +45,10 @@ .await .expect("Failed to get root client"); - producer.create_stream(STREAM_NAME).await.unwrap(); + producer + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); producer .create_topic( &Identifier::named(STREAM_NAME).unwrap(),
diff --git a/core/integration/tests/server/scenarios/stale_client_consumer_group_scenario.rs b/core/integration/tests/server/scenarios/stale_client_consumer_group_scenario.rs index 6787665..e94f299 100644 --- a/core/integration/tests/server/scenarios/stale_client_consumer_group_scenario.rs +++ b/core/integration/tests/server/scenarios/stale_client_consumer_group_scenario.rs
@@ -358,7 +358,10 @@ .await .unwrap(); - client.create_stream(STREAM_NAME).await.unwrap(); + client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); client .create_topic(
diff --git a/core/integration/tests/server/scenarios/stream_size_validation_scenario.rs b/core/integration/tests/server/scenarios/stream_size_validation_scenario.rs index e0b134d..bf618c5 100644 --- a/core/integration/tests/server/scenarios/stream_size_validation_scenario.rs +++ b/core/integration/tests/server/scenarios/stream_size_validation_scenario.rs
@@ -149,7 +149,10 @@ async fn create_stream_assert_empty(client: &IggyClient, stream_name: &str) { // 1. Create stream - client.create_stream(stream_name).await.unwrap(); + client + .create_stream(stream_name, &StreamCreateOptions::default()) + .await + .unwrap(); // 2. Validate stream size and number of messages validate_stream(client, stream_name, 0, 0).await;
diff --git a/core/integration/tests/server/scenarios/system_scenario.rs b/core/integration/tests/server/scenarios/system_scenario.rs index 1bf1ed6..3559ca6 100644 --- a/core/integration/tests/server/scenarios/system_scenario.rs +++ b/core/integration/tests/server/scenarios/system_scenario.rs
@@ -86,7 +86,10 @@ assert!(streams.is_empty()); // 3. Create the stream - let stream = client.create_stream(STREAM_NAME).await.unwrap(); + let stream = client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); let stream_id = stream.id; assert_eq!(stream.name, STREAM_NAME); @@ -126,7 +129,9 @@ assert_eq!(stream.name, STREAM_NAME); // 7. Try to create the stream with the same name and validate that it fails - let create_stream_result = client.create_stream(STREAM_NAME).await; + let create_stream_result = client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await; assert!(create_stream_result.is_err()); // 9. Create the topic @@ -156,11 +161,20 @@ assert_eq!(topic.id, topic_id); assert_eq!(topic.name, TOPIC_NAME); assert_eq!(topic.partitions_count, PARTITIONS_COUNT); - assert_eq!(topic.compression_algorithm, CompressionAlgorithm::default()); + assert_eq!( + topic.resolved_options().compression_algorithm, + CompressionAlgorithm::default() + ); assert_eq!(topic.size, 0); assert_eq!(topic.messages_count, 0); - assert_eq!(topic.message_expiry, IggyExpiry::NeverExpire); - assert_eq!(topic.max_topic_size, MaxTopicSize::Unlimited); + assert_eq!( + topic.resolved_options().message_expiry, + IggyExpiry::NeverExpire + ); + assert_eq!( + topic.resolved_options().max_topic_size, + MaxTopicSize::Unlimited + ); // 11. Get topic details by ID. The owning shards materialize fresh // partitions asynchronously after the commit, but the reply reports the @@ -619,9 +633,8 @@ .update_topic( &Identifier::named(STREAM_NAME).unwrap(), &Identifier::named(TOPIC_NAME).unwrap(), - &updated_topic_name, &TopicUpdateOptions { - compression_algorithm: Some(CompressionAlgorithm::Gzip), + name: Some(updated_topic_name.to_owned()), message_expiry: Some(IggyExpiry::ExpireDuration(message_expiry_duration)), max_topic_size: Some(updated_max_topic_size), ..TopicUpdateOptions::default() @@ -641,20 +654,15 @@ assert_eq!(updated_topic.name, updated_topic_name); assert_eq!( - updated_topic.message_expiry, + updated_topic.resolved_options().message_expiry, IggyExpiry::ExpireDuration(message_expiry_duration) ); assert_eq!( - updated_topic.compression_algorithm, - CompressionAlgorithm::Gzip + updated_topic.resolved_options().max_topic_size, + updated_max_topic_size ); - assert_eq!(updated_topic.max_topic_size, updated_max_topic_size); - // The three settings the update carries as fixed fields are mirrored into - // the stored map, so `options` cannot report a value the typed field has - // already moved past. Compared through the rendered value rather than the - // raw bytes: this scenario runs on every transport, and HTTP renders option - // values as readable strings where the binary transports carry the kind the - // server stored. + // The updated settings read back from the options map as explicit entries, + // the same on every transport. let expiry_key = HeaderKey::from_str(topic_option_keys::MESSAGE_EXPIRY).unwrap(); let expiry = updated_topic .options @@ -663,7 +671,7 @@ assert!(expiry.explicit, "an updated key is explicit"); assert_eq!( expiry.value.to_string_value(), - u64::from(updated_topic.message_expiry).to_string() + u64::from(updated_topic.resolved_options().message_expiry).to_string() ); let max_size_key = HeaderKey::from_str(topic_option_keys::MAX_TOPIC_SIZE).unwrap(); let max_size = updated_topic @@ -672,7 +680,7 @@ .expect("max topic size echoes back as an option"); assert_eq!( max_size.value.to_string_value(), - u64::from(updated_topic.max_topic_size).to_string() + u64::from(updated_topic.resolved_options().max_topic_size).to_string() ); // 40. Update the existing stream and ensure it's updated @@ -681,8 +689,10 @@ client .update_stream( &Identifier::named(STREAM_NAME).unwrap(), - &updated_stream_name, - &StreamUpdateOptions::default(), + &StreamUpdateOptions { + name: Some(updated_stream_name.to_owned()), + ..StreamUpdateOptions::default() + }, ) .await .unwrap(); @@ -711,7 +721,10 @@ // 43. Create the stream let stream_name = format!("{STREAM_NAME}-auto"); - let _ = client.create_stream(&stream_name).await.unwrap(); + let _ = client + .create_stream(&stream_name, &StreamCreateOptions::default()) + .await + .unwrap(); let stream = client .get_stream(&Identifier::named(&stream_name).unwrap())
diff --git a/core/integration/tests/server/scenarios/tcp_tls_scenario.rs b/core/integration/tests/server/scenarios/tcp_tls_scenario.rs index 7a21f03..852a3c3 100644 --- a/core/integration/tests/server/scenarios/tcp_tls_scenario.rs +++ b/core/integration/tests/server/scenarios/tcp_tls_scenario.rs
@@ -21,7 +21,10 @@ pub async fn run(client: &IggyClient) { let stream_name = "test-tls-stream"; - client.create_stream(stream_name).await.unwrap(); + client + .create_stream(stream_name, &StreamCreateOptions::default()) + .await + .unwrap(); let stream = client .get_stream(&Identifier::named(stream_name).unwrap())
diff --git a/core/integration/tests/server/scenarios/timestamp_scenario.rs b/core/integration/tests/server/scenarios/timestamp_scenario.rs index 6a592bb..23508fb 100644 --- a/core/integration/tests/server/scenarios/timestamp_scenario.rs +++ b/core/integration/tests/server/scenarios/timestamp_scenario.rs
@@ -147,7 +147,10 @@ topic_name: &str, topic_options: &TopicCreateOptions, ) { - client.create_stream(stream_name).await.unwrap(); + client + .create_stream(stream_name, &StreamCreateOptions::default()) + .await + .unwrap(); client .create_topic( &Identifier::named(stream_name).unwrap(),
diff --git a/core/integration/tests/server/scenarios/user_scenario.rs b/core/integration/tests/server/scenarios/user_scenario.rs index 91ef52e..618975d 100644 --- a/core/integration/tests/server/scenarios/user_scenario.rs +++ b/core/integration/tests/server/scenarios/user_scenario.rs
@@ -18,6 +18,7 @@ use crate::server::scenarios::create_client; use iggy::prelude::Identifier; use iggy::prelude::PersonalAccessTokenExpiry; +use iggy::prelude::UserCreateOptions; use iggy::prelude::UserStatus; use iggy::prelude::UserUpdateOptions; use iggy::prelude::defaults::DEFAULT_ROOT_USERNAME; @@ -85,6 +86,7 @@ }, streams: None, }), + &UserCreateOptions::default(), ) .await .unwrap(); @@ -106,7 +108,13 @@ // 10. Trying to create a new user (or any other resource) without the appropriate permissions should fail let create_user_result = client - .create_user("user2", "secret", UserStatus::Active, None) + .create_user( + "user2", + "secret", + UserStatus::Active, + None, + &UserCreateOptions::default(), + ) .await; assert!(create_user_result.is_err()); @@ -197,7 +205,13 @@ // 20. Trying to create a new user with the same username should fail let create_duplicated_user = client - .create_user(test_user, test_password, UserStatus::Active, None) + .create_user( + test_user, + test_password, + UserStatus::Active, + None, + &UserCreateOptions::default(), + ) .await; assert!(create_duplicated_user.is_err()); @@ -208,9 +222,11 @@ client .update_user( &Identifier::named(test_user).unwrap(), - Some(updated_test_user), - Some(UserStatus::Inactive), - &UserUpdateOptions::default(), + &UserUpdateOptions { + username: Some(updated_test_user.to_owned()), + status: Some(UserStatus::Inactive), + ..UserUpdateOptions::default() + }, ) .await .unwrap();
diff --git a/core/integration/tests/server/scenarios/websocket_tls_scenario.rs b/core/integration/tests/server/scenarios/websocket_tls_scenario.rs index b0d3f64..4c08a6e 100644 --- a/core/integration/tests/server/scenarios/websocket_tls_scenario.rs +++ b/core/integration/tests/server/scenarios/websocket_tls_scenario.rs
@@ -21,7 +21,10 @@ pub async fn run(client: &IggyClient) { let stream_name = "test-websocket-tls-stream"; - client.create_stream(stream_name).await.unwrap(); + client + .create_stream(stream_name, &StreamCreateOptions::default()) + .await + .unwrap(); let stream = client .get_stream(&Identifier::named(stream_name).unwrap())
diff --git a/core/integration/tests/server/topic_admission_vsr.rs b/core/integration/tests/server/topic_admission_vsr.rs index c4a1458..19ce1be 100644 --- a/core/integration/tests/server/topic_admission_vsr.rs +++ b/core/integration/tests/server/topic_admission_vsr.rs
@@ -65,7 +65,7 @@ async fn given_out_of_bounds_topic_when_creating_should_reject_typed(harness: &TestHarness) { let client = harness.tcp_root_client().await.expect("tcp root client"); client - .create_stream("admission-stream") + .create_stream("admission-stream", &StreamCreateOptions::default()) .await .expect("create stream"); let stream_id = Identifier::from_str_value("admission-stream").expect("stream identifier"); @@ -124,7 +124,7 @@ ) { let client = harness.tcp_root_client().await.expect("tcp root client"); client - .create_stream("update-bounds-stream") + .create_stream("update-bounds-stream", &StreamCreateOptions::default()) .await .expect("create stream"); let stream_id = Identifier::from_str_value("update-bounds-stream").expect("stream identifier"); @@ -155,8 +155,8 @@ .update_topic( stream_id, topic_id, - "update-bounds-topic", &TopicUpdateOptions { + name: Some("update-bounds-topic".to_owned()), max_topic_size: Some(max_topic_size), ..TopicUpdateOptions::default() }, @@ -178,7 +178,8 @@ .expect("get topic") .expect("topic exists"); assert_eq!( - topic.max_topic_size, at_floor, + topic.resolved_options().max_topic_size, + at_floor, "the rejected update must not reach consensus, so the stored cap is unchanged" ); @@ -201,7 +202,7 @@ .expect("get topic") .expect("topic exists"); assert_eq!( - topic.max_topic_size, + topic.resolved_options().max_topic_size, MaxTopicSize::Unlimited, "a sentinel update carries no cap, so the stored one survives" ); @@ -213,7 +214,7 @@ async fn given_updated_topic_when_getting_topic_should_echo_stored_values(harness: &TestHarness) { let client = harness.tcp_root_client().await.expect("tcp root client"); client - .create_stream("echo-stream") + .create_stream("echo-stream", &StreamCreateOptions::default()) .await .expect("create stream"); let stream_id = Identifier::from_str_value("echo-stream").expect("stream identifier"); @@ -237,8 +238,8 @@ .update_topic( stream_id, topic_id, - "echo-topic", &TopicUpdateOptions { + name: Some("echo-topic".to_owned()), message_expiry: Some(message_expiry), max_topic_size: Some(max_topic_size), ..TopicUpdateOptions::default() @@ -252,9 +253,18 @@ .expect("get topic") .expect("topic exists"); let reported_options = TopicCreateOptions::from_resource_options(&topic.options); - assert_eq!(reported_options.max_topic_size, Some(topic.max_topic_size)); - assert_eq!(reported_options.message_expiry, Some(topic.message_expiry)); - (topic.max_topic_size, topic.message_expiry) + assert_eq!( + reported_options.max_topic_size, + Some(topic.resolved_options().max_topic_size) + ); + assert_eq!( + reported_options.message_expiry, + Some(topic.resolved_options().message_expiry) + ); + ( + topic.resolved_options().max_topic_size, + topic.resolved_options().message_expiry, + ) } }; @@ -286,7 +296,7 @@ ) { let client = harness.tcp_root_client().await.expect("tcp root client"); client - .create_stream("sentinel-stream") + .create_stream("sentinel-stream", &StreamCreateOptions::default()) .await .expect("create stream"); let stream_id = Identifier::from_str_value("sentinel-stream").expect("stream identifier"); @@ -322,7 +332,7 @@ .get(&HeaderKey::from_str("max_topic_size").expect("option key")) .expect("max_topic_size is stored"); assert_eq!( - u64::from(topic.max_topic_size), + u64::from(topic.resolved_options().max_topic_size), iggy_common::DEFAULT_MAX_TOPIC_SIZE, "the typed field resolves to the node default" ); @@ -339,7 +349,7 @@ ) { let client = harness.tcp_root_client().await.expect("tcp root client"); client - .create_stream("partitions-stream") + .create_stream("partitions-stream", &StreamCreateOptions::default()) .await .expect("create stream"); let stream_id = Identifier::from_str_value("partitions-stream").expect("stream identifier"); @@ -409,7 +419,7 @@ ) { let client = harness.tcp_root_client().await.expect("tcp root client"); client - .create_stream("over-count-stream") + .create_stream("over-count-stream", &StreamCreateOptions::default()) .await .expect("create stream"); let stream_id = Identifier::from_str_value("over-count-stream").expect("stream identifier");
diff --git a/core/integration/tests/server/topic_metrics_vsr.rs b/core/integration/tests/server/topic_metrics_vsr.rs index c94d6a6..26be62f 100644 --- a/core/integration/tests/server/topic_metrics_vsr.rs +++ b/core/integration/tests/server/topic_metrics_vsr.rs
@@ -59,7 +59,10 @@ harness: &TestHarness, ) { let client = harness.tcp_root_client().await.expect("tcp root client"); - client.create_stream(STREAM).await.expect("create stream"); + client + .create_stream(STREAM, &StreamCreateOptions::default()) + .await + .expect("create stream"); let stream_id = Identifier::from_str_value(STREAM).expect("stream identifier"); let topic_id = Identifier::from_str_value(TOPIC).expect("topic identifier"); client @@ -196,6 +199,7 @@ METRICS_PASSWORD, UserStatus::Active, Some(permissions), + &UserCreateOptions::default(), ) .await .expect("create metrics user"); @@ -242,7 +246,10 @@ } async fn create_topic(client: &IggyClient, max_topic_size: MaxTopicSize) { - client.create_stream(STREAM).await.expect("create stream"); + client + .create_stream(STREAM, &StreamCreateOptions::default()) + .await + .expect("create stream"); let stream_id = Identifier::from_str_value(STREAM).expect("stream identifier"); client .create_topic(
diff --git a/core/metadata/src/impls/metadata.rs b/core/metadata/src/impls/metadata.rs index 47cfbaf..b328ffb 100644 --- a/core/metadata/src/impls/metadata.rs +++ b/core/metadata/src/impls/metadata.rs
@@ -28,6 +28,7 @@ use crate::stm::stream::{Streams, TruncatePartitionRequest}; use crate::stm::user::{CreatePersonalAccessTokenRequest, DeletePersonalAccessTokenRequest, Users}; use crate::stm::{ConsensusGroupAllocator, StateMachine}; +use bytes::Bytes; use compio::io::AsyncWriteAtExt; use consensus::{ CLIENTS_TABLE_MAX, Canceled, ClientTable, ClientTableSnapshot, CommitLogEvent, CommitReply, @@ -49,11 +50,18 @@ use iggy_binary_protocol::primitives::partition_assignment::CreatedPartitionAssignment; use iggy_binary_protocol::requests::partitions::CreatePartitionsRequest as WireCreatePartitionsRequest; use iggy_binary_protocol::requests::partitions::CreatePartitionsWithAssignmentsRequest as PersistedCreatePartitionsRequest; -use iggy_binary_protocol::requests::system::{FinalizeSessionRequest, SessionIdentity}; +use iggy_binary_protocol::requests::streams::{ + CreateStreamRequest as WireCreateStreamRequest, RewrapStreamKeysRequest, + UpdateStreamResolvedRequest, +}; +use iggy_binary_protocol::requests::system::{ + FinalizeSessionRequest, ResourceTarget, SessionIdentity, UpdateResourceRequest, +}; use iggy_binary_protocol::requests::topics::CreateTopicRequest as WireCreateTopicRequest; use iggy_binary_protocol::requests::topics::CreateTopicWithAssignmentsRequest as PersistedCreateTopicRequest; -use iggy_binary_protocol::requests::users::CreateUserRequest; +use iggy_binary_protocol::requests::topics::UpdateTopicResolvedRequest; use iggy_binary_protocol::requests::users::login_register::RegisterRequest; +use iggy_binary_protocol::requests::users::{CreateUserRequest, UpdateUserResolvedRequest}; use iggy_binary_protocol::{ Command, ConsensusHeader, EvictionReason, GenericHeader, Operation, PrepareHeader, PrepareOkHeader, ProtocolVersion, ReplyHeader, RoutedRequestHeader, WireDecode, WireEncode, @@ -61,9 +69,12 @@ }; use iggy_common::calculate_checksum; use iggy_common::variadic; +use iggy_common::wire_conversions::{resource_options_from_wire, resource_options_to_wire}; use iggy_common::{ - IggyByteSize, IggyError, IggyExpiry, MaxTopicSize, TopicCreateOptions, TopicRuntimeDefaults, - UserId, topic_option_keys, validate_topic_segment_size, + HeaderKey, IggyError, OptionAdmissionContext, OptionKey, OptionOperation, OptionValue, + OptionVisibility, OptionsProvenance, OptionsScope, ResourceOptions, ServerKeyContext, + TopicCreateOptions, TopicResolvedOptions, UserId, admit, critical_keys_marker, + derived_defaults, stream_wrap_set, topic_option_keys, validate_topic_segment_size, }; use journal::local_gate::LocalGate; use journal::superblock::{ @@ -1163,6 +1174,9 @@ /// the reads parked on it. Shared by every shard; see /// [`AppliedFrontier`] for the ordering and the wake contract. applied_frontier: Arc<AppliedFrontier>, + /// This node's server keys. The primary refuses to prepare a + /// `CreateStream` whose wrap set none of them opens. + stream_key_context: Arc<ServerKeyContext>, } impl<B, J, S, M, SB> IggyMetadata<VsrConsensus<B>, J, S, M, SB> @@ -1230,6 +1244,7 @@ partitions_max: Cell::new(0), max_tokens_per_user: Cell::new(0), applied_frontier: Arc::default(), + stream_key_context: Arc::default(), } } } @@ -1253,6 +1268,15 @@ self } + /// Admit `CreateStream` wrap sets against this node's server keys, the + /// ones its partitions open stream keys with. Without them every wrapped + /// create is refused. + #[must_use] + pub fn with_stream_key_context(mut self, stream_key_context: Arc<ServerKeyContext>) -> Self { + self.stream_key_context = stream_key_context; + self + } + /// The node-wide applied frontier, readable on every shard. Reads gate on /// it so a client cannot be served state older than a write it already saw /// acked, and park on its wait when it is behind. @@ -3309,6 +3333,59 @@ } } + /// Submit a replicated `RewrapStreamKeys` originated by the server (the + /// stream key rewrapper during a key-encryption key rotation). + /// + /// Same in-process path as the PAT cleaner's delete: the reserved + /// internal client `0`, no request preflight. Apply skips any entry that + /// no longer matches its stream, so a re-proposal on the next tick is + /// harmless. + /// + /// # Errors + /// `NotPrimary` / `NotCaughtUp` when this node cannot replicate, + /// `PipelineFull` under pipeline pressure, `Canceled` if the prepare is + /// dropped before commit. + #[allow(clippy::future_not_send)] + pub async fn submit_rewrap_stream_keys_in_process( + &self, + request: RewrapStreamKeysRequest, + ) -> Result<u64, MetadataSubmitError> { + let consensus = self + .consensus + .as_ref() + .ok_or(MetadataSubmitError::NotPrimary)?; + if !is_caught_up_primary(consensus) { + return Err( + if consensus.is_primary() && consensus.is_normal() && !consensus.is_transferring() { + MetadataSubmitError::NotCaughtUp + } else { + MetadataSubmitError::NotPrimary + }, + ); + } + if consensus.pipeline_is_full() { + return Err(MetadataSubmitError::PipelineFull); + } + let header = RoutedRequestHeader { + client: RESERVED_CLIENT_ID, + group: server_common::sharding::METADATA_GROUP, + ..RoutedRequestHeader::default() + }; + let prepare = consensus::seal_prepare_capacity( + build_prepare_message( + consensus, + &header, + Operation::RewrapStreamKeys, + &request.to_bytes(), + ), + consensus.retry_capacity(&self.client_table.borrow()), + ); + match self.dispatch_prepare_and_await(consensus, prepare).await { + Ok(reply) => Ok(reply.header().commit), + Err(Canceled) => Err(MetadataSubmitError::Canceled), + } + } + /// Submit a replicated client request from in-process and await the /// committed reply. /// @@ -4618,74 +4695,48 @@ Operation::CreateTopic => { let mut request = WireCreateTopicRequest::decode_from(body) .map_err(|_| IggyError::InvalidCommand)?; - // Resolve every absent catalog key against server config here, + // Resolve every absent catalog key to its catalog default here, // at primary admission, so the replicated payload carries // concrete values and every replica commits the same state - // regardless of local config. Resolved defaults ride a separate - // derived block, preserving per-key provenance for `GetTopic`. - let explicit = TopicCreateOptions::parse(&request.options)?; - // Re-encode the explicit block from the parse rather than - // forwarding the client's bytes. Parsing normalizes a zero - // sentinel to "absent", so the resolved value goes in the - // derived block -- but apply merges with explicit winning, so a - // forwarded literal `0` would land back on top as the stored - // effective value. `GetTopic` would report 0, and a restart - // would re-parse that 0 to absent and fall back to whatever the - // node default is by then, not the value resolved at creation. - // Re-encoding also canonicalizes kinds (a `"128MiB"` string - // becomes `Uint64`), so the stored map reads back uniformly. - let supplied_options = request.options.clone(); - request.options = explicit.to_explicit_wire(|key| { - supplied_options - .into_iter() - .any(|entry| entry.key == key.as_bytes()) - })?; - let resolved_segment_size = explicit - .segment_size - .unwrap_or_else(|| IggyByteSize::from(iggy_common::DEFAULT_SEGMENT_SIZE)); - let resolved_max_topic_size = explicit - .max_topic_size - .unwrap_or_else(|| MaxTopicSize::from(iggy_common::DEFAULT_MAX_TOPIC_SIZE)); + // whatever its build. Resolved defaults ride a separate derived + // block, preserving per-key provenance for `GetTopic`. + let admitted = admit( + OptionsScope::Topic, + OptionOperation::Create, + &request.options, + &[], + &OptionAdmissionContext::default(), + )?; + // Re-encode the admitted block rather than forwarding the + // client's bytes. Admission drops a zero sentinel, so the + // resolved value goes in the derived block, but apply merges + // with explicit winning, so a forwarded literal `0` would land + // back on top as the stored effective value. Re-encoding also + // canonicalizes kinds (a `"128MiB"` string becomes `Uint64`). + request.options = + resource_options_to_wire(&admitted.explicit, OptionsProvenance::All)?; + let derived = derived_defaults(OptionsScope::Topic, &admitted.explicit)?; + let mut stored = derived.clone(); + stored.extend(admitted.explicit.clone()); + let resolved = TopicResolvedOptions::from(&stored); + let resolved_segment_size = resolved.runtime.effective_segment_size(); // Backstop for the transport-side typed checks: an explicit // segment size outside its bounds, or a topic cap below one // segment, must never enter the WAL. - if let Some(segment_size) = explicit.segment_size { + if let Some(segment_size) = + TopicCreateOptions::from_resource_options(&admitted.explicit).segment_size + { validate_topic_segment_size( segment_size.as_bytes_u64(), iggy_common::MAX_TOPIC_SEGMENT_SIZE, )?; } - if resolved_max_topic_size.as_bytes_u64() < resolved_segment_size.as_bytes_u64() { + if resolved.max_topic_size.as_bytes_u64() < resolved_segment_size.as_bytes_u64() { return Err(IggyError::InvalidOptionValue( topic_option_keys::MAX_TOPIC_SIZE.to_string(), )); } - let derived_options = explicit.derived_block( - explicit.compression_algorithm.unwrap_or_default(), - explicit - .message_expiry - .unwrap_or_else(|| IggyExpiry::from(iggy_common::DEFAULT_MESSAGE_EXPIRY)), - resolved_max_topic_size, - TopicRuntimeDefaults { - segment_size: resolved_segment_size, - durability: iggy_common::Durability::default(), - consumer_offset_durability: iggy_common::Durability::default(), - messages_required_to_save: explicit - .messages_required_to_save - .unwrap_or(iggy_common::DEFAULT_MESSAGES_REQUIRED_TO_SAVE), - size_of_messages_required_to_save: explicit - .size_of_messages_required_to_save - .unwrap_or_else(|| { - IggyByteSize::from( - iggy_common::DEFAULT_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE, - ) - }), - preallocate_segments: explicit - .preallocate_segments - .unwrap_or(iggy_common::DEFAULT_PREALLOCATE_SEGMENTS), - }, - &supplied_options, - )?; + let derived_options = resource_options_to_wire(&derived, OptionsProvenance::All)?; let mut counts = self.placement_counts()?; let partitions = self .allocator @@ -4755,10 +4806,33 @@ &body, )) } - // `UpdateTopic` deliberately takes the default arm: unlike create, - // an update stores `ServerDefault` sentinels verbatim (legacy - // parity), so a later get echoes `ServerDefault` instead of the - // node default frozen at update time. + Operation::CreateStream => { + let request = WireCreateStreamRequest::decode_from(body) + .map_err(|_| IggyError::InvalidCommand)?; + let options = resource_options_from_wire(&request.options, true)?; + // Apply refuses a plaintext key too, but only after every replica journaled it. + if let Some(secret) = options + .keys() + .filter_map(|key| OptionKey::try_from(key).ok()) + .find(|key| key.descriptor().visibility == OptionVisibility::Secret) + { + return Err(IggyError::InvalidOptionValue(secret.to_string())); + } + // An edge whose keys no peer holds wraps only under its own, and + // data committed under such a wrap opens nowhere else. + if let Some(set) = stream_wrap_set(&options)? + && self.stream_key_context.open(&set).is_err() + { + return Err(IggyError::StreamEncryptionUnavailable); + } + Ok(message.project(consensus)) + } + Operation::UpdateResource => { + let request = UpdateResourceRequest::decode_from(body) + .map_err(|_| IggyError::InvalidCommand)?; + let (operation, body) = resolve_update_resource(&request)?; + Ok(build_prepare_message(consensus, &header, operation, &body)) + } _ => Ok(message.project(consensus)), }?; let prepare = consensus::seal_prepare_capacity( @@ -5163,6 +5237,104 @@ ) } +/// Rewrite a generic patch into the resolved operation of its scope: the +/// column values extracted, the set keys canonical and explicit, each reset key +/// resolved to its catalog default in the derived block. +/// +/// The defaults come from the catalog, never from node configuration, so the +/// replicated body is the same whichever primary admits it. The capacity of +/// the patched map is checked at apply, against committed state. +fn resolve_update_resource( + request: &UpdateResourceRequest, +) -> Result<(Operation, Bytes), IggyError> { + let scope = OptionsScope::from_code(request.target.scope_code())?; + let reset: Vec<String> = request + .reset + .iter() + .map(|name| name.as_str().to_owned()) + .collect(); + let admitted = admit( + scope, + OptionOperation::Update, + &request.options, + &reset, + &OptionAdmissionContext::default(), + )?; + let set_keys: Vec<OptionKey> = admitted + .explicit + .keys() + .filter_map(|key| OptionKey::try_from(key).ok()) + .collect(); + // No update path wraps a secret, so one would be replicated as sent. + if let Some(secret) = set_keys + .iter() + .find(|key| key.descriptor().visibility == OptionVisibility::Secret) + { + return Err(IggyError::OptionNotUpdatable(secret.to_string())); + } + let critical: Vec<OptionKey> = set_keys + .into_iter() + .chain(admitted.reset.iter().copied()) + .filter(|key| key.descriptor().critical) + .collect(); + let mut explicit = admitted.explicit; + if !critical.is_empty() { + explicit.insert( + HeaderKey::from(OptionKey::CriticalKeys), + OptionValue::explicit(critical_keys_marker(&critical)?), + ); + } + let explicit_options = resource_options_to_wire(&explicit, OptionsProvenance::All)?; + let mut derived = ResourceOptions::new(); + for key in &admitted.reset { + if let Some(default) = key.descriptor().default_value()? { + derived.insert(HeaderKey::from(*key), OptionValue::derived(default)); + } + } + let derived_options = resource_options_to_wire(&derived, OptionsProvenance::All)?; + let column_name = |name: Option<String>, error: IggyError| { + name.map(|name| WireName::new(name).map_err(|_| error)) + .transpose() + }; + Ok(match &request.target { + ResourceTarget::Topic { + stream_id, + topic_id, + } => ( + Operation::UpdateTopicResolved, + UpdateTopicResolvedRequest { + stream_id: stream_id.clone(), + topic_id: topic_id.clone(), + name: column_name(admitted.columns.name, IggyError::InvalidTopicName)?, + explicit_options, + derived_options, + } + .to_bytes(), + ), + ResourceTarget::Stream { stream_id } => ( + Operation::UpdateStreamResolved, + UpdateStreamResolvedRequest { + stream_id: stream_id.clone(), + name: column_name(admitted.columns.name, IggyError::InvalidStreamName)?, + explicit_options, + derived_options, + } + .to_bytes(), + ), + ResourceTarget::User { user_id } => ( + Operation::UpdateUserResolved, + UpdateUserResolvedRequest { + user_id: user_id.clone(), + username: column_name(admitted.columns.username, IggyError::InvalidUsername)?, + status: admitted.columns.status.map(|status| status.as_code()), + explicit_options, + derived_options, + } + .to_bytes(), + ), + }) +} + fn build_prepare_message<B, P>( consensus: &VsrConsensus<B, P>, request: &RoutedRequestHeader, @@ -5453,29 +5625,226 @@ use crate::stm::stream::{Streams, StreamsInner}; use crate::stm::user::{PAT_TOKEN_HASH_BYTES, Users, UsersInner}; use consensus::{HistoryRecovery, LocalPipeline}; + use iggy_binary_protocol::codes::CREATE_USER_CODE; use iggy_binary_protocol::primitives::permissions::{ WireGlobalPermissions, WirePermissions, WireStreamPermissions, }; use iggy_binary_protocol::requests::consumer_groups::CreateConsumerGroupRequest; use iggy_binary_protocol::requests::streams::CreateStreamRequest; use iggy_binary_protocol::requests::topics::{CreateTopicRequest, DeleteTopicRequest}; - use iggy_binary_protocol::requests::users::{ - CreateUserRequest, DeleteUserRequest, UpdateUserRequest, + use iggy_binary_protocol::requests::users::{CreateUserRequest, DeleteUserRequest}; + use iggy_binary_protocol::{RequestHeader, StartViewHeader, WireOptions}; + use iggy_common::{ + HeaderValue, IggyExpiry, IggyTimestamp, OptionKey, OptionStorage, OptionValueCodec, + ServerKeyContext, StreamEncryptionKey, StreamKeyOrigin, TopicUpdateOptions, UserStatus, + UserUpdateOptions, WrapSet, critical_keys_marker, descriptors_for, stream_wrap_set, + variadic, }; - use iggy_binary_protocol::{StartViewHeader, WireOptions}; - use iggy_common::{IggyTimestamp, UserStatus, variadic}; use journal::prepare_journal::{DEFAULT_SLOT_COUNT, PrepareJournal}; use journal::superblock::SuperblockContents; use message_bus::{BusMessage, ConnectionLostFn, JoinHandle, ReplicaForwardFn, SendError}; use server_common::MESSAGE_ALIGN; use server_common::iobuf::Frozen; use std::cell::RefCell; + use std::collections::{BTreeMap, BTreeSet}; use std::rc::Rc; use std::sync::mpsc; use std::thread; use std::time::{Duration, Instant}; #[test] + fn given_create_user_when_building_prepare_should_not_store_body_hash_as_request_identifier() { + let body = CreateUserRequest { + username: WireName::new("alice").expect("valid username"), + password: "guessable-password".to_owned(), + status: UserStatus::Active.as_code(), + permissions: None, + options: WireOptions::empty(), + } + .to_bytes(); + let client = + RequestHeader::for_request(CREATE_USER_CODE, 7, 1, 3, &body).expect("request header"); + let consensus = VsrConsensus::new( + 1, + 0, + 1, + server_common::sharding::METADATA_GROUP, + NoopBus, + LocalPipeline::new(), + ); + consensus.init(); + let routed = RoutedRequestHeader { + request_checksum: client.request_checksum, + ..RoutedRequestHeader::default() + }; + let prepare = build_prepare_message(&consensus, &routed, Operation::CreateUser, &body); + assert_ne!(prepare.header().request_checksum, 0); + assert_eq!(prepare.header().request_checksum, client.request_checksum); + assert_ne!( + prepare.header().request_checksum, + u128::from(iggy_common::calculate_checksum(&body)) + ); + } + + #[test] + fn given_topic_patch_when_resolving_should_extract_name_and_resolve_reset_defaults() { + let patch = TopicUpdateOptions { + name: Some("renamed".to_owned()), + message_expiry: Some(IggyExpiry::from(9_000_000u64)), + reset: BTreeSet::from([OptionKey::MaxTopicSize]), + ..TopicUpdateOptions::default() + } + .to_patch() + .unwrap(); + let request = UpdateResourceRequest { + target: ResourceTarget::Topic { + stream_id: WireIdentifier::numeric(1), + topic_id: WireIdentifier::named("orders").unwrap(), + }, + reset: patch.reset, + options: patch.set, + }; + + let (operation, body) = resolve_update_resource(&request).unwrap(); + + assert_eq!(operation, Operation::UpdateTopicResolved); + let resolved = UpdateTopicResolvedRequest::decode_from(&body).unwrap(); + let mut explicit = ResourceOptions::new(); + explicit.insert( + HeaderKey::from(OptionKey::MessageExpiry), + OptionValue::explicit(IggyExpiry::from(9_000_000u64).encode().unwrap()), + ); + let mut derived = ResourceOptions::new(); + derived.insert( + HeaderKey::from(OptionKey::MaxTopicSize), + OptionValue::derived( + OptionKey::MaxTopicSize + .descriptor() + .default_value() + .unwrap() + .unwrap(), + ), + ); + assert_eq!( + resolved, + UpdateTopicResolvedRequest { + stream_id: WireIdentifier::numeric(1), + topic_id: WireIdentifier::named("orders").unwrap(), + name: Some(WireName::new("renamed").unwrap()), + explicit_options: resource_options_to_wire(&explicit, OptionsProvenance::All) + .unwrap(), + derived_options: resource_options_to_wire(&derived, OptionsProvenance::All) + .unwrap(), + } + ); + } + + #[test] + fn given_default_sentinel_when_resolving_patch_should_leave_the_key_unchanged() { + let patch = TopicUpdateOptions { + name: Some("renamed".to_owned()), + raw: BTreeMap::from([( + OptionKey::MessageExpiry.to_string(), + "server_default".to_owned(), + )]), + ..TopicUpdateOptions::default() + } + .to_patch() + .unwrap(); + let request = UpdateResourceRequest { + target: ResourceTarget::Topic { + stream_id: WireIdentifier::numeric(1), + topic_id: WireIdentifier::numeric(2), + }, + reset: patch.reset, + options: patch.set, + }; + + let (_, body) = resolve_update_resource(&request).unwrap(); + + let resolved = UpdateTopicResolvedRequest::decode_from(&body).unwrap(); + assert_eq!(resolved.name, Some(WireName::new("renamed").unwrap())); + assert!(resolved.explicit_options.is_empty()); + assert!(resolved.derived_options.is_empty()); + } + + #[test] + fn given_create_only_or_foreign_key_when_resolving_patch_should_refuse_before_replication() { + let topic = ResourceTarget::Topic { + stream_id: WireIdentifier::numeric(1), + topic_id: WireIdentifier::numeric(2), + }; + let stream = ResourceTarget::Stream { + stream_id: WireIdentifier::numeric(1), + }; + let raw = |key: OptionKey, value: &str| TopicUpdateOptions { + raw: BTreeMap::from([(key.to_string(), value.to_owned())]), + ..TopicUpdateOptions::default() + }; + let cases = [ + ( + topic.clone(), + raw(OptionKey::PartitionResizePolicy, "fixed"), + IggyError::OptionNotUpdatable(OptionKey::PartitionResizePolicy.to_string()), + ), + ( + topic, + raw(OptionKey::EncryptionKeySource, "stream"), + IggyError::UnsupportedOptionKey(OptionKey::EncryptionKeySource.to_string()), + ), + ( + stream, + raw(OptionKey::MessageExpiry, "1000"), + IggyError::UnsupportedOptionKey(OptionKey::MessageExpiry.to_string()), + ), + ]; + for (target, options, expected) in cases { + let patch = options.to_patch().unwrap(); + let request = UpdateResourceRequest { + target, + reset: patch.reset, + options: patch.set, + }; + assert_eq!( + resolve_update_resource(&request).unwrap_err().as_code(), + expected.as_code() + ); + } + } + + #[test] + fn given_user_patch_when_resolving_should_carry_username_and_status_columns() { + let patch = UserUpdateOptions { + username: Some("alicia".to_owned()), + status: Some(UserStatus::Inactive), + ..UserUpdateOptions::default() + } + .to_patch() + .unwrap(); + let request = UpdateResourceRequest { + target: ResourceTarget::User { + user_id: WireIdentifier::numeric(3), + }, + reset: patch.reset, + options: patch.set, + }; + + let (operation, body) = resolve_update_resource(&request).unwrap(); + + assert_eq!(operation, Operation::UpdateUserResolved); + assert_eq!( + UpdateUserResolvedRequest::decode_from(&body).unwrap(), + UpdateUserResolvedRequest { + user_id: WireIdentifier::numeric(3), + username: Some(WireName::new("alicia").unwrap()), + status: Some(UserStatus::Inactive.as_code()), + explicit_options: WireOptions::empty(), + derived_options: WireOptions::empty(), + } + ); + } + + #[test] fn given_cancelled_snapshot_writer_when_replaced_should_keep_the_new_snapshot() { let runtime = server_common::executor::create_shard_executor().unwrap(); runtime.block_on(async { @@ -6870,6 +7239,164 @@ ); } + #[test] + fn given_keyless_create_topic_when_preparing_should_derive_every_catalog_default() { + const CLIENT: u128 = 1; + const SESSION: u64 = 10; + const ACTING_USER: u32 = 7; + let plane = metadata_plane(); + plane + .client_table + .borrow_mut() + .commit_register( + CLIENT, + ACTING_USER, + [0x5a; 32], + register_reply(CLIENT, SESSION), + ) + .expect("register commits"); + + let prepare = plane + .prepare_request(create_topic_request(CLIENT, ACTING_USER)) + .expect("CreateTopic is client-allowed"); + let body = &prepare.as_slice()[size_of::<PrepareHeader>()..prepare.header().size as usize]; + let persisted = PersistedCreateTopicRequest::decode_from(body) + .expect("create topic with assignments prepare must decode"); + let derived = resource_options_from_wire(&persisted.derived_options, false) + .expect("derived block decodes"); + let mut expected = ResourceOptions::new(); + for descriptor in descriptors_for(OptionsScope::Topic) + .filter(|descriptor| descriptor.storage == OptionStorage::Map) + { + if let Some(default) = descriptor.default_value().expect("default encodes") { + expected.insert( + HeaderKey::from(descriptor.key), + OptionValue::derived(default), + ); + } + } + assert!(!expected.is_empty(), "the topic catalog has defaulted rows"); + assert_eq!(derived, expected); + } + + #[test] + fn given_plaintext_stream_key_when_preparing_create_stream_should_refuse_before_replication() { + const CLIENT: u128 = 1; + const SESSION: u64 = 10; + const ACTING_USER: u32 = 7; + let plane = metadata_plane(); + plane + .client_table + .borrow_mut() + .commit_register( + CLIENT, + ACTING_USER, + [0x5a; 32], + register_reply(CLIENT, SESSION), + ) + .expect("register commits"); + let mut options = ResourceOptions::new(); + options.insert( + HeaderKey::from(OptionKey::EncryptionKey), + OptionValue::explicit( + HeaderValue::try_from([7u8; 32].as_slice()).expect("raw key encodes"), + ), + ); + let options = + resource_options_to_wire(&options, OptionsProvenance::All).expect("options encode"); + + let Err(refusal) = + plane.prepare_request(create_stream_request_with(CLIENT, 1, "secrets", options)) + else { + panic!("a plaintext stream key must not be sealed into a prepare"); + }; + + assert!( + matches!( + &refusal, + IggyError::InvalidOptionValue(key) if *key == OptionKey::EncryptionKey.to_string() + ), + "expected the plaintext key refusal, got {refusal:?}" + ); + } + + #[test] + fn given_wrap_set_no_primary_key_opens_when_preparing_create_stream_should_refuse_before_replication() + { + const CLIENT: u128 = 1; + const SESSION: u64 = 10; + const ACTING_USER: u32 = 7; + let primary_keys = Arc::new( + ServerKeyContext::from_config( + "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=", + &[], + true, + ) + .expect("valid key"), + ); + let stranger_keys = ServerKeyContext::from_config( + "AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA=", + &[], + true, + ) + .expect("valid key"); + let plane = metadata_plane().with_stream_key_context(Arc::clone(&primary_keys)); + plane + .client_table + .borrow_mut() + .commit_register( + CLIENT, + ACTING_USER, + [0x5a; 32], + register_reply(CLIENT, SESSION), + ) + .expect("register commits"); + let wrapped_under = |keys: &ServerKeyContext| { + let set = keys + .wrap_set( + &StreamEncryptionKey::try_from([7u8; 32].as_slice()).expect("32 bytes"), + StreamKeyOrigin::Server, + ) + .expect("wraps"); + let mut options = ResourceOptions::new(); + options.insert( + HeaderKey::from(OptionKey::EncryptionKeyWrapped), + OptionValue::explicit(HeaderValue::try_from(set.as_ref()).expect("fits")), + ); + options.insert( + HeaderKey::from(OptionKey::CriticalKeys), + OptionValue::explicit( + critical_keys_marker(&[OptionKey::EncryptionKeyWrapped]).expect("marker"), + ), + ); + resource_options_to_wire(&options, OptionsProvenance::All).expect("options encode") + }; + + let Err(refusal) = plane.prepare_request(create_stream_request_with( + CLIENT, + 1, + "stranger", + wrapped_under(&stranger_keys), + )) else { + panic!("a wrap set the primary cannot open must not be sealed into a prepare"); + }; + + assert_eq!( + refusal.as_code(), + IggyError::StreamEncryptionUnavailable.as_code() + ); + assert!( + plane + .prepare_request(create_stream_request_with( + CLIENT, + 2, + "own", + wrapped_under(&primary_keys), + )) + .is_ok() + ); + } + /// Bus whose `send_to_client` parks forever while `stall` is set, /// recording each parked send in `stall_hits`. Models a client whose /// connection writer stalled, so an `on_ack` driver suspends at a wire @@ -8012,9 +8539,18 @@ request: u64, name: &str, ) -> Message<RoutedRequestHeader> { + create_stream_request_with(client, request, name, WireOptions::empty()) + } + + fn create_stream_request_with( + client: u128, + request: u64, + name: &str, + options: WireOptions, + ) -> Message<RoutedRequestHeader> { let body = iggy_binary_protocol::requests::streams::CreateStreamRequest { name: WireName::new(name).unwrap(), - options: WireOptions::empty(), + options, } .to_bytes(); let header_size = size_of::<RoutedRequestHeader>(); @@ -9781,7 +10317,7 @@ header.session = session; }) }; - for revocation in [Operation::DeleteUser, Operation::UpdateUser] { + for revocation in [Operation::DeleteUser, Operation::UpdateResource] { let directory = tempfile::tempdir().unwrap(); let metadata_directory = directory.path().join(crate::impls::METADATA_DIR); std::fs::create_dir_all(&metadata_directory).unwrap(); @@ -9819,11 +10355,18 @@ } .to_bytes() } else { - UpdateUserRequest { - user_id: WireIdentifier::numeric(VICTIM_USER), - username: None, - status: Some(UserStatus::Inactive.as_code()), - options: WireOptions::empty(), + let patch = UserUpdateOptions { + status: Some(UserStatus::Inactive), + ..UserUpdateOptions::default() + } + .to_patch() + .unwrap(); + UpdateResourceRequest { + target: ResourceTarget::User { + user_id: WireIdentifier::numeric(VICTIM_USER), + }, + reset: patch.reset, + options: patch.set, } .to_bytes() }; @@ -10357,6 +10900,87 @@ } #[compio::test] + async fn given_retried_keyless_create_when_committed_should_store_one_envelope() { + const CLIENT: u128 = 9; + const USER: u32 = 0; + let directory = tempfile::tempdir().unwrap(); + let context = Arc::new( + ServerKeyContext::from_config( + "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=", + &[], + true, + ) + .unwrap(), + ); + let mut metadata = metadata_plane().with_stream_key_context(Arc::clone(&context)); + metadata.journal = Some( + PrepareJournal::open(&directory.path().join("journal.wal"), 0) + .await + .unwrap(), + ); + metadata.mux_stm.users().ensure_root_user("root", "hash"); + metadata + .submit_register_in_process( + CLIENT, + USER, + RegisterRequest { + user_created_at: 1, + bind_verifier: [0x5a; 32], + }, + ) + .await + .unwrap(); + // Two edge rewrites of the same keyless create generate two keys. + let envelope_block = |seed: u8| { + let set = WrapSet::from( + context + .wrap( + &StreamEncryptionKey::try_from([seed; 32].as_slice()).unwrap(), + StreamKeyOrigin::Server, + ) + .unwrap(), + ); + let mut options = ResourceOptions::new(); + options.insert( + HeaderKey::from(OptionKey::EncryptionKeyWrapped), + OptionValue::explicit(HeaderValue::try_from(set.as_ref()).unwrap()), + ); + options.insert( + HeaderKey::from(OptionKey::CriticalKeys), + OptionValue::explicit( + critical_keys_marker(&[OptionKey::EncryptionKeyWrapped]).unwrap(), + ), + ); + ( + set, + resource_options_to_wire(&options, OptionsProvenance::All).unwrap(), + ) + }; + let (first_set, first) = envelope_block(1); + let (_, retry) = envelope_block(2); + + let committed = metadata + .submit_request_in_process(create_stream_request_with(CLIENT, 1, "orders", first)) + .await + .unwrap(); + let replayed = metadata + .submit_request_in_process(create_stream_request_with(CLIENT, 1, "orders", retry)) + .await + .unwrap(); + + assert_eq!(replayed.as_slice(), committed.as_slice()); + let stored = metadata.mux_stm.streams().read(|inner| { + let (_, stream) = inner.items.iter().next().expect("one stream"); + stream_wrap_set(&stream.options).unwrap() + }); + assert_eq!(stored, Some(first_set)); + assert_eq!( + metadata.mux_stm.streams().read(|inner| inner.items.len()), + 1 + ); + } + + #[compio::test] async fn given_newer_local_state_when_installing_stale_transfer_should_preserve_sessions() { const CLIENT: u128 = 9; const NEW_CLIENT: u128 = 10;
diff --git a/core/metadata/src/impls/recovery.rs b/core/metadata/src/impls/recovery.rs index 7fdfdd6..13aeb1e 100644 --- a/core/metadata/src/impls/recovery.rs +++ b/core/metadata/src/impls/recovery.rs
@@ -912,7 +912,7 @@ use iggy_binary_protocol::primitives::permissions::{WireGlobalPermissions, WirePermissions}; use iggy_binary_protocol::requests::streams::CreateStreamRequest; use iggy_binary_protocol::requests::users::{ - CreateUserRequest, DeleteUserRequest, UpdateUserRequest, + CreateUserRequest, DeleteUserRequest, UpdateUserResolvedRequest, }; use iggy_binary_protocol::{WireEncode, WireIdentifier, WireName, WireOptions, result_code}; use iggy_common::UserStatus; @@ -2382,9 +2382,9 @@ type UserStm = MuxStateMachine<iggy_common::variadic!(Users, Streams)>; for (operation, register_first, replace) in [ (Operation::DeleteUser, true, false), - (Operation::UpdateUser, true, false), + (Operation::UpdateUserResolved, true, false), (Operation::DeleteUser, false, false), - (Operation::UpdateUser, false, false), + (Operation::UpdateUserResolved, false, false), (Operation::DeleteUser, false, true), ] { let directory = tempdir().unwrap(); @@ -2411,11 +2411,12 @@ } .to_bytes() } else { - UpdateUserRequest { + UpdateUserResolvedRequest { user_id: WireIdentifier::numeric(USER), username: None, status: Some(UserStatus::Inactive.as_code()), - options: WireOptions::empty(), + explicit_options: WireOptions::empty(), + derived_options: WireOptions::empty(), } .to_bytes() };
diff --git a/core/metadata/src/stm/authz.rs b/core/metadata/src/stm/authz.rs index 7c4f022..ec2ada6 100644 --- a/core/metadata/src/stm/authz.rs +++ b/core/metadata/src/stm/authz.rs
@@ -48,9 +48,9 @@ use iggy_binary_protocol::requests::partitions::{ CreatePartitionsWithAssignmentsRequest, DeletePartitionsRequest, }; -use iggy_binary_protocol::requests::streams::{DeleteStreamRequest, UpdateStreamRequest}; +use iggy_binary_protocol::requests::streams::{DeleteStreamRequest, UpdateStreamResolvedRequest}; use iggy_binary_protocol::requests::topics::{ - CreateTopicWithAssignmentsRequest, DeleteTopicRequest, UpdateTopicRequest, + CreateTopicWithAssignmentsRequest, DeleteTopicRequest, UpdateTopicResolvedRequest, }; use iggy_binary_protocol::requests::users::{ ChangePasswordRequest, CreateUserRequest, UpdatePermissionsRequest, @@ -188,8 +188,10 @@ match header.operation { // Streams. `create_stream` is unscoped; the rest resolve the stream id. Operation::CreateStream => check(users, |perm| perm.create_stream(user_id)), - Operation::UpdateStream => { - let Ok(request) = UpdateStreamRequest::decode_from(body) else { + // The generic patch reaches apply only in its resolved form, which + // keeps the permission of the scope it patches. + Operation::UpdateStreamResolved => { + let Ok(request) = UpdateStreamResolvedRequest::decode_from(body) else { return None; }; stream_scoped(users, streams, &request.stream_id, |perm, sid| { @@ -216,8 +218,8 @@ perm.create_topic(user_id, sid) }) } - Operation::UpdateTopic => { - let Ok(request) = UpdateTopicRequest::decode_from(body) else { + Operation::UpdateTopicResolved => { + let Ok(request) = UpdateTopicResolvedRequest::decode_from(body) else { return None; }; topic_scoped( @@ -336,7 +338,7 @@ // legacy execution layer exempts when the target is the acting user. Operation::CreateUser => check(users, |perm| perm.create_user(user_id)), Operation::DeleteUser => check(users, |perm| perm.delete_user(user_id)), - Operation::UpdateUser => check(users, |perm| perm.update_user(user_id)), + Operation::UpdateUserResolved => check(users, |perm| perm.update_user(user_id)), Operation::UpdatePermissions => check(users, |perm| perm.update_permissions(user_id)), Operation::ChangePassword => { let Ok(request) = ChangePasswordRequest::decode_from(body) else { @@ -365,6 +367,7 @@ | Operation::RetireSession | Operation::CreateTopic | Operation::CreatePartitions + | Operation::UpdateResource | Operation::DeleteSegments | Operation::CreatePersonalAccessToken | Operation::DeletePersonalAccessToken @@ -372,6 +375,7 @@ | Operation::StoreConsumerOffset | Operation::DeleteConsumerOffset | Operation::CompleteLifecycle + | Operation::RewrapStreamKeys | Operation::TransitionPartitionHistory | Operation::InstallConsumerGroupOwner | Operation::RetireConsumerGroupOwners => None, @@ -548,7 +552,7 @@ }; use iggy_binary_protocol::requests::streams::CreateStreamRequest; use iggy_binary_protocol::requests::topics::CreateTopicRequest; - use iggy_binary_protocol::requests::users::CreateUserRequest; + use iggy_binary_protocol::requests::users::{CreateUserRequest, UpdateUserResolvedRequest}; use iggy_binary_protocol::{Command, WireEncode, WireName, WireOptions}; use iggy_common::{StreamPermissions, UserStatus}; use server_common::iobuf::Owned; @@ -787,6 +791,91 @@ } #[test] + fn given_unprivileged_user_when_applying_resolved_patch_should_deny_all_scopes() { + let mux = mux(); + commit_as_root( + &mux, + 1, + Operation::CreateUser, + &create_user_body("alice", no_grants(), Vec::new()), + ); + commit_as_root(&mux, 2, Operation::CreateStream, &create_stream_body("s0")); + commit_as_root( + &mux, + 3, + Operation::CreateTopicWithAssignments, + &create_topic_body(0, "t0"), + ); + let patches = [ + ( + Operation::UpdateStreamResolved, + UpdateStreamResolvedRequest { + stream_id: WireIdentifier::numeric(0), + name: Some(WireName::new("s0-renamed").unwrap()), + explicit_options: WireOptions::empty(), + derived_options: WireOptions::empty(), + } + .to_bytes(), + ), + ( + Operation::UpdateTopicResolved, + UpdateTopicResolvedRequest { + stream_id: WireIdentifier::numeric(0), + topic_id: WireIdentifier::numeric(0), + name: Some(WireName::new("t0-renamed").unwrap()), + explicit_options: WireOptions::empty(), + derived_options: WireOptions::empty(), + } + .to_bytes(), + ), + ( + Operation::UpdateUserResolved, + UpdateUserResolvedRequest { + user_id: WireIdentifier::numeric(ALICE), + username: Some(WireName::new("alice-renamed").unwrap()), + status: None, + explicit_options: WireOptions::empty(), + derived_options: WireOptions::empty(), + } + .to_bytes(), + ), + ]; + let names = |mux: &MuxStateMachine<variadic!(Users, Streams)>| { + let stream_and_topic = mux.streams().read(|inner| { + let stream = &inner.items[0]; + (stream.name.to_string(), stream.topics[0].name.to_string()) + }); + let user = mux.users().read(|inner| { + inner + .items + .get(ALICE as usize) + .map(|user| user.username.to_string()) + }); + (stream_and_topic, user) + }; + let before = names(&mux); + + for (index, (operation, body)) in patches.iter().enumerate() { + let reply = mux + .gated_update(make_prepare(4 + index as u64, *operation, ALICE, body)) + .unwrap(); + assert_eq!(reply.code, UNAUTHORIZED, "{operation:?} must be denied"); + } + assert_eq!(names(&mux), before); + + for (index, (operation, body)) in patches.iter().enumerate() { + commit_as_root(&mux, 7 + index as u64, *operation, body); + } + assert_eq!( + names(&mux), + ( + ("s0-renamed".to_owned(), "t0-renamed".to_owned()), + Some("alice-renamed".to_owned()) + ) + ); + } + + #[test] fn given_manage_streams_grant_when_create_stream_should_allow() { let mux = mux(); let grants = WireGlobalPermissions {
diff --git a/core/metadata/src/stm/lifecycle.rs b/core/metadata/src/stm/lifecycle.rs index 85261ec..fe87e86 100644 --- a/core/metadata/src/stm/lifecycle.rs +++ b/core/metadata/src/stm/lifecycle.rs
@@ -28,6 +28,7 @@ DeletePartitionsRequest, InstallConsumerGroupOwnerRequest, TransitionPartitionHistoryRequest, }; use iggy_binary_protocol::requests::streams::DeleteStreamRequest; +use iggy_binary_protocol::requests::system::{OPTIONS_SCOPE_STREAM, OPTIONS_SCOPE_TOPIC}; use iggy_binary_protocol::requests::topics::DeleteTopicRequest; use iggy_binary_protocol::{ LIFECYCLE_BATCH_HEADER_SIZE, LIFECYCLE_COMPLETION_SIZE, Operation, WireDecode, WireEncode, @@ -235,16 +236,23 @@ } pub fn request_lifecycle_blocked(&self, operation: Operation, body: &[u8]) -> bool { - let topic_scoped = match operation { - Operation::UpdateStream | Operation::DeleteStream | Operation::CreateTopic => false, - Operation::UpdateTopic - | Operation::DeleteTopic + // The generic patch names its scope in the first byte. A user patch + // touches no stream, a stream or topic patch is gated like the + // per-resource update it replaces. + let (topic_scoped, body) = match operation { + Operation::UpdateResource => match body.split_first() { + Some((&OPTIONS_SCOPE_STREAM, identifiers)) => (false, identifiers), + Some((&OPTIONS_SCOPE_TOPIC, identifiers)) => (true, identifiers), + _ => return false, + }, + Operation::DeleteStream | Operation::CreateTopic => (false, body), + Operation::DeleteTopic | Operation::CreatePartitions | Operation::DeletePartitions | Operation::CreateConsumerGroup | Operation::DeleteConsumerGroup | Operation::JoinConsumerGroup - | Operation::LeaveConsumerGroup => true, + | Operation::LeaveConsumerGroup => (true, body), _ => return false, }; let Ok((stream, consumed)) = WireIdentifier::decode(body) else { @@ -781,6 +789,9 @@ use iggy_binary_protocol::requests::consumer_groups::{ CreateConsumerGroupRequest, JoinConsumerGroupRequest, }; + use iggy_binary_protocol::requests::system::{ + ResourceTarget, UpdateResourceRequest, WireResetKeys, + }; use iggy_binary_protocol::requests::topics::{ CreateTopicRequest, CreateTopicWithAssignmentsRequest, }; @@ -804,6 +815,7 @@ client_id: 17, session: 5, request: 8, + user_id: 0, } } @@ -1377,6 +1389,56 @@ } #[test] + fn given_pending_lifecycle_when_preflighting_generic_patch_should_gate_like_the_scope() { + let streams = streams(); + streams.seed_namespace(IggyNamespace::new(0, 1, 0), 3, 0); + streams.seed_namespace(IggyNamespace::new(1, 0, 0), 4, 0); + assert_eq!( + apply( + &streams, + Operation::DeleteTopic, + &DeleteTopicRequest { + stream_id: WireIdentifier::numeric(0), + topic_id: WireIdentifier::numeric(0), + }, + context() + ) + .code, + 0 + ); + assert!(streams.lifecycle_pending(INTENT_OP)); + let blocked = |target| { + let body = UpdateResourceRequest { + target, + reset: WireResetKeys::try_from(vec![WireName::new("message_expiry").unwrap()]) + .unwrap(), + options: WireOptions::empty(), + } + .to_bytes(); + streams.request_lifecycle_blocked(Operation::UpdateResource, &body) + }; + let topic = |topic_id| ResourceTarget::Topic { + stream_id: WireIdentifier::numeric(0), + topic_id: WireIdentifier::numeric(topic_id), + }; + let stream = |stream_id| ResourceTarget::Stream { + stream_id: WireIdentifier::numeric(stream_id), + }; + + assert!(blocked(topic(0))); + assert!(!blocked(topic(1))); + // A stream patch waits for every intent in its stream, as the + // per-stream update it replaces did. + assert!(blocked(stream(0))); + assert!(!blocked(stream(1))); + // A user identifier that names stream 0 if the gate read an identifier + // at byte 0 instead of the scope. + assert!(!blocked(ResourceTarget::User { + user_id: WireIdentifier::numeric(0), + })); + } + + #[test] fn given_pending_group_deletion_when_other_consumers_arrive_should_isolate_the_group() { let streams = streams(); let partitions = streams.read(|inner| inner.items[0].topics[0].partitions.clone());
diff --git a/core/metadata/src/stm/mod.rs b/core/metadata/src/stm/mod.rs index e3c8c0e..68f55b5 100644 --- a/core/metadata/src/stm/mod.rs +++ b/core/metadata/src/stm/mod.rs
@@ -41,6 +41,7 @@ pub client_id: u128, pub session: u64, pub request: u64, + pub user_id: u32, } impl From<&PrepareHeader> for ApplyContext { @@ -50,6 +51,7 @@ client_id: header.client, session: header.session, request: header.request, + user_id: header.user_id, } } }
diff --git a/core/metadata/src/stm/result.rs b/core/metadata/src/stm/result.rs index d54a97d..9c999ad 100644 --- a/core/metadata/src/stm/result.rs +++ b/core/metadata/src/stm/result.rs
@@ -175,6 +175,7 @@ StreamNotFound = 1009, NameAlreadyExists = 1012, InvalidOptionValue = 4042, + OptionsBlockTooLarge = 4043, }); result_enum!(DeleteStreamResult { StreamNotFound = 1009 }); @@ -189,10 +190,11 @@ }); result_enum!(UpdateTopicResult { StreamNotFound = 1009, + InvalidTopicSize = 1019, TopicNotFound = 2010, NameAlreadyExists = 2013, - UnsupportedOptionKey = 4041, InvalidOptionValue = 4042, + OptionsBlockTooLarge = 4043, }); result_enum!(DeleteTopicResult { StreamNotFound = 1009, @@ -238,8 +240,11 @@ result_enum!(UpdateUserResult { UserNotFound = 20, InvalidUsername = 43, + InvalidUserStatus = 45, UsernameAlreadyExists = 46, InvalidOptionValue = 4042, + OptionsBlockTooLarge = 4043, + CannotModifyRootUser = 4108, }); result_enum!(DeleteUserResult { UserNotFound = 20, @@ -328,12 +333,19 @@ } match operation { Operation::CreateStream => CreateStreamResult::from_u32(code).is_some(), - Operation::UpdateStream => UpdateStreamResult::from_u32(code).is_some(), + Operation::UpdateStreamResolved => UpdateStreamResult::from_u32(code).is_some(), Operation::DeleteStream => DeleteStreamResult::from_u32(code).is_some(), Operation::CreateTopic | Operation::CreateTopicWithAssignments => { CreateTopicResult::from_u32(code).is_some() } - Operation::UpdateTopic => UpdateTopicResult::from_u32(code).is_some(), + Operation::UpdateTopicResolved => UpdateTopicResult::from_u32(code).is_some(), + // A client resend can carry the generic code before the reply names + // the resolved operation, so accept a result of any scope. + Operation::UpdateResource => { + UpdateStreamResult::from_u32(code).is_some() + || UpdateTopicResult::from_u32(code).is_some() + || UpdateUserResult::from_u32(code).is_some() + } Operation::DeleteTopic => DeleteTopicResult::from_u32(code).is_some(), Operation::CreatePartitions | Operation::CreatePartitionsWithAssignments => { CreatePartitionsResult::from_u32(code).is_some() @@ -341,7 +353,7 @@ Operation::DeletePartitions => DeletePartitionsResult::from_u32(code).is_some(), Operation::TruncatePartition => TruncatePartitionResult::from_u32(code).is_some(), Operation::CreateUser => CreateUserResult::from_u32(code).is_some(), - Operation::UpdateUser => UpdateUserResult::from_u32(code).is_some(), + Operation::UpdateUserResolved => UpdateUserResult::from_u32(code).is_some(), Operation::DeleteUser => DeleteUserResult::from_u32(code).is_some(), Operation::ChangePassword => ChangePasswordResult::from_u32(code).is_some(), Operation::UpdatePermissions => UpdatePermissionsResult::from_u32(code).is_some(), @@ -364,11 +376,12 @@ const fn lifecycle_gated(operation: Operation) -> bool { matches!( operation, - Operation::UpdateStream + Operation::UpdateResource + | Operation::UpdateStreamResolved | Operation::DeleteStream | Operation::CreateTopic | Operation::CreateTopicWithAssignments - | Operation::UpdateTopic + | Operation::UpdateTopicResolved | Operation::DeleteTopic | Operation::CreatePartitions | Operation::CreatePartitionsWithAssignments @@ -396,7 +409,7 @@ #[cfg(test)] mod tests { use super::*; - use iggy_common::{Identifier, IggyError}; + use iggy_common::{Identifier, IggyByteSize, IggyError, MaxTopicSize}; #[test] fn from_u32_round_trips_known_codes_and_rejects_unknown() { @@ -516,7 +529,7 @@ for operation in [ Operation::CreateUser, Operation::CreateStream, - Operation::UpdateTopic, + Operation::UpdateTopicResolved, Operation::JoinConsumerGroup, ] { assert!( @@ -637,6 +650,12 @@ topic_name_exists ); + // InvalidTopicSize (1019). + assert_eq!( + u32::from(UpdateTopicResult::InvalidTopicSize), + IggyError::InvalidTopicSize(MaxTopicSize::default(), IggyByteSize::default()).as_code(), + ); + // InvalidPartitionsCount (2019). assert_eq!( u32::from(CreatePartitionsResult::InvalidPartitionsCount), @@ -668,10 +687,24 @@ IggyError::PartitionNotFound(0, id(), id()).as_code(), ); - // UnsupportedOptionKey (4041) / InvalidOptionValue (4042). + // InvalidOptionValue (4042) / OptionsBlockTooLarge (4043). + let too_large = IggyError::OptionsBlockTooLarge(String::new()).as_code(); assert_eq!( - u32::from(UpdateTopicResult::UnsupportedOptionKey), - IggyError::UnsupportedOptionKey(String::new()).as_code(), + u32::from(UpdateStreamResult::OptionsBlockTooLarge), + too_large + ); + assert_eq!( + u32::from(UpdateTopicResult::OptionsBlockTooLarge), + too_large + ); + assert_eq!(u32::from(UpdateUserResult::OptionsBlockTooLarge), too_large); + assert_eq!( + u32::from(UpdateUserResult::InvalidUserStatus), + IggyError::InvalidUserStatus.as_code() + ); + assert_eq!( + u32::from(UpdateUserResult::CannotModifyRootUser), + IggyError::CannotModifyRootUser.as_code() ); let invalid_option_value = IggyError::InvalidOptionValue(String::new()).as_code(); assert_eq!(
diff --git a/core/metadata/src/stm/snapshot.rs b/core/metadata/src/stm/snapshot.rs index a311178..8d24c23 100644 --- a/core/metadata/src/stm/snapshot.rs +++ b/core/metadata/src/stm/snapshot.rs
@@ -59,7 +59,11 @@ /// /// Version 12: pending consumer group owner generations equal their metadata /// operation, replacing independently incremented generations. -pub const SNAPSHOT_FORMAT_VERSION: u32 = 12; +/// +/// Version 13: `TopicSnapshot` dropped its typed expiry, compression and size +/// fields, which the options map already holds, and `StreamSnapshot` gained +/// `created_op`, the identity of a stream incarnation. +pub const SNAPSHOT_FORMAT_VERSION: u32 = 13; /// Oldest format version [`MetadataSnapshot::decode`] still reads. /// @@ -592,10 +596,12 @@ #[cfg(test)] mod tests { use super::*; - use crate::stm::stream::{PartitionSnapshot, StatsSnapshot, StreamSnapshot, TopicSnapshot}; + use crate::stm::stream::{ + PartitionSnapshot, StatsSnapshot, StreamSnapshot, StreamsSnapshot, TopicSnapshot, + }; use crate::stm::user::UserSnapshot; use iggy_common::{ - CompressionAlgorithm, IggyExpiry, IggyTimestamp, MaxTopicSize, ResourceOptions, + HeaderKey, HeaderValue, IggyTimestamp, OptionKey, OptionValue, ResourceOptions, }; #[test] @@ -622,7 +628,7 @@ // operator's boot reading one field's bytes as another's. Changing either // number is the reminder to change the other. const FIELD_COUNT: u32 = 7; - const PINNED_VERSION: u32 = 12; + const PINNED_VERSION: u32 = 13; let encoded = MetadataSnapshot::new(0).encode().unwrap(); let mut cursor = encoded.as_slice(); @@ -644,8 +650,8 @@ // the same element count: a version 2 file then decoded field-by-field // one position out of place instead of failing. The types the STM // actually grows need their own pins. - const TOPIC_FIELD_COUNT: u32 = 12; - const STREAM_FIELD_COUNT: u32 = 6; + const TOPIC_FIELD_COUNT: u32 = 9; + const STREAM_FIELD_COUNT: u32 = 7; const STREAMS_FIELD_COUNT: u32 = 4; const USER_FIELD_COUNT: u32 = 7; // Version 4 appended `fences`; it is defaulted on read, which is what @@ -672,9 +678,6 @@ id: 0, name: String::new(), created_at: IggyTimestamp::default(), - message_expiry: IggyExpiry::default(), - compression_algorithm: CompressionAlgorithm::default(), - max_topic_size: MaxTopicSize::default(), stats: StatsSnapshot { size_bytes: 0, messages_count: 0, @@ -719,6 +722,7 @@ }, topics: Vec::new(), options: ResourceOptions::new(), + created_op: 0, }; let encoded = rmp_serde::to_vec(&stream).unwrap(); assert_eq!( @@ -829,6 +833,49 @@ } #[test] + fn given_format_thirteen_snapshot_when_restoring_should_preserve_options_and_created_op() { + const PREVIOUS_FORMAT: u32 = 12; + let mut options = ResourceOptions::new(); + options.insert( + HeaderKey::from(OptionKey::MessageExpiry), + OptionValue::explicit(HeaderValue::from(60_000_000u64)), + ); + options.insert( + HeaderKey::from(OptionKey::MaxTopicSize), + OptionValue::derived(HeaderValue::from(u64::MAX)), + ); + let mut stream = stream_snapshot_fixture( + 3, + "orders", + StatsSnapshot { + size_bytes: 0, + messages_count: 0, + segments_count: 0, + }, + ); + stream.created_op = 41; + stream.options = options.clone(); + let mut snapshot = MetadataSnapshot::new(50); + snapshot.streams = Some(StreamsSnapshot { + lifecycle_intents: std::collections::BTreeMap::default(), + revision: 0, + namespace_revision: 0, + items: vec![(3, stream)], + }); + + let decoded = MetadataSnapshot::decode(&snapshot.encode().unwrap()).unwrap(); + let (_, restored) = &decoded.streams.as_ref().unwrap().items[0]; + assert_eq!(restored.created_op, 41); + assert_eq!(restored.options, options); + + snapshot.version = PREVIOUS_FORMAT; + assert!(matches!( + MetadataSnapshot::decode(&snapshot.encode().unwrap()), + Err(SnapshotError::UnsupportedFormatVersion { found, .. }) if found == PREVIOUS_FORMAT + )); + } + + #[test] fn decode_refuses_any_other_format_version() { // Newer and older alike: there is no accepted window, so both directions are // the same refusal. Version 0 is what a zeroed or absent stamp reads as. @@ -906,9 +953,6 @@ id: 0, name: "topic".to_string(), created_at: ts, - message_expiry: IggyExpiry::default(), - compression_algorithm: CompressionAlgorithm::default(), - max_topic_size: MaxTopicSize::default(), stats: StatsSnapshot { size_bytes: 256, messages_count: 12, @@ -932,6 +976,7 @@ }, )], options: ResourceOptions::default(), + created_op: 9, }, )], }); @@ -983,6 +1028,7 @@ stats, topics: vec![], options: ResourceOptions::default(), + created_op: 0, } }
diff --git a/core/metadata/src/stm/stream.rs b/core/metadata/src/stm/stream.rs index 6416bb5..c1ae327 100644 --- a/core/metadata/src/stm/stream.rs +++ b/core/metadata/src/stm/stream.rs
@@ -35,11 +35,6 @@ use ahash::{AHashMap, AHashSet}; use bytes::{BufMut, Bytes, BytesMut}; use iggy_binary_protocol::codec::{WireDecode, WireEncode}; -use std::collections::{BTreeMap, BTreeSet, VecDeque}; -// Only `seed_namespace` (sim/test-gated) uses this at module scope, so keep the -// import under the same gate. The test module re-imports it independently. -#[cfg(any(test, feature = "simulator"))] -use iggy_binary_protocol::primitives::options::WireOptions; use iggy_binary_protocol::primitives::partition_assignment::CreatedPartitionAssignment; use iggy_binary_protocol::requests::consumer_groups::{ CreateConsumerGroupRequest, DeleteConsumerGroupRequest, @@ -50,8 +45,9 @@ CreatePartitionsWithAssignmentsRequest, DeletePartitionsRequest, }; use iggy_binary_protocol::requests::streams::{ - CreateStreamRequest, DeleteStreamRequest, UpdateStreamRequest, + CreateStreamRequest, DeleteStreamRequest, RewrapStreamKeysRequest, UpdateStreamResolvedRequest, }; +use std::collections::{BTreeMap, BTreeSet, VecDeque}; // Only the slab-seeding helpers build a bare `CreateTopicRequest`; without // their cfg the import is dead and `-p <crate>` clippy (which skips the // simulator feature) rejects it. @@ -59,7 +55,7 @@ #[cfg(any(test, feature = "simulator"))] use iggy_binary_protocol::requests::topics::CreateTopicRequest; use iggy_binary_protocol::requests::topics::{ - CreateTopicWithAssignmentsRequest, DeleteTopicRequest, UpdateTopicRequest, + CreateTopicWithAssignmentsRequest, DeleteTopicRequest, UpdateTopicResolvedRequest, }; use iggy_binary_protocol::responses::consumer_groups::consumer_group_response::ConsumerGroupResponse; use iggy_binary_protocol::responses::consumer_groups::get_consumer_group::{ @@ -68,12 +64,15 @@ use iggy_binary_protocol::responses::streams::StreamResponse; use iggy_binary_protocol::responses::streams::get_stream::{GetStreamResponse, TopicHeader}; use iggy_binary_protocol::responses::topics::get_topic::PartitionResponse; -use iggy_binary_protocol::{WireIdentifier, WireName}; -use iggy_common::wire_conversions::{resource_options_from_wire, resource_options_to_wire_split}; +use iggy_binary_protocol::{WireIdentifier, WireName, WireOptions}; +use iggy_common::wire_conversions::{ + resource_options_from_wire, resource_options_to_wire, resource_options_to_wire_split, +}; use iggy_common::{ - CompressionAlgorithm, IggyByteSize, IggyExpiry, IggyTimestamp, MaxTopicSize, - PartitionResizePolicy, PartitionStats, ResourceOptions, StreamStats, TopicCreateOptions, - TopicRuntimeOptions, TopicStats, topic_option_keys, + HeaderKey, HeaderValue, IggyByteSize, IggyExpiry, IggyTimestamp, MaxTopicSize, OptionKey, + OptionValue, OptionsProvenance, OptionsScope, PartitionResizePolicy, PartitionStats, + ResourceOptions, StreamStats, TopicCreateOptions, TopicResolvedOptions, TopicStats, WrapSet, + public_options, stream_wrap_set, }; use serde::{Deserialize, Serialize}; use server_common::sharding::{IggyNamespace, MAX_PARTITIONS, MAX_STREAMS, MAX_TOPICS}; @@ -164,9 +163,6 @@ pub id: usize, pub name: String, pub created_at: IggyTimestamp, - pub message_expiry: IggyExpiry, - pub compression_algorithm: CompressionAlgorithm, - pub max_topic_size: MaxTopicSize, pub stats: StatsSnapshot, pub partitions: Vec<PartitionSnapshot>, // `round_robin_counter` is intentionally NOT snapshotted. It is a local @@ -188,13 +184,13 @@ pub id: usize, pub name: Arc<str>, pub created_at: IggyTimestamp, - pub message_expiry: IggyExpiry, - pub compression_algorithm: CompressionAlgorithm, - pub max_topic_size: MaxTopicSize, - /// Resolved creation options: the client's explicit keys plus the - /// defaults derived at admission. `partitions_count` is never stored - /// here; the partitions vec is the authority. + /// The client's explicit keys plus the defaults derived at admission, the + /// single source of truth for the topic's settings. `partitions_count` is + /// never stored here, the partitions vec is the authority. Replace it + /// through [`Topic::set_options`] so `resolved` follows. pub options: ResourceOptions, + /// Typed view of `options`, rebuilt whenever they change. Not persisted. + pub resolved: TopicResolvedOptions, pub stats: Arc<TopicStats>, pub partitions: Vec<Partition>, @@ -227,10 +223,8 @@ id: 0, name: Arc::from(""), created_at: IggyTimestamp::default(), - message_expiry: IggyExpiry::default(), - compression_algorithm: CompressionAlgorithm::default(), - max_topic_size: MaxTopicSize::default(), options: ResourceOptions::new(), + resolved: TopicResolvedOptions::from(&ResourceOptions::new()), stats: Arc::new(TopicStats::default()), partitions: Vec::new(), round_robin_counter: Arc::new(AtomicUsize::new(0)), @@ -243,22 +237,13 @@ } impl Topic { - pub fn new( - name: Arc<str>, - created_at: IggyTimestamp, - message_expiry: IggyExpiry, - compression_algorithm: CompressionAlgorithm, - max_topic_size: MaxTopicSize, - stream_stats: Arc<StreamStats>, - ) -> Self { + pub fn new(name: Arc<str>, created_at: IggyTimestamp, stream_stats: Arc<StreamStats>) -> Self { Self { id: 0, name, created_at, - message_expiry, - compression_algorithm, - max_topic_size, options: ResourceOptions::new(), + resolved: TopicResolvedOptions::from(&ResourceOptions::new()), stats: Arc::new(TopicStats::new(stream_stats)), partitions: Vec::new(), round_robin_counter: Arc::new(AtomicUsize::new(0)), @@ -269,6 +254,12 @@ } } + /// Replace the stored options and rebuild the typed view from them. + pub fn set_options(&mut self, options: ResourceOptions) { + self.resolved = TopicResolvedOptions::from(&options); + self.options = options; + } + pub fn rebalance_consumer_groups(&mut self, metadata_op: u64, timestamp: IggyTimestamp) { for group in self.consumer_groups.values_mut() { group.rebalance_members(&self.partitions, metadata_op, timestamp.as_micros()); @@ -299,6 +290,9 @@ pub topics: Vec<(usize, TopicSnapshot)>, #[serde(default)] pub options: ResourceOptions, + /// Metadata op that created this stream incarnation. A reused stream id + /// gets a new value, so per-stream key material never carries over. + pub created_op: u64, } #[derive(Debug)] @@ -307,6 +301,8 @@ pub name: Arc<str>, pub created_at: IggyTimestamp, pub options: ResourceOptions, + /// Metadata op that created this stream incarnation. + pub created_op: u64, pub stats: Arc<StreamStats>, pub topics: IdSlab<Topic>, @@ -320,6 +316,7 @@ name: Arc::from(""), created_at: IggyTimestamp::default(), options: ResourceOptions::default(), + created_op: 0, stats: Arc::new(StreamStats::default()), topics: IdSlab::new(), topic_index: AHashMap::default(), @@ -334,6 +331,7 @@ name: self.name.clone(), created_at: self.created_at, options: self.options.clone(), + created_op: self.created_op, stats: self.stats.clone(), topics: self.topics.clone(), topic_index: self.topic_index.clone(), @@ -349,6 +347,7 @@ name, created_at, options: ResourceOptions::new(), + created_op: 0, stats: Arc::new(StreamStats::default()), topics: IdSlab::new(), topic_index: AHashMap::default(), @@ -362,6 +361,7 @@ name, created_at, options: ResourceOptions::new(), + created_op: 0, stats, topics: IdSlab::new(), topic_index: AHashMap::default(), @@ -986,10 +986,10 @@ collect_handlers! { Streams { CreateStream, - UpdateStream, + UpdateStreamResolved, DeleteStream, CreateTopicWithAssignments, - UpdateTopic, + UpdateTopicResolved, DeleteTopic, CreatePartitionsWithAssignments, DeletePartitions, @@ -1005,6 +1005,7 @@ CompleteConsumerGroupRevocation, TruncatePartition, CompleteLifecycle, + RewrapStreamKeys, } internal { RefreshConsumerGroupSession } } @@ -1293,10 +1294,10 @@ self.inner.read(|inner| { let topic = inner.items.get(stream_id)?.topics.get(topic_id)?; Some(( - topic.message_expiry, - topic.max_topic_size, + topic.resolved.message_expiry, + topic.resolved.max_topic_size, topic.partitions.len(), - TopicRuntimeOptions::from_resource_options(&topic.options).segment_size, + topic.resolved.runtime.segment_size, )) }) } @@ -1363,7 +1364,7 @@ let stream_slab = inner.resolve_stream_id(stream_id)?; let topic_slab = inner.resolve_topic_id(stream_slab, topic_id)?; let topic = inner.items.get(stream_slab)?.topics.get(topic_slab)?; - TopicRuntimeOptions::from_resource_options(&topic.options).segment_size + topic.resolved.runtime.segment_size }) } @@ -2177,6 +2178,11 @@ let Ok(options) = resource_options_from_wire(&self.options, true) else { return ApplyReply::err(CreateStreamResult::InvalidOptionValue); }; + // The edge replaces a client key with its wrapped form. A plaintext key + // reaching apply means that step was skipped, and it is never stored. + if options.contains_key(&HeaderKey::from(OptionKey::EncryptionKey)) { + return ApplyReply::err(CreateStreamResult::InvalidOptionValue); + } // Share one `Arc<StreamStats>` across both left-right buffers via the // registry (see `StatsRegistry`). The id the next insert will use is @@ -2185,12 +2191,19 @@ if !admits_slab_key(id, MAX_STREAMS) { return ApplyReply::err(CreateStreamResult::TooManyStreams); } + // The reply is cached for retries and folded into snapshots, so it is + // projected here, before caching: internal keys such as the wrapped + // stream key never leave the state machine. The projections come from + // committed state alone, so every node caches the same reply. A replay + // keeps the create-time KEK fingerprints, which a later rewrap outdates. + let reply_options = public_reply_options(OptionsScope::Stream, &options); let stream_stats = state.stats_registry.stream(id); let stream = Stream { id, name: name_arc.clone(), created_at: timestamp, options, + created_op: state.apply_context.metadata_op, stats: stream_stats, topics: IdSlab::new(), topic_index: AHashMap::default(), @@ -2211,7 +2224,8 @@ size_bytes: 0, messages_count: 0, name: self.name.clone(), - options: self.options.clone(), + options: reply_options.0, + derived_options: reply_options.1, }, topics: Vec::new(), } @@ -2220,39 +2234,50 @@ } } -impl StateHandler for UpdateStreamRequest { +impl StateHandler for UpdateStreamResolvedRequest { type State = StreamsInner; fn apply(&self, state: &mut StreamsInner, _timestamp: IggyTimestamp) -> ApplyReply { let Some(stream_id) = state.resolve_stream_id(&self.stream_id) else { return ApplyReply::err(UpdateStreamResult::StreamNotFound); }; - if state.lifecycle_blocks(stream_id, None) { return ApplyReply::err(iggy_common::IggyError::LifecycleBusy.as_code()); } - let Some(stream) = state.items.get_mut(stream_id) else { + let Some(stream) = state.items.get(stream_id) else { return ApplyReply::err(UpdateStreamResult::StreamNotFound); }; - - let new_name_arc: Arc<str> = Arc::from(self.name.as_str()); - if let Some(&existing_id) = state.index.get(&new_name_arc) + // Every check runs before any mutation, so a refused patch leaves the + // stream exactly as it was: no half-applied rename or option change. + let new_name: Option<Arc<str>> = self.name.as_ref().map(|name| Arc::from(name.as_str())); + if let Some(name) = &new_name + && let Some(&existing_id) = state.index.get(name) && existing_id != stream_id { return ApplyReply::err(UpdateStreamResult::NameAlreadyExists); } - - // Decoded before any mutation: a malformed block must leave the stream - // untouched rather than half-renamed. - let Ok(updated_options) = resource_options_from_wire(&self.options, true) else { - return ApplyReply::err(UpdateStreamResult::InvalidOptionValue); + let options = match patched_options( + &stream.options, + &self.explicit_options, + &self.derived_options, + ) { + Ok(options) => options, + Err(PatchError::InvalidValue) => { + return ApplyReply::err(UpdateStreamResult::InvalidOptionValue); + } + Err(PatchError::TooLarge) => { + return ApplyReply::err(UpdateStreamResult::OptionsBlockTooLarge); + } }; - state.index.remove(&stream.name); - stream.name = new_name_arc.clone(); - // Patch, never replace: keys the client did not send keep their - // current value, so a client that predates a key cannot erase it. - stream.options.extend(updated_options); - state.index.insert(new_name_arc, stream_id); + let Some(stream) = state.items.get_mut(stream_id) else { + return ApplyReply::err(UpdateStreamResult::StreamNotFound); + }; + if let Some(name) = new_name { + state.index.remove(&stream.name); + stream.name = name.clone(); + state.index.insert(name, stream_id); + } + stream.options = options; ApplyReply::ok(Bytes::new()) } } @@ -2387,11 +2412,7 @@ id: topic_id, name: name_arc.clone(), created_at: timestamp, - message_expiry: resolved.message_expiry.unwrap_or(IggyExpiry::ServerDefault), - compression_algorithm: resolved.compression_algorithm.unwrap_or_default(), - max_topic_size: resolved - .max_topic_size - .unwrap_or(MaxTopicSize::ServerDefault), + resolved: TopicResolvedOptions::from(&options), options, stats: topic_stats, partitions: Vec::new(), @@ -2436,6 +2457,18 @@ } } +/// A create reply's option blocks: the public projection, or none for a +/// resource this build cannot serve. The resource is stored either way, so the +/// committed verdict never depends on the local catalog. +pub(crate) fn public_reply_options( + scope: OptionsScope, + stored: &ResourceOptions, +) -> (WireOptions, WireOptions) { + public_options(scope, stored) + .and_then(|public| resource_options_to_wire_split(&public)) + .unwrap_or_else(|_| (WireOptions::empty(), WireOptions::empty())) +} + /// Encode the `CreateTopic` reply as `[TopicHeader][PartitionResponse]*`, /// the `GetTopicResponse` shape the SDK already decodes, so the create /// reply deserializes without a schema break. Returns empty bytes on a @@ -2453,16 +2486,11 @@ let Ok(partitions_count_u32) = u32::try_from(topic.partitions.len()) else { return Bytes::new(); }; - let Ok((options, derived_options)) = resource_options_to_wire_split(&topic.options) else { - return Bytes::new(); - }; + let (options, derived_options) = public_reply_options(OptionsScope::Topic, &topic.options); let header = TopicHeader { id: topic_id_u32, created_at: topic.created_at.into(), partitions_count: partitions_count_u32, - message_expiry: u64::from(topic.message_expiry), - compression_algorithm: topic.compression_algorithm.as_code(), - max_topic_size: u64::from(topic.max_topic_size), size_bytes: 0, messages_count: 0, name: name.clone(), @@ -2506,7 +2534,7 @@ buf.freeze() } -impl StateHandler for UpdateTopicRequest { +impl StateHandler for UpdateTopicResolvedRequest { type State = StreamsInner; fn apply(&self, state: &mut StreamsInner, _timestamp: IggyTimestamp) -> ApplyReply { let Some(stream_id) = state.resolve_stream_id(&self.stream_id) else { @@ -2518,63 +2546,155 @@ if state.lifecycle_blocks(stream_id, Some(topic_id)) { return ApplyReply::err(iggy_common::IggyError::LifecycleBusy.as_code()); } - let Some(stream) = state.items.get_mut(stream_id) else { return ApplyReply::err(UpdateTopicResult::StreamNotFound); }; - let Some(topic) = stream.topics.get_mut(topic_id) else { + let Some(topic) = stream.topics.get(topic_id) else { return ApplyReply::err(UpdateTopicResult::TopicNotFound); }; - - let new_name_arc: Arc<str> = Arc::from(self.name.as_str()); - if let Some(&existing_id) = stream.topic_index.get(&new_name_arc) + // Every check runs before any mutation, so a refused patch leaves the + // topic exactly as it was: no half-applied rename or option change. + // The blocks are read leniently, a key this build does not know is + // stored rather than refused, so the verdict never depends on the + // build applying it. + let new_name: Option<Arc<str>> = self.name.as_ref().map(|name| Arc::from(name.as_str())); + if let Some(name) = &new_name + && let Some(&existing_id) = stream.topic_index.get(name) && existing_id != topic_id { return ApplyReply::err(UpdateTopicResult::NameAlreadyExists); } - - // Decoded before any mutation: a malformed block must leave the topic - // untouched rather than half-renamed. - let Ok(mut updated_options) = resource_options_from_wire(&self.options, true) else { - return ApplyReply::err(UpdateTopicResult::InvalidOptionValue); + let options = match patched_options( + &topic.options, + &self.explicit_options, + &self.derived_options, + ) { + Ok(options) => options, + Err(PatchError::InvalidValue) => { + return ApplyReply::err(UpdateTopicResult::InvalidOptionValue); + } + Err(PatchError::TooLarge) => { + return ApplyReply::err(UpdateTopicResult::OptionsBlockTooLarge); + } }; - if self - .options - .into_iter() - .any(|entry| entry.key == topic_option_keys::PARTITION_RESIZE_POLICY.as_bytes()) + // The edge floor ran on a view a delete and recreate can outrun, and + // `segment_size` is create-only, so the committed topic's value decides. + let segment_size = topic.resolved.runtime.effective_segment_size(); + if let Some(max_topic_size) = + TopicCreateOptions::parse_committed(&self.explicit_options).max_topic_size + && !matches!(max_topic_size, MaxTopicSize::ServerDefault) + && max_topic_size.as_bytes_u64() < segment_size.as_bytes_u64() { - return ApplyReply::err(UpdateTopicResult::UnsupportedOptionKey); + return ApplyReply::err(UpdateTopicResult::InvalidTopicSize); } - // Read leniently, like every other committed op: a key this build does - // not know is skipped rather than failing an operation its peers - // accepted. - let updated = TopicCreateOptions::parse_committed(&self.options); - // Default sentinels leave both the effective value and its provenance - // unchanged, just as an omitted key does. - updated_options.retain(|key, _| match key.as_str() { - Ok(topic_option_keys::MESSAGE_EXPIRY) => updated.message_expiry.is_some(), - Ok(topic_option_keys::MAX_TOPIC_SIZE) => updated.max_topic_size.is_some(), - _ => true, - }); - stream.topic_index.remove(&topic.name); - topic.name = new_name_arc.clone(); - if let Some(compression_algorithm) = updated.compression_algorithm { - topic.compression_algorithm = compression_algorithm; + let Some(topic) = stream.topics.get_mut(topic_id) else { + return ApplyReply::err(UpdateTopicResult::TopicNotFound); + }; + if let Some(name) = new_name { + stream.topic_index.remove(&topic.name); + topic.name = name.clone(); + stream.topic_index.insert(name, topic_id); } - if let Some(message_expiry) = updated.message_expiry { - topic.message_expiry = message_expiry; - } - if let Some(max_topic_size) = updated.max_topic_size { - topic.max_topic_size = max_topic_size; - } - // Patch, never replace, for the stored map too. - topic.options.extend(updated_options); - stream.topic_index.insert(new_name_arc, topic_id); + topic.set_options(options); ApplyReply::ok(Bytes::new()) } } +impl StateHandler for RewrapStreamKeysRequest { + type State = StreamsInner; + /// Replace the wrap set of every stream incarnation an entry still names. + /// An entry for a recreated stream, another key or a malformed set is + /// skipped, so a late or repeated rewrap never touches the wrong stream. + /// Reply body: the number of sets replaced, `u32` LE. + fn apply(&self, state: &mut StreamsInner, _timestamp: IggyTimestamp) -> ApplyReply { + let mut replaced = 0u32; + for entry in &self.entries { + let Some(stream) = state.items.get_mut(entry.stream_id as usize) else { + continue; + }; + if stream.created_op != entry.created_op { + continue; + } + let Ok(Some(stored)) = stream_wrap_set(&stream.options) else { + continue; + }; + let Ok(set) = WrapSet::try_from(entry.wrap_set.as_ref()) else { + continue; + }; + if stored.key_fingerprint_bytes() != entry.key_fingerprint + || set.key_fingerprint_bytes() != entry.key_fingerprint + || set.origin() != stored.origin() + { + continue; + } + let Ok(value) = HeaderValue::try_from(set.as_ref()) else { + continue; + }; + stream.options.insert( + HeaderKey::from(OptionKey::EncryptionKeyWrapped), + OptionValue::explicit(value), + ); + replaced += 1; + } + ApplyReply::ok(Bytes::copy_from_slice(&replaced.to_le_bytes())) + } +} + +/// Why a committed patch could not be applied. +pub(crate) enum PatchError { + /// A block held an entry of a known kind with the wrong payload. + InvalidValue, + /// The patched map would exceed the options block limits. + TooLarge, +} + +/// `current` with the patch applied: derived resets first, then the explicit +/// set entries, each replacing the stored entry of the same key. The patch's +/// `critical_keys` names are added to the stored marker instead. +/// +/// The size check runs on the result, against the same caps a response +/// encoder enforces, so a stored map can always be read back. It depends only +/// on the committed blocks and state, so every replica reaches the same +/// verdict. +pub(crate) fn patched_options( + current: &ResourceOptions, + explicit: &WireOptions, + derived: &WireOptions, +) -> Result<ResourceOptions, PatchError> { + let mut explicit = + resource_options_from_wire(explicit, true).map_err(|_| PatchError::InvalidValue)?; + let derived = + resource_options_from_wire(derived, false).map_err(|_| PatchError::InvalidValue)?; + let marker_key = HeaderKey::from(OptionKey::CriticalKeys); + let mut options = current.clone(); + options.extend(derived); + if let Some(added) = explicit.remove(&marker_key) { + let merged = merged_critical_marker(current.get(&marker_key), &added)?; + options.insert(marker_key, OptionValue::explicit(merged)); + } + options.extend(explicit); + resource_options_to_wire(&options, OptionsProvenance::All).map_err(|_| PatchError::TooLarge)?; + Ok(options) +} + +/// The stored marker's names joined with the patch's, sorted and unique. +/// +/// Plain text work, never a catalog lookup, so a name this build does not know +/// survives and every replica stores the same marker. +fn merged_critical_marker( + stored: Option<&OptionValue>, + added: &OptionValue, +) -> Result<HeaderValue, PatchError> { + let mut names = BTreeSet::new(); + for marker in stored.into_iter().chain([added]) { + let text = str::from_utf8(marker.value.as_bytes()).map_err(|_| PatchError::InvalidValue)?; + names.extend(text.split(',')); + } + let merged = names.into_iter().collect::<Vec<_>>().join(","); + HeaderValue::try_from(merged.as_str()).map_err(|_| PatchError::TooLarge) +} + impl StateHandler for DeleteTopicRequest { type State = StreamsInner; fn apply(&self, state: &mut StreamsInner, _timestamp: IggyTimestamp) -> ApplyReply { @@ -2650,9 +2770,7 @@ let Some(topic) = stream.topics.get_mut(topic_id) else { return ApplyReply::err(CreatePartitionsResult::TopicNotFound); }; - if TopicCreateOptions::from_resource_options(&topic.options).partition_resize_policy - == Some(PartitionResizePolicy::Fixed) - { + if topic.resolved.partition_resize_policy == PartitionResizePolicy::Fixed { return ApplyReply::err(CreatePartitionsResult::PartitionResizeDisabled); } @@ -2748,9 +2866,7 @@ }; let count_to_delete = self.partitions_count as usize; - if TopicCreateOptions::from_resource_options(&topic.options).partition_resize_policy - == Some(PartitionResizePolicy::Fixed) - { + if topic.resolved.partition_resize_policy == PartitionResizePolicy::Fixed { return ApplyReply::err(DeletePartitionsResult::PartitionResizeDisabled); } if count_to_delete > topic.partitions.len() { @@ -2836,9 +2952,6 @@ id: topic.id, name: topic.name.to_string(), created_at: topic.created_at, - message_expiry: topic.message_expiry, - compression_algorithm: topic.compression_algorithm, - max_topic_size: topic.max_topic_size, options: topic.options.clone(), stats: StatsSnapshot { size_bytes: t_size, @@ -2883,6 +2996,7 @@ }, topics, options: stream.options.clone(), + created_op: stream.created_op, }, ) }) @@ -2972,9 +3086,7 @@ id: topic_snap.id, name: topic_name.clone(), created_at: topic_snap.created_at, - message_expiry: topic_snap.message_expiry, - compression_algorithm: topic_snap.compression_algorithm, - max_topic_size: topic_snap.max_topic_size, + resolved: TopicResolvedOptions::from(&topic_snap.options), options: topic_snap.options, stats: topic_stats, partitions: topic_snap @@ -3023,6 +3135,7 @@ name: stream_name.clone(), created_at: stream_snap.created_at, options: stream_snap.options, + created_op: stream_snap.created_op, stats: stream_stats, topics, topic_index, @@ -3080,11 +3193,16 @@ CreatePartitionsRequest as WireCreatePartitionsRequest, CreatePartitionsWithAssignmentsRequest, }; + use iggy_binary_protocol::requests::streams::RewrapStreamKeyEntry; use iggy_binary_protocol::requests::topics::{ CreateTopicRequest as WireCreateTopicRequest, CreateTopicWithAssignmentsRequest, }; use iggy_binary_protocol::responses::topics::get_topic::GetTopicResponse; - use iggy_common::{HeaderKey, HeaderKind, TopicUpdateOptions, topic_option_keys}; + use iggy_common::{ + CompressionAlgorithm, DEFAULT_SEGMENT_SIZE, HeaderKey, HeaderKind, OptionKey, OptionValue, + OptionValueCodec, ServerKeyContext, StreamEncryptionKey, StreamKeyOrigin, WrappedStreamKey, + topic_option_keys, + }; use std::str::FromStr; #[test] @@ -3164,9 +3282,18 @@ let stream = inner.items.get(0).unwrap(); let (_, topic) = stream.topics.iter().next().unwrap(); - assert_eq!(topic.message_expiry, IggyExpiry::from(5_000_000u64)); - assert_eq!(topic.max_topic_size, MaxTopicSize::from(10_000_000_000u64)); - assert_eq!(topic.compression_algorithm, CompressionAlgorithm::None); + assert_eq!( + topic.resolved.message_expiry, + IggyExpiry::from(5_000_000u64) + ); + assert_eq!( + topic.resolved.max_topic_size, + MaxTopicSize::from(10_000_000_000u64) + ); + assert_eq!( + topic.resolved.compression_algorithm, + CompressionAlgorithm::None + ); // partitions_count is create-consumed, never persisted. assert_eq!(topic.options.len(), 5); @@ -3331,7 +3458,10 @@ let stream = inner.items.get(0).unwrap(); let (_, topic) = stream.topics.iter().next().unwrap(); - assert_eq!(topic.max_topic_size, MaxTopicSize::from(10_000_000_000u64)); + assert_eq!( + topic.resolved.max_topic_size, + MaxTopicSize::from(10_000_000_000u64) + ); let size_key = HeaderKey::from_str(topic_option_keys::MAX_TOPIC_SIZE).unwrap(); let stored = topic.options.get(&size_key).unwrap(); assert_eq!( @@ -3341,8 +3471,498 @@ ); } + const OLD_CONFIG_KEY: &str = "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE="; + const NEW_CONFIG_KEY: &str = "AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA="; + const THIRD_CONFIG_KEY: &str = "ICEiIyQlJicoKSorLC0uLzAxMjM0NTY3ODk6Ozw9Pj8="; + + fn key_context(key: &str, previous: &[&str]) -> ServerKeyContext { + let previous: Vec<String> = previous.iter().map(|key| (*key).to_owned()).collect(); + ServerKeyContext::from_config(key, &previous, true).unwrap() + } + + fn envelope(context: &ServerKeyContext, seed: u8, origin: StreamKeyOrigin) -> WrappedStreamKey { + context + .wrap( + &StreamEncryptionKey::try_from([seed; 32].as_slice()).unwrap(), + origin, + ) + .unwrap() + } + + fn create_encrypted_stream(inner: &mut StreamsInner, name: &str, set: &WrapSet, op: u64) { + let mut options = ResourceOptions::new(); + options.insert( + HeaderKey::from(OptionKey::EncryptionKeyWrapped), + OptionValue::explicit(HeaderValue::try_from(set.as_ref()).unwrap()), + ); + inner.apply_context.metadata_op = op; + let request = CreateStreamRequest { + name: WireName::new(name).unwrap(), + options: resource_options_to_wire(&options, OptionsProvenance::All).unwrap(), + }; + assert_eq!( + StateHandler::apply(&request, inner, IggyTimestamp::now()).code, + 0 + ); + } + + fn rewrap_entry( + stream_id: u32, + created_op: u64, + stored: &WrapSet, + set: &[u8], + ) -> RewrapStreamKeyEntry { + RewrapStreamKeyEntry { + stream_id, + created_op, + key_fingerprint: stored.key_fingerprint_bytes().try_into().unwrap(), + wrap_set: Bytes::copy_from_slice(set), + } + } + + fn replaced_count(reply: &ApplyReply) -> u32 { + u32::from_le_bytes(reply.body.as_ref().try_into().unwrap()) + } + #[test] - fn update_topic_sentinels_preserve_effective_options_and_provenance() { + fn given_stale_rewrap_entry_for_recreated_stream_when_applying_should_skip_it() { + let old = key_context(OLD_CONFIG_KEY, &[]); + let rotated = key_context(NEW_CONFIG_KEY, &[OLD_CONFIG_KEY]); + let mut inner = StreamsInner::new(); + let first = WrapSet::from(envelope(&old, 1, StreamKeyOrigin::Server)); + create_encrypted_stream(&mut inner, "orders", &first, 5); + let delete = DeleteStreamRequest { + stream_id: WireIdentifier::numeric(0), + }; + assert_eq!( + crate::stm::lifecycle::apply_with_lifecycle_completion( + &delete, + &mut inner, + IggyTimestamp::now() + ) + .code, + 0 + ); + let second = WrapSet::from(envelope(&old, 2, StreamKeyOrigin::Server)); + create_encrypted_stream(&mut inner, "orders", &second, 9); + let stale = rotated.target_set(&first).unwrap().unwrap(); + let fresh = rotated.target_set(&second).unwrap().unwrap(); + + for skipped in [ + rewrap_entry(0, 5, &first, stale.as_ref()), + rewrap_entry(0, 9, &first, fresh.as_ref()), + rewrap_entry(7, 9, &second, fresh.as_ref()), + ] { + let reply = StateHandler::apply( + &RewrapStreamKeysRequest { + entries: vec![skipped], + }, + &mut inner, + IggyTimestamp::now(), + ); + assert_eq!(replaced_count(&reply), 0); + assert_eq!( + stream_wrap_set(&inner.items[0].options).unwrap(), + Some(second.clone()) + ); + } + + let reply = StateHandler::apply( + &RewrapStreamKeysRequest { + entries: vec![rewrap_entry(0, 9, &second, fresh.as_ref())], + }, + &mut inner, + IggyTimestamp::now(), + ); + assert_eq!(replaced_count(&reply), 1); + assert_eq!( + stream_wrap_set(&inner.items[0].options).unwrap(), + Some(fresh) + ); + } + + #[test] + fn given_rewrap_entry_of_earlier_incarnation_with_same_key_when_applying_should_skip_it() { + let old = key_context(OLD_CONFIG_KEY, &[]); + let rotated = key_context(NEW_CONFIG_KEY, &[OLD_CONFIG_KEY]); + let mut inner = StreamsInner::new(); + let first = WrapSet::from(envelope(&old, 1, StreamKeyOrigin::Client)); + create_encrypted_stream(&mut inner, "orders", &first, 5); + let delete = DeleteStreamRequest { + stream_id: WireIdentifier::numeric(0), + }; + assert_eq!( + crate::stm::lifecycle::apply_with_lifecycle_completion( + &delete, + &mut inner, + IggyTimestamp::now() + ) + .code, + 0 + ); + let second = WrapSet::from(envelope(&old, 1, StreamKeyOrigin::Client)); + create_encrypted_stream(&mut inner, "orders", &second, 9); + let stale = rotated + .target_set(&first) + .expect("rewrap target builds") + .expect("first set lacks the new KEK"); + + let reply = StateHandler::apply( + &RewrapStreamKeysRequest { + entries: vec![rewrap_entry(0, 5, &first, stale.as_ref())], + }, + &mut inner, + IggyTimestamp::now(), + ); + + assert_eq!(replaced_count(&reply), 0); + assert_eq!( + stream_wrap_set(&inner.items[0].options).expect("stored set decodes"), + Some(second) + ); + } + + #[test] + fn given_rewrap_entry_of_another_key_when_applying_should_skip_it() { + let old = key_context(OLD_CONFIG_KEY, &[]); + let rotated = key_context(NEW_CONFIG_KEY, &[OLD_CONFIG_KEY]); + let mut inner = StreamsInner::new(); + let stored = WrapSet::from(envelope(&old, 3, StreamKeyOrigin::Server)); + create_encrypted_stream(&mut inner, "payments", &stored, 4); + let before = inner.items[0].options.clone(); + let other = WrapSet::from(envelope(&old, 9, StreamKeyOrigin::Server)); + let other_rewrapped = rotated + .target_set(&other) + .expect("rewrap target builds") + .expect("other set lacks the new KEK"); + + let reply = StateHandler::apply( + &RewrapStreamKeysRequest { + entries: vec![rewrap_entry(0, 4, &other, other_rewrapped.as_ref())], + }, + &mut inner, + IggyTimestamp::now(), + ); + + assert_eq!(replaced_count(&reply), 0); + assert_eq!(inner.items[0].options, before); + } + + #[test] + fn given_malformed_rewrap_set_when_applying_should_skip_without_mutation() { + let old = key_context(OLD_CONFIG_KEY, &[]); + let new = key_context(NEW_CONFIG_KEY, &[]); + let third = key_context(THIRD_CONFIG_KEY, &[]); + let mut inner = StreamsInner::new(); + let stored = WrapSet::from(envelope(&old, 3, StreamKeyOrigin::Client)); + create_encrypted_stream(&mut inner, "payments", &stored, 4); + let before = inner.items[0].options.clone(); + let same_key = |context: &ServerKeyContext| envelope(context, 3, StreamKeyOrigin::Client); + let mut sorted = vec![same_key(&old), same_key(&new), same_key(&third)]; + sorted.sort_by_key(WrappedStreamKey::kek_fingerprint); + let concat = |envelopes: &[WrappedStreamKey]| -> Vec<u8> { + envelopes + .iter() + .flat_map(|envelope| envelope.as_ref().to_vec()) + .collect() + }; + let mut reversed = sorted[..2].to_vec(); + reversed.reverse(); + let malformed = [ + Vec::new(), + concat(&reversed), + concat(&[same_key(&old), same_key(&old)]), + concat(&[same_key(&old), envelope(&new, 9, StreamKeyOrigin::Client)]), + concat(&[same_key(&old), envelope(&new, 3, StreamKeyOrigin::Server)]), + concat( + &[ + sorted.clone(), + vec![envelope(&new, 3, StreamKeyOrigin::Client)], + ] + .concat(), + ), + ]; + for set in malformed { + let reply = StateHandler::apply( + &RewrapStreamKeysRequest { + entries: vec![rewrap_entry(0, 4, &stored, &set)], + }, + &mut inner, + IggyTimestamp::now(), + ); + assert_eq!(replaced_count(&reply), 0, "{} bytes", set.len()); + assert_eq!(inner.items[0].options, before); + } + } + + #[test] + fn given_well_formed_set_of_other_key_or_origin_when_rewrapping_should_skip_it() { + let old = key_context(OLD_CONFIG_KEY, &[]); + let new = key_context(NEW_CONFIG_KEY, &[]); + let mut inner = StreamsInner::new(); + let stored = WrapSet::from(envelope(&old, 3, StreamKeyOrigin::Client)); + create_encrypted_stream(&mut inner, "payments", &stored, 4); + let before = inner.items[0].options.clone(); + let other_key = WrapSet::from(envelope(&new, 9, StreamKeyOrigin::Client)); + let relabeled = WrapSet::from(envelope(&new, 3, StreamKeyOrigin::Server)); + let rewrapped = WrapSet::from(envelope(&new, 3, StreamKeyOrigin::Client)); + + let mut replaced = Vec::new(); + for set in [&other_key, &relabeled] { + let reply = StateHandler::apply( + &RewrapStreamKeysRequest { + entries: vec![rewrap_entry(0, 4, &stored, set.as_ref())], + }, + &mut inner, + IggyTimestamp::now(), + ); + replaced.push(replaced_count(&reply)); + assert_eq!(inner.items[0].options, before); + } + let reply = StateHandler::apply( + &RewrapStreamKeysRequest { + entries: vec![rewrap_entry(0, 4, &stored, rewrapped.as_ref())], + }, + &mut inner, + IggyTimestamp::now(), + ); + replaced.push(replaced_count(&reply)); + assert_eq!(replaced, [0, 0, 1]); + assert_eq!( + stream_wrap_set(&inner.items[0].options).unwrap(), + Some(rewrapped) + ); + } + + #[test] + fn given_plaintext_stream_key_when_applying_create_should_refuse_without_storing() { + let mut options = ResourceOptions::new(); + options.insert( + HeaderKey::from(OptionKey::EncryptionKey), + OptionValue::explicit(HeaderValue::try_from([7u8; 32].as_slice()).unwrap()), + ); + let request = CreateStreamRequest { + name: WireName::new("secrets").unwrap(), + options: resource_options_to_wire(&options, OptionsProvenance::All).unwrap(), + }; + let mut inner = StreamsInner::new(); + + let reply = StateHandler::apply(&request, &mut inner, IggyTimestamp::now()); + + assert_eq!( + reply.code, + u32::from(CreateStreamResult::InvalidOptionValue) + ); + assert!(inner.items.is_empty()); + assert!(inner.index.is_empty()); + } + + #[test] + fn given_topic_with_retention_when_restoring_snapshot_should_rebuild_resolved_view() { + let mut inner = StreamsInner::new(); + create_stream(&mut inner, "stream"); + let message_expiry = IggyExpiry::from(5_000_000u64); + let max_topic_size = MaxTopicSize::from(10_000_000_000u64); + let segment_size = IggyByteSize::from(64 * 1024 * 1024); + let create = CreateTopicWithAssignmentsRequest { + request: WireCreateTopicRequest { + stream_id: WireIdentifier::numeric(0), + partitions_count: 1, + name: WireName::new("topic").unwrap(), + options: TopicCreateOptions { + message_expiry: Some(message_expiry), + ..TopicCreateOptions::default() + } + .to_explicit_wire(|key| key == topic_option_keys::MESSAGE_EXPIRY) + .unwrap(), + }, + derived_options: TopicCreateOptions { + max_topic_size: Some(max_topic_size), + segment_size: Some(segment_size), + ..TopicCreateOptions::default() + } + .to_wire() + .unwrap(), + partitions: vec![CreatedPartitionAssignment { + initial_view: 0, + partition_id: 0, + consensus_group_id: 1, + }], + }; + assert_eq!( + StateHandler::apply(&create, &mut inner, IggyTimestamp::from(1)).code, + 0 + ); + let streams: Streams = inner.into(); + let mut snapshot = MetadataSnapshot::new(1); + snapshot.streams = Some(streams.to_snapshot()); + let decoded = MetadataSnapshot::decode(&snapshot.encode().unwrap()).unwrap(); + + let restored = Streams::from_snapshot(decoded.streams.unwrap()).unwrap(); + + assert_eq!( + restored.topic_retention_config(0, 0).unwrap(), + (message_expiry, max_topic_size, 1, Some(segment_size)) + ); + assert_eq!( + restored.topic_segment_size(&WireIdentifier::numeric(0), &WireIdentifier::numeric(0)), + Some(segment_size) + ); + } + + #[test] + fn given_stored_critical_marker_when_patch_declares_more_keys_should_store_the_bounded_union() { + let marker_key = HeaderKey::from(OptionKey::CriticalKeys); + let marker = |text: &str| OptionValue::explicit(HeaderValue::try_from(text).unwrap()); + let update = |names: &str| { + let mut explicit = ResourceOptions::new(); + explicit.insert(marker_key.clone(), marker(names)); + UpdateStreamResolvedRequest { + stream_id: WireIdentifier::numeric(0), + name: None, + explicit_options: resource_options_to_wire(&explicit, OptionsProvenance::All) + .unwrap(), + derived_options: WireOptions::empty(), + } + }; + let mut inner = StreamsInner::new(); + create_stream(&mut inner, "stream"); + inner.items[0] + .options + .insert(marker_key.clone(), marker("encryption_key_wrapped")); + + let reply = StateHandler::apply(&update("a_future_key"), &mut inner, IggyTimestamp::now()); + assert_eq!(reply.code, 0); + let merged = marker("a_future_key,encryption_key_wrapped"); + assert_eq!(inner.items[0].options.get(&marker_key), Some(&merged)); + + let reply = + StateHandler::apply(&update(&"z".repeat(230)), &mut inner, IggyTimestamp::now()); + assert_eq!( + reply.code, + u32::from(UpdateStreamResult::OptionsBlockTooLarge) + ); + assert_eq!(inner.items[0].options.get(&marker_key), Some(&merged)); + } + + #[test] + fn given_internal_or_unservable_topic_options_when_creating_should_not_echo_them() { + let marker_key = HeaderKey::from(OptionKey::CriticalKeys); + let wrapped_key = HeaderKey::from(OptionKey::EncryptionKeyWrapped); + let future_key = HeaderKey::try_from("a_future_key").unwrap(); + let marker = |text: &str| OptionValue::explicit(HeaderValue::try_from(text).unwrap()); + let set = WrapSet::from(envelope( + &key_context(OLD_CONFIG_KEY, &[]), + 1, + StreamKeyOrigin::Server, + )); + let mut servable = ResourceOptions::new(); + servable.insert(marker_key.clone(), marker("encryption_key_wrapped")); + servable.insert( + wrapped_key.clone(), + OptionValue::explicit(HeaderValue::try_from(set.as_ref()).unwrap()), + ); + let mut unservable = ResourceOptions::new(); + unservable.insert(marker_key.clone(), marker("a_future_key")); + unservable.insert( + future_key.clone(), + OptionValue::explicit(HeaderValue::from(7u64)), + ); + + for stored in [servable, unservable] { + let mut inner = StreamsInner::new(); + create_stream(&mut inner, "stream"); + let create = CreateTopicWithAssignmentsRequest { + request: make_topic_request(0, 1, "topic"), + derived_options: resource_options_to_wire(&stored, OptionsProvenance::All).unwrap(), + partitions: vec![CreatedPartitionAssignment { + partition_id: 0, + consensus_group_id: 1, + initial_view: 0, + }], + }; + + let apply = StateHandler::apply(&create, &mut inner, IggyTimestamp::now()); + + assert_eq!(apply.code, 0); + assert!(inner.items[0].topics[0].options.contains_key(&marker_key)); + let (reply, _) = GetTopicResponse::decode(&apply.body).unwrap(); + let echoed: Vec<HeaderKey> = resource_options_from_wire(&reply.topic.options, true) + .unwrap() + .into_keys() + .chain( + resource_options_from_wire(&reply.topic.derived_options, false) + .unwrap() + .into_keys(), + ) + .collect(); + for hidden in [&marker_key, &wrapped_key, &future_key] { + assert!(!echoed.contains(hidden), "{echoed:?}"); + } + } + } + + #[test] + fn given_unknown_critical_key_when_applying_create_stream_should_store_it_without_echoing_options() + { + let mut options = ResourceOptions::new(); + options.insert( + HeaderKey::from(OptionKey::CriticalKeys), + OptionValue::explicit(HeaderValue::try_from("a_future_key").unwrap()), + ); + options.insert( + HeaderKey::try_from("a_future_key").unwrap(), + OptionValue::explicit(HeaderValue::from(7u64)), + ); + let request = CreateStreamRequest { + name: WireName::new("future").unwrap(), + options: resource_options_to_wire(&options, OptionsProvenance::All).unwrap(), + }; + let mut inner = StreamsInner::new(); + + let reply = StateHandler::apply(&request, &mut inner, IggyTimestamp::now()); + + assert_eq!(reply.code, 0); + assert_eq!(inner.items[0].options, options); + let (response, _) = GetStreamResponse::decode(&reply.body).unwrap(); + assert_eq!( + (response.stream.options, response.stream.derived_options), + (WireOptions::empty(), WireOptions::empty()) + ); + } + + #[test] + fn given_reused_stream_id_when_creating_should_have_distinct_created_op() { + let mut inner = StreamsInner::new(); + let timestamp = IggyTimestamp::from(1); + let create = CreateStreamRequest { + name: WireName::new("orders").unwrap(), + options: WireOptions::empty(), + }; + inner.apply_context.metadata_op = 5; + assert_eq!(StateHandler::apply(&create, &mut inner, timestamp).code, 0); + let first = inner.items[0].created_op; + + inner.apply_context.metadata_op = 6; + let delete = DeleteStreamRequest { + stream_id: WireIdentifier::numeric(0), + }; + assert_eq!( + crate::stm::lifecycle::apply_with_lifecycle_completion(&delete, &mut inner, timestamp) + .code, + 0 + ); + assert!(inner.items.get(0).is_none()); + + inner.apply_context.metadata_op = 7; + assert_eq!(StateHandler::apply(&create, &mut inner, timestamp).code, 0); + let recreated = &inner.items[0]; + assert_eq!(recreated.created_at, timestamp); + assert_eq!((first, recreated.created_op), (5, 7)); + } + + #[test] + fn given_rename_only_patch_when_applying_should_preserve_options_and_provenance() { let mut inner = StreamsInner::new(); create_stream(&mut inner, "stream"); let message_expiry = IggyExpiry::from(5_000_000u64); @@ -3366,62 +3986,243 @@ .to_wire() .unwrap(), partitions: vec![CreatedPartitionAssignment { - initial_view: 0, partition_id: 0, consensus_group_id: 1, + initial_view: 0, }], }; assert_eq!( StateHandler::apply(&create, &mut inner, IggyTimestamp::from(1)).code, 0 ); - let original_options = inner - .items - .get(0) - .unwrap() - .topics - .get(0) - .unwrap() - .options - .clone(); + let original_options = inner.items[0].topics[0].options.clone(); - for options in [ - TopicUpdateOptions::default(), - TopicUpdateOptions { - message_expiry: Some(IggyExpiry::ServerDefault), - max_topic_size: Some(MaxTopicSize::ServerDefault), - ..TopicUpdateOptions::default() - }, - TopicUpdateOptions { - raw: [ - topic_option_keys::MESSAGE_EXPIRY, - topic_option_keys::MAX_TOPIC_SIZE, - ] - .into_iter() - .map(|key| (key.to_owned(), "server_default".to_owned())) - .collect(), - ..TopicUpdateOptions::default() - }, - ] { - let update = UpdateTopicRequest { + let update = UpdateTopicResolvedRequest { + stream_id: WireIdentifier::numeric(0), + topic_id: WireIdentifier::numeric(0), + name: Some(WireName::new("renamed").unwrap()), + explicit_options: WireOptions::empty(), + derived_options: WireOptions::empty(), + }; + assert_eq!( + StateHandler::apply(&update, &mut inner, IggyTimestamp::from(2)).code, + 0 + ); + let topic = &inner.items[0].topics[0]; + assert_eq!(topic.name.as_ref(), "renamed"); + assert_eq!(topic.resolved.message_expiry, message_expiry); + assert_eq!(topic.resolved.max_topic_size, max_topic_size); + assert_eq!(topic.options, original_options); + } + + #[test] + fn given_topic_patch_without_name_when_applying_should_preserve_name_and_other_keys() { + let mut inner = streams_with_resize_policy(Some(PartitionResizePolicy::Mutable)); + let before = inner.items[0].topics[0].options.clone(); + let expiry = IggyExpiry::from(9_000_000u64); + let mut explicit = ResourceOptions::new(); + explicit.insert( + HeaderKey::from(OptionKey::MessageExpiry), + OptionValue::explicit(expiry.encode().unwrap()), + ); + let update = UpdateTopicResolvedRequest { + stream_id: WireIdentifier::numeric(0), + topic_id: WireIdentifier::numeric(0), + name: None, + explicit_options: resource_options_to_wire(&explicit, OptionsProvenance::All).unwrap(), + derived_options: WireOptions::empty(), + }; + assert_eq!( + StateHandler::apply(&update, &mut inner, IggyTimestamp::now()).code, + 0 + ); + let topic = &inner.items[0].topics[0]; + assert_eq!(topic.name.as_ref(), "topic"); + assert_eq!(topic.resolved.message_expiry, expiry); + let mut expected = before; + expected.extend(explicit); + assert_eq!(topic.options, expected); + } + + #[test] + fn given_explicit_topic_key_when_resetting_should_store_primary_default_as_derived() { + let mut inner = StreamsInner::new(); + create_stream(&mut inner, "stream"); + let create = CreateTopicWithAssignmentsRequest { + request: WireCreateTopicRequest { stream_id: WireIdentifier::numeric(0), - topic_id: WireIdentifier::numeric(0), - name: WireName::new("renamed").unwrap(), - options: options.to_wire().unwrap(), + partitions_count: 1, + name: WireName::new("topic").unwrap(), + options: TopicCreateOptions { + max_topic_size: Some(MaxTopicSize::from(10_000_000_000u64)), + ..TopicCreateOptions::default() + } + .to_explicit_wire(|key| key == topic_option_keys::MAX_TOPIC_SIZE) + .unwrap(), + }, + derived_options: WireOptions::empty(), + partitions: vec![CreatedPartitionAssignment { + partition_id: 0, + consensus_group_id: 1, + initial_view: 0, + }], + }; + assert_eq!( + StateHandler::apply(&create, &mut inner, IggyTimestamp::from(1)).code, + 0 + ); + let default = OptionKey::MaxTopicSize + .descriptor() + .default_value() + .unwrap() + .unwrap(); + let mut derived = ResourceOptions::new(); + derived.insert( + HeaderKey::from(OptionKey::MaxTopicSize), + OptionValue::derived(default.clone()), + ); + let update = UpdateTopicResolvedRequest { + stream_id: WireIdentifier::numeric(0), + topic_id: WireIdentifier::numeric(0), + name: None, + explicit_options: WireOptions::empty(), + derived_options: resource_options_to_wire(&derived, OptionsProvenance::All).unwrap(), + }; + assert_eq!( + StateHandler::apply(&update, &mut inner, IggyTimestamp::from(2)).code, + 0 + ); + let topic = &inner.items[0].topics[0]; + let stored = &topic.options[&HeaderKey::from(OptionKey::MaxTopicSize)]; + assert_eq!(stored, &OptionValue::derived(default)); + assert_eq!( + topic.resolved.max_topic_size, + MaxTopicSize::from(iggy_common::DEFAULT_MAX_TOPIC_SIZE) + ); + } + + #[test] + fn given_conflicting_rename_with_patch_when_applying_should_leave_state_unchanged() { + let mut inner = StreamsInner::new(); + create_stream(&mut inner, "stream"); + for name in ["first", "second"] { + let create = CreateTopicWithAssignmentsRequest { + request: make_topic_request(0, 1, name), + derived_options: WireOptions::empty(), + partitions: vec![CreatedPartitionAssignment { + partition_id: 0, + consensus_group_id: 1, + initial_view: 0, + }], }; assert_eq!( - StateHandler::apply(&update, &mut inner, IggyTimestamp::from(2)).code, + StateHandler::apply(&create, &mut inner, IggyTimestamp::now()).code, 0 ); - let topic = inner.items.get(0).unwrap().topics.get(0).unwrap(); - assert_eq!(topic.name.as_ref(), "renamed"); - assert_eq!(topic.message_expiry, message_expiry); - assert_eq!(topic.max_topic_size, max_topic_size); - assert_eq!( - topic.options, original_options, - "update {options:?} must preserve values and provenance" - ); } + let before = inner.items[0].topics[0].options.clone(); + let index_before = inner.items[0].topic_index.clone(); + let mut explicit = ResourceOptions::new(); + explicit.insert( + HeaderKey::from(OptionKey::MessageExpiry), + OptionValue::explicit(IggyExpiry::from(1_000_000u64).encode().unwrap()), + ); + let update = UpdateTopicResolvedRequest { + stream_id: WireIdentifier::numeric(0), + topic_id: WireIdentifier::numeric(0), + name: Some(WireName::new("second").unwrap()), + explicit_options: resource_options_to_wire(&explicit, OptionsProvenance::All).unwrap(), + derived_options: WireOptions::empty(), + }; + assert_eq!( + StateHandler::apply(&update, &mut inner, IggyTimestamp::now()).code, + u32::from(UpdateTopicResult::NameAlreadyExists) + ); + let topic = &inner.items[0].topics[0]; + assert_eq!(topic.name.as_ref(), "first"); + assert_eq!(topic.options, before); + assert_eq!(inner.items[0].topic_index, index_before); + + let rename_stream = UpdateStreamResolvedRequest { + stream_id: WireIdentifier::numeric(0), + name: Some(WireName::new("renamed-stream").unwrap()), + explicit_options: WireOptions::empty(), + derived_options: WireOptions::empty(), + }; + assert_eq!( + StateHandler::apply(&rename_stream, &mut inner, IggyTimestamp::now()).code, + 0 + ); + assert_eq!(inner.items[0].name.as_ref(), "renamed-stream"); + assert_eq!( + inner.resolve_stream_id(&WireIdentifier::named("renamed-stream").unwrap()), + Some(0) + ); + assert_eq!( + inner.resolve_stream_id(&WireIdentifier::named("stream").unwrap()), + None + ); + } + + #[test] + fn given_cap_below_topic_segment_size_when_applying_update_should_refuse_it_unchanged() { + let segment_size = 64 * 1024 * 1024; + let mut inner = StreamsInner::new(); + create_stream(&mut inner, "stream"); + let mut request = make_topic_request(0, 1, "topic"); + request.options = TopicCreateOptions { + segment_size: Some(IggyByteSize::from(segment_size)), + ..TopicCreateOptions::default() + } + .to_explicit_wire(|key| key == topic_option_keys::SEGMENT_SIZE) + .unwrap(); + let create = CreateTopicWithAssignmentsRequest { + request, + derived_options: WireOptions::empty(), + partitions: vec![CreatedPartitionAssignment { + partition_id: 0, + consensus_group_id: 1, + initial_view: 0, + }], + }; + assert_eq!( + StateHandler::apply(&create, &mut inner, IggyTimestamp::from(1)).code, + 0 + ); + let before = inner.items[0].topics[0].options.clone(); + + let below = topic_cap_update(segment_size - 1, Some("renamed")); + assert_eq!( + StateHandler::apply(&below, &mut inner, IggyTimestamp::from(2)).code, + u32::from(UpdateTopicResult::InvalidTopicSize) + ); + let topic = &inner.items[0].topics[0]; + assert_eq!(topic.name.as_ref(), "topic"); + assert_eq!(topic.options, before); + + let at_floor = topic_cap_update(segment_size, Some("renamed")); + assert_eq!( + StateHandler::apply(&at_floor, &mut inner, IggyTimestamp::from(3)).code, + 0 + ); + let topic = &inner.items[0].topics[0]; + assert_eq!(topic.name.as_ref(), "renamed"); + assert_eq!( + topic.resolved.max_topic_size, + MaxTopicSize::from(segment_size) + ); + } + + #[test] + fn given_topic_without_segment_size_when_applying_cap_below_default_should_refuse_it() { + let mut inner = streams_with_resize_policy(None); + let before = inner.items[0].topics[0].options.clone(); + let update = topic_cap_update(DEFAULT_SEGMENT_SIZE - 1, None); + assert_eq!( + StateHandler::apply(&update, &mut inner, IggyTimestamp::now()).code, + u32::from(UpdateTopicResult::InvalidTopicSize) + ); + assert_eq!(inner.items[0].topics[0].options, before); } #[test] @@ -3510,6 +4311,21 @@ inner } + fn topic_cap_update(max_topic_size: u64, name: Option<&str>) -> UpdateTopicResolvedRequest { + UpdateTopicResolvedRequest { + stream_id: WireIdentifier::numeric(0), + topic_id: WireIdentifier::numeric(0), + name: name.map(|name| WireName::new(name).unwrap()), + explicit_options: TopicCreateOptions { + max_topic_size: Some(MaxTopicSize::from(max_topic_size)), + ..TopicCreateOptions::default() + } + .to_explicit_wire(|key| key == topic_option_keys::MAX_TOPIC_SIZE) + .unwrap(), + derived_options: WireOptions::empty(), + } + } + #[test] fn given_resize_policy_when_resizing_after_snapshot_restore_should_preserve_the_contract() { for policy in [ @@ -3587,41 +4403,6 @@ } } - #[test] - fn given_resize_policy_when_updating_in_ordered_apply_should_reject_without_renaming() { - for policy in [PartitionResizePolicy::Mutable, PartitionResizePolicy::Fixed] { - let mut inner = streams_with_resize_policy(Some(policy)); - let replacement = if policy == PartitionResizePolicy::Fixed { - PartitionResizePolicy::Mutable - } else { - PartitionResizePolicy::Fixed - }; - let update = UpdateTopicRequest { - stream_id: WireIdentifier::numeric(0), - topic_id: WireIdentifier::numeric(0), - name: WireName::new("renamed").unwrap(), - options: TopicCreateOptions { - partition_resize_policy: Some(replacement), - ..TopicCreateOptions::default() - } - .to_wire() - .unwrap(), - }; - let revision = inner.revision; - assert_eq!( - StateHandler::apply(&update, &mut inner, IggyTimestamp::now()).code, - u32::from(UpdateTopicResult::UnsupportedOptionKey), - ); - assert_eq!(inner.items[0].topics[0].name.as_ref(), "topic"); - assert_eq!( - TopicCreateOptions::from_resource_options(&inner.items[0].topics[0].options) - .partition_resize_policy, - Some(policy), - ); - assert_eq!(inner.revision, revision); - } - } - /// Regression guard for the [`StreamsSnapshot`] serialized-form invariant: a /// populated snapshot must re-encode byte-identically after a decode, or the /// checkpoint checksum cross-check (`recovery::verify_checkpoint_pairing`) would
diff --git a/core/metadata/src/stm/user.rs b/core/metadata/src/stm/user.rs index 4492f36..b4f9767 100644 --- a/core/metadata/src/stm/user.rs +++ b/core/metadata/src/stm/user.rs
@@ -23,6 +23,7 @@ UpdateUserResult, }; use crate::stm::snapshot::Snapshotable; +use crate::stm::stream::{PatchError, patched_options, public_reply_options}; use crate::stm::{ApplyContext, StateHandler}; use crate::{collect_handlers, define_state, impl_fill_restore}; use ahash::AHashMap; @@ -35,7 +36,7 @@ use iggy_binary_protocol::primitives::permissions::{WireGlobalPermissions, WirePermissions}; use iggy_binary_protocol::requests::users::{ ChangePasswordRequest, CreateUserRequest, DeleteUserRequest, UpdatePermissionsRequest, - UpdateUserRequest, + UpdateUserResolvedRequest, }; use iggy_binary_protocol::responses::users::get_user::UserDetailsResponse; use iggy_binary_protocol::responses::users::user_response::UserResponse; @@ -43,8 +44,8 @@ use iggy_common::defaults::{DEFAULT_ROOT_USER_ID, MAX_USERNAME_LENGTH, MIN_USERNAME_LENGTH}; use iggy_common::wire_conversions::resource_options_from_wire; use iggy_common::{ - GlobalPermissions, IggyError, IggyExpiry, IggyTimestamp, Permissions, PersonalAccessToken, - ResourceOptions, StreamPermissions, UserId, UserStatus, + GlobalPermissions, IggyError, IggyExpiry, IggyTimestamp, OptionsScope, Permissions, + PersonalAccessToken, ResourceOptions, StreamPermissions, UserId, UserStatus, }; use serde::{Deserialize, Serialize}; use std::collections::BTreeSet; @@ -127,7 +128,7 @@ collect_handlers! { Users { CreateUser, - UpdateUser, + UpdateUserResolved, DeleteUser, ChangePassword, UpdatePermissions, @@ -182,13 +183,13 @@ .and_then(|&id| self.items.get(id as usize)) } - /// Stored password hash of the user named by `identifier`, `None` when - /// the user does not resolve. + /// Id and stored password hash of the user named by `identifier`, `None` + /// when the user does not resolve. #[must_use] - pub fn password_hash_of(&self, identifier: &WireIdentifier) -> Option<Arc<str>> { + pub fn resolve_password_hash(&self, identifier: &WireIdentifier) -> Option<(UserId, Arc<str>)> { self.resolve_user_id(identifier) .and_then(|user_id| self.items.get(user_id)) - .map(|user| Arc::clone(&user.password_hash)) + .map(|user| (user.id, Arc::clone(&user.password_hash))) } /// Clone the committed permissions of the user at slab `id`, if any. @@ -540,6 +541,8 @@ Ok(options) => options, Err(result) => return ApplyReply::err(result), }; + // The reply is cached for retries, so it is projected before caching. + let reply_options = public_reply_options(OptionsScope::User, &options); let username_arc: Arc<str> = Arc::from(self.username.as_str()); let status = UserStatus::from_code(self.status).unwrap_or_default(); @@ -584,7 +587,8 @@ created_at: timestamp.as_micros(), status: self.status, username: self.username.clone(), - options: self.options.clone(), + options: reply_options.0, + derived_options: reply_options.1, }, permissions: self.permissions.clone(), } @@ -593,51 +597,67 @@ } } -impl StateHandler for UpdateUserRequest { +impl StateHandler for UpdateUserResolvedRequest { type State = UsersInner; #[allow(clippy::cast_possible_truncation)] fn apply(&self, state: &mut UsersInner, _timestamp: IggyTimestamp) -> ApplyReply { let Some(user_id) = state.resolve_user_id(&self.user_id) else { return ApplyReply::err(UpdateUserResult::UserNotFound); }; - - let Some(user) = state.items.get_mut(user_id) else { + let Some(user) = state.items.get(user_id) else { return ApplyReply::err(UpdateUserResult::UserNotFound); }; - // Decoded before any mutation: a malformed block must leave the user - // untouched rather than half-renamed. - let Ok(updated_options) = resource_options_from_wire(&self.options, true) else { - return ApplyReply::err(UpdateUserResult::InvalidOptionValue); - }; + if user_id as u32 == DEFAULT_ROOT_USER_ID + && (self.username.is_some() || self.status.is_some()) + && (state.apply_context.user_id != DEFAULT_ROOT_USER_ID + || self.status == Some(UserStatus::Inactive.as_code())) + { + return ApplyReply::err(UpdateUserResult::CannotModifyRootUser); + } - if let Some(new_username) = &self.username { + // Every check runs before any mutation, so a refused patch leaves the + // user exactly as it was: no half-applied rename or status change. + let new_username: Option<Arc<str>> = + self.username.as_ref().map(|name| Arc::from(name.as_str())); + if let Some(new_username) = &new_username { // Same bound as CreateUser apply: a rename must not smuggle in a - // username the edges reject. Rejected before any mutation. - if !(MIN_USERNAME_LENGTH..=MAX_USERNAME_LENGTH).contains(&new_username.as_str().len()) { + // username the edges reject. + if !(MIN_USERNAME_LENGTH..=MAX_USERNAME_LENGTH).contains(&new_username.len()) { return ApplyReply::err(UpdateUserResult::InvalidUsername); } - - let new_username_arc: Arc<str> = Arc::from(new_username.as_str()); - if let Some(&existing_id) = state.index.get(&new_username_arc) + if let Some(&existing_id) = state.index.get(new_username) && existing_id != user_id as UserId { return ApplyReply::err(UpdateUserResult::UsernameAlreadyExists); } - - state.index.remove(&user.username); - user.username = new_username_arc.clone(); - state.index.insert(new_username_arc, user_id as UserId); } + let Ok(new_status) = self.status.map(UserStatus::from_code).transpose() else { + return ApplyReply::err(UpdateUserResult::InvalidUserStatus); + }; + let options = + match patched_options(&user.options, &self.explicit_options, &self.derived_options) { + Ok(options) => options, + Err(PatchError::InvalidValue) => { + return ApplyReply::err(UpdateUserResult::InvalidOptionValue); + } + Err(PatchError::TooLarge) => { + return ApplyReply::err(UpdateUserResult::OptionsBlockTooLarge); + } + }; - if let Some(status_code) = self.status - && let Ok(new_status) = UserStatus::from_code(status_code) - { + let Some(user) = state.items.get_mut(user_id) else { + return ApplyReply::err(UpdateUserResult::UserNotFound); + }; + if let Some(new_username) = new_username { + state.index.remove(&user.username); + user.username = new_username.clone(); + state.index.insert(new_username, user_id as UserId); + } + if let Some(new_status) = new_status { user.status = new_status; } - // Patch, never replace: keys the client did not send keep their - // current value, so a client that predates a key cannot erase it. - user.options.extend(updated_options); + user.options = options; ApplyReply { revoked_user: (user.status != UserStatus::Active).then_some(user_id as UserId), ..ApplyReply::ok(Bytes::new()) @@ -686,11 +706,12 @@ }; // An empty `new_password` is the primary's signal that the caller's - // current password did not match (see the server - // `verify_and_rewrite_change_password`): the accept path always - // replicates a non-empty Argon2 hash, so this is unambiguous. Rejecting - // here (rather than denying pre-consensus) commits the op as a no-op, - // recording the request id in the ClientTable so a retry of it dedups. + // current password did not match or that the target did not resolve + // there (see the server `verify_and_rewrite_change_password`). The + // accept path always replicates a non-empty Argon2 hash, so this is + // unambiguous. Rejecting here (rather than denying pre-consensus) + // commits the op as a no-op, recording the request id in the + // ClientTable so a retry of it dedups. if self.new_password.is_empty() { return ApplyReply::err(ChangePasswordResult::InvalidCredentials); } @@ -852,7 +873,9 @@ return ApplyReply::err(CreatePersonalAccessTokenResult::AlreadyExists); } - let expiry_at = PersonalAccessToken::calculate_expiry_at(timestamp, expiry); + let Ok(expiry_at) = PersonalAccessToken::calculate_expiry_at(timestamp, expiry) else { + return ApplyReply::err(CreatePersonalAccessTokenResult::InvalidExpiry); + }; if let Some(expiry_at) = expiry_at && expiry_at.as_micros() <= timestamp.as_micros() { @@ -1143,7 +1166,10 @@ use iggy_binary_protocol::primitives::permissions::{ WireStreamPermissions, WireTopicPermissions, }; - use iggy_common::IggyError; + use iggy_common::wire_conversions::resource_options_to_wire; + use iggy_common::{ + HeaderKey, HeaderValue, IggyError, OptionKey, OptionValue, OptionsProvenance, + }; #[test] fn create_pat_request_roundtrip_preserves_user_id() { @@ -1232,6 +1258,29 @@ } #[test] + fn given_expiry_past_the_end_of_the_clock_when_creating_token_should_refuse_it() { + let mut users = UsersInner::new(); + let now = IggyTimestamp::now(); + for expiry in [u64::MAX - now.as_micros() + 1, 0xFFFF_FFFF_0000_0000] { + let reply = CreatePersonalAccessTokenRequest { + user_id: 4, + name: WireName::new("far").unwrap(), + expiry, + token_hash: [b'a'; PAT_TOKEN_HASH_BYTES], + } + .apply(&mut users, now); + + assert_eq!( + reply.code, + u32::from(CreatePersonalAccessTokenResult::InvalidExpiry) + ); + } + assert_eq!(users.pat_count_of(4), 0); + assert!(users.personal_access_token_index.is_empty()); + assert!(users.personal_access_token_expiry_index.is_empty()); + } + + #[test] fn delete_keeps_expiry_index_in_sync() { let mut users = UsersInner::new(); // Created at the epoch with a 1-unit expiry: a `Some(expiry_at)`, so @@ -1409,6 +1458,37 @@ } #[test] + fn given_unknown_critical_key_when_apply_create_user_should_store_it_without_echoing_options() { + let mut options = ResourceOptions::new(); + options.insert( + HeaderKey::from(OptionKey::CriticalKeys), + OptionValue::explicit(HeaderValue::try_from("a_future_key").unwrap()), + ); + options.insert( + HeaderKey::try_from("a_future_key").unwrap(), + OptionValue::explicit(HeaderValue::from(7u64)), + ); + let request = CreateUserRequest { + username: WireName::new("alice").unwrap(), + password: "hash".to_owned(), + status: 1, + permissions: None, + options: resource_options_to_wire(&options, OptionsProvenance::All).unwrap(), + }; + let mut users = UsersInner::new(); + + let apply = StateHandler::apply(&request, &mut users, IggyTimestamp::now()); + + assert_eq!(apply.code, 0); + assert_eq!(users.items.get(0).unwrap().options, options); + let (reply, _) = UserDetailsResponse::decode(&apply.body).unwrap(); + assert_eq!( + (reply.user.options, reply.user.derived_options), + (WireOptions::empty(), WireOptions::empty()) + ); + } + + #[test] fn given_duplicate_username_when_apply_create_user_should_return_user_already_exists() { let mut users = UsersInner::new(); create_user(&mut users, "alice"); @@ -1986,11 +2066,12 @@ let alice_id = create_user_with(&mut users, "alice", None); let short = "a".repeat(MIN_USERNAME_LENGTH - 1); - let rename = UpdateUserRequest { + let rename = UpdateUserResolvedRequest { user_id: WireIdentifier::numeric(alice_id), username: Some(WireName::new(&short).unwrap()), status: None, - options: WireOptions::empty(), + explicit_options: WireOptions::empty(), + derived_options: WireOptions::empty(), }; let reply = StateHandler::apply(&rename, &mut users, IggyTimestamp::now()); assert_eq!(reply.code, u32::from(UpdateUserResult::InvalidUsername)); @@ -2006,4 +2087,113 @@ assert_eq!(users.index.get("alice"), Some(&alice_id)); assert!(!users.index.contains_key(short.as_str())); } + + #[test] + fn given_user_status_patch_when_applying_should_revoke_inactive_sessions() { + let mut users = UsersInner::new(); + create_user_with(&mut users, "iggy", None); + let alice_id = create_user_with(&mut users, "alice", None); + let patch = + |username: Option<&str>, status: Option<UserStatus>| UpdateUserResolvedRequest { + user_id: WireIdentifier::numeric(alice_id), + username: username.map(|name| WireName::new(name).unwrap()), + status: status.map(|status| status.as_code()), + explicit_options: WireOptions::empty(), + derived_options: WireOptions::empty(), + }; + + let rename = StateHandler::apply( + &patch(Some("alicia"), None), + &mut users, + IggyTimestamp::now(), + ); + assert_eq!(rename.code, 0); + assert_eq!(rename.revoked_user, None); + assert_eq!(users.index.get("alicia"), Some(&alice_id)); + assert!(!users.index.contains_key("alice")); + + let deactivate = StateHandler::apply( + &patch(None, Some(UserStatus::Inactive)), + &mut users, + IggyTimestamp::now(), + ); + assert_eq!(deactivate.code, 0); + assert_eq!(deactivate.revoked_user, Some(alice_id)); + assert_eq!( + users.items.get(alice_id as usize).map(|user| user.status), + Some(UserStatus::Inactive) + ); + + let taken = + StateHandler::apply(&patch(Some("iggy"), None), &mut users, IggyTimestamp::now()); + assert_eq!( + taken.code, + u32::from(UpdateUserResult::UsernameAlreadyExists) + ); + assert_eq!(taken.revoked_user, None); + + let invalid_status = UpdateUserResolvedRequest { + status: Some(9), + ..patch(None, None) + }; + assert_eq!( + StateHandler::apply(&invalid_status, &mut users, IggyTimestamp::now()).code, + u32::from(UpdateUserResult::InvalidUserStatus) + ); + } + + #[test] + fn given_root_patch_when_applying_should_enforce_root_identity_and_active_status() { + let mut users = UsersInner::new(); + create_user_with(&mut users, "iggy", None); + let manager_id = create_user_with(&mut users, "manager", None); + let patch = + |username: Option<&str>, status: Option<UserStatus>| UpdateUserResolvedRequest { + user_id: WireIdentifier::numeric(DEFAULT_ROOT_USER_ID), + username: username.map(|name| WireName::new(name).unwrap()), + status: status.map(|value| value.as_code()), + explicit_options: WireOptions::empty(), + derived_options: WireOptions::empty(), + }; + + users.apply_context.user_id = manager_id; + for request in [ + patch(Some("root-renamed"), None), + patch(None, Some(UserStatus::Inactive)), + ] { + let reply = StateHandler::apply(&request, &mut users, IggyTimestamp::zero()); + assert_eq!(reply.code, IggyError::CannotModifyRootUser.as_code()); + assert_eq!( + users.items.get(0).map(|user| user.username.as_ref()), + Some("iggy") + ); + assert_eq!( + users.items.get(0).map(|user| user.status), + Some(UserStatus::Active) + ); + } + + users.apply_context.user_id = DEFAULT_ROOT_USER_ID; + let renamed = StateHandler::apply( + &patch(Some("root-renamed"), None), + &mut users, + IggyTimestamp::zero(), + ); + assert_eq!(renamed.code, 0); + assert_eq!( + users.items.get(0).map(|user| user.username.as_ref()), + Some("root-renamed") + ); + + let deactivated = StateHandler::apply( + &patch(None, Some(UserStatus::Inactive)), + &mut users, + IggyTimestamp::zero(), + ); + assert_eq!(deactivated.code, IggyError::CannotModifyRootUser.as_code()); + assert_eq!( + users.items.get(0).map(|user| user.status), + Some(UserStatus::Active) + ); + } }
diff --git a/core/partitions/src/iggy_partition.rs b/core/partitions/src/iggy_partition.rs index 9530c43..00a016c 100644 --- a/core/partitions/src/iggy_partition.rs +++ b/core/partitions/src/iggy_partition.rs
@@ -73,7 +73,7 @@ use iggy_binary_protocol::{PrepareOkHeader, ReplyHeader, RoutedRequestHeader}; use iggy_common::{ ConsumerGroupId, ConsumerGroupOffsets, ConsumerKind, ConsumerOffset, ConsumerOffsets, - IggyByteSize, IggyError, IggyExpiry, IggyTimestamp, PartitionStats, PollingKind, + EncryptorKind, IggyByteSize, IggyError, IggyExpiry, IggyTimestamp, PartitionStats, PollingKind, TopicRuntimeOptions, }; use journal::Journal as _; @@ -185,6 +185,11 @@ /// `PartitionsConfig` value (simulator and tests build partitions with /// no resolved options at all). pub(crate) runtime_options: TopicRuntimeOptions, + /// The at-rest encryptor of this partition's stream incarnation, `None` + /// for a plaintext stream. Set before the partition becomes servable and + /// never swapped, so a poll answered after the stream id is reused still + /// decrypts with the key its bytes were written under. + encryptor: Option<Arc<EncryptorKind>>, pub(crate) persistence: Option<Rc<PartitionPersistence>>, pending_retry_checkpoint: Option<u64>, pub(crate) materialization_missing: bool, @@ -450,6 +455,10 @@ /// Assigned prepare to replicate after releasing the reply. `None` can also /// mean the automatic commit is queued and has no prepare slot yet. pub replication: Option<PollReplication>, + /// The key the fragments were written under, `None` for a plaintext + /// stream or an empty reply. Captured with the fragments, so a reply that + /// outlives the stream never decrypts with a later stream's key. + pub encryptor: Option<Arc<EncryptorKind>>, } /// A prepare assigned by completion, with capacity held through journal staging. @@ -864,6 +873,7 @@ partition_dir: None, segment_names_dirty: Cell::new(true), runtime_options: TopicRuntimeOptions::default(), + encryptor: None, persistence: None, pending_retry_checkpoint: None, materialization_missing: false, @@ -3544,6 +3554,15 @@ self.runtime_options } + pub fn set_encryptor(&mut self, encryptor: Option<Arc<EncryptorKind>>) { + self.encryptor = encryptor; + } + + #[must_use] + pub const fn encryptor(&self) -> Option<&Arc<EncryptorKind>> { + self.encryptor.as_ref() + } + /// Segment size used consistently by live writes, recovery, and transfer. #[must_use] pub fn effective_segment_size(&self) -> IggyByteSize { @@ -4149,6 +4168,13 @@ owner_generation: result.context.owner.map_or(0, |owner| owner.generation), metadata_op: self.required_metadata_frontier, }, + // An empty reply has nothing to decrypt, so it skips the cross-core + // refcount bump of the owner's encryptor. + encryptor: if result.fragments.is_empty() { + None + } else { + self.encryptor.clone() + }, fragments: result.fragments, current_offset: result.commit_offset, replication, @@ -12787,6 +12813,7 @@ use iggy_binary_protocol::{ Command, CommitHeader, ReplyHeader, StartViewHeader, WireConsumer, WireEncode, WireName, }; + use iggy_common::Aes256GcmEncryptor; use journal::DurableAppend; use journal::durable_storage::{ DiskStorage, DurableStorage, OpenMode, RegularFiles, StorageEntry, @@ -17772,6 +17799,61 @@ assert_eq!(recovered.required_metadata_frontier, METADATA_FRONTIER); } + #[test] + fn given_delayed_poll_after_stream_recreation_when_replying_should_keep_original_encryptor() { + let encryptor = |seed: u8| { + Arc::new(EncryptorKind::Aes256Gcm( + Aes256GcmEncryptor::new(&[seed; 32]).expect("valid key"), + )) + }; + let (original_key, replacement_key) = (encryptor(1), encryptor(2)); + let mut original = test_partition(); + original.set_encryptor(Some(Arc::clone(&original_key))); + let read = poll_read_result(&original, PollingConsumer::Consumer(1, 0), false, Some(0)); + let completion = original.complete_poll(read).expect("completes"); + + // The stream is deleted and its id reused before the reply is built. + drop(original); + let mut recreated = test_partition(); + recreated.set_encryptor(Some(Arc::clone(&replacement_key))); + + let pinned = completion.encryptor.expect("an encrypted stream's poll"); + assert!(Arc::ptr_eq(&pinned, &original_key)); + assert!(!Arc::ptr_eq(&pinned, &replacement_key)); + let mut plaintext = test_partition(); + let read = poll_read_result(&plaintext, PollingConsumer::Consumer(1, 0), false, None); + assert!( + plaintext + .complete_poll(read) + .expect("completes") + .encryptor + .is_none() + ); + } + + #[test] + fn given_empty_poll_of_encrypted_stream_when_replying_should_not_carry_its_key() { + let encryptor = Arc::new(EncryptorKind::Aes256Gcm( + Aes256GcmEncryptor::new(&[3; 32]).expect("valid key"), + )); + let mut partition = test_partition(); + partition.set_encryptor(Some(Arc::clone(&encryptor))); + + let empty = poll_read_result(&partition, PollingConsumer::Consumer(1, 0), false, None); + let completion = partition.complete_poll(empty).expect("completes"); + + assert!(completion.fragments.is_empty()); + assert!(completion.encryptor.is_none()); + assert_eq!(Arc::strong_count(&encryptor), 2); + let read = poll_read_result(&partition, PollingConsumer::Consumer(1, 0), false, Some(0)); + let pinned = partition + .complete_poll(read) + .expect("completes") + .encryptor + .expect("a poll with fragments carries its stream's key"); + assert!(Arc::ptr_eq(&pinned, &encryptor)); + } + #[compio::test] async fn given_idle_owner_when_polled_or_view_changed_should_require_drain_evidence() { const GROUP_ID: u64 = 7; @@ -29763,7 +29845,7 @@ validate_checksum: true, segment_size: IggyByteSize::from(1024 * 1024), preallocate_segments: false, - encryptor: None, + key_context: Arc::default(), path_layout: crate::PartitionPathLayout::default(), } }
diff --git a/core/partitions/src/iggy_partitions.rs b/core/partitions/src/iggy_partitions.rs index f4901ab..ab103a5 100644 --- a/core/partitions/src/iggy_partitions.rs +++ b/core/partitions/src/iggy_partitions.rs
@@ -18,6 +18,7 @@ #![allow(dead_code)] use crate::poll_plan::{PollPlan, PollReadResult}; +use crate::stream_keyring::{StreamKeyIdentity, StreamKeyring}; use crate::types::PartitionsConfig; use crate::{IggyPartition, Partition, PollingArgs, PollingConsumer}; use crate::{PollCompletion, PollReplication}; @@ -29,7 +30,7 @@ Command, ConsensusHeader, Operation, PrepareHeader, PrepareOkHeader, ReplyHeader, RoutedRequestHeader, }; -use iggy_common::IggyError; +use iggy_common::{EncryptorKind, IggyError, ServerKeyContext, StreamEncryption}; use journal::superblock::{PingPongSuperblock, SuperblockStore}; use message_bus::MessageBus; use server_common::Message; @@ -39,6 +40,7 @@ use std::cell::{RefCell, UnsafeCell}; use std::collections::BTreeMap; use std::rc::Rc; +use std::sync::Arc; use std::task::Waker; use tracing::warn; @@ -110,6 +112,9 @@ /// per-shard runtime is single-threaded, so runtime borrow checks /// suffice; callers must not hold a borrow across `.await`. tombstoned: RefCell<AHashMap<IggyNamespace, Option<u64>>>, + /// Stream encryptors by stream incarnation, opened with this node's keys. + /// Same single-threaded borrow rules as `tombstoned`. + keyring: RefCell<StreamKeyring>, /// Reconciler fences must not borrow a partition held by the pump across await. teardown: RefCell<AHashMap<IggyNamespace, crate::PartitionTeardown>>, consumer_group_offsets_reconcile_epoch: Rc<Cell<u64>>, @@ -149,12 +154,14 @@ { #[must_use] pub fn new(shard_id: ShardId, config: PartitionsConfig) -> Self { + let keyring = RefCell::new(StreamKeyring::new(Arc::clone(&config.key_context))); Self { shard_id, config, partitions: UnsafeCell::new(Vec::new()), namespace_to_local: UnsafeCell::new(BTreeMap::new()), tombstoned: RefCell::new(AHashMap::new()), + keyring, teardown: RefCell::new(AHashMap::new()), consumer_group_offsets_reconcile_epoch: Rc::new(Cell::new(0)), fence_published: Rc::default(), @@ -168,6 +175,7 @@ #[must_use] pub fn with_capacity(shard_id: ShardId, config: PartitionsConfig, capacity: usize) -> Self { + let keyring = RefCell::new(StreamKeyring::new(Arc::clone(&config.key_context))); Self { shard_id, config, @@ -175,6 +183,7 @@ // BTreeMap has no capacity hint; the Vec above absorbs the sizing. namespace_to_local: UnsafeCell::new(BTreeMap::new()), tombstoned: RefCell::new(AHashMap::new()), + keyring, teardown: RefCell::new(AHashMap::new()), consumer_group_offsets_reconcile_epoch: Rc::new(Cell::new(0)), fence_published: Rc::default(), @@ -186,6 +195,37 @@ } } + /// This node's key-encryption keys, for wrapping new stream keys. + #[must_use] + pub fn key_context(&self) -> Arc<ServerKeyContext> { + Arc::clone(&self.config.key_context) + } + + /// The encryptor of one stream incarnation, `None` only for a plaintext + /// stream. + /// + /// # Errors + /// + /// See [`StreamKeyring::resolve`]: the stream is unavailable on this node. + pub fn resolve_stream_encryptor( + &self, + identity: StreamKeyIdentity, + encryption: &StreamEncryption, + ) -> Result<Option<Arc<EncryptorKind>>, IggyError> { + self.keyring.borrow_mut().resolve(identity, encryption) + } + + /// Whether this shard holds the key of any stream incarnation. + #[must_use] + pub fn has_stream_keys(&self) -> bool { + !self.keyring.borrow().is_empty() + } + + /// Drop the encryptors of the stream incarnations `is_live` rejects. + pub fn retain_stream_keys(&self, is_live: impl FnMut(StreamKeyIdentity) -> bool) { + self.keyring.borrow_mut().retain(is_live); + } + pub fn set_persistence_notifier(&self, notifier: crate::PersistenceNotifier) { for namespace in self.namespaces() { if let Some(partition) = self.get_by_ns(namespace) { @@ -796,14 +836,31 @@ ); return; } + let Some(partition) = self.get_mut_by_ns(&namespace) else { + warn!( + target: "iggy.partitions.diag", + plane = "partitions", + namespace_raw = namespace.inner(), + operation = ?message.header().operation, + "partition not initialized for namespace" + ); + Self::answer_waiter( + reply.take(), + message.header(), + IggyError::TransientNotAccepted.as_code(), + ); + return; + }; // At-rest encryption happens HERE, once, before the op enters // consensus: canonicalize the wire form first so both the legacy and // v2 request encodings encrypt identically, then encrypt payload + // user headers per message. The ciphertext is what gets journaled, // replicated, checksummed, and persisted -- every replica stores // identical bytes -- and the poll reply is the single decrypt point. + // The partition's own encryptor: each stream has its own key, pinned + // to the partition when it was built. let message = if message.header().operation == Operation::SendMessages - && let Some(encryptor) = &self.config().encryptor + && let Some(encryptor) = partition.encryptor() { // Compute the batch checksum: this canonical output is validated by // `encrypt_batch_request`'s decode before re-encryption, and the @@ -830,21 +887,6 @@ } else { message }; - let Some(partition) = self.get_mut_by_ns(&namespace) else { - warn!( - target: "iggy.partitions.diag", - plane = "partitions", - namespace_raw = namespace.inner(), - operation = ?message.header().operation, - "partition not initialized for namespace" - ); - Self::answer_waiter( - reply.take(), - message.header(), - IggyError::TransientNotAccepted.as_code(), - ); - return; - }; partition.on_request(message, reply).await; } @@ -990,7 +1032,7 @@ validate_checksum: true, segment_size: IggyByteSize::from(1024 * 1024_u64), preallocate_segments: false, - encryptor: None, + key_context: Arc::default(), path_layout: crate::PartitionPathLayout::default(), }; let partitions = IggyPartitions::new(ShardId::new(0), config); @@ -1336,7 +1378,7 @@ validate_checksum: false, segment_size: IggyByteSize::from(1024 * 1024), preallocate_segments: false, - encryptor: None, + key_context: Arc::default(), path_layout: crate::PartitionPathLayout::default(), }; let partitions = IggyPartitions::new(ShardId::new(0), config); @@ -1395,7 +1437,7 @@ validate_checksum: false, segment_size, preallocate_segments: false, - encryptor: None, + key_context: Arc::default(), path_layout: crate::PartitionPathLayout::default(), }; let partitions = IggyPartitions::new(ShardId::new(0), config);
diff --git a/core/partitions/src/lib.rs b/core/partitions/src/lib.rs index 3e75451..29268f2 100644 --- a/core/partitions/src/lib.rs +++ b/core/partitions/src/lib.rs
@@ -42,6 +42,7 @@ pub mod segment_anchor; mod segment_recovery; pub mod state_transfer; +mod stream_keyring; mod types; #[cfg(feature = "simulator")] mod wal_storage; @@ -107,6 +108,7 @@ use server_common::Message; pub use server_common::send_messages::{IggyMessage, IggyMessageHeader, IggyMessages}; pub use state_transfer::CONSUMER_OFFSETS_ENTRIES_MAX; +pub use stream_keyring::{StreamKeyIdentity, StreamKeyring}; pub use types::{ AppendResult, COMMIT_WALK_OPS_MAX, FatalCommit, Fragment, PartitionOffsets, PartitionPathLayout, PartitionsConfig, PollFragments, PollQueryResult, PollingArgs,
diff --git a/core/partitions/src/segment.rs b/core/partitions/src/segment.rs index 538ccd8..5673f67 100644 --- a/core/partitions/src/segment.rs +++ b/core/partitions/src/segment.rs
@@ -82,7 +82,7 @@ match expiry { IggyExpiry::NeverExpire | IggyExpiry::ServerDefault => false, IggyExpiry::ExpireDuration(duration) => { - self.max_timestamp + duration.as_micros() <= now.as_micros() + self.max_timestamp.saturating_add(duration.as_micros()) <= now.as_micros() } } } @@ -132,6 +132,14 @@ } #[test] + fn expiry_past_the_end_of_the_clock_is_never_reached() { + let now = IggyTimestamp::now(); + let segment = sealed_segment(now.as_micros()); + let expiry = IggyExpiry::ExpireDuration(IggyDuration::from(u64::MAX - 1)); + assert!(!segment.is_expired(now, expiry)); + } + + #[test] fn never_expire_and_server_default_never_expire() { let segment = sealed_segment(1); assert!(!segment.is_expired(IggyTimestamp::now(), IggyExpiry::NeverExpire));
diff --git a/core/partitions/src/segment_recovery.rs b/core/partitions/src/segment_recovery.rs index 97f1d8e..decdd41 100644 --- a/core/partitions/src/segment_recovery.rs +++ b/core/partitions/src/segment_recovery.rs
@@ -3066,6 +3066,7 @@ use server_common::sharding::IggyNamespace; use std::fs; use std::os::unix::fs::symlink; + use std::sync::Arc; use tempfile::{TempDir, tempdir}; const STREAM_ID: usize = 1; @@ -3087,7 +3088,7 @@ validate_checksum: true, segment_size: IggyByteSize::from(SEGMENT_MAX_SIZE), preallocate_segments: iggy_common::DEFAULT_PREALLOCATE_SEGMENTS, - encryptor: None, + key_context: Arc::default(), path_layout: PartitionPathLayout { streams_root: tmp.path().join("streams").to_string_lossy().into_owned(), },
diff --git a/core/partitions/src/stream_keyring.rs b/core/partitions/src/stream_keyring.rs new file mode 100644 index 0000000..436277a --- /dev/null +++ b/core/partitions/src/stream_keyring.rs
@@ -0,0 +1,178 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use iggy_common::{EncryptorKind, IggyError, ServerKeyContext, StreamEncryption}; +use std::collections::BTreeMap; +use std::fmt::{Debug, Formatter}; +use std::sync::Arc; + +/// One stream incarnation. A stream id reused after a delete gets a new +/// `created_op`, so its key is never mistaken for the previous stream's. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct StreamKeyIdentity { + pub stream_id: usize, + pub created_op: u64, +} + +/// A shard's stream encryptors by stream incarnation, opened from each +/// stream's committed wrap set with this node's key-encryption keys. +pub struct StreamKeyring { + context: Arc<ServerKeyContext>, + encryptors: BTreeMap<StreamKeyIdentity, Arc<EncryptorKind>>, +} + +impl StreamKeyring { + #[must_use] + pub const fn new(context: Arc<ServerKeyContext>) -> Self { + Self { + context, + encryptors: BTreeMap::new(), + } + } + + /// This node's key-encryption keys, for wrapping new stream keys. + #[must_use] + pub const fn key_context(&self) -> &Arc<ServerKeyContext> { + &self.context + } + + /// The encryptor of one stream incarnation, `None` only for a plaintext + /// stream. + /// + /// # Errors + /// + /// The stream's critical state is unreadable, or no envelope of its wrap + /// set opens on this node. The stream is unavailable then, it never falls + /// back to plaintext or to another key. + pub fn resolve( + &mut self, + identity: StreamKeyIdentity, + encryption: &StreamEncryption, + ) -> Result<Option<Arc<EncryptorKind>>, IggyError> { + let set = match encryption { + StreamEncryption::Plaintext => return Ok(None), + StreamEncryption::Unavailable(error) => return Err(error.clone()), + StreamEncryption::Wrapped(set) => set, + }; + if let Some(encryptor) = self.encryptors.get(&identity) { + return Ok(Some(Arc::clone(encryptor))); + } + let encryptor = self.context.open(set)?.encryptor()?; + self.encryptors.insert(identity, Arc::clone(&encryptor)); + Ok(Some(encryptor)) + } + + #[must_use] + pub fn is_empty(&self) -> bool { + self.encryptors.is_empty() + } + + /// Drop the encryptors of the stream incarnations `is_live` rejects. + pub fn retain(&mut self, mut is_live: impl FnMut(StreamKeyIdentity) -> bool) { + self.encryptors.retain(|identity, _| is_live(*identity)); + } +} + +impl Debug for StreamKeyring { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("StreamKeyring") + .field("context", &self.context) + .field("streams", &self.encryptors.len()) + .finish() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use iggy_common::{StreamEncryptionKey, StreamKeyOrigin, WrapSet}; + + const CONFIG_KEY: &str = "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE="; + const OTHER_CONFIG_KEY: &str = "AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA="; + + #[test] + fn given_reused_stream_id_when_resolving_should_never_use_previous_key() { + let context = context(CONFIG_KEY); + let mut keyring = StreamKeyring::new(Arc::clone(&context)); + let old = StreamKeyIdentity { + stream_id: 0, + created_op: 5, + }; + let recreated = StreamKeyIdentity { + stream_id: 0, + created_op: 9, + }; + let old_encryptor = keyring + .resolve(old, &wrapped(&context, 1)) + .expect("opens") + .expect("encrypted"); + let new_encryptor = keyring + .resolve(recreated, &wrapped(&context, 2)) + .expect("opens") + .expect("encrypted"); + + let ciphertext = old_encryptor.encrypt(b"payload").expect("encrypts"); + assert!(new_encryptor.decrypt(&ciphertext).is_err()); + keyring.retain(|identity| identity == recreated); + assert_eq!(keyring.encryptors.len(), 1); + let again = keyring + .resolve(recreated, &wrapped(&context, 2)) + .expect("opens") + .expect("encrypted"); + assert!(Arc::ptr_eq(&again, &new_encryptor)); + } + + #[test] + fn given_plaintext_and_wrapped_streams_when_resolving_should_never_fall_back() { + let context = context(CONFIG_KEY); + let mut keyring = StreamKeyring::new(context); + let identity = StreamKeyIdentity { + stream_id: 3, + created_op: 1, + }; + assert!( + keyring + .resolve(identity, &StreamEncryption::Plaintext) + .expect("plaintext") + .is_none() + ); + let foreign = wrapped(&self::context(OTHER_CONFIG_KEY), 4); + assert!(keyring.resolve(identity, &foreign).is_err()); + assert!( + keyring + .resolve( + identity, + &StreamEncryption::Unavailable(IggyError::UnsupportedCriticalOption( + "future".to_owned() + )) + ) + .is_err() + ); + assert!(keyring.encryptors.is_empty()); + } + + fn context(config_key: &str) -> Arc<ServerKeyContext> { + Arc::new(ServerKeyContext::from_config(config_key, &[], true).expect("valid config")) + } + + fn wrapped(context: &ServerKeyContext, seed: u8) -> StreamEncryption { + let key = StreamEncryptionKey::try_from([seed; 32].as_slice()).expect("32 bytes"); + let envelope = context.wrap(&key, StreamKeyOrigin::Server).expect("wraps"); + StreamEncryption::Wrapped(WrapSet::from(envelope)) + } +}
diff --git a/core/partitions/src/types.rs b/core/partitions/src/types.rs index e7df848..27a6466 100644 --- a/core/partitions/src/types.rs +++ b/core/partitions/src/types.rs
@@ -16,7 +16,7 @@ // under the License. use iggy_binary_protocol::Operation; -use iggy_common::{EncryptorKind, IggyByteSize, PollingKind, PollingStrategy}; +use iggy_common::{IggyByteSize, PollingKind, PollingStrategy, ServerKeyContext}; use server_common::iobuf::Frozen; use smallvec::SmallVec; use std::sync::Arc; @@ -387,13 +387,11 @@ pub segment_size: IggyByteSize, /// Whether local message files reserve the configured segment size on open. pub preallocate_segments: bool, - /// Server-side at-rest encryption. Applied ONCE, on the primary at - /// ingestion, so the ciphertext replicates verbatim: every replica - /// journals, acks, and persists identical bytes (checksums and the - /// deterministic segment rolls both depend on that), and the poll path - /// decrypts uniformly whether a fragment came from the resident journal - /// or from disk. - pub encryptor: Option<Arc<EncryptorKind>>, + /// This node's key-encryption keys. Each stream's own key is opened from + /// its committed wrap set and pinned to its partitions. Encryption runs + /// once, on the primary at ingestion, so every replica journals, acks and + /// persists identical ciphertext. + pub key_context: Arc<ServerKeyContext>, /// On-disk location scheme for partition directories. pub path_layout: PartitionPathLayout, }
diff --git a/core/partitions/tests/consumer_offset_history.rs b/core/partitions/tests/consumer_offset_history.rs index a8ee4a9..5d82075 100644 --- a/core/partitions/tests/consumer_offset_history.rs +++ b/core/partitions/tests/consumer_offset_history.rs
@@ -267,7 +267,7 @@ validate_checksum: true, segment_size, preallocate_segments: false, - encryptor: None, + key_context: Arc::default(), path_layout: PartitionPathLayout::default(), }; let consensus = VsrConsensus::new(
diff --git a/core/sdk/Cargo.toml b/core/sdk/Cargo.toml index 4361c7c..f5553e0 100644 --- a/core/sdk/Cargo.toml +++ b/core/sdk/Cargo.toml
@@ -17,7 +17,7 @@ [package] name = "iggy" -version = "0.11.1-edge.1" +version = "0.11.1-edge.2" description = "Iggy is the persistent message streaming platform written in Rust, supporting QUIC, TCP and HTTP transport protocols, capable of processing millions of messages per second." edition = "2024" rust-version.workspace = true @@ -49,6 +49,7 @@ reqwest-middleware = { workspace = true } reqwest-retry = { workspace = true } reqwest-tracing = { workspace = true } +ring = { workspace = true } rustls = { workspace = true } secrecy = { workspace = true } serde = { workspace = true }
diff --git a/core/sdk/README.md b/core/sdk/README.md index 08bc6be..0838b49 100644 --- a/core/sdk/README.md +++ b/core/sdk/README.md
@@ -39,6 +39,7 @@ - **Reliability**: managed replica connections, heartbeat, ordered mutation lanes, explicit recovery of uncertain mutations, and offset auto-commit. - **Message features**: optional headers (`HeaderKey` / `HeaderValue`), client-side AES-256-GCM encryption (via `Aes256GcmEncryptor`), topic compression metadata (`None` and `Gzip`; no runtime compression yet), server-honored message expiry, and server-side deduplication. - **Admin**: stream/topic/partition CRUD, consumer-group management, server-side consumer offsets, system stats. +- **Resource options**: typed create and update options for streams, topics and users (`StreamCreateOptions`, `TopicUpdateOptions`, `UserUpdateOptions` and friends). An update sends only the settings it names and resets others by key. A stream can carry its own at-rest key (`StreamCreateOptions::encryption_key`), reads report only its source and fingerprints. ## Installation
diff --git a/core/sdk/src/client_wrappers/binary_stream_client.rs b/core/sdk/src/client_wrappers/binary_stream_client.rs index 1540ef8..d16f6d4 100644 --- a/core/sdk/src/client_wrappers/binary_stream_client.rs +++ b/core/sdk/src/client_wrappers/binary_stream_client.rs
@@ -18,8 +18,9 @@ use crate::client_wrappers::client_wrapper::ClientWrapper; use async_trait::async_trait; use iggy_common::StreamClient; -use iggy_common::StreamUpdateOptions; -use iggy_common::{Identifier, IggyError, Stream, StreamDetails}; +use iggy_common::{ + Identifier, IggyError, Stream, StreamCreateOptions, StreamDetails, StreamUpdateOptions, +}; #[async_trait] impl StreamClient for ClientWrapper { @@ -43,30 +44,31 @@ } } - async fn create_stream(&self, name: &str) -> Result<StreamDetails, IggyError> { + async fn create_stream( + &self, + name: &str, + options: &StreamCreateOptions, + ) -> Result<StreamDetails, IggyError> { match self { - ClientWrapper::Iggy(client) => client.create_stream(name).await, - ClientWrapper::Http(client) => client.create_stream(name).await, - ClientWrapper::Tcp(client) => client.create_stream(name).await, - ClientWrapper::Quic(client) => client.create_stream(name).await, - ClientWrapper::WebSocket(client) => client.create_stream(name).await, + ClientWrapper::Iggy(client) => client.create_stream(name, options).await, + ClientWrapper::Http(client) => client.create_stream(name, options).await, + ClientWrapper::Tcp(client) => client.create_stream(name, options).await, + ClientWrapper::Quic(client) => client.create_stream(name, options).await, + ClientWrapper::WebSocket(client) => client.create_stream(name, options).await, } } async fn update_stream( &self, stream_id: &Identifier, - name: &str, options: &StreamUpdateOptions, ) -> Result<(), IggyError> { match self { - ClientWrapper::Iggy(client) => client.update_stream(stream_id, name, options).await, - ClientWrapper::Http(client) => client.update_stream(stream_id, name, options).await, - ClientWrapper::Tcp(client) => client.update_stream(stream_id, name, options).await, - ClientWrapper::Quic(client) => client.update_stream(stream_id, name, options).await, - ClientWrapper::WebSocket(client) => { - client.update_stream(stream_id, name, options).await - } + ClientWrapper::Iggy(client) => client.update_stream(stream_id, options).await, + ClientWrapper::Http(client) => client.update_stream(stream_id, options).await, + ClientWrapper::Tcp(client) => client.update_stream(stream_id, options).await, + ClientWrapper::Quic(client) => client.update_stream(stream_id, options).await, + ClientWrapper::WebSocket(client) => client.update_stream(stream_id, options).await, } }
diff --git a/core/sdk/src/client_wrappers/binary_topic_client.rs b/core/sdk/src/client_wrappers/binary_topic_client.rs index 982e21b..f7cffd8 100644 --- a/core/sdk/src/client_wrappers/binary_topic_client.rs +++ b/core/sdk/src/client_wrappers/binary_topic_client.rs
@@ -67,34 +67,15 @@ &self, stream_id: &Identifier, topic_id: &Identifier, - name: &str, options: &TopicUpdateOptions, ) -> Result<(), IggyError> { match self { - ClientWrapper::Iggy(client) => { - client - .update_topic(stream_id, topic_id, name, options) - .await - } - ClientWrapper::Http(client) => { - client - .update_topic(stream_id, topic_id, name, options) - .await - } - ClientWrapper::Tcp(client) => { - client - .update_topic(stream_id, topic_id, name, options) - .await - } - ClientWrapper::Quic(client) => { - client - .update_topic(stream_id, topic_id, name, options) - .await - } + ClientWrapper::Iggy(client) => client.update_topic(stream_id, topic_id, options).await, + ClientWrapper::Http(client) => client.update_topic(stream_id, topic_id, options).await, + ClientWrapper::Tcp(client) => client.update_topic(stream_id, topic_id, options).await, + ClientWrapper::Quic(client) => client.update_topic(stream_id, topic_id, options).await, ClientWrapper::WebSocket(client) => { - client - .update_topic(stream_id, topic_id, name, options) - .await + client.update_topic(stream_id, topic_id, options).await } } }
diff --git a/core/sdk/src/client_wrappers/binary_user_client.rs b/core/sdk/src/client_wrappers/binary_user_client.rs index dc38d26..66297b2 100644 --- a/core/sdk/src/client_wrappers/binary_user_client.rs +++ b/core/sdk/src/client_wrappers/binary_user_client.rs
@@ -18,10 +18,10 @@ use crate::client_wrappers::client_wrapper::ClientWrapper; use async_trait::async_trait; use iggy_common::UserClient; -use iggy_common::UserUpdateOptions; use iggy_common::{ Identifier, IdentityInfo, IggyError, Permissions, UserInfo, UserInfoDetails, UserStatus, }; +use iggy_common::{UserCreateOptions, UserUpdateOptions}; #[async_trait] impl UserClient for ClientWrapper { @@ -51,31 +51,32 @@ password: &str, status: UserStatus, permissions: Option<Permissions>, + options: &UserCreateOptions, ) -> Result<UserInfoDetails, IggyError> { match self { ClientWrapper::Http(client) => { client - .create_user(username, password, status, permissions) + .create_user(username, password, status, permissions, options) .await } ClientWrapper::Tcp(client) => { client - .create_user(username, password, status, permissions) + .create_user(username, password, status, permissions, options) .await } ClientWrapper::Quic(client) => { client - .create_user(username, password, status, permissions) + .create_user(username, password, status, permissions, options) .await } ClientWrapper::Iggy(client) => { client - .create_user(username, password, status, permissions) + .create_user(username, password, status, permissions, options) .await } ClientWrapper::WebSocket(client) => { client - .create_user(username, password, status, permissions) + .create_user(username, password, status, permissions, options) .await } } @@ -94,26 +95,14 @@ async fn update_user( &self, user_id: &Identifier, - username: Option<&str>, - status: Option<UserStatus>, options: &UserUpdateOptions, ) -> Result<(), IggyError> { match self { - ClientWrapper::Http(client) => { - client.update_user(user_id, username, status, options).await - } - ClientWrapper::Tcp(client) => { - client.update_user(user_id, username, status, options).await - } - ClientWrapper::Quic(client) => { - client.update_user(user_id, username, status, options).await - } - ClientWrapper::Iggy(client) => { - client.update_user(user_id, username, status, options).await - } - ClientWrapper::WebSocket(client) => { - client.update_user(user_id, username, status, options).await - } + ClientWrapper::Http(client) => client.update_user(user_id, options).await, + ClientWrapper::Tcp(client) => client.update_user(user_id, options).await, + ClientWrapper::Quic(client) => client.update_user(user_id, options).await, + ClientWrapper::Iggy(client) => client.update_user(user_id, options).await, + ClientWrapper::WebSocket(client) => client.update_user(user_id, options).await, } }
diff --git a/core/sdk/src/clients/binary_streams.rs b/core/sdk/src/clients/binary_streams.rs index 5c555fb..16ac9fd 100644 --- a/core/sdk/src/clients/binary_streams.rs +++ b/core/sdk/src/clients/binary_streams.rs
@@ -18,9 +18,9 @@ use crate::prelude::IggyClient; use async_trait::async_trait; use iggy_common::StreamClient; -use iggy_common::StreamUpdateOptions; use iggy_common::locking::IggyRwLockFn; use iggy_common::{Identifier, IggyError, Stream, StreamDetails}; +use iggy_common::{StreamCreateOptions, StreamUpdateOptions}; #[async_trait] impl StreamClient for IggyClient { @@ -32,17 +32,20 @@ super::with_client!(self, client => client.get_streams().await) } - async fn create_stream(&self, name: &str) -> Result<StreamDetails, IggyError> { - super::with_client!(self, client => client.create_stream(name).await) + async fn create_stream( + &self, + name: &str, + options: &StreamCreateOptions, + ) -> Result<StreamDetails, IggyError> { + super::with_client!(self, client => client.create_stream(name, options).await) } async fn update_stream( &self, stream_id: &Identifier, - name: &str, options: &StreamUpdateOptions, ) -> Result<(), IggyError> { - super::with_client!(self, client => client.update_stream(stream_id, name, options).await) + super::with_client!(self, client => client.update_stream(stream_id, options).await) } async fn delete_stream(&self, stream_id: &Identifier) -> Result<(), IggyError> {
diff --git a/core/sdk/src/clients/binary_topics.rs b/core/sdk/src/clients/binary_topics.rs index f62e66f..92a7cec 100644 --- a/core/sdk/src/clients/binary_topics.rs +++ b/core/sdk/src/clients/binary_topics.rs
@@ -50,10 +50,9 @@ &self, stream_id: &Identifier, topic_id: &Identifier, - name: &str, options: &TopicUpdateOptions, ) -> Result<(), IggyError> { - super::with_client!(self, client => client.update_topic(stream_id, topic_id, name, options).await) + super::with_client!(self, client => client.update_topic(stream_id, topic_id, options).await) } async fn delete_topic(
diff --git a/core/sdk/src/clients/binary_users.rs b/core/sdk/src/clients/binary_users.rs index 8049eee..5ba389d 100644 --- a/core/sdk/src/clients/binary_users.rs +++ b/core/sdk/src/clients/binary_users.rs
@@ -18,11 +18,11 @@ use crate::prelude::IggyClient; use async_trait::async_trait; use iggy_common::UserClient; -use iggy_common::UserUpdateOptions; use iggy_common::locking::IggyRwLockFn; use iggy_common::{ Identifier, IdentityInfo, IggyError, Permissions, UserInfo, UserInfoDetails, UserStatus, }; +use iggy_common::{UserCreateOptions, UserUpdateOptions}; #[async_trait] impl UserClient for IggyClient { @@ -40,8 +40,9 @@ password: &str, status: UserStatus, permissions: Option<Permissions>, + options: &UserCreateOptions, ) -> Result<UserInfoDetails, IggyError> { - super::with_client!(self, client => client.create_user(username, password, status, permissions).await) + super::with_client!(self, client => client.create_user(username, password, status, permissions, options).await) } async fn delete_user(&self, user_id: &Identifier) -> Result<(), IggyError> { @@ -51,11 +52,9 @@ async fn update_user( &self, user_id: &Identifier, - username: Option<&str>, - status: Option<UserStatus>, options: &UserUpdateOptions, ) -> Result<(), IggyError> { - super::with_client!(self, client => client.update_user(user_id, username, status, options).await) + super::with_client!(self, client => client.update_user(user_id, options).await) } async fn update_permissions(
diff --git a/core/sdk/src/clients/client.rs b/core/sdk/src/clients/client.rs index 46f7a93..f1b5033 100644 --- a/core/sdk/src/clients/client.rs +++ b/core/sdk/src/clients/client.rs
@@ -1112,7 +1112,9 @@ use iggy_binary_protocol::codes::{ CREATE_STREAM_CODE, GET_STATS_CODE, SYNC_CONSUMER_GROUP_CODE, }; - use iggy_common::{StreamClient, TcpClientConfig, TcpClientReconnectionConfig}; + use iggy_common::{ + StreamClient, StreamCreateOptions, TcpClientConfig, TcpClientReconnectionConfig, + }; use std::iter::repeat_n; use std::time::Duration; use tokio::net::TcpListener; @@ -1157,7 +1159,8 @@ let client_lock = client.client.clone(); let guard = client_lock.write().await; let start = tokio::time::Instant::now(); - let (result, ()) = tokio::join!(client.create_stream("stream"), async { + let options = StreamCreateOptions::default(); + let (result, ()) = tokio::join!(client.create_stream("stream", &options), async { sleep(Duration::from_secs(20)).await; drop(guard); });
diff --git a/core/sdk/src/clients/producer.rs b/core/sdk/src/clients/producer.rs index 44e7b05..b68bf67 100644 --- a/core/sdk/src/clients/producer.rs +++ b/core/sdk/src/clients/producer.rs
@@ -24,7 +24,10 @@ use crate::driver::operation_deadline; use bytes::Bytes; use iggy_common::locking::{IggyRwLock, IggyRwLockFn}; -use iggy_common::{BinarySendPartitioning, Client, StreamClient, TopicClient, TopicCreateOptions}; +use iggy_common::{ + BinarySendPartitioning, Client, StreamClient, StreamCreateOptions, TopicClient, + TopicCreateOptions, +}; use iggy_common::{ Durability, EncryptorKind, IdKind, Identifier, IggyError, IggyExpiry, IggyMessage, IggyTimestamp, MaxTopicSize, Partitioner, Partitioning, SendMessagesConfirmationResponse, @@ -115,7 +118,9 @@ } info!("Creating stream: {name}"); - client.create_stream(&name).await?; + client + .create_stream(&name, &StreamCreateOptions::default()) + .await?; } if client.get_topic(&stream_id, &topic_id).await?.is_none() { @@ -1143,6 +1148,7 @@ messages_count: 0, name: WireName::new("stream").unwrap(), options: WireOptions::empty(), + derived_options: WireOptions::empty(), } .to_bytes() .to_vec(),
diff --git a/core/sdk/src/driver.rs b/core/sdk/src/driver.rs index 4977eca..6c630ef 100644 --- a/core/sdk/src/driver.rs +++ b/core/sdk/src/driver.rs
@@ -27,7 +27,7 @@ use iggy_binary_protocol::codes::{ BIND_SESSION_CODE, CHANGE_PASSWORD_CODE, DELETE_CONSUMER_OFFSET_CODE, GET_CONSUMER_OFFSET_CODE, GET_PARTITION_ROUTE_CODE, LOGIN_REGISTER_CODE, LOGIN_REGISTER_WITH_PAT_CODE, - POLL_MESSAGES_CODE, SEND_MESSAGES_CODE, STORE_CONSUMER_OFFSET_CODE, UPDATE_USER_CODE, + POLL_MESSAGES_CODE, SEND_MESSAGES_CODE, STORE_CONSUMER_OFFSET_CODE, UPDATE_RESOURCE_CODE, }; use iggy_binary_protocol::consensus::{Operation, operation_for_code}; use iggy_binary_protocol::primitives::partition_history::PartitionContext; @@ -37,12 +37,16 @@ use iggy_binary_protocol::requests::messages::{ GetPartitionRouteRequest, PollMessagesRequest, SendMessagesHeader, }; -use iggy_binary_protocol::requests::users::{ChangePasswordRequest, UpdateUserRequest}; +use iggy_binary_protocol::requests::system::{ResourceTarget, UpdateResourceRequest}; +use iggy_binary_protocol::requests::users::ChangePasswordRequest; use iggy_binary_protocol::responses::messages::PartitionRouteResponse; use iggy_binary_protocol::responses::users::LoginRegisterResponse; -use iggy_binary_protocol::{WireDecode, WireEncode, WirePartitioning, WirePollingStrategy}; +use iggy_binary_protocol::{ + WireDecode, WireEncode, WireOptions, WirePartitioning, WirePollingStrategy, +}; use iggy_common::{ - BinaryOperationLineage, ClientState, Credentials, DiagnosticEvent, Identifier, IggyError, + BinaryOperationLineage, ClientState, Credentials, DiagnosticEvent, HeaderKind, Identifier, + IggyError, OptionKey, }; use route_cache::RouteCache; use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; @@ -329,15 +333,16 @@ self.session.refresh_password(&user, &request.new_password); } } - UPDATE_USER_CODE => { - if let Ok(UpdateUserRequest { - user_id, - username: Some(username), + UPDATE_RESOURCE_CODE => { + if let Ok(UpdateResourceRequest { + target: ResourceTarget::User { user_id }, + options, .. - }) = UpdateUserRequest::decode_from(&frame.body) + }) = UpdateResourceRequest::decode_from(&frame.body) + && let Some(username) = patched_username(&options) && let Ok(user) = Identifier::try_from(&user_id) { - self.session.refresh_username(&user, username.as_str()); + self.session.refresh_username(&user, username); } } _ => {} @@ -1109,6 +1114,15 @@ } } +/// The new username a user patch sets, if it sets one. +fn patched_username(options: &WireOptions) -> Option<&str> { + let key = <&str>::from(OptionKey::Username).as_bytes(); + options + .into_iter() + .find(|entry| entry.key == key && entry.value_kind.0 == HeaderKind::String.as_code()) + .and_then(|entry| std::str::from_utf8(entry.value).ok()) +} + fn is_sign_in(code: u32) -> bool { matches!(code, LOGIN_REGISTER_CODE | LOGIN_REGISTER_WITH_PAT_CODE) }
diff --git a/core/sdk/src/http/messages.rs b/core/sdk/src/http/messages.rs index 1a04866..4d929ae 100644 --- a/core/sdk/src/http/messages.rs +++ b/core/sdk/src/http/messages.rs
@@ -231,8 +231,8 @@ use crate::http::scripted_server::{Recorded, Reply, ScriptedServer}; use crate::prelude::{ Identifier, IggyError, IggyMessage, MessageClient, PartitionClient, PartitionContext, - Partitioning, SendMessagesConfirmationResponse, StreamClient, TopicClient, - TopicCreateOptions, + Partitioning, SendMessagesConfirmationResponse, StreamClient, StreamCreateOptions, + TopicClient, TopicCreateOptions, }; use bytes::Bytes; @@ -560,7 +560,10 @@ let first = send(&client, &partitioning).await; let changed = match change { - Change::CreateStream => client.create_stream("stream").await.map(drop), + Change::CreateStream => client + .create_stream("stream", &StreamCreateOptions::default()) + .await + .map(drop), Change::CreateTopic => client .create_topic(&stream_id, "topic", &TopicCreateOptions::default()) .await
diff --git a/core/sdk/src/http/scripted_server.rs b/core/sdk/src/http/scripted_server.rs index f8e1c32..2333e7b 100644 --- a/core/sdk/src/http/scripted_server.rs +++ b/core/sdk/src/http/scripted_server.rs
@@ -20,8 +20,8 @@ use crate::http::http_client::HttpClient; use crate::prelude::{ - CompressionAlgorithm, HttpClientConfig, IggyByteSize, IggyError, IggyExpiry, IggyTimestamp, - MaxTopicSize, Partition, PartitionContext, StreamDetails, TopicDetails, + HttpClientConfig, IggyByteSize, IggyError, IggyTimestamp, Partition, PartitionContext, + StreamDetails, TopicDetails, }; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -204,9 +204,6 @@ created_at: IggyTimestamp::zero(), name: "topic".to_owned(), size: IggyByteSize::default(), - message_expiry: IggyExpiry::NeverExpire, - compression_algorithm: CompressionAlgorithm::None, - max_topic_size: MaxTopicSize::Unlimited, messages_count: 0, partitions_count: u32::try_from(partitions.len()).unwrap(), partitions,
diff --git a/core/sdk/src/http/streams.rs b/core/sdk/src/http/streams.rs index 7c14964..cc7801d 100644 --- a/core/sdk/src/http/streams.rs +++ b/core/sdk/src/http/streams.rs
@@ -21,11 +21,10 @@ use crate::prelude::Identifier; use crate::prelude::IggyError; use async_trait::async_trait; -use iggy_common::StreamClient; -use iggy_common::StreamUpdateOptions; use iggy_common::create_stream::CreateStream; -use iggy_common::update_stream::UpdateStream; -use iggy_common::{Stream, StreamDetails}; +use iggy_common::{ + OptionInputMap, Stream, StreamClient, StreamCreateOptions, StreamDetails, StreamUpdateOptions, +}; const PATH: &str = "/streams"; @@ -57,12 +56,18 @@ Ok(streams) } - async fn create_stream(&self, name: &str) -> Result<StreamDetails, IggyError> { + async fn create_stream( + &self, + name: &str, + options: &StreamCreateOptions, + ) -> Result<StreamDetails, IggyError> { + // The options erase on drop, the JSON body the HTTP client serializes does not. let response = self .post( PATH, &CreateStream { name: name.to_string(), + options: OptionInputMap(options.to_input_map()?), }, ) .await?; @@ -76,16 +81,11 @@ async fn update_stream( &self, stream_id: &Identifier, - name: &str, options: &StreamUpdateOptions, ) -> Result<(), IggyError> { self.put( &get_details_path(&stream_id.as_cow_str()), - &UpdateStream { - stream_id: stream_id.clone(), - name: name.to_string(), - options: options.raw.clone(), - }, + &options.to_update_body()?, ) .await?; Ok(())
diff --git a/core/sdk/src/http/topics.rs b/core/sdk/src/http/topics.rs index f154903..496caa8 100644 --- a/core/sdk/src/http/topics.rs +++ b/core/sdk/src/http/topics.rs
@@ -18,13 +18,12 @@ use crate::http::http_client::HttpClient; use crate::http::http_transport::HttpTransport; use crate::http::path::encode_segment; -use crate::prelude::{Identifier, IggyError, IggyExpiry, MaxTopicSize}; +use crate::prelude::{Identifier, IggyError}; use async_trait::async_trait; -use iggy_common::TopicClient; use iggy_common::create_topic::CreateTopic; -use iggy_common::update_topic::UpdateTopic; use iggy_common::{ - DEFAULT_PARTITIONS_COUNT, Topic, TopicCreateOptions, TopicDetails, TopicUpdateOptions, + DEFAULT_PARTITIONS_COUNT, OptionInputMap, Topic, TopicClient, TopicCreateOptions, TopicDetails, + TopicUpdateOptions, }; #[async_trait] @@ -63,16 +62,7 @@ stream_id: stream_id.clone(), name: name.to_string(), partitions_count: options.partitions_count.unwrap_or(DEFAULT_PARTITIONS_COUNT), - compression_algorithm: options.compression_algorithm.unwrap_or_default(), - message_expiry: options.message_expiry.unwrap_or(IggyExpiry::ServerDefault), - max_topic_size: options - .max_topic_size - .unwrap_or(MaxTopicSize::ServerDefault), - // Carries the runtime knobs too, not just `raw`: this body - // has no dedicated field for them, and dropping them here - // gave one transport a topic without the fsync the caller - // asked for while the other honored it. - options: options.to_string_options()?, + options: OptionInputMap(options.to_input_map()?), }, ) .await?; @@ -87,20 +77,11 @@ &self, stream_id: &Identifier, topic_id: &Identifier, - name: &str, options: &TopicUpdateOptions, ) -> Result<(), IggyError> { self.put( &get_details_path(&stream_id.as_cow_str(), &topic_id.as_cow_str()), - &UpdateTopic { - stream_id: stream_id.clone(), - topic_id: topic_id.clone(), - name: name.to_string(), - compression_algorithm: options.compression_algorithm, - message_expiry: options.message_expiry, - max_topic_size: options.max_topic_size, - options: options.raw.clone(), - }, + &options.to_update_body()?, ) .await?; Ok(())
diff --git a/core/sdk/src/http/users.rs b/core/sdk/src/http/users.rs index 0d6b82b..fffc393 100644 --- a/core/sdk/src/http/users.rs +++ b/core/sdk/src/http/users.rs
@@ -19,14 +19,14 @@ use crate::http::http_transport::HttpTransport; use crate::prelude::{Identifier, IggyError}; use async_trait::async_trait; -use iggy_common::UserClient; -use iggy_common::UserUpdateOptions; use iggy_common::change_password::ChangePassword; use iggy_common::create_user::CreateUser; use iggy_common::login_user::LoginUser; use iggy_common::update_permissions::UpdatePermissions; -use iggy_common::update_user::UpdateUser; -use iggy_common::{IdentityInfo, Permissions, UserInfo, UserInfoDetails, UserStatus}; +use iggy_common::{ + IdentityInfo, OptionInputMap, Permissions, UserClient, UserCreateOptions, UserInfo, + UserInfoDetails, UserStatus, UserUpdateOptions, +}; use secrecy::SecretString; const PATH: &str = "/users"; @@ -65,6 +65,7 @@ password: &str, status: UserStatus, permissions: Option<Permissions>, + options: &UserCreateOptions, ) -> Result<UserInfoDetails, IggyError> { let response = self .post( @@ -74,6 +75,7 @@ password: SecretString::from(password), status, permissions, + options: OptionInputMap(options.to_input_map()?), }, ) .await?; @@ -93,18 +95,11 @@ async fn update_user( &self, user_id: &Identifier, - username: Option<&str>, - status: Option<UserStatus>, options: &UserUpdateOptions, ) -> Result<(), IggyError> { self.put( &format!("{PATH}/{}", user_id.as_cow_str()), - &UpdateUser { - user_id: user_id.clone(), - username: username.map(|s| s.to_string()), - status, - options: options.raw.clone(), - }, + &options.to_update_body()?, ) .await?; Ok(())
diff --git a/core/sdk/src/links/fake_cluster.rs b/core/sdk/src/links/fake_cluster.rs index 8e0505f..1358b31 100644 --- a/core/sdk/src/links/fake_cluster.rs +++ b/core/sdk/src/links/fake_cluster.rs
@@ -38,9 +38,7 @@ use iggy_binary_protocol::responses::topics::{GetTopicResponse, PartitionResponse}; use iggy_binary_protocol::responses::users::LoginRegisterResponse; use iggy_binary_protocol::{PartitionContext, WireDecode, WireEncode, WireName, WireOptions}; -use iggy_common::{ - AutoLogin, ClusterNodeRole, ClusterNodeStatus, CompressionAlgorithm, Credentials, IggyError, -}; +use iggy_common::{AutoLogin, ClusterNodeRole, ClusterNodeStatus, Credentials, IggyError}; use secrecy::SecretString; use std::collections::{HashMap, HashSet, VecDeque}; use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; @@ -458,9 +456,6 @@ id: 1, created_at: 0, partitions_count, - message_expiry: 0, - compression_algorithm: CompressionAlgorithm::None.as_code(), - max_topic_size: 0, size_bytes: 0, messages_count: 0, name: WireName::new("topic").unwrap(),
diff --git a/core/sdk/src/prelude.rs b/core/sdk/src/prelude.rs index 18cc280..2863c90 100644 --- a/core/sdk/src/prelude.rs +++ b/core/sdk/src/prelude.rs
@@ -58,17 +58,19 @@ HttpMethod, IdKind, Identifier, IdentityInfo, IggyByteSize, IggyDuration, IggyError, IggyExpiry, IggyIndexView, IggyMessage, IggyMessageHeader, IggyMessageHeaderView, IggyMessageView, IggyMessageViewIterator, IggyTimestamp, MaxTopicSize, NonZeroDurationError, - NonZeroIggyDuration, OptionSpec, OptionValue, OptionsScope, Partition, PartitionContext, - PartitionResizePolicy, Partitioner, Partitioning, PendingOperationsConfig, Permissions, - PersonalAccessTokenExpiry, PollMessages, PolledMessages, PollingKind, PollingStrategy, - QuicClientConfig, QuicClientConfigBuilder, QuicClientReconnectionConfig, ResourceOptions, - SendMessages, SendMessagesConfirmationResponse, SendMessagesResponse, Sizeable, - SnapshotCompression, Stats, Stream, StreamDetails, StreamPermissions, StreamUpdateOptions, + NonZeroIggyDuration, OptionKey, OptionSpec, OptionValue, OptionsScope, Partition, + PartitionContext, PartitionResizePolicy, Partitioner, Partitioning, PendingOperationsConfig, + Permissions, PersonalAccessTokenExpiry, PollMessages, PolledMessages, PollingKind, + PollingStrategy, QuicClientConfig, QuicClientConfigBuilder, QuicClientReconnectionConfig, + ResourceOptions, SendMessages, SendMessagesConfirmationResponse, SendMessagesResponse, + Sizeable, SnapshotCompression, Stats, Stream, StreamCreateOptions, StreamDetails, + StreamEncryptionKey, StreamEncryptionSource, StreamPermissions, StreamUpdateOptions, SystemSnapshotType, TcpClientConfig, TcpClientConfigBuilder, TcpClientReconnectionConfig, - Topic, TopicCreateOptions, TopicDetails, TopicPermissions, TopicUpdateOptions, - TransportEndpoints, TransportProtocol, UserId, UserInfo, UserInfoDetails, UserStatus, - UserUpdateOptions, Validatable, WebSocketClientConfig, WebSocketClientConfigBuilder, - WebSocketClientReconnectionConfig, WebSocketConfig, defaults, locking, topic_option_keys, + Topic, TopicCreateOptions, TopicDetails, TopicPermissions, TopicResolvedOptions, + TopicUpdateOptions, TransportEndpoints, TransportProtocol, UserCreateOptions, UserId, UserInfo, + UserInfoDetails, UserStatus, UserUpdateOptions, Validatable, WebSocketClientConfig, + WebSocketClientConfigBuilder, WebSocketClientReconnectionConfig, WebSocketConfig, defaults, + locking, topic_option_keys, }; pub use iggy_common::{ Client, ClusterClient, ConsumerGroupClient, ConsumerOffsetClient, MessageClient,
diff --git a/core/sdk/src/session.rs b/core/sdk/src/session.rs index ec39b7e..0fa7973 100644 --- a/core/sdk/src/session.rs +++ b/core/sdk/src/session.rs
@@ -52,6 +52,7 @@ /// itself always uses request=0. request_counter: u64, bind_secret: BindSecret, + register_request_identifier: Option<u128>, } impl ConsensusSession { @@ -69,6 +70,7 @@ session: None, request_counter: 1, bind_secret: BindSecret::new(Box::new(rand::random())), + register_request_identifier: None, } } @@ -125,6 +127,23 @@ 0 } + pub(crate) fn register_request_identifier(&mut self) -> Result<u128, IggyError> { + if let Some(identifier) = self.register_request_identifier { + return Ok(identifier); + } + let random = ring::rand::SystemRandom::new(); + loop { + let mut bytes = [0u8; 16]; + ring::rand::SecureRandom::fill(&random, &mut bytes) + .map_err(|_| IggyError::InvalidConfiguration)?; + let identifier = u128::from_le_bytes(bytes); + if identifier != 0 { + self.register_request_identifier = Some(identifier); + return Ok(identifier); + } + } + } + /// Get the next application request ID and advance the counter. /// /// Returns 1, 2, 3, ... (request 0 is reserved for register).
diff --git a/core/sdk/src/stream_builder/build/build_stream_topic.rs b/core/sdk/src/stream_builder/build/build_stream_topic.rs index d4a0555..b77aff7 100644 --- a/core/sdk/src/stream_builder/build/build_stream_topic.rs +++ b/core/sdk/src/stream_builder/build/build_stream_topic.rs
@@ -17,7 +17,8 @@ */ use crate::prelude::{ - Identifier, IggyClient, IggyError, StreamClient, TopicClient, TopicCreateOptions, + Identifier, IggyClient, IggyError, StreamClient, StreamCreateOptions, TopicClient, + TopicCreateOptions, }; use crate::stream_builder::IggyConsumerConfig; @@ -61,7 +62,9 @@ } trace!("Creating stream: {stream_name}"); - client.create_stream(stream_name).await?; + client + .create_stream(stream_name, &StreamCreateOptions::default()) + .await?; } trace!("Check if topic exists.");
diff --git a/core/sdk/src/tcp/tcp_client.rs b/core/sdk/src/tcp/tcp_client.rs index e89243c..25dd570 100644 --- a/core/sdk/src/tcp/tcp_client.rs +++ b/core/sdk/src/tcp/tcp_client.rs
@@ -123,7 +123,7 @@ BIND_SESSION_CODE, CHANGE_PASSWORD_CODE, GET_CLUSTER_METADATA_CODE, GET_STREAMS_CODE, GET_TOPIC_CODE, LOGIN_REGISTER_CODE, LOGIN_REGISTER_WITH_PAT_CODE, LOGOUT_USER_CODE, PING_CODE, POLL_MESSAGES_CODE, SEND_MESSAGES_CODE, SYNC_CONSUMER_GROUP_CODE, - UPDATE_USER_CODE, + UPDATE_RESOURCE_CODE, }; use iggy_binary_protocol::consensus::EvictionReason; use iggy_binary_protocol::requests::users::{ @@ -137,8 +137,8 @@ use iggy_common::{ BinaryTransport, Client, ClientState, Consumer, Credentials, Identifier, IggyDuration, IggyMessage, MessageClient, Partitioning, PersonalAccessTokenClient, PollingStrategy, - SystemClient, TcpClientReconnectionConfig, UserClient, UserUpdateOptions, - with_binary_deadline, + SystemClient, TcpClientReconnectionConfig, TopicClient, TopicUpdateOptions, UserClient, + UserUpdateOptions, with_binary_deadline, }; use secrecy::{ExposeSecret, SecretString}; use std::time::Duration; @@ -1248,21 +1248,21 @@ } #[tokio::test(start_paused = true)] - async fn given_a_committed_rename_of_the_session_user_when_signing_in_again_should_send_the_new_username() - { + async fn given_generic_user_rename_when_reconnecting_should_use_updated_username() { for resolved in [false, true] { let cluster = FakeCluster::new(1); let client = signed_in_client(&cluster).await; if resolved { - end_uncertain(&cluster, UPDATE_USER_CODE, Answer::Commit(COMMIT)); + end_uncertain(&cluster, UPDATE_RESOURCE_CODE, Answer::Commit(COMMIT)); } let renamed = client .update_user( &Identifier::named(USERNAME).unwrap(), - Some(NEW_USERNAME), - None, - &UserUpdateOptions::default(), + &UserUpdateOptions { + username: Some(NEW_USERNAME.to_owned()), + ..UserUpdateOptions::default() + }, ) .await; proved(&client, renamed).await.unwrap(); @@ -1270,12 +1270,51 @@ let case = format!("resolved={resolved}"); let attempts = if resolved { 3 } else { 1 }; - assert_eq!(cluster.count(UPDATE_USER_CODE), attempts, "{case}"); + assert_eq!(cluster.count(UPDATE_RESOURCE_CODE), attempts, "{case}"); assert_eq!(sign_in.username.as_str(), NEW_USERNAME, "{case}"); } } #[tokio::test(start_paused = true)] + async fn given_rejected_rename_or_topic_rename_when_reconnecting_should_keep_the_username() { + let cluster = FakeCluster::new(1); + let client = signed_in_client(&cluster).await; + cluster.script( + &endpoint(0), + UPDATE_RESOURCE_CODE, + [Answer::CommitRejected(COMMIT, IggyError::UserAlreadyExists)], + ); + let rejected = client + .update_user( + &Identifier::named(USERNAME).unwrap(), + &UserUpdateOptions { + username: Some(NEW_USERNAME.to_owned()), + ..UserUpdateOptions::default() + }, + ) + .await; + assert_eq!( + proved(&client, rejected).await, + Err(IggyError::UserAlreadyExists) + ); + client + .update_topic( + &Identifier::numeric(1).unwrap(), + &Identifier::numeric(1).unwrap(), + &TopicUpdateOptions { + name: Some(NEW_USERNAME.to_owned()), + ..TopicUpdateOptions::default() + }, + ) + .await + .unwrap(); + + let sign_in = sign_in_again(&cluster).await; + + assert_eq!(sign_in.username.as_str(), USERNAME); + } + + #[tokio::test(start_paused = true)] async fn given_a_committed_rejection_of_a_password_change_when_signing_in_again_should_send_the_old_password() { for resolved in [false, true] {
diff --git a/core/sdk/src/vsr.rs b/core/sdk/src/vsr.rs index e73c246..3797eca 100644 --- a/core/sdk/src/vsr.rs +++ b/core/sdk/src/vsr.rs
@@ -71,12 +71,15 @@ } } }; - // The header rules (checksum stamp, size, reserved code) live in the + // The header rules (request identifier, size, reserved code) live in the // protocol crate so every client and the cross-SDK fixtures share them; // only the session identity is decided here. - let header = + let mut header = RequestHeader::for_request(code, session.client_id(), request_id, session_id, payload) .map_err(|_| IggyError::InvalidConfiguration)?; + if operation == Operation::Register { + header.request_checksum = session.register_request_identifier()?; + } debug_assert_eq!(header.operation, operation); Ok((header, header.size as usize)) } @@ -180,18 +183,9 @@ if let Some(request) = request.filter(|_| header.has_committed_result()) { let matches_request = header.client == request.client && header.request_checksum == request.request_checksum - && (header.operation == request.operation - || matches!( - (request.operation, header.operation), - // The primary resolves these commands before committing their prepare. - ( - Operation::CreateTopic, - Operation::CreateTopicWithAssignments - ) | ( - Operation::CreatePartitions, - Operation::CreatePartitionsWithAssignments - ) | (Operation::DeleteSegments, Operation::TruncatePartition) - )) + // The primary resolves some commands into an internal + // operation before committing their prepare. + && header.operation.request_operation() == request.operation && header.partition_incarnation == request.partition_incarnation && header.op != 0 && header.op == header.commit; @@ -345,7 +339,7 @@ use bytes::{BufMut, BytesMut}; use iggy_binary_protocol::codes::{ CREATE_PARTITIONS_CODE, CREATE_STREAM_CODE, CREATE_TOPIC_CODE, DELETE_SEGMENTS_CODE, - GET_STREAM_CODE, LOGOUT_USER_CODE, PING_CODE, SEND_MESSAGES_CODE, + GET_STREAM_CODE, LOGOUT_USER_CODE, PING_CODE, SEND_MESSAGES_CODE, UPDATE_RESOURCE_CODE, }; use iggy_binary_protocol::consensus::{NON_REPLICATED_CODE_RANGE, PrepareHeader}; use iggy_binary_protocol::requests::streams::CreateStreamRequest; @@ -412,6 +406,40 @@ } #[test] + fn given_resolved_update_reply_when_correlating_should_accept_only_matching_request_identity() { + let request = + RequestHeader::for_request(UPDATE_RESOURCE_CODE, 1, 2, 3, b"request").unwrap(); + let body = rejection_body(0); + for resolved in [ + Operation::UpdateStreamResolved, + Operation::UpdateTopicResolved, + Operation::UpdateUserResolved, + ] { + let mut reply = committed_header(&request, &body); + reply.operation = resolved; + let outcome = decode_for_test(&request, &reply, body.clone()).unwrap(); + assert!(outcome.committed, "{resolved:?}"); + assert!(outcome.result.is_ok(), "{resolved:?}"); + let mismatches: [fn(&mut ReplyHeader); 3] = [ + |reply| reply.client ^= 1, + |reply| reply.request_checksum ^= 1, + |reply| reply.operation = Operation::CreateTopicWithAssignments, + ]; + for mismatch in mismatches { + let mut unrelated = reply; + mismatch(&mut unrelated); + assert!( + matches!( + decode_for_test(&request, &unrelated, body.clone()), + Err(IggyError::InvalidCommand) + ), + "{resolved:?}" + ); + } + } + } + + #[test] fn given_a_committed_rejection_when_identity_or_marker_differs_should_not_prove_the_result() { let request = RequestHeader::for_request(CREATE_STREAM_CODE, 1, 2, 3, b"request").unwrap(); assert_ne!(request.request_checksum, request.checksum); @@ -661,8 +689,8 @@ #[test] fn request_checksum_is_stamped_for_metadata_but_not_partition_operations() { - // Metadata dedup compares the stamp against cached replies. Partition - // dedup checks request ids without stamps; NonReplicated bypasses dedup. + // Metadata replies carry the request identifier. Partition operations + // leave it zero and NonReplicated bypasses dedup. let mut session = ConsensusSession::with_client_id(42); session.bind(99).unwrap(); let payload = Bytes::from_static(b"payload"); @@ -670,11 +698,26 @@ let metadata = encode_request_header(&mut session, CREATE_STREAM_CODE, &payload) .unwrap() .0; - assert_eq!( + assert_ne!(metadata.request_checksum, 0); + assert_ne!( metadata.request_checksum, u128::from(calculate_checksum(&payload)), ); + let repeated = encode_request_header(&mut session, CREATE_STREAM_CODE, &payload) + .unwrap() + .0; + assert_ne!(metadata.request_checksum, repeated.request_checksum); + let request_frame = crate::links::Frame { + header: metadata, + body: payload.clone(), + }; + let replay_frame = request_frame.clone(); + assert_eq!( + replay_frame.header.request_checksum, + request_frame.header.request_checksum + ); + let partition = encode_request_header(&mut session, SEND_MESSAGES_CODE, &payload) .unwrap() .0; @@ -687,6 +730,21 @@ } #[test] + fn given_register_retry_when_encoding_again_should_keep_request_identifier() { + let mut session = ConsensusSession::with_client_id(42); + let payload = Bytes::from_static(b"login-body"); + let first = encode_request_header(&mut session, LOGIN_REGISTER_CODE, &payload) + .expect("first register") + .0; + let retry = encode_request_header(&mut session, LOGIN_REGISTER_CODE, &payload) + .expect("register retry") + .0; + assert_ne!(first.request_checksum, 0); + assert_eq!(retry.request_checksum, first.request_checksum); + assert_eq!(retry.request, first.request); + } + + #[test] fn partition_request_consumes_the_request_counter() { // Dedup identity requires each send to carry a distinct id, so // partition ops advance the counter exactly like metadata ops and
diff --git a/core/server/Cargo.toml b/core/server/Cargo.toml index 8620905..7c47bdf 100644 --- a/core/server/Cargo.toml +++ b/core/server/Cargo.toml
@@ -17,7 +17,7 @@ [package] name = "server" -version = "0.9.1-edge.1" +version = "0.9.1-edge.2" edition = "2024" license = "Apache-2.0" publish = false
diff --git a/core/server/README.md b/core/server/README.md index ab38fb2..20b4cde 100644 --- a/core/server/README.md +++ b/core/server/README.md
@@ -47,13 +47,14 @@ Deploy matching server and SDK releases together. Rolling upgrades and mixed server versions are unsupported. Partition control frames from older servers carry incarnation zero and are rejected, so a mixed-version cluster cannot -complete partition view changes. The `IGGY-BOUND-GRANTS-5` storage gate +complete partition view changes. The `IGGY-RESOURCE-OPTIONS-1` storage gate requires fresh storage and rejects older data directories without modifying them. Cached replies now preserve a committed-result marker, which older receipts lack. Partition creation records carry an initial view in each 16-byte assignment, replacing the 12-byte assignments and single request-wide view in older metadata -WAL entries. This boundary also rejects `IGGY-BOUND-GRANTS-4` directories from -upstream and earlier development builds, unchanged before replay. +WAL entries. This boundary also rejects `IGGY-BOUND-GRANTS-4` and +`IGGY-BOUND-GRANTS-5` directories from upstream and earlier development builds, +unchanged before replay. No in-place upgrade, storage conversion tool or retained-data migration is provided. Use an offline cutover with logical re-ingestion: @@ -223,7 +224,7 @@ ### Sessions and retry receipts -Protocol 0.11.1 uses client-owned registration proofs and `BindSession` (15) to +Protocol 0.11.1 and later use client-owned registration proofs and `BindSession` (15) to share a logical session across connections. Commands 14, 103, 104, 123 and 124 are retired. Partition routing uses 105, polling uses 100 and offset reads use 120. Disconnecting a connection does not log out its logical session. Server-observed activity renews @@ -306,6 +307,50 @@ Set the option through `TopicCreateOptions::partition_resize_policy`, HTTP's `options` map, or the CLI's `--set partition_resize_policy=fixed`. +## Resource options + +Streams, topics and users take their settings from one option catalog. The binary `DescribeOptions` command (13), `GET /options/{scope}` over HTTP (`stream`, `topic` or `user`) and `iggy options <scope>` list every key a client can see with its kind, default, description and five flags: updatable, resettable, secret, read-only and critical. + +Most keys are fixed at create. A topic's `compression_algorithm`, `segment_size`, `durability`, `consumer_offset_durability`, `partition_resize_policy` and save thresholds, and a stream's `encryption_key`, are refused by an update with `OptionNotUpdatable` (4103). `compression_algorithm` is create-only because the server never applies it and stored batches carry no codec marker. + +The updatable keys are `name` for streams and topics, `username` and `status` for users, and a topic's `message_expiry` and `max_topic_size`. Only those two topic keys can be reset to their default, any other reset is refused with `OptionNotResettable` (4104). Read-only keys, such as the stream key projections below, are refused with `ReadOnlyOption` (4105). + +Only root may rename the root user or change its status. Root cannot deactivate itself. These requests return `CannotModifyRootUser` (4108), and a rejected update leaves root active with its existing username. + +`UpdateResource` (16) replaces `UpdateStream` (204), `UpdateTopic` (304) and `UpdateUser` (35). It names a scope and a target, sets the keys it carries, resets the keys it lists and keeps every other key as stored. A key may not be both set and reset. + +Reads return every option with its provenance: explicit when a client set it, derived when the server filled it in at create. Topic responses no longer carry fixed `message_expiry`, `compression_algorithm` and `max_topic_size` fields, those values are in the option blocks. + +Over HTTP, `POST /streams`, `POST /streams/{stream_id}/topics` and `POST /users` take an optional nested `options` object. `PUT` on the stream, topic and user paths takes one flat JSON object: members other than `reset` are option keys of the scope, `name`, `username` and `status` included. A value sets the key, an absent key or a `null` value leaves it unchanged, and `reset` is an optional array of key names that return to their defaults. + +`null` never resets a key. The body is not JSON Merge Patch and there is no `PATCH` route. Request values are strings in the config-file syntax (`"7 days"`, `"1 GiB"`, `"gzip"`), or numbers and booleans where the kind allows. See [server-api.http](../../web/docs/server-api.http) for examples. + +## Stream encryption + +With encryption, every encrypted stream has its own 32-byte data key. The server `[encryption] key` never encrypts messages: HKDF-SHA256 derives a key-encryption key from it, and each stream key is stored in the stream's options wrapped under that key with AES-256-GCM. Messages and user headers of a stream are encrypted with that stream's key using AES-256-GCM-SIV. The stored form is a 12-byte random nonce, ciphertext and a 16-byte tag. Client-side payload encryption remains AES-256-GCM. + +A client may give a stream its own key at create, as `encryption_key` (32 random bytes, base64 over HTTP). The key travels inside the create request, so protecting it in transit with TLS is the caller's job. Never derive it from a password, because the reported fingerprint would then allow offline guessing. Without a client key, a server with `enabled = true` generates one for the stream. + +With `enabled = false` and a valid `key`, client keys are accepted and keyless streams stay plaintext. Without a server `key`, a client key is refused with `StreamEncryptionUnavailable` (4106). The choice is made once at create: a stream created without a key stays plaintext for life, and enabling encryption later affects only new streams. + +The server never returns a stream key. Stream reads report `encryption_key_source` (`stream` for a client key, `server` for a generated one, `none` for a plaintext stream), `encryption_key_fingerprint` and `encryption_kek_fingerprint`, the fingerprints of the server keys that wrap it, comma-separated during a rotation. Every SDK has a helper that computes the fingerprint of a key the caller holds. + +Boot opens every stored stream key. A standalone node whose `key` and `previous_keys` open none of a stream's wraps refuses to start and names the stream and both sets of fingerprints. A node without a `key` refuses the same way over any encrypted stream, in a cluster too. + +A cluster node holding a key may have metadata that trails the cluster until it catches up, so it starts, logs the stream and both sets of fingerprints, and builds that stream's partitions once a wrap it can open has committed. Every node of a cluster must hold the same `[encryption]` section. Losing a server key that still wraps a stream makes that stream unreadable. + +Rotating the server key takes two phases, and messages are never re-encrypted: + +1. Set `key` to the new key and `previous_keys` to the old one on every node (`IGGY_ENCRYPTION_PREVIOUS_KEYS`). The metadata primary adds a wrap under the new key to every encrypted stream, and a stream created on a phase 1 node is wrapped under both keys. Old wraps stay, so nodes still on the old key keep serving and booting during the rollout, whichever node is primary. +2. Once every node runs the phase 1 config and every encrypted stream reports the new fingerprint in `encryption_kek_fingerprint`, empty `previous_keys` and set `retired_kek_fingerprints` to the old key's fingerprint (`IGGY_ENCRYPTION_RETIRED_KEK_FINGERPRINTS`). The primary then drops the wraps under it. +3. Once every encrypted stream has dropped the old fingerprint, clear `retired_kek_fingerprints` on every node. + + A node given only the new key before phase 1 finished serves none of the streams still wrapped only under the old key, and a standalone node refuses to boot. Without the retired fingerprint a node never drops a wrap. + +The primary logs each sweep that rewraps streams, each stream it cannot rewrap, and once that every set is settled. `previous_keys` holds at most two keys, and a stream holds at most three wraps. Retire an old fingerprint before adding a fourth key. + +A stream created during phase 1 also carries a wrap under the old key until phase 2 drops it. Rotation changes only the wraps, so whoever already holds a stream's data key, or an old wrap together with the old server key, can still read that stream. Changing a stream's own data key is not supported. + ## Upgrade recovery This release changes protocol and storage formats. Servers and compatible @@ -314,11 +359,11 @@ Executable packaging does not affect that identity, so stripping or rebuilding the same compatible release does not by itself prevent a replica from joining. -This checkout uses development protocol 0.11.1. Finalize its version and minimum -before the next stable release. Released `server-0.9.0` uses protocol 0.11.0 and -accepts every 0.11.x patch. An incompatible release therefore needs a new minor -protocol version, such as 0.12.0, for that released server to reject new clients -before decoding their login. +This checkout uses protocol 0.12.0, and 0.12.0 is also its minimum. It is a breaking change: released `server-0.9.0` uses protocol 0.11.0 and accepts every 0.11.x patch, so it rejects 0.12.0 clients before decoding their login, and this server rejects 0.11.x clients the same way. The retired update commands and the removed fixed topic fields mean older clients cannot work against it in any case. + +REST has no version handshake, so a 0.11 HTTP client is not refused at login. A create body that sends an option beside `options`, such as a top-level `message_expiry`, and an update body with a nested member, such as the `{"name": ..., "options": {}}` stream update older SDKs send, are refused with `InvalidCommand` (3), and the error's `field` names the member. + +Metadata snapshot format 13 is both the current and the oldest readable format. A data directory holding an older snapshot is refused, and journals written by older builds hold operations this build no longer knows. Start this release from fresh storage on every replica. The protocol number is independent of server and SDK package versions. Its current and minimum values are maintained manually for stable releases. The @@ -332,7 +377,7 @@ procedure above for an upgrade. A recovery from a backup must use a separately verified, compatible backup; copying a format marker is not a migration. -Within compatible `IGGY-BOUND-GRANTS-5` storage, this release reads superblock +Within compatible `IGGY-RESOURCE-OPTIONS-1` storage, this release reads superblock envelope versions 1 and 2 and writes version 2. Legacy 66- and 74-byte consensus records remain readable. New records atomically retain an adopted view, its advertised head and exact prepare headers alongside the existing consensus state. @@ -405,7 +450,7 @@ `partition-initialization/<namespace>/created.revision` under the system data directory. This atomic record remains outside the partition directory, including after that directory is lost. Preserve it with metadata and partition backups. -Storage format `IGGY-BOUND-GRANTS-5` requires this initialization contract; +Storage format `IGGY-RESOURCE-OPTIONS-1` requires this initialization contract; older data directories are refused before mutation. Retirement also writes `retirement.fence` in that external namespace directory
diff --git a/core/server/config.toml b/core/server/config.toml index aec2d0a..b207f8f 100644 --- a/core/server/config.toml +++ b/core/server/config.toml
@@ -424,15 +424,39 @@ # Encryption configuration [encryption] # Encrypt message payloads and user headers. Metadata and structural headers remain unencrypted. -# `true` enables encryption for stored data using AES-256-GCM. +# `true` enables encryption for stored payloads and user headers using AES-256-GCM-SIV. # `false` means data is stored without encryption. enabled = false -# The encryption key used when encryption is enabled (string). -# Should be a 32 bytes length key, provided as a base64 encoded string. -# This key is required and used only if encryption is enabled. +# The server encryption key (string), 32 bytes as standard base64. +# Every encrypted stream has its own data key, stored wrapped under a key derived from this one. +# Required when encryption is enabled. With encryption disabled a key may still be set: +# streams created with their own key are then encrypted, keyless streams stay plaintext. +# A stream created without encryption stays plaintext for life. key = "" +# Earlier server keys (list of strings) a key rotation still needs, at most two. +# A node unwraps stream keys with them and wraps every new stream key under them too. +# Rotation runs in two phases: +# 1. Set `key` to the new key and `previous_keys` to the old one on every node. The metadata +# primary adds a wrap under the new key to every encrypted stream and keeps the old wraps, +# so nodes still on the old key keep serving and booting, new streams included. +# 2. Once every node runs the phase 1 config and every encrypted stream reports the new +# key's fingerprint in `encryption_kek_fingerprint`, empty `previous_keys` and put the +# old key's fingerprint in `retired_kek_fingerprints`. The primary drops the wraps under it. +# 3. Once every stream has dropped the old wrap, clear `retired_kek_fingerprints` on every node. +# A stream holds at most three wraps. Retire an old fingerprint before adding a fourth key. +# Messages are never re-encrypted. Losing a key that still wraps a stream makes it unreadable. +# As an environment variable: IGGY_ENCRYPTION_PREVIOUS_KEYS='["<old base64 key>"]', a single key may also be given bare. +previous_keys = [] + +# Fingerprints of retired server keys (list of strings), 16 lowercase hex characters each, as +# `encryption_kek_fingerprint` reports them. The metadata primary drops stream key wraps under +# these keys and never drops a wrap under a key that is not listed, even one this node lacks. +# Needs `key`, and may not name `key` or a key in `previous_keys`. Not secret. +# As an environment variable: IGGY_ENCRYPTION_RETIRED_KEK_FINGERPRINTS='["<fingerprint>"]'. +retired_kek_fingerprints = [] + # Memory pool configuration [memory_pool] # Enables or disables the memory pool (boolean).
diff --git a/core/server/src/boot/mod.rs b/core/server/src/boot/mod.rs index e6d6356..aa10cb5 100644 --- a/core/server/src/boot/mod.rs +++ b/core/server/src/boot/mod.rs
@@ -87,7 +87,7 @@ const STORAGE_FORMAT_FILE: &str = "storage-format"; const STORAGE_FORMAT_TEMP_FILE: &str = "storage-format.tmp"; -const STORAGE_FORMAT: &[u8] = b"IGGY-BOUND-GRANTS-5\n"; +const STORAGE_FORMAT: &[u8] = b"IGGY-RESOURCE-OPTIONS-1\n"; /// Load the server configuration from the active config provider. /// @@ -740,6 +740,7 @@ sessions, consumer_group_liveness, shard_handle, + unbuilt, } = Box::pin(build_shard_for_thread( shard_id, total_shards, @@ -858,6 +859,7 @@ topology.self_replica_id, topology.replica_count, )); + reconciler_ctx.mark_unbuilt(unbuilt); let reconcile_periodic = config.sharding.reconcile_periodic_interval.get_duration(); let reconciler_handle = compio::runtime::spawn({ let ctx = Rc::clone(&reconciler_ctx); @@ -919,6 +921,39 @@ None }; + // Stream key rewrapper: shard 0 only, and only while this node's keys call + // for a rotation, since otherwise no stored set can differ from its target. + // Retired fingerprints matter only to the rewrapper, so only its context + // holds them. Each sweep no-ops unless this node is the caught-up metadata + // primary, so a rotation is replicated once to every replica. + let rewrap_context = if shard_id == 0 { + Some( + (*shard.plane.partitions().key_context()) + .clone() + .with_retired_kek_fingerprints(&config.encryption.retired_kek_fingerprints) + .map_err(|error| ServerError::Iggy(Box::new(error)))?, + ) + } else { + None + }; + let stream_key_rewrapper_stop = match rewrap_context { + Some(rewrap_context) if rewrap_context.rotating() => { + let (rewrapper_stop_tx, rewrapper_stop_rx) = channel(1); + let rewrapper_shard = Rc::clone(&shard); + let rewrapper_handle = compio::runtime::spawn(async move { + crate::stream_key_rewrapper::run_stream_key_rewrapper( + rewrapper_shard, + rewrap_context, + rewrapper_stop_rx, + ) + .await; + }); + bus.track_background(rewrapper_handle); + Some(rewrapper_stop_tx) + } + _ => None, + }; + // Segment cleaner: runs on every shard (each replica trims its own log, // primary and backup alike). Local and unreplicated; gated by the shared // data-maintenance config. @@ -939,6 +974,7 @@ reconciler: reconcile_stop_tx, heartbeat: heartbeat_stop_tx, pat_cleaner: pat_cleaner_stop, + stream_key_rewrapper: stream_key_rewrapper_stop, segment_cleaner: segment_cleaner_stop, sysinfo_printer: None, consumer_group_liveness: None, @@ -1251,6 +1287,7 @@ b"IGGY-BOUND-GRANTS-2\n", b"IGGY-BOUND-GRANTS-3\n", b"IGGY-BOUND-GRANTS-4\n", + b"IGGY-BOUND-GRANTS-5\n", b"IGGY-DURABLE-SESSIONS-3\n", &STORAGE_FORMAT[..STORAGE_FORMAT.len() - 1], ] { @@ -1281,7 +1318,11 @@ #[test] fn given_different_storage_formats_when_identifying_replicas_should_reject_mixing() { - for older_format in [b"IGGY-BOUND-GRANTS-3\n", b"IGGY-BOUND-GRANTS-4\n"] { + for older_format in [ + b"IGGY-BOUND-GRANTS-3\n".as_slice(), + b"IGGY-BOUND-GRANTS-4\n", + b"IGGY-BOUND-GRANTS-5\n", + ] { assert_ne!( message_bus::replica::handshake::binary_identity(crate::VERSION, STORAGE_FORMAT), message_bus::replica::handshake::binary_identity(crate::VERSION, older_format),
diff --git a/core/server/src/boot/recovery.rs b/core/server/src/boot/recovery.rs index 0e5bac6..a7f33bb 100644 --- a/core/server/src/boot/recovery.rs +++ b/core/server/src/boot/recovery.rs
@@ -32,14 +32,18 @@ ClientTable, HistoryRecovery, JoinMode, LocalPipeline, PipelineEntry, Sequencer, VsrConsensus, VsrRestore, VsrState, }; -use iggy_common::{Aes256GcmEncryptor, EncryptorKind, IggyByteSize, TopicRuntimeOptions}; +use iggy_common::{ + EncryptorKind, IggyByteSize, IggyError, PartitionStats, ServerKeyContext, StreamEncryption, + TopicRuntimeOptions, check_critical_options, +}; use journal::Journal; use journal::prepare_journal::PrepareJournal; use journal::superblock::PingPongSuperblock; use message_bus::IggyMessageBus; use metadata::impls::metadata::{IggySnapshot, StreamsFrontend}; use metadata::stm::snapshot::Snapshot; -use partitions::{IggyPartitions, PartitionsConfig}; +use metadata::stm::stream::{Partition, Streams}; +use partitions::{IggyPartitions, PartitionsConfig, StreamKeyIdentity}; use server_common::sharding::{IggyNamespace, PartitionLocation, ShardId}; use shard::builder::IggyShardBuilder; use shard::metrics::ShardMetrics; @@ -49,11 +53,32 @@ ShardIdentity, TaggedSender, }; use std::cell::RefCell; +use std::collections::BTreeSet; use std::rc::Rc; use std::sync::Arc; use std::time::Duration; use tracing::{error, info, warn}; +/// A partition this shard owns, with what building it needs from metadata. +struct OwnedPartition { + stream_id: usize, + topic_id: usize, + stats: Arc<PartitionStats>, + metadata: Partition, + runtime: TopicRuntimeOptions, + identity: StreamKeyIdentity, + encryption: StreamEncryption, +} + +/// How boot treats an owned partition, by whether its stream key opens here. +enum OwnedPartitionKey { + /// Build it with this encryptor, `None` for a plaintext stream. + Build(Option<Arc<EncryptorKind>>), + /// Leave it unbuilt, hosted nowhere on this shard, until catch-up commits + /// a wrap this node opens. + Defer, +} + /// A shard built for its thread, with what `shard_main` wires after the /// build: the session manager its request plane shares and the weak /// self-reference the host's handlers upgrade per frame, already @@ -63,6 +88,9 @@ pub sessions: Rc<RefCell<SessionManager>>, pub consumer_group_liveness: Rc<RefCell<ConsumerGroupLiveness>>, pub shard_handle: ShellShardHandle<Rc<IggyMessageBus>, PrepareJournal, IggySnapshot>, + /// Owned namespaces left unbuilt, for the reconciler to clean up if their + /// delete commits before they build. + pub unbuilt: Vec<IggyNamespace>, } #[allow(clippy::too_many_arguments, clippy::too_many_lines)] @@ -127,16 +155,22 @@ let owned_partitions_capacity = total_partitions .div_ceil(usize::from(total_shards).max(1)) .saturating_mul(2); - // At-rest encryption: built once per shard from the shared config; the - // ingestion path encrypts on the primary and the poll reply decrypts. - // A bad key fails the boot rather than silently serving plaintext. - let encryptor = if config.encryption.enabled { - let aes = Aes256GcmEncryptor::from_base64_key(&config.encryption.key) - .map_err(|error| ServerError::Iggy(Box::new(error)))?; - Some(Arc::new(EncryptorKind::Aes256Gcm(aes))) - } else { - None - }; + // Each encrypted stream holds its own key, wrapped under the server key. + // The shards owning its partitions open it below, shard 0 opens the keys of + // streams that own no partition. + let key_context = Arc::new( + ServerKeyContext::from_config( + &config.encryption.key, + &config.encryption.previous_keys, + config.encryption.enabled, + ) + .map_err(|error| ServerError::Iggy(Box::new(error)))?, + ); + let solo = topology.replica_count == 1; + if shard_id == 0 { + refuse_partitionless_stream_keys(metadata.mux_stm.streams(), &key_context, solo)?; + } + refuse_unreadable_topics(metadata.mux_stm.streams())?; let partitions = IggyPartitions::with_capacity( shard_local_id, PartitionsConfig { @@ -147,7 +181,7 @@ validate_checksum: config.partition.validate_checksum, segment_size: IggyByteSize::from(iggy_common::DEFAULT_SEGMENT_SIZE), preallocate_segments: iggy_common::DEFAULT_PREALLOCATE_SEGMENTS, - encryptor, + key_context, path_layout: partitions::PartitionPathLayout { streams_root: config.get_streams_path(), }, @@ -164,6 +198,11 @@ let owned = metadata.mux_stm.streams().read(|inner| { let mut owned = Vec::with_capacity(owned_partitions_capacity); for (_, stream) in &inner.items { + let identity = StreamKeyIdentity { + stream_id: stream.id, + created_op: stream.created_op, + }; + let mut encryption = None; for (topic_id, topic) in &stream.topics { for partition in &topic.partitions { let namespace = IggyNamespace::new(stream.id, topic_id, partition.id); @@ -178,13 +217,17 @@ partition, topic.stats.clone(), ); - owned.push(( - stream.id, + owned.push(OwnedPartition { + stream_id: stream.id, topic_id, stats, - partition.clone(), - TopicRuntimeOptions::from_resource_options(&topic.options), - )); + metadata: partition.clone(), + runtime: TopicRuntimeOptions::from_resource_options(&topic.options), + identity, + encryption: encryption + .get_or_insert_with(|| StreamEncryption::from(&stream.options)) + .clone(), + }); } else { shards_table.insert( namespace, @@ -204,8 +247,33 @@ // bundle was broadcast (see `MetadataHandoff::Owner`). All shards // here only add their per-partition deltas, so the shared // `Arc<TopicStats>` atomics race only against other atomic adds. - for (stream_id, topic_id, partition_stats, partition_metadata, topic_runtime) in owned { + let mut deferred_streams = BTreeSet::new(); + let mut unbuilt = Vec::new(); + for OwnedPartition { + stream_id, + topic_id, + stats: partition_stats, + metadata: partition_metadata, + runtime: topic_runtime, + identity, + encryption, + } in owned + { let namespace = IggyNamespace::new(stream_id, topic_id, partition_metadata.id); + let encryptor = match owned_partition_key( + &partitions, + &mut deferred_streams, + stream_id, + identity, + &encryption, + solo, + )? { + OwnedPartitionKey::Build(encryptor) => encryptor, + OwnedPartitionKey::Defer => { + unbuilt.push(namespace); + continue; + } + }; let loaded = load_partition_or_fence( config, namespace, @@ -235,10 +303,13 @@ "skipping this partition at boot; the reconciler retries its first \ offset reservation claim" ); + unbuilt.push(namespace); continue; } Err(error) => return Err(error), }; + let mut partition = partition; + partition.set_encryptor(encryptor); partitions.insert(namespace, partition); shards_table.insert( namespace, @@ -327,9 +398,39 @@ sessions, consumer_group_liveness, shard_handle, + unbuilt, }) } +/// The key decision for one owned partition. A stream deferred once stays +/// deferred for its other partitions, so its key is resolved once. +fn owned_partition_key( + partitions: &IggyPartitions<Rc<IggyMessageBus>>, + deferred_streams: &mut BTreeSet<usize>, + stream_id: usize, + identity: StreamKeyIdentity, + encryption: &StreamEncryption, + solo: bool, +) -> Result<OwnedPartitionKey, ServerError> { + if deferred_streams.contains(&stream_id) { + return Ok(OwnedPartitionKey::Defer); + } + match partitions.resolve_stream_encryptor(identity, encryption) { + Ok(encryptor) => Ok(OwnedPartitionKey::Build(encryptor)), + Err(source) => { + defer_or_refuse_stream_key( + stream_id, + encryption, + &partitions.key_context(), + source, + solo, + )?; + deferred_streams.insert(stream_id); + Ok(OwnedPartitionKey::Defer) + } + } +} + // Pin the configs-crate default literals (duplicated there to avoid a // build-time edge onto the runtime crates) against the runtime constants, // mirroring the message_bus IOV_MAX pin. A drift on either side fails this @@ -398,6 +499,98 @@ assert!(consensus::DVC_HEADERS_MAX == iggy_binary_protocol::consensus::DVC_HEADERS_MAX); const _: () = assert!(consensus::DVC_HEADERS_MAX == u128::BITS as usize); +/// The boot check of encrypted streams that own no partition, whose keys no +/// shard opens while building its partitions. +fn refuse_partitionless_stream_keys( + streams: &Streams, + key_context: &ServerKeyContext, + solo: bool, +) -> Result<(), ServerError> { + let unopenable: Vec<(usize, StreamEncryption, IggyError)> = streams.read(|inner| { + inner + .items + .iter() + .filter(|(_, stream)| { + stream + .topics + .iter() + .all(|(_, topic)| topic.partitions.is_empty()) + }) + .filter_map(|(_, stream)| { + let encryption = StreamEncryption::from(&stream.options); + let source = match &encryption { + StreamEncryption::Plaintext => None, + StreamEncryption::Unavailable(error) => Some(error.clone()), + StreamEncryption::Wrapped(set) => key_context.open(set).err(), + }?; + Some((stream.id, encryption, source)) + }) + .collect() + }); + for (stream_id, encryption, source) in unopenable { + defer_or_refuse_stream_key(stream_id, &encryption, key_context, source, solo)?; + } + Ok(()) +} + +/// A stream key that does not open at boot. A cluster replica's metadata can +/// trail the commit point until catch-up or state transfer, so the stream waits +/// for the reconciler, which builds its partitions once a wrap this node opens +/// commits. Catch-up heals neither a node without a key nor unreadable critical +/// state, and a solo replay is the whole journal, so these refuse. +fn defer_or_refuse_stream_key( + stream_id: usize, + encryption: &StreamEncryption, + key_context: &ServerKeyContext, + source: IggyError, + solo: bool, +) -> Result<(), ServerError> { + let stored_kek_fingerprints = match encryption { + StreamEncryption::Wrapped(set) => set.kek_fingerprints().join(","), + StreamEncryption::Plaintext | StreamEncryption::Unavailable(_) => String::new(), + }; + let error = ServerError::StreamKeyUnavailable { + stream_id, + stored_kek_fingerprints, + node_kek_fingerprints: key_context.kek_fingerprints().join(","), + source: Box::new(source), + }; + if solo + || !key_context.accepts_stream_keys() + || !matches!(encryption, StreamEncryption::Wrapped(_)) + { + return Err(error); + } + warn!( + stream_id, + %error, + "Stream {stream_id} is not served on this node until metadata catch-up commits a wrap its keys open" + ); + Ok(()) +} + +/// A topic whose critical options this build cannot read is never served, the +/// way unreadable stream state refuses boot. +fn refuse_unreadable_topics(streams: &Streams) -> Result<(), ServerError> { + let unreadable = streams.read(|inner| { + inner.items.iter().find_map(|(_, stream)| { + stream.topics.iter().find_map(|(topic_id, topic)| { + check_critical_options(&topic.options) + .err() + .map(|error| (stream.id, topic_id, error)) + }) + }) + }); + match unreadable { + Some((stream_id, topic_id, source)) => Err(ServerError::TopicOptionsUnavailable { + stream_id, + topic_id, + source: Box::new(source), + }), + None => Ok(()), + } +} + /// `[cluster] superblock_wedged_fatal_timeout` as a consecutive-failure count, /// which is the only shape the shard's `superblock_wedged` can compare. Zero /// stays zero (fail-stop disabled). @@ -695,6 +888,225 @@ #[cfg(test)] mod tests { use super::*; + use crate::dispatch::test_support::{TestMux, prepare_message}; + use iggy_binary_protocol::primitives::partition_assignment::CreatedPartitionAssignment; + use iggy_binary_protocol::requests::streams::CreateStreamRequest; + use iggy_binary_protocol::requests::topics::{ + CreateTopicRequest, CreateTopicWithAssignmentsRequest, + }; + use iggy_binary_protocol::{Operation, WireEncode, WireIdentifier, WireName, WireOptions}; + use iggy_common::wire_conversions::resource_options_to_wire; + use iggy_common::{ + HeaderKey, HeaderValue, OptionKey, OptionValue, OptionsProvenance, ResourceOptions, + StreamEncryptionKey, StreamKeyOrigin, WrapSet, + }; + use metadata::stm::StateMachine as _; + + const CONFIG_KEY: &str = "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE="; + const OTHER_CONFIG_KEY: &str = "AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA="; + + #[test] + fn given_topic_with_unknown_critical_option_when_booting_should_refuse_it() { + let future_key = HeaderKey::try_from("a_future_key").expect("key fits"); + let unreadable = ResourceOptions::from([ + ( + HeaderKey::from(OptionKey::CriticalKeys), + OptionValue::explicit(HeaderValue::try_from("a_future_key").expect("marker fits")), + ), + (future_key, OptionValue::explicit(HeaderValue::from(7u64))), + ]); + for (derived, refused) in [(ResourceOptions::new(), false), (unreadable, true)] { + let mux = TestMux::default(); + mux.users().ensure_root_user("iggy", "hash"); + mux.update(prepare_message( + Operation::CreateStream, + 1, + 1, + &CreateStreamRequest { + name: WireName::new("stream").expect("name fits"), + options: WireOptions::empty(), + } + .to_bytes(), + )) + .expect("stream create applies"); + mux.update(prepare_message( + Operation::CreateTopicWithAssignments, + 1, + 2, + &CreateTopicWithAssignmentsRequest { + request: CreateTopicRequest { + stream_id: WireIdentifier::numeric(0), + partitions_count: 1, + name: WireName::new("topic").expect("name fits"), + options: WireOptions::empty(), + }, + derived_options: resource_options_to_wire(&derived, OptionsProvenance::All) + .expect("options encode"), + partitions: vec![CreatedPartitionAssignment { + partition_id: 0, + consensus_group_id: 1, + initial_view: 0, + }], + } + .to_bytes(), + )) + .expect("topic create applies"); + + let result = refuse_unreadable_topics(mux.streams()); + + assert_eq!( + matches!( + result, + Err(ServerError::TopicOptionsUnavailable { + stream_id: 0, + topic_id: 0, + .. + }) + ), + refused, + "{result:?}" + ); + } + } + + #[test] + fn given_stream_key_that_does_not_open_when_booting_should_refuse_only_a_complete_replay() { + let stream_context = ServerKeyContext::from_config(CONFIG_KEY, &[], true).expect("key"); + let key = StreamEncryptionKey::try_from([1; 32].as_slice()).expect("32 bytes"); + let set = WrapSet::from( + stream_context + .wrap(&key, StreamKeyOrigin::Server) + .expect("wraps"), + ); + let wrapped = ResourceOptions::from([( + HeaderKey::from(OptionKey::EncryptionKeyWrapped), + OptionValue::explicit(HeaderValue::try_from(set.as_ref()).expect("set fits")), + )]); + let unreadable = ResourceOptions::from([( + HeaderKey::from(OptionKey::CriticalKeys), + OptionValue::explicit(HeaderValue::try_from("a_future_key").expect("marker fits")), + )]); + let plaintext = ResourceOptions::new(); + let same_key = ServerKeyContext::from_config(CONFIG_KEY, &[], true).expect("key"); + let other_key = ServerKeyContext::from_config(OTHER_CONFIG_KEY, &[], true).expect("key"); + let no_key = ServerKeyContext::disabled(); + for (options, node_context, solo, refused) in [ + (&wrapped, &same_key, true, false), + (&wrapped, &same_key, false, false), + (&wrapped, &other_key, true, true), + (&wrapped, &other_key, false, false), + (&wrapped, &no_key, true, true), + (&wrapped, &no_key, false, true), + (&plaintext, &no_key, true, false), + (&plaintext, &no_key, false, false), + (&unreadable, &same_key, true, true), + (&unreadable, &same_key, false, true), + ] { + let mux = TestMux::default(); + mux.users().ensure_root_user("iggy", "hash"); + mux.update(prepare_message( + Operation::CreateStream, + 1, + 1, + &CreateStreamRequest { + name: WireName::new("stream").expect("name fits"), + options: resource_options_to_wire(options, OptionsProvenance::All) + .expect("options encode"), + } + .to_bytes(), + )) + .expect("stream create applies"); + + let result = refuse_partitionless_stream_keys(mux.streams(), node_context, solo); + + assert_eq!( + matches!( + result, + Err(ServerError::StreamKeyUnavailable { stream_id: 0, .. }) + ), + refused, + "node keys {:?}, solo {solo}: {result:?}", + node_context.kek_fingerprints() + ); + } + } + + #[test] + fn given_stream_key_that_does_not_open_when_deciding_owned_partitions_should_leave_them_unbuilt() + { + let stream_context = ServerKeyContext::from_config(CONFIG_KEY, &[], true).expect("key"); + let key = StreamEncryptionKey::try_from([1; 32].as_slice()).expect("32 bytes"); + let set = WrapSet::from( + stream_context + .wrap(&key, StreamKeyOrigin::Server) + .expect("wraps"), + ); + let wrapped = StreamEncryption::from(&ResourceOptions::from([( + HeaderKey::from(OptionKey::EncryptionKeyWrapped), + OptionValue::explicit(HeaderValue::try_from(set.as_ref()).expect("set fits")), + )])); + let identity = StreamKeyIdentity { + stream_id: 0, + created_op: 1, + }; + let node_partitions = |config_key: &str| { + IggyPartitions::new( + ShardId::new(0), + PartitionsConfig { + messages_required_to_save: 1, + size_of_messages_required_to_save: IggyByteSize::from(1024_u64), + validate_checksum: true, + segment_size: IggyByteSize::from(iggy_common::DEFAULT_SEGMENT_SIZE), + preallocate_segments: false, + key_context: Arc::new( + ServerKeyContext::from_config(config_key, &[], true).expect("key"), + ), + path_layout: partitions::PartitionPathLayout { + streams_root: "/var/lib/iggy/streams".to_owned(), + }, + }, + ) + }; + + let other_key = node_partitions(OTHER_CONFIG_KEY); + let mut deferred = BTreeSet::new(); + for _ in 0..2 { + let decision = + owned_partition_key(&other_key, &mut deferred, 0, identity, &wrapped, false); + assert!(matches!(decision, Ok(OwnedPartitionKey::Defer))); + } + assert_eq!(deferred, BTreeSet::from([0])); + assert!(matches!( + owned_partition_key( + &other_key, + &mut BTreeSet::new(), + 0, + identity, + &wrapped, + true + ), + Err(ServerError::StreamKeyUnavailable { stream_id: 0, .. }) + )); + + let same_key = node_partitions(CONFIG_KEY); + let mut deferred = BTreeSet::new(); + assert!(matches!( + owned_partition_key(&same_key, &mut deferred, 0, identity, &wrapped, false), + Ok(OwnedPartitionKey::Build(Some(_))) + )); + assert!(matches!( + owned_partition_key( + &same_key, + &mut deferred, + 1, + identity, + &StreamEncryption::Plaintext, + false + ), + Ok(OwnedPartitionKey::Build(None)) + )); + assert!(deferred.is_empty()); + } #[test] fn partition_runtime_and_server_use_the_same_fixed_directory_layout() { @@ -710,7 +1122,7 @@ validate_checksum: server.partition.validate_checksum, segment_size: IggyByteSize::from(iggy_common::DEFAULT_SEGMENT_SIZE), preallocate_segments: iggy_common::DEFAULT_PREALLOCATE_SEGMENTS, - encryptor: None, + key_context: Arc::default(), path_layout: partitions::PartitionPathLayout { streams_root: server.get_streams_path(), },
diff --git a/core/server/src/boot/threads.rs b/core/server/src/boot/threads.rs index 79ecef5..a5e72f9 100644 --- a/core/server/src/boot/threads.rs +++ b/core/server/src/boot/threads.rs
@@ -858,6 +858,7 @@ pub(in crate::boot) reconciler: Sender<()>, pub(in crate::boot) heartbeat: Option<Sender<()>>, pub(in crate::boot) pat_cleaner: Option<Sender<()>>, + pub(in crate::boot) stream_key_rewrapper: Option<Sender<()>>, pub(in crate::boot) segment_cleaner: Option<Sender<()>>, pub(in crate::boot) sysinfo_printer: Option<Sender<()>>, pub(in crate::boot) consumer_group_liveness: Option<Sender<()>>, @@ -871,6 +872,7 @@ for stop in [ &self.heartbeat, &self.pat_cleaner, + &self.stream_key_rewrapper, &self.segment_cleaner, &self.sysinfo_printer, &self.consumer_group_liveness,
diff --git a/core/server/src/consumer_group.rs b/core/server/src/consumer_group.rs index bb832b5..df0d05c 100644 --- a/core/server/src/consumer_group.rs +++ b/core/server/src/consumer_group.rs
@@ -359,7 +359,7 @@ validate_checksum: true, segment_size: IggyByteSize::from(1_048_576_u64), preallocate_segments: false, - encryptor: None, + key_context: Arc::default(), path_layout: PartitionPathLayout::default(), }, );
diff --git a/core/server/src/dispatch/authz.rs b/core/server/src/dispatch/authz.rs index 66be6d0..a5d05e2 100644 --- a/core/server/src/dispatch/authz.rs +++ b/core/server/src/dispatch/authz.rs
@@ -121,14 +121,15 @@ | Operation::FinalizeSession | Operation::RetireSession | Operation::CompleteLifecycle + | Operation::RewrapStreamKeys | Operation::TransitionPartitionHistory | Operation::InstallConsumerGroupOwner | Operation::RetireConsumerGroupOwners | Operation::CreateStream - | Operation::UpdateStream + | Operation::UpdateStreamResolved | Operation::DeleteStream | Operation::CreateTopic - | Operation::UpdateTopic + | Operation::UpdateTopicResolved | Operation::DeleteTopic | Operation::CreatePartitions | Operation::DeletePartitions @@ -136,7 +137,8 @@ | Operation::CreateConsumerGroup | Operation::DeleteConsumerGroup | Operation::CreateUser - | Operation::UpdateUser + | Operation::UpdateUserResolved + | Operation::UpdateResource | Operation::DeleteUser | Operation::ChangePassword | Operation::UpdatePermissions
diff --git a/core/server/src/dispatch/mod.rs b/core/server/src/dispatch/mod.rs index 037f7e7..f8b75b5 100644 --- a/core/server/src/dispatch/mod.rs +++ b/core/server/src/dispatch/mod.rs
@@ -59,7 +59,6 @@ use crate::rewrite::{RewriteDeny, tcp_chain}; use crate::session_manager::{ConnectionContext, SessionManager}; use crate::shell::{ShellBus, ShellShard, ShellShardHandle}; -use crate::wire::verify_request_checksum; use ahash::{AHashMap, AHashSet}; use configs::server::ServerConfig; use consensus::MetadataHandle; @@ -758,19 +757,6 @@ // resolution sites below stamp it before anything routes on it. let request = request.into_routed(); - // The last point that still sees the body the CLIENT sent; every rewrite below - // substitutes server-chosen bytes and carries the stamp through unchanged. - if let Err(error) = verify_request_checksum(&request) { - warn!( - transport_client_id, - operation = ?request.header().operation, - request = request.header().request, - "dropping client request whose body does not match its own checksum" - ); - send_deny_reply(shard, origin, request.header(), error.as_code()).await; - return; - } - // ONE `connections` walk for the whole prologue. Any request is liveness // proof, not just PING: an idle-but-active client (e.g. an admin issuing // reads between long sleeps) must not be evicted by the heartbeat @@ -1245,17 +1231,20 @@ }; use iggy_binary_protocol::requests::messages::{GetPartitionRouteRequest, PollMessagesRequest}; use iggy_binary_protocol::requests::personal_access_tokens::CreatePersonalAccessTokenRequest as WireCreatePersonalAccessTokenRequest; - use iggy_binary_protocol::requests::streams::{CreateStreamRequest, UpdateStreamRequest}; - use iggy_binary_protocol::requests::system::{ - BindSessionRequest, GetClientRequest, GetClientsRequest, GetStatsRequest, + use iggy_binary_protocol::requests::streams::{ + CreateStreamRequest, UpdateStreamResolvedRequest, }; - use iggy_binary_protocol::requests::topics::{DeleteTopicRequest, UpdateTopicRequest}; + use iggy_binary_protocol::requests::system::{ + BindSessionRequest, GetClientRequest, GetClientsRequest, GetStatsRequest, ResourceTarget, + UpdateResourceRequest, + }; + use iggy_binary_protocol::requests::topics::{DeleteTopicRequest, UpdateTopicResolvedRequest}; use iggy_binary_protocol::requests::users::login_register::{ LoginRegisterRequest, RegisterRequest, }; use iggy_binary_protocol::requests::users::login_register_with_pat::LoginRegisterWithPatRequest; use iggy_binary_protocol::requests::users::{ - CreateUserRequest, DeleteUserRequest, UpdatePermissionsRequest, UpdateUserRequest, + CreateUserRequest, DeleteUserRequest, UpdatePermissionsRequest, UpdateUserResolvedRequest, }; use iggy_binary_protocol::{ AckLevel, ClientVersionInfo, EvictionHeader, IGGY_PROTOCOL_VERSION, ReplyHeader, @@ -1508,31 +1497,35 @@ let bus = SpyBus::default(); let shard = Rc::new(test_shard(&bus, 0, 1, FIRST_BOOT)); let metadata = shard.plane.metadata(); + let account_id = u32::from(status == Some(UserStatus::Inactive)); if let Some(status) = status { metadata.mux_stm.users().ensure_root_user("iggy", "hash"); - metadata - .mux_stm - .update(prepare_message( - Operation::UpdateUser, - CLIENT, - SESSION, - &UpdateUserRequest { - user_id: WireIdentifier::numeric(0), - username: None, - status: Some(status.as_code()), - options: WireOptions::empty(), - } - .to_bytes(), - )) - .unwrap(); + if status == UserStatus::Inactive { + metadata + .mux_stm + .update(prepare_message( + Operation::CreateUser, + CLIENT, + SESSION, + &CreateUserRequest { + username: WireName::new("inactive-user").unwrap(), + password: "hash".to_owned(), + status: status.as_code(), + permissions: None, + options: WireOptions::empty(), + } + .to_bytes(), + )) + .unwrap(); + } } metadata .client_table .borrow_mut() .commit_register( client_id, - 0, - bind_verifier(client_id, 0, &SECRET), + account_id, + bind_verifier(client_id, account_id, &SECRET), register_reply(client_id, SESSION), ) .unwrap(); @@ -1559,7 +1552,7 @@ if status == Some(UserStatus::Active) && !message_bus::is_server_internal_client(client_id) { - assert_eq!(outcome.unwrap(), (0, SESSION)); + assert_eq!(outcome.unwrap(), (account_id, SESSION)); assert_eq!( sessions.borrow().get_session(TRANSPORT), Some((client_id, SESSION)) @@ -1749,7 +1742,10 @@ let stream_id = WireIdentifier::named("stream").unwrap(); let topic_id = WireIdentifier::named("topic").unwrap(); for (code, body) in partition_namespace_reads(&stream_id, &topic_id) { - for rename in [Operation::UpdateStream, Operation::UpdateTopic] { + for rename in [ + Operation::UpdateStreamResolved, + Operation::UpdateTopicResolved, + ] { assert_read_refused_after_shard_wait( code, &body, @@ -2012,19 +2008,21 @@ WireIdentifier::numeric(namespace.stream_id().try_into().unwrap()); let name = WireName::new("renamed").unwrap(); let body = match operation { - Operation::UpdateStream => UpdateStreamRequest { + Operation::UpdateStreamResolved => UpdateStreamResolvedRequest { stream_id, - name, - options: WireOptions::empty(), + name: Some(name), + explicit_options: WireOptions::empty(), + derived_options: WireOptions::empty(), } .to_bytes(), - Operation::UpdateTopic => UpdateTopicRequest { + Operation::UpdateTopicResolved => UpdateTopicResolvedRequest { stream_id, topic_id: WireIdentifier::numeric( namespace.topic_id().try_into().unwrap(), ), - name, - options: WireOptions::empty(), + name: Some(name), + explicit_options: WireOptions::empty(), + derived_options: WireOptions::empty(), } .to_bytes(), _ => panic!("{operation:?} renames neither a stream nor a topic"), @@ -2185,6 +2183,7 @@ context: attachment.metadata.context(0), fragments: Vec::new().into(), current_offset: STORED_OFFSET, + encryptor: None, }) .unwrap(), LifecycleFrame::PartitionRead { @@ -2211,7 +2210,7 @@ #[compio::test] async fn given_deactivated_user_when_reactivated_should_reject_retained_session() { for primary in [false, true] { - assert_revoked_user_cannot_resume(Operation::UpdateUser, primary).await; + assert_revoked_user_cannot_resume(Operation::UpdateUserResolved, primary).await; } } @@ -2289,11 +2288,12 @@ } .to_bytes() } else { - UpdateUserRequest { + UpdateUserResolvedRequest { user_id: WireIdentifier::numeric(USER), username: None, status: Some(UserStatus::Inactive.as_code()), - options: WireOptions::empty(), + explicit_options: WireOptions::empty(), + derived_options: WireOptions::empty(), } .to_bytes() }; @@ -2306,6 +2306,28 @@ let consensus = metadata.consensus.as_ref().unwrap(); consensus.restore_commit_state(SESSION, SESSION); consensus.sequencer().set_sequence(SESSION); + // A client sends the generic patch, the primary resolves it. + let (operation, body) = if operation == Operation::UpdateUserResolved { + let patch = iggy_common::UserUpdateOptions { + status: Some(UserStatus::Inactive), + ..iggy_common::UserUpdateOptions::default() + } + .to_patch() + .unwrap(); + ( + Operation::UpdateResource, + UpdateResourceRequest { + target: ResourceTarget::User { + user_id: WireIdentifier::numeric(USER), + }, + reset: patch.reset, + options: patch.set, + } + .to_bytes(), + ) + } else { + (operation, body) + }; let request = request_message(operation, ROOT_CLIENT, 1, 1, &body).transmute_header( |old, header: &mut RoutedRequestHeader| { *header = old; @@ -2343,14 +2365,15 @@ ) } else { prepare_message( - Operation::UpdateUser, + Operation::UpdateUserResolved, 0, 4, - &UpdateUserRequest { + &UpdateUserResolvedRequest { user_id: WireIdentifier::numeric(USER), username: None, status: Some(UserStatus::Active.as_code()), - options: WireOptions::empty(), + explicit_options: WireOptions::empty(), + derived_options: WireOptions::empty(), } .to_bytes(), ) @@ -2397,7 +2420,7 @@ validate_checksum: true, segment_size: iggy_common::IggyByteSize::from(1_048_576_u64), preallocate_segments: false, - encryptor: None, + key_context: Arc::default(), path_layout: PartitionPathLayout::default(), }, ); @@ -3014,55 +3037,6 @@ assert_eq!(reply_status(frame), 0, "pre-auth ping must succeed"); } - /// The request-checksum gate runs before every probe: a replicated op - /// from an UNBOUND transport with a bad stamp must get the checksum deny - /// Reply, not the `Eviction(NoSession)` the session gate would send. - #[compio::test] - async fn checksum_mismatch_denies_before_everything() { - const TRANSPORT: u128 = 97; - const BODY: &[u8] = b"stream-body"; - let bus = SpyBus::default(); - let shard = Rc::new(test_shard(&bus, 0, 1, FIRST_BOOT)); - let sessions = Rc::new(RefCell::new(SessionManager::new())); - let server_config = Arc::new(ServerConfig::default()); - - let mut message = wire_request(Operation::CreateStream, TRANSPORT, 1, 1, BODY); - { - let header_size = size_of::<RequestHeader>(); - let header = bytemuck::checked::from_bytes_mut::<RequestHeader>( - &mut message.as_mut_slice()[..header_size], - ); - // Nonzero (zero means unstamped and skips the check) and never - // the body's real checksum. - header.request_checksum = u128::from(iggy_common::calculate_checksum(BODY)) + 1; - } - handle_client_request( - &shard, - &sessions, - &server_config, - &ClientRequestQueues::default(), - &Rc::new(ClientDrain::new(&server_config)), - ClientOrigin::capture(&shard.bus, TRANSPORT), - message.into_generic(), - None, - ) - .await; - let replies = bus.client_replies.borrow(); - assert_eq!(replies.len(), 1, "bad stamp must be answered"); - let (client, frame) = &replies[0]; - assert_eq!(*client, TRANSPORT); - assert_eq!( - frame_command(frame), - Command::Reply as u8, - "a bad stamp must get the checksum deny, not the NoSession eviction" - ); - assert_eq!( - reply_status(frame), - IggyError::InvalidFormat.as_code(), - "the deny status must carry the checksum error" - ); - } - /// The late-bound self-reference as the shard build creates it: unset /// until the shard exists. fn unset_shard_handle() -> ShellShardHandle<SpyBus, PrepareJournal, IggySnapshot> {
diff --git a/core/server/src/dispatch/partition.rs b/core/server/src/dispatch/partition.rs index d33c303..0c067be 100644 --- a/core/server/src/dispatch/partition.rs +++ b/core/server/src/dispatch/partition.rs
@@ -41,8 +41,8 @@ use crate::shell::{ShellBus, ShellShard}; use crate::wire::request_body; use bytes::Bytes; +use consensus::MetadataHandle; use consensus::client_table::SessionAttachment; -use consensus::{MetadataHandle, PartitionsHandle}; use iggy_binary_protocol::PrepareHeader; use iggy_binary_protocol::codes::{GET_CONSUMER_OFFSET_CODE, POLL_MESSAGES_CODE}; use iggy_binary_protocol::primitives::consumer::WireConsumer; @@ -673,8 +673,8 @@ /// Run the resolved poll on the owning shard and re-encode the stored /// batches into the wire `PolledMessages` reply. Owner rejections preserve /// their error; a missing reply reports a communication error without claiming -/// that the owner rejected the read. Unexpected replies and encoding failures -/// retain the empty poll fallback. +/// that the owner rejected the read. Decrypt and encoding failures deny with +/// their error. Unexpected replies retain the empty poll fallback. #[allow(clippy::future_not_send)] async fn read_polled_messages<B, MJ, S, SB>( shard: &Rc<ShellShard<B, MJ, S, SB>>, @@ -750,6 +750,7 @@ context, fragments, current_offset, + encryptor, }) => build_polled_messages_reply( request.header(), current_metadata_commit(shard), @@ -757,15 +758,17 @@ current_offset, context, fragments, - shard.plane.partitions().config().encryptor.as_deref(), + encryptor.as_deref(), ) .map_err(|error| { + // An empty poll would read as caught up, so a batch that cannot be + // decrypted or re-encoded must surface as an error. warn!( transport_client_id, error = %error, - "failed to re-encode polled batches; replying empty poll" + "failed to decrypt or re-encode polled batches; replying denied" ); - ReadPolledMessagesError::Fallback(empty_poll_fallback(partition_id)) + ReadPolledMessagesError::Rejected(error) }), Some(PartitionReadReply::Rejected(error)) => Err(ReadPolledMessagesError::Rejected(error)), None => { @@ -1515,8 +1518,8 @@ use super::*; use crate::dispatch::test_support::{ SpyBus, TestMux, TestShard, create_consumer_group, create_one_partition, - create_reader_request, prepare_message, register_reply, request_message, test_shard, - test_shard_with_frontier, + create_reader_request, prepare_message, register_reply, request_message, + stored_batch_record, test_shard, test_shard_with_frontier, }; use crate::dispatch::tests::non_replicated_request; use crate::dispatch::{ @@ -1527,6 +1530,7 @@ use crate::shell::ShellShardHandle; use bytes::BytesMut; use configs::server::ServerConfig; + use consensus::PartitionsHandle; #[cfg(target_os = "linux")] use consensus::Sequencer; use iggy_binary_protocol::codes::PING_CODE; @@ -1545,8 +1549,8 @@ Command, ConsensusHeader, GenericHeader, PrepareOkHeader, ReplyHeader, SegmentDeleteHeader, SegmentDeleteKind, WireEncode, WireIdentifier, WireName, WireOptions, WirePartitioning, }; - use iggy_common::Identifier; use iggy_common::defaults::DEFAULT_ROOT_USER_ID; + use iggy_common::{Aes256GcmEncryptor, EncryptorKind, Identifier}; use journal::prepare_journal::PrepareJournal; use message_bus::{BusReceiver, IggyMessageBus, Shutdown, ShutdownToken}; use metadata::IggyMetadata; @@ -1556,7 +1560,9 @@ CompleteConsumerGroupRevocationRequest, JoinConsumerGroupRequest, }; use metadata::stm::lifecycle::{CompleteLifecycleRequest, LifecycleCompletion}; - use partitions::{IggyPartitions, PartitionPathLayout, PartitionsConfig}; + use partitions::{ + Fragment, IggyPartitions, PartitionPathLayout, PartitionsConfig, PollFragments, + }; use server_common::MessageBag; use server_common::sharding::{PartitionLocation, ShardId}; use shard::metrics::{ShardMetrics, frame_drop_reason, frame_drop_variant}; @@ -2826,6 +2832,55 @@ } #[compio::test] + async fn given_decrypt_failure_when_building_poll_reply_should_return_rejected_error() { + let bus = SpyBus::default(); + let encryptor = Arc::new(EncryptorKind::Aes256Gcm( + Aes256GcmEncryptor::new(&[7; 32]).expect("valid key"), + )); + let mut shard = test_shard(&bus, 0, 1, 1); + let (sender, owner_inbox, _owner_replies) = shard_channel(0, 1, 1); + shard.attach_senders(vec![sender]); + let shard = Rc::new(shard); + let namespace = route_one_partition(&shard); + + // The first stored record was written in plaintext, the second with + // the partition's key: only the second may be served. + let owner_task = compio::runtime::spawn(async move { + for record in [ + stored_batch_record(None), + stored_batch_record(Some(&encryptor)), + ] { + let ShardFrame::Lifecycle(LifecycleFrame::PartitionRead { reply, .. }) = + owner_inbox.recv().await.unwrap() + else { + panic!("the poll must have reached the owner"); + }; + reply + .try_send(PartitionReadReply::Poll { + context: PartitionContext::default(), + fragments: PollFragments::from_iter([Fragment::whole(record)]), + current_offset: 1, + encryptor: Some(Arc::clone(&encryptor)), + }) + .expect("the caller still awaits the reply"); + } + }); + + let attachment = attach_root_session(&shard, 1, 1); + assert!(matches!( + poll_attached(&shard, namespace, &attachment).await, + Err(ReadPolledMessagesError::Rejected( + IggyError::CannotDecryptData + )) + )); + let reply = poll_attached(&shard, namespace, &attachment) + .await + .unwrap_or_else(|_| panic!("a record encrypted with the shard key must decrypt")); + assert!(reply.total_len() > size_of::<ReplyHeader>()); + owner_task.await.expect("the owner task must finish"); + } + + #[compio::test] async fn given_pending_poll_when_owner_reply_times_out_should_report_unknown_acceptance() { const TRANSPORT_CLIENT_ID: u128 = 91; const VSR_CLIENT_ID: u128 = 1; @@ -3799,7 +3854,7 @@ validate_checksum: true, segment_size: iggy_common::IggyByteSize::from(1_048_576_u64), preallocate_segments: false, - encryptor: None, + key_context: Arc::default(), path_layout: PartitionPathLayout::default(), }, ); @@ -3920,7 +3975,7 @@ validate_checksum: true, segment_size: iggy_common::IggyByteSize::from(1_048_576_u64), preallocate_segments: false, - encryptor: None, + key_context: Arc::default(), path_layout: PartitionPathLayout::default(), }, ); @@ -3964,7 +4019,7 @@ validate_checksum: true, segment_size: iggy_common::IggyByteSize::from(1_048_576_u64), preallocate_segments: false, - encryptor: None, + key_context: Arc::default(), path_layout: PartitionPathLayout::default(), }, ); @@ -4070,7 +4125,7 @@ validate_checksum: true, segment_size: iggy_common::IggyByteSize::from(1_048_576_u64), preallocate_segments: false, - encryptor: None, + key_context: Arc::default(), path_layout: PartitionPathLayout::default(), }, ); @@ -4241,12 +4296,35 @@ /// `namespace`, so the poll metadata gate and the context fence pass for /// an existing partition. async fn poll_and_expect_error(shard: &Rc<TestShard>, namespace: IggyNamespace) -> IggyError { + match poll_once(shard, namespace).await { + Err(ReadPolledMessagesError::Rejected(error)) => error, + Err(ReadPolledMessagesError::Fallback(_)) => { + panic!("poll must not return an empty reply") + } + Ok(_) => panic!("poll must not succeed without an owner reply"), + } + } + + /// One root-session poll of partition 0 of `namespace`, as the binary + /// transports resolve it. + async fn poll_once( + shard: &Rc<TestShard>, + namespace: IggyNamespace, + ) -> Result<BusMessage, ReadPolledMessagesError> { + let attachment = attach_root_session(shard, 1, 1); + poll_attached(shard, namespace, &attachment).await + } + + async fn poll_attached( + shard: &Rc<TestShard>, + namespace: IggyNamespace, + attachment: &SessionAttachment, + ) -> Result<BusMessage, ReadPolledMessagesError> { let vsr_client_id = 1; let session = 1; let transport_client_id = 91; let consumer_id = 1; let partition_id = 0; - let attachment = attach_root_session(shard, vsr_client_id, session); let request = with_partition_incarnation( shard, namespace, @@ -4258,22 +4336,15 @@ PollingConsumer::Consumer(consumer_id, usize::try_from(partition_id).unwrap()), PollingArgs::new(PollingStrategy::offset(0), 1, true), ); - match read_polled_messages( + read_polled_messages( shard, ClientOrigin::capture(&shard.bus, transport_client_id), &request, resolved_poll, - &attachment, + attachment, transport_client_id, ) .await - { - Err(ReadPolledMessagesError::Rejected(error)) => error, - Err(ReadPolledMessagesError::Fallback(_)) => { - panic!("poll must not return an empty reply") - } - Ok(_) => panic!("poll must not succeed without an owner reply"), - } } /// The control of a routing refusal: the same poll reaches the owner @@ -4505,7 +4576,7 @@ validate_checksum: true, segment_size: iggy_common::IggyByteSize::from(1_048_576_u64), preallocate_segments: false, - encryptor: None, + key_context: Arc::default(), path_layout: PartitionPathLayout::default(), }, );
diff --git a/core/server/src/dispatch/session_ops.rs b/core/server/src/dispatch/session_ops.rs index f09610b..9b42bd5 100644 --- a/core/server/src/dispatch/session_ops.rs +++ b/core/server/src/dispatch/session_ops.rs
@@ -57,7 +57,9 @@ use iggy_common::defaults::{ MAX_PASSWORD_LENGTH, MAX_USERNAME_LENGTH, MIN_PASSWORD_LENGTH, MIN_USERNAME_LENGTH, }; -use iggy_common::{IggyError, IggyTimestamp, PersonalAccessToken, UserStatus}; +use iggy_common::{ + IggyError, IggyTimestamp, PersonalAccessToken, UserStatus, check_critical_options, +}; use journal::superblock::SuperblockStore; use journal::{Journal, JournalHandle}; use metadata::MetadataSubmitError; @@ -125,9 +127,11 @@ // Verify before the status check and collapse inactive to // InvalidCredentials: an inactive account must answer exactly like a // wrong password (same error, same Argon2 cost), or login could probe - // which accounts exist but are disabled. + // which accounts exist but are disabled. An account holding critical + // options this build cannot read answers the same way. if !crypto::verify_password(password, user.password_hash.as_ref()) || user.status != UserStatus::Active + || check_critical_options(&user.options).is_err() { return Err(LoginRegisterError::InvalidCredentials); } @@ -200,7 +204,7 @@ let Some(user) = users.items.get(*user_id as usize) else { return Err(LoginRegisterError::InvalidToken); }; - if user.status != UserStatus::Active { + if user.status != UserStatus::Active || check_critical_options(&user.options).is_err() { return Err(LoginRegisterError::UserInactive); } // `expiry_at == None` is a never-expiring PAT; map it to `u64::MAX` so @@ -1282,13 +1286,21 @@ }; use consensus::{LocalPipeline, Plane as _, PlaneKind, VsrConsensus}; use iggy_binary_protocol::Operation; + use iggy_binary_protocol::WireName; use iggy_binary_protocol::requests::streams::CreateStreamRequest; + use iggy_binary_protocol::requests::users::CreateUserRequest; use iggy_binary_protocol::{PrepareOkHeader, ReplyHeader, WireEncode, WireOptions}; use iggy_common::eviction_reason_to_error; + use iggy_common::wire_conversions::resource_options_to_wire; + use iggy_common::{ + HeaderKey, HeaderValue, OptionKey, OptionValue, OptionsProvenance, ResourceOptions, + }; use journal::prepare_journal::PrepareJournal; use message_bus::installer::conn_info::ClientTransportKind; use metadata::IggyMetadata; use metadata::impls::metadata::IggySnapshot; + use metadata::stm::StateMachine as _; + use metadata::stm::user::CreatePersonalAccessTokenRequest; use partitions::{IggyPartitions, PartitionPathLayout, PartitionsConfig}; use server_common::MessageBag; use server_common::sharding::ShardId; @@ -1296,6 +1308,7 @@ use shard::{PartitionConsensusConfig, ReplicaTopology, ShardIdentity}; use std::future::Future; use std::mem::size_of; + use std::sync::Arc; #[test] fn given_uninitialized_root_when_verifying_credentials_should_return_a_retryable_failure() { @@ -1315,6 +1328,75 @@ } #[test] + fn given_user_with_unknown_critical_option_when_logging_in_should_refuse_it() { + const PASSWORD: &str = "user-password"; + const TOKEN: &str = "user-personal-access-token"; + let unreadable = ResourceOptions::from([ + ( + HeaderKey::from(OptionKey::CriticalKeys), + OptionValue::explicit(HeaderValue::try_from("a_future_key").expect("marker fits")), + ), + ( + HeaderKey::try_from("a_future_key").expect("key fits"), + OptionValue::explicit(HeaderValue::from(7u64)), + ), + ]); + for (options, refused) in [(ResourceOptions::new(), false), (unreadable, true)] { + let bus = SpyBus::default(); + let shard = Rc::new(test_shard(&bus, 0, 1, FIRST_BOOT)); + let mux_stm = &shard.plane.metadata().mux_stm; + mux_stm.users().ensure_root_user("root", "hash"); + mux_stm + .update(prepare_message( + Operation::CreateUser, + 0, + 1, + &CreateUserRequest { + username: WireName::new("user").expect("name fits"), + password: server_common::crypto::hash_password(PASSWORD), + status: UserStatus::Active.as_code(), + permissions: None, + options: resource_options_to_wire(&options, OptionsProvenance::All) + .expect("options encode"), + } + .to_bytes(), + )) + .expect("user create applies"); + mux_stm + .update(prepare_message( + Operation::CreatePersonalAccessToken, + 0, + 2, + &CreatePersonalAccessTokenRequest { + user_id: 1, + name: WireName::new("token").expect("name fits"), + expiry: 0, + token_hash: PersonalAccessToken::hash_token(TOKEN) + .as_bytes() + .try_into() + .expect("hash fits"), + } + .to_bytes(), + )) + .expect("token create applies"); + + let password_login = verify_login_credentials(&shard, "user", PASSWORD); + let token_login = verify_pat_credentials(&shard, TOKEN); + + if refused { + assert!(matches!( + password_login, + Err(LoginRegisterError::InvalidCredentials) + )); + assert!(matches!(token_login, Err(LoginRegisterError::UserInactive))); + } else { + assert!(password_login.is_ok(), "{password_login:?}"); + assert!(token_login.is_ok(), "{token_login:?}"); + } + } + } + + #[test] fn terminal_vs_transient() { assert!(LoginRegisterError::InvalidCredentials.is_terminal()); assert!(LoginRegisterError::InvalidToken.is_terminal()); @@ -1451,7 +1533,7 @@ validate_checksum: true, segment_size: iggy_common::IggyByteSize::from(1_048_576_u64), preallocate_segments: false, - encryptor: None, + key_context: Arc::default(), path_layout: PartitionPathLayout::default(), }, );
diff --git a/core/server/src/dispatch/test_support.rs b/core/server/src/dispatch/test_support.rs index 7155743..62299d6 100644 --- a/core/server/src/dispatch/test_support.rs +++ b/core/server/src/dispatch/test_support.rs
@@ -33,7 +33,7 @@ Command, Operation, PrepareHeader, ReplyHeader, RoutedRequestHeader, WireEncode, WireGlobalPermissions, WireIdentifier, WireName, WireOptions, WirePermissions, }; -use iggy_common::{UserStatus, variadic}; +use iggy_common::{EncryptorKind, ServerKeyContext, UserStatus, variadic}; use journal::prepare_journal::PrepareJournal; use message_bus::client_listener::RequestHandler; use message_bus::fd_transfer::DupedFd; @@ -52,7 +52,11 @@ use metadata::stm::user::Users; use metadata::{IggyMetadata, MuxStateMachine}; use partitions::{IggyPartitions, PartitionPathLayout, PartitionsConfig}; -use server_common::iobuf::Frozen; +use server_common::iobuf::{Frozen, Owned}; +use server_common::send_messages::{ + COMMAND_HEADER_SIZE, IggyMessage, IggyMessageHeader, IggyMessages, PREPARE_SPLIT_POINT, + SendMessagesOwned, encrypt_batch_request, +}; use server_common::sharding::{IggyNamespace, ShardId}; use server_common::{MESSAGE_ALIGN, Message}; use shard::shards_table::PapayaShardsTable; @@ -61,6 +65,7 @@ use std::future::Future; use std::mem::size_of; use std::rc::Rc; +use std::sync::Arc; pub type TestMux = MuxStateMachine<variadic!(Users, Streams)>; pub type TestShard = IggyShard<SpyBus, PrepareJournal, IggySnapshot, TestMux, PapayaShardsTable>; @@ -280,12 +285,24 @@ /// forward-nonce counter from it, so passing a different value models a /// restart. pub fn test_shard(bus: &SpyBus, replica: u8, replica_count: u8, incarnation: u128) -> TestShard { - test_shard_with_frontier( + test_shard_with_key_context(bus, replica, replica_count, incarnation, Arc::default()) +} + +/// [`test_shard`] holding `key_context` as its server keys. +pub fn test_shard_with_key_context( + bus: &SpyBus, + replica: u8, + replica_count: u8, + incarnation: u128, + key_context: Arc<ServerKeyContext>, +) -> TestShard { + build_test_shard( bus, replica, replica_count, incarnation, - std::sync::Arc::default(), + key_context, + Arc::default(), ) } @@ -294,7 +311,25 @@ replica: u8, replica_count: u8, incarnation: u128, - frontier: std::sync::Arc<metadata::AppliedFrontier>, + frontier: Arc<metadata::AppliedFrontier>, +) -> TestShard { + build_test_shard( + bus, + replica, + replica_count, + incarnation, + Arc::default(), + frontier, + ) +} + +fn build_test_shard( + bus: &SpyBus, + replica: u8, + replica_count: u8, + incarnation: u128, + key_context: Arc<ServerKeyContext>, + frontier: Arc<metadata::AppliedFrontier>, ) -> TestShard { let consensus = VsrConsensus::new( 1, @@ -317,7 +352,7 @@ validate_checksum: true, segment_size: iggy_common::IggyByteSize::from(1_048_576_u64), preallocate_segments: false, - encryptor: None, + key_context, path_layout: PartitionPathLayout::default(), }, ); @@ -351,6 +386,51 @@ reply } +/// A stored batch record holding two messages, encrypted the way the primary +/// encrypts at ingestion when `encryptor` is set, plaintext otherwise. +pub fn stored_batch_record(encryptor: Option<&EncryptorKind>) -> Frozen<MESSAGE_ALIGN> { + let namespace = IggyNamespace::new(1, 1, 3); + let mut messages = IggyMessages::with_capacity(2); + for (id, payload) in [(7u128, &b"first-payload"[..]), (8, &b"second-payload"[..])] { + messages.push(IggyMessage { + header: IggyMessageHeader { + id, + origin_timestamp: 1_000, + ..Default::default() + }, + payload: Bytes::copy_from_slice(payload), + user_headers: None, + }); + } + let owned = SendMessagesOwned::from_messages(namespace, &messages).expect("build batch"); + let header_size = size_of::<RoutedRequestHeader>(); + let total = header_size + owned.header.total_size(); + let mut buffer = Owned::<MESSAGE_ALIGN>::zeroed(total); + { + let header: &mut RoutedRequestHeader = + bytemuck::checked::try_from_bytes_mut(&mut buffer.as_mut_slice()[..header_size]) + .expect("zeroed bytes form a valid RoutedRequestHeader"); + header.command = Command::Request; + header.operation = Operation::SendMessages; + header.client = 1; + header.session = 1; + header.request = 1; + header.size = u32::try_from(total).expect("size fits u32"); + } + let bytes = buffer.as_mut_slice(); + owned + .header + .encode_into(&mut bytes[header_size..header_size + COMMAND_HEADER_SIZE]); + bytes[PREPARE_SPLIT_POINT..].copy_from_slice(&owned.blob); + let canonical = Message::try_from(buffer).expect("request message is valid"); + let stored = match encryptor { + Some(encryptor) => encrypt_batch_request(canonical, encryptor).expect("encrypt batch"), + None => canonical, + }; + let record = &stored.as_slice()[header_size..stored.header().size as usize]; + Owned::<MESSAGE_ALIGN>::copy_from_slice(record).into() +} + pub fn request_message( operation: Operation, client: u128,
diff --git a/core/server/src/http/error.rs b/core/server/src/http/error.rs index 1851322..dbf719e 100644 --- a/core/server/src/http/error.rs +++ b/core/server/src/http/error.rs
@@ -77,6 +77,8 @@ | IggyError::InvalidPersonalAccessToken => StatusCode::UNAUTHORIZED, IggyError::Unauthorized => StatusCode::FORBIDDEN, IggyError::PartitionHistoryChanged => StatusCode::CONFLICT, + // Stored state this build cannot read, not a caller error. + IggyError::UnsupportedCriticalOption(_) => StatusCode::INTERNAL_SERVER_ERROR, // The write path ran out of replays of a request id whose // op may still commit (see `submit::submit_committed`). // HTTP mints a new id per request, so a resend could apply @@ -234,6 +236,10 @@ /// business error or, worse, a 200 with a stale body. pub(in crate::http) enum WriteError { Rejected(IggyError), + /// A body member the request shape refuses, rendered as `InvalidCommand` + /// with the member named in `field`, because REST has no version handshake + /// to tell an older client why its body no longer fits. + MalformedMember(String), /// The VSR session was evicted (its client slot was reclaimed cluster-side, /// e.g. LRU-evicted from the full client table). Renders identically to a /// terminal `Rejected` (401 -> re-authenticate), but is a distinct variant @@ -249,6 +255,14 @@ Self::Rejected(error) | Self::Evicted(error) => { CustomError::from(error).into_response() } + Self::MalformedMember(member) => ( + StatusCode::BAD_REQUEST, + Json(ErrorResponse { + field: Some(member), + ..ErrorResponse::from_error(&IggyError::InvalidCommand) + }), + ) + .into_response(), Self::Unavailable => service_unavailable(), } } @@ -686,6 +700,22 @@ } #[test] + fn given_unreadable_critical_option_when_rendering_read_error_should_answer_500() { + let response = ReadError::Rejected(IggyError::UnsupportedCriticalOption( + "future_key".to_owned(), + )) + .into_response(); + + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + } + + #[test] + fn given_root_guard_error_when_rendering_http_should_answer_400() { + let response = CustomError::Error(IggyError::CannotModifyRootUser).into_response(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + } + + #[test] fn primary_redirect_location_targets_primary_http_addr_with_path_passthrough() { let roster = roster(vec![ node(0, "10.0.0.1", Some(8080)),
diff --git a/core/server/src/http/extractor.rs b/core/server/src/http/extractor.rs index 576a3b6..130a63b 100644 --- a/core/server/src/http/extractor.rs +++ b/core/server/src/http/extractor.rs
@@ -15,21 +15,32 @@ // specific language governing permissions and limitations // under the License. -//! Bearer-credential extractor for protected shard-0 HTTP routes. +//! Bearer-credential and JSON body extractors for protected shard-0 HTTP +//! routes. +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt::Formatter; use std::net::IpAddr; use std::rc::Rc; -use axum::extract::{ConnectInfo, FromRequestParts}; +use axum::Json; +use axum::body::{Body, Bytes}; +use axum::extract::{ConnectInfo, FromRequest, FromRequestParts, Request}; use axum::http::HeaderMap; use axum::http::header::AUTHORIZATION; use axum::http::request::Parts; -use iggy_common::{IggyError, IggyTimestamp, PersonalAccessToken}; +use axum::response::{IntoResponse, Response}; +use iggy_common::{ + IggyError, IggyTimestamp, OptionInputValue, OptionsUpdate, PersonalAccessToken, RESET_FIELD, + erase_buffer, +}; use send_wrapper::SendWrapper; +use serde::Deserializer as _; +use serde::de::{DeserializeOwned, Error as _, IgnoredAny, MapAccess, Visitor}; use tracing::debug; use super::HttpState; -use super::error::AuthError; +use super::error::{AuthError, WriteError}; use super::jwt::TokenSubject; use super::session::HttpSession; use super::state::HttpInner; @@ -69,6 +80,26 @@ pub session: SendWrapper<Rc<HttpSession>>, } +/// A JSON body with the names of its top-level members, so a create handler +/// can refuse an option sent beside `options` instead of silently dropping it. +pub struct JsonMembers<T> { + pub value: T, + pub members: BTreeSet<String>, +} + +/// An update body. A body the update shape refuses is `InvalidCommand`, the +/// category the binary decoder gives the same patch, naming the first member +/// at fault where one is. +pub(in crate::http) struct JsonUpdate(pub OptionsUpdate); + +/// Walks an update body with the member types [`OptionsUpdate`] parses and +/// records the first member that does not parse: a repeated name, a `reset` +/// that is not a list of distinct names, or another member that is not a +/// scalar or `null`. +struct RefusedMemberVisitor<'a> { + refused: &'a mut Option<String>, +} + impl FromRequestParts<HttpState> for Authenticated { type Rejection = AuthError; @@ -91,6 +122,90 @@ } } +impl<T, S> FromRequest<S> for JsonMembers<T> +where + T: DeserializeOwned, + S: Send + Sync, +{ + type Rejection = Response; + + /// Runs axum's own `Json` extraction, so content type checks, body limits + /// and rejections stay what every other route answers. + async fn from_request(request: Request, state: &S) -> Result<Self, Self::Rejection> { + let (parts, body) = request.into_parts(); + let bytes = Bytes::from_request(Request::from_parts(parts.clone(), body), state) + .await + .map_err(IntoResponse::into_response)?; + let value = + Json::<T>::from_request(Request::from_parts(parts, Body::from(bytes.clone())), state) + .await; + let members = Json::<BTreeMap<String, IgnoredAny>>::from_bytes(&bytes); + erase_buffer(bytes); + let Json(value) = value.map_err(IntoResponse::into_response)?; + let Json(members) = members.map_err(IntoResponse::into_response)?; + Ok(Self { + value, + members: members.into_keys().collect(), + }) + } +} + +impl<S> FromRequest<S> for JsonUpdate +where + S: Send + Sync, +{ + type Rejection = WriteError; + + async fn from_request(request: Request, state: &S) -> Result<Self, Self::Rejection> { + let (parts, body) = request.into_parts(); + let bytes = Bytes::from_request(Request::from_parts(parts.clone(), body), state) + .await + .map_err(|_| WriteError::Rejected(IggyError::InvalidCommand))?; + let update = match Json::<OptionsUpdate>::from_request( + Request::from_parts(parts, Body::from(bytes.clone())), + state, + ) + .await + { + Ok(Json(update)) => Ok(Self(update)), + Err(_) => Err(refused_member(&bytes).map_or( + WriteError::Rejected(IggyError::InvalidCommand), + WriteError::MalformedMember, + )), + }; + erase_buffer(bytes); + update + } +} + +impl<'de> Visitor<'de> for RefusedMemberVisitor<'_> { + type Value = (); + + fn expecting(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + formatter.write_str("an update body object") + } + + fn visit_map<A: MapAccess<'de>>(self, mut access: A) -> Result<Self::Value, A::Error> { + let mut seen = BTreeSet::new(); + while let Some(key) = access.next_key::<String>()? { + let fits = seen.insert(key.clone()) + && if key == RESET_FIELD { + access.next_value::<Vec<String>>().is_ok_and(|names| { + let mut distinct = BTreeSet::new(); + names.into_iter().all(|name| distinct.insert(name)) + }) + } else { + access.next_value::<Option<OptionInputValue>>().is_ok() + }; + if !fits { + *self.refused = Some(key); + return Err(A::Error::custom("refused update member")); + } + } + Ok(()) + } +} + /// A verified bearer with what its VSR session is keyed and leased by, but no /// session yet. A partition write resolves the session only where it is /// served ([`Self::session`]): a relayed write runs under the serving node's @@ -317,6 +432,20 @@ ) } +/// The member [`RefusedMemberVisitor`] finds in `bytes`, `None` for a body that +/// is not a JSON object or has no member at fault. +fn refused_member(bytes: &[u8]) -> Option<String> { + let mut refused = None; + let walked = + serde_json::Deserializer::from_slice(bytes).deserialize_map(RefusedMemberVisitor { + refused: &mut refused, + }); + match walked { + Ok(()) => None, + Err(_) => refused, + } +} + #[cfg(test)] mod tests { use super::*; @@ -367,6 +496,38 @@ ); } + #[test] + fn given_update_bodies_when_finding_refused_member_should_name_the_first_misfit() { + let cases = [ + (r#"{"name":"orders","reset":["message_expiry"]}"#, None), + (r#"{"name":"orders","message_expiry":null}"#, None), + (r#"{"name":["orders"]}"#, Some("name")), + ( + r#"{"name":"a","message_expiry":{"value":"1 day"}}"#, + Some("message_expiry"), + ), + (r#"{"reset":null}"#, Some("reset")), + ( + r#"{"message_expiry":1.5,"name":"a","name":"b"}"#, + Some("message_expiry"), + ), + (r#"{"name":"a","name":"b"}"#, Some("name")), + ( + r#"{"reset":["message_expiry","message_expiry"]}"#, + Some("reset"), + ), + (r#"["name"]"#, None), + (r#"{"name":"a""#, None), + ]; + for (body, expected) in cases { + assert_eq!( + refused_member(body.as_bytes()).as_deref(), + expected, + "{body}" + ); + } + } + fn unverifiable_bearer_parts() -> Parts { Request::builder() .uri("/streams/1/topics/1/messages")
diff --git a/core/server/src/http/forward.rs b/core/server/src/http/forward.rs index bbac57a..1bc48c6 100644 --- a/core/server/src/http/forward.rs +++ b/core/server/src/http/forward.rs
@@ -101,7 +101,7 @@ use cyper::IntoUrl; use futures::{Stream, StreamExt}; use iggy_binary_protocol::primitives::partition_history::PartitionContext; -use iggy_common::{IggyError, IggyTimestamp}; +use iggy_common::{IggyError, IggyTimestamp, erase_buffer}; use message_bus::MessageBus; use message_bus::transports::tls::load_pem; use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}; @@ -920,9 +920,12 @@ ); }; let Some(_guard) = ForwardGuard::admit(&state.forward.in_flight) else { + erase_buffer(body); return forward_busy(); }; let response = forward(&state, &parts, &body, deadline).await; + // Kept for retries, so it may hold a create body's plaintext stream key. + erase_buffer(body); record_relayed_floor(&state, user_id, &response); response } @@ -2383,6 +2386,7 @@ context: metadata.context(READER_CREATED_AT), fragments: Vec::new().into(), current_offset: 0, + encryptor: None, }) .expect("the poll waits for its reply"); assert_eq!(
diff --git a/core/server/src/http/handlers.rs b/core/server/src/http/handlers.rs index 33c98d5..76f7059 100644 --- a/core/server/src/http/handlers.rs +++ b/core/server/src/http/handlers.rs
@@ -19,6 +19,7 @@ //! the control-plane writes, and the data-plane produce / poll / consumer-offset //! routes the router binds. +use std::collections::BTreeSet; use std::str::FromStr; use std::sync::Arc; @@ -27,8 +28,9 @@ use axum::http::{HeaderMap, HeaderValue, StatusCode, header}; use axum::response::{IntoResponse, Response}; use axum::{Extension, Json}; +use bytes::{Bytes, BytesMut}; use chrono::Local; -use consensus::{MetadataHandle, PartitionsHandle}; +use consensus::MetadataHandle; use iggy_binary_protocol::codes::{ DESCRIBE_OPTIONS_CODE, GET_CLIENT_CODE, GET_CLIENTS_CODE, GET_CONSUMER_GROUP_CODE, GET_CONSUMER_GROUPS_CODE, GET_CONSUMER_OFFSET_CODE, GET_PERSONAL_ACCESS_TOKENS_CODE, @@ -50,16 +52,16 @@ use iggy_binary_protocol::requests::segments::DeleteSegmentsRequest; use iggy_binary_protocol::requests::streams::{ CreateStreamRequest, DeleteStreamRequest, GetStreamRequest, GetStreamsRequest, - UpdateStreamRequest, }; -use iggy_binary_protocol::requests::system::DescribeOptionsRequest; -use iggy_binary_protocol::requests::system::GetStatsRequest; +use iggy_binary_protocol::requests::system::{ + DescribeOptionsRequest, GetStatsRequest, ResourceTarget, UpdateResourceRequest, +}; use iggy_binary_protocol::requests::topics::{ - CreateTopicRequest, DeleteTopicRequest, GetTopicRequest, GetTopicsRequest, UpdateTopicRequest, + CreateTopicRequest, DeleteTopicRequest, GetTopicRequest, GetTopicsRequest, }; use iggy_binary_protocol::requests::users::{ ChangePasswordRequest, CreateUserRequest, DeleteUserRequest, GetUserRequest, GetUsersRequest, - UpdatePermissionsRequest, UpdateUserRequest, + UpdatePermissionsRequest, }; use iggy_binary_protocol::responses::clients::client_response::ConsumerGroupInfoResponse; use iggy_binary_protocol::responses::clients::get_client::ClientDetailsResponse; @@ -75,9 +77,7 @@ use iggy_binary_protocol::responses::topics::get_topics::GetTopicsResponse; use iggy_binary_protocol::responses::users::get_user::UserDetailsResponse; use iggy_binary_protocol::responses::users::get_users::GetUsersResponse; -use iggy_binary_protocol::{ - Operation, WireDecode, WireEncode, WireIdentifier, WireName, WireOptions, -}; +use iggy_binary_protocol::{Operation, WireDecode, WireEncode, WireIdentifier, WireName}; use iggy_common::change_password::ChangePassword; use iggy_common::create_consumer_group::CreateConsumerGroup; use iggy_common::create_partitions::CreatePartitions; @@ -94,27 +94,23 @@ use iggy_common::login_with_personal_access_token::LoginWithPersonalAccessToken; use iggy_common::store_consumer_offset::StoreConsumerOffset; use iggy_common::update_permissions::UpdatePermissions; -use iggy_common::update_stream::UpdateStream; -use iggy_common::update_topic::UpdateTopic; -use iggy_common::update_user::UpdateUser; use iggy_common::wire_conversions::{ clients_from_wire, consumer_groups_from_wire, identifier_to_wire, option_specs_from_wire, permissions_to_wire, personal_access_tokens_from_wire, streams_from_wire, topics_from_wire, users_from_wire, }; use iggy_common::{ - ClientInfo, ClientInfoDetails, ClusterMetadata, CompressionAlgorithm, Consumer, ConsumerGroup, - ConsumerGroupDetails, ConsumerOffsetInfo, DEFAULT_MAX_TOPIC_SIZE, Identifier, IdentityInfo, - IggyError, IggyExpiry, IggyTimestamp, MaxTopicSize, OptionSpec, OptionsScope, Partitioning, - PersonalAccessTokenInfo, PollMessages, PolledMessages, RawPersonalAccessToken, SendMessages, - SendMessagesConfirmations, Stats, Stream, StreamDetails, StreamUpdateOptions, TokenInfo, Topic, - TopicCreateOptions, TopicDetails, TopicUpdateOptions, UPDATABLE_STREAM_OPTION_KEYS, - UPDATABLE_TOPIC_OPTION_KEYS, UPDATABLE_USER_OPTION_KEYS, UserInfo, UserInfoDetails, - UserUpdateOptions, Validatable, validate_preallocated_topic_bytes, validate_topic_segment_size, + ClientInfo, ClientInfoDetails, ClusterMetadata, Consumer, ConsumerGroup, ConsumerGroupDetails, + ConsumerOffsetInfo, DEFAULT_MAX_TOPIC_SIZE, Identifier, IdentityInfo, IggyError, IggyTimestamp, + OptionSpec, OptionStorage, OptionsScope, Partitioning, PersonalAccessTokenInfo, PollMessages, + PolledMessages, RawPersonalAccessToken, SendMessages, SendMessagesConfirmations, Stats, Stream, + StreamDetails, TokenInfo, Topic, TopicDetails, UserInfo, UserInfoDetails, Validatable, + descriptors_for, erase_buffer, }; use metadata::impls::metadata::StreamsFrontend; use metadata::permissioner::Permissioner; use secrecy::ExposeSecret; +use secrecy::zeroize::Zeroizing; use send_wrapper::SendWrapper; use serde::Deserialize; use shard::{PartitionRead, PartitionReadReply}; @@ -128,7 +124,7 @@ Consistency, ConsistencyQuery, CustomError, PartitionWriteError, ProduceAck, ProduceQuery, ReadError, WriteError, }; -use crate::http::extractor::{Authenticated, Credential, Identity}; +use crate::http::extractor::{Authenticated, Credential, Identity, JsonMembers, JsonUpdate}; use crate::http::forward::{ FirstAttempt, PartitionEffect, PartitionHop, PartitionRequest, require_local_primary, serve_partition, @@ -155,10 +151,6 @@ }; use crate::reply_frame::{build_polled_messages_body, build_raw_pat_reply}; use crate::responses::connected_client_to_response; -use crate::rewrite::{ - validate_option_keys, validate_topic_bounds, validate_topic_size_floor, - warn_unenforceable_topic_size, warn_unenforceable_topic_size_on_partition_add, -}; use crate::snapshot; /// `GET /ping` response body, matching the legacy HTTP server's health probe. @@ -696,7 +688,10 @@ topic: Arc::clone(&topic.name), size_bytes: topic.stats.size_bytes_inconsistent(), messages: topic.stats.messages_count_inconsistent(), - max_size_bytes: topic.max_topic_size.resolve(DEFAULT_MAX_TOPIC_SIZE), + max_size_bytes: topic + .resolved + .max_topic_size + .resolve(DEFAULT_MAX_TOPIC_SIZE), }); } } @@ -900,20 +895,24 @@ /// `POST /streams`: create a stream and render the committed reply as the same /// `StreamDetails` JSON the legacy server returns. /// -/// The accepted body is name-only (`{"name": ...}`), matching the legacy -/// request; server's wire `CreateStreamRequest` is likewise name-only and -/// auto-assigns the id, so there is no client-supplied stream id to honor. +/// The body carries the name and an optional `options` object. The server +/// assigns the id, so there is no client-supplied stream id to honor. pub(in crate::http) async fn create_stream( State(state): State<HttpState>, identity: Authenticated, - Json(command): Json<CreateStream>, + JsonMembers { + value: command, + members, + }: JsonMembers<CreateStream>, ) -> Result<Json<StreamDetails>, WriteError> { + refuse_misplaced_options(OptionsScope::Stream, &members)?; let request = CreateStreamRequest { name: WireName::new(command.name) .map_err(|_| WriteError::Rejected(IggyError::InvalidStreamName))?, - options: WireOptions::empty(), + options: command.options.to_wire().map_err(WriteError::Rejected)?, }; - let body = request.to_bytes(); + let body = erasing_body(&request); + erase_buffer(request.options.into_bytes()); let payload = SendWrapper::new(submit_write( &state, &identity.session, @@ -924,39 +923,20 @@ Ok(Json(decode_stream_details(&payload)?)) } -/// `PUT /streams/{stream_id}`: rename a stream. A committed write returns 204 -/// with no body, matching the legacy server. +/// `PUT /streams/{stream_id}`: update the stream's name and options. An absent +/// or `null` key keeps its value, a key in `reset` returns to its default. +/// Returns 204 on commit. pub(in crate::http) async fn update_stream( State(state): State<HttpState>, identity: Authenticated, Path(stream_id): Path<String>, - Json(command): Json<UpdateStream>, + update: Result<JsonUpdate, WriteError>, ) -> Result<StatusCode, WriteError> { let stream_id = Identifier::from_str_value(&stream_id).map_err(WriteError::Rejected)?; - let wire_options = StreamUpdateOptions { - raw: command.options, - } - .to_wire() - .map_err(WriteError::Rejected)?; - // Same pre-consensus gate the TCP ingress applies: a key this command may - // not change is denied here by name rather than riding a log entry. - validate_option_keys(&wire_options, UPDATABLE_STREAM_OPTION_KEYS) - .map_err(WriteError::Rejected)?; - let request = UpdateStreamRequest { + let target = ResourceTarget::Stream { stream_id: identifier_to_wire(&stream_id).map_err(WriteError::Rejected)?, - name: WireName::new(command.name) - .map_err(|_| WriteError::Rejected(IggyError::InvalidStreamName))?, - options: wire_options, }; - let body = request.to_bytes(); - SendWrapper::new(submit_write( - &state, - &identity.session, - Operation::UpdateStream, - body, - )) - .await?; - Ok(StatusCode::NO_CONTENT) + submit_update(&state, &identity, target, update).await } /// `DELETE /streams/{stream_id}`: delete a stream. Returns 204 on commit. @@ -991,76 +971,17 @@ State(state): State<HttpState>, identity: Authenticated, Path(stream_id): Path<String>, - Json(command): Json<CreateTopic>, + JsonMembers { + value: command, + members, + }: JsonMembers<CreateTopic>, ) -> Result<Json<TopicDetails>, WriteError> { let stream_id = Identifier::from_str_value(&stream_id).map_err(WriteError::Rejected)?; - // Rejects empty/oversized name and partitions_count > MAX. + refuse_misplaced_options(OptionsScope::Topic, &members)?; + // Rejects empty/oversized name and partitions_count > MAX. The options are + // admitted by the submit chain, the same gate the binary transports use. command.validate().map_err(WriteError::Rejected)?; - let durability = command - .options - .get(iggy_common::topic_option_keys::DURABILITY) - .map(|value| value.parse()) - .transpose() - .map_err(|_| WriteError::Rejected(IggyError::InvalidOptionValue("durability".to_string())))? - .unwrap_or_default(); - let consumer_offset_durability = command - .options - .get(iggy_common::topic_option_keys::CONSUMER_OFFSET_DURABILITY) - .map(|value| value.parse()) - .transpose() - .map_err(|_| { - WriteError::Rejected(IggyError::InvalidOptionValue( - "consumer_offset_durability".to_string(), - )) - })? - .unwrap_or_default(); - let options = TopicCreateOptions { - durability, - consumer_offset_durability, - partitions_count: Some(command.partitions_count), - compression_algorithm: (command.compression_algorithm != CompressionAlgorithm::default()) - .then_some(command.compression_algorithm), - message_expiry: (command.message_expiry != IggyExpiry::ServerDefault) - .then_some(command.message_expiry), - max_topic_size: (command.max_topic_size != MaxTopicSize::ServerDefault) - .then_some(command.max_topic_size), - raw: command.options, - ..TopicCreateOptions::default() - }; - let wire_options = options - .to_explicit_wire(|key| options.raw.contains_key(key)) - .map_err(WriteError::Rejected)?; - // Re-parse the encoded block so `--set`-style raw string entries get the - // same typed pre-consensus checks as native fields; unknown keys deny - // here with the key name. - let parsed = TopicCreateOptions::parse(&wire_options).map_err(WriteError::Rejected)?; - if let Some(segment_size) = parsed.segment_size { - validate_topic_segment_size( - segment_size.as_bytes_u64(), - iggy_common::MAX_TOPIC_SEGMENT_SIZE, - ) - .map_err(WriteError::Rejected)?; - } - let segment_size = parsed.segment_size.map_or_else( - || iggy_common::DEFAULT_SEGMENT_SIZE, - |segment_size| segment_size.as_bytes_u64(), - ); - if parsed - .preallocate_segments - .unwrap_or(iggy_common::DEFAULT_PREALLOCATE_SEGMENTS) - { - validate_preallocated_topic_bytes(segment_size, command.partitions_count) - .map_err(WriteError::Rejected)?; - } - let max_topic_size = parsed.max_topic_size.unwrap_or(MaxTopicSize::ServerDefault); - validate_topic_bounds(command.partitions_count, max_topic_size, segment_size) - .map_err(WriteError::Rejected)?; - warn_unenforceable_topic_size( - max_topic_size, - segment_size, - state.shard.bus_max_message_size(), - command.partitions_count, - ); + let wire_options = command.options.to_wire().map_err(WriteError::Rejected)?; let request = CreateTopicRequest { stream_id: identifier_to_wire(&stream_id).map_err(WriteError::Rejected)?, partitions_count: command.partitions_count, @@ -1079,73 +1000,22 @@ Ok(Json(decode_topic_details(&payload)?)) } -/// `PUT /streams/{stream_id}/topics/{topic_id}`: update a topic. Returns 204. +/// `PUT /streams/{stream_id}/topics/{topic_id}`: update the topic's name and +/// updatable options. An absent or `null` key keeps its value, a key in `reset` +/// returns to its default. Returns 204 on commit. pub(in crate::http) async fn update_topic( State(state): State<HttpState>, identity: Authenticated, Path((stream_id, topic_id)): Path<(String, String)>, - Json(command): Json<UpdateTopic>, + update: Result<JsonUpdate, WriteError>, ) -> Result<StatusCode, WriteError> { let stream_id = Identifier::from_str_value(&stream_id).map_err(WriteError::Rejected)?; let topic_id = Identifier::from_str_value(&topic_id).map_err(WriteError::Rejected)?; - command.validate().map_err(WriteError::Rejected)?; - // The named JSON fields fold into the same option keys the binary protocol - // uses, so REST keeps its ergonomics without giving a setting two homes. - let wire_options = TopicUpdateOptions { - compression_algorithm: command.compression_algorithm, - message_expiry: command.message_expiry, - max_topic_size: command.max_topic_size, - raw: command.options, - } - .to_wire() - .map_err(WriteError::Rejected)?; - // Same pre-consensus gates the TCP ingress applies: a key this command may - // not change is denied here by name rather than riding a log entry, and a - // cap below one of this topic's segments is denied for the same reason it is - // on create. - validate_option_keys(&wire_options, UPDATABLE_TOPIC_OPTION_KEYS) - .map_err(WriteError::Rejected)?; - let stream_wire = identifier_to_wire(&stream_id).map_err(WriteError::Rejected)?; - let topic_wire = identifier_to_wire(&topic_id).map_err(WriteError::Rejected)?; - if let Some(max_topic_size) = TopicCreateOptions::parse(&wire_options) - .map_err(WriteError::Rejected)? - .max_topic_size - { - let metadata = state.shard.plane.metadata(); - let streams = metadata.mux_stm.streams(); - let segment_size = streams - .topic_segment_size(&stream_wire, &topic_wire) - .map_or_else( - || iggy_common::DEFAULT_SEGMENT_SIZE, - |segment_size| segment_size.as_bytes_u64(), - ); - validate_topic_size_floor(max_topic_size, segment_size).map_err(WriteError::Rejected)?; - let partitions_count = streams - .topic_partitions_count(&stream_wire, &topic_wire) - .unwrap_or(0); - warn_unenforceable_topic_size( - max_topic_size, - segment_size, - state.shard.bus_max_message_size(), - u32::try_from(partitions_count).unwrap_or(u32::MAX), - ); - } - let request = UpdateTopicRequest { - stream_id: stream_wire, - topic_id: topic_wire, - name: WireName::new(command.name) - .map_err(|_| WriteError::Rejected(IggyError::InvalidTopicName))?, - options: wire_options, + let target = ResourceTarget::Topic { + stream_id: identifier_to_wire(&stream_id).map_err(WriteError::Rejected)?, + topic_id: identifier_to_wire(&topic_id).map_err(WriteError::Rejected)?, }; - let body = request.to_bytes(); - SendWrapper::new(submit_write( - &state, - &identity.session, - Operation::UpdateTopic, - body, - )) - .await?; - Ok(StatusCode::NO_CONTENT) + submit_update(&state, &identity, target, update).await } /// `DELETE /streams/{stream_id}/topics/{topic_id}`: delete a topic. Returns 204. @@ -1190,14 +1060,6 @@ topic_id: identifier_to_wire(&topic_id).map_err(WriteError::Rejected)?, partitions_count: command.partitions_count, }; - let metadata = state.shard.plane.metadata(); - warn_unenforceable_topic_size_on_partition_add( - metadata.mux_stm.streams(), - &request.stream_id, - &request.topic_id, - state.shard.bus_max_message_size(), - request.partitions_count, - ); let body = request.to_bytes(); SendWrapper::new(submit_write( &state, @@ -1422,6 +1284,7 @@ context, fragments, current_offset, + encryptor, }) => { // TCP parity (`build_polled_messages_reply`): the served context // carries at least the metadata this node applied, so a later poll @@ -1437,7 +1300,7 @@ current_offset, context, fragments, - state.shard.plane.partitions().config().encryptor.as_deref(), + encryptor.as_deref(), ) .map_err(ReadError::Rejected)?; Ok( @@ -1955,8 +1818,12 @@ pub(in crate::http) async fn create_user( State(state): State<HttpState>, identity: Authenticated, - Json(command): Json<CreateUser>, + JsonMembers { + value: command, + members, + }: JsonMembers<CreateUser>, ) -> Result<Json<UserInfoDetails>, WriteError> { + refuse_misplaced_options(OptionsScope::User, &members)?; // Rejects empty/oversized username or password before any consensus work. command.validate().map_err(WriteError::Rejected)?; let request = CreateUserRequest { @@ -1965,7 +1832,7 @@ password: command.password.expose_secret().to_string(), status: command.status.as_code(), permissions: command.permissions.as_ref().map(permissions_to_wire), - options: WireOptions::empty(), + options: command.options.to_wire().map_err(WriteError::Rejected)?, }; let body = request.to_bytes(); let payload = SendWrapper::new(submit_write( @@ -1978,45 +1845,73 @@ Ok(Json(decode_user_details(&payload)?)) } -/// `PUT /users/{user_id}`: update a user's username and/or status. Returns 204. +/// `PUT /users/{user_id}`: update the user's username, status and options. An +/// absent or `null` key keeps its value, a key in `reset` returns to its +/// default. Returns 204 on commit. pub(in crate::http) async fn update_user( State(state): State<HttpState>, identity: Authenticated, Path(user_id): Path<String>, - Json(command): Json<UpdateUser>, + update: Result<JsonUpdate, WriteError>, ) -> Result<StatusCode, WriteError> { let user_id = Identifier::from_str_value(&user_id).map_err(WriteError::Rejected)?; - // Rejects an oversized replacement username; a no-op when username is absent. - command.validate().map_err(WriteError::Rejected)?; - let user_update_options = UserUpdateOptions { - raw: command.options, - } - .to_wire() - .map_err(WriteError::Rejected)?; - validate_option_keys(&user_update_options, UPDATABLE_USER_OPTION_KEYS) - .map_err(WriteError::Rejected)?; - let request = UpdateUserRequest { + let target = ResourceTarget::User { user_id: identifier_to_wire(&user_id).map_err(WriteError::Rejected)?, - username: command - .username - .as_deref() - .map(WireName::new) - .transpose() - .map_err(|_| WriteError::Rejected(IggyError::InvalidUsername))?, - status: command.status.map(|status| status.as_code()), - options: user_update_options, }; - let body = request.to_bytes(); + submit_update(&state, &identity, target, update).await +} + +/// Submit an update body as the generic `UpdateResource` the binary transports +/// send, so admission and apply are shared. +async fn submit_update( + state: &HttpState, + identity: &Authenticated, + target: ResourceTarget, + update: Result<JsonUpdate, WriteError>, +) -> Result<StatusCode, WriteError> { + let JsonUpdate(update) = update?; + let parts = update.to_patch().map_err(WriteError::Rejected)?; + let request = UpdateResourceRequest { + target, + reset: parts.reset, + options: parts.set, + }; + let body = erasing_body(&request); + erase_buffer(request.options.into_bytes()); SendWrapper::new(submit_write( - &state, + state, &identity.session, - Operation::UpdateUser, + Operation::UpdateResource, body, )) .await?; Ok(StatusCode::NO_CONTENT) } +/// Encode `request` into a body that zeroes itself when its last handle drops, +/// since its options may hold the stream key. +fn erasing_body(request: &impl WireEncode) -> Bytes { + let mut buffer = BytesMut::with_capacity(request.encoded_size()); + request.encode(&mut buffer); + Bytes::from_owner(Zeroizing::new(Vec::from(buffer))) +} + +/// Refuses a create body member that names a stored option of `scope`, an +/// option the client placed beside `options` instead of inside it, and names +/// that member. +fn refuse_misplaced_options( + scope: OptionsScope, + members: &BTreeSet<String>, +) -> Result<(), WriteError> { + descriptors_for(scope) + .filter(|descriptor| descriptor.storage != OptionStorage::Column) + .map(|descriptor| <&str>::from(descriptor.key)) + .find(|key| members.contains(*key)) + .map_or(Ok(()), |key| { + Err(WriteError::MalformedMember(key.to_owned())) + }) +} + /// `DELETE /users/{user_id}`: delete a user. Returns 204. pub(in crate::http) async fn delete_user( State(state): State<HttpState>, @@ -2223,16 +2118,47 @@ mod tests { use super::*; + use std::collections::BTreeMap; + use axum::body::to_bytes; use iggy_binary_protocol::responses::messages::{ SendMessagesConfirmationResponse, SendMessagesResponse, }; + use iggy_common::{OptionInputMap, OptionInputValue, OptionKey}; use serde_json::Value; use crate::http::reply::classify_partition_reply; use crate::http::wire::build_request_message; use crate::reply_frame::build_deny_reply; + /// A create body can carry the stream key, so it zeroes itself on drop and + /// leaves its options block for the handler to erase. + #[test] + fn given_create_with_stream_key_when_encoding_body_should_zero_on_drop_and_free_its_options() { + let options = OptionInputMap(BTreeMap::from([( + OptionKey::EncryptionKey.to_string(), + OptionInputValue::String( + "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=" + .to_owned() + .into(), + ), + )])); + let request = CreateStreamRequest { + name: WireName::new("orders").expect("valid name"), + options: options.to_wire().expect("encodes"), + }; + let body = erasing_body(&request); + assert_eq!(body, request.to_bytes(), "the plain encoding"); + assert!( + body.try_into_mut().is_err(), + "an owned buffer that zeroes on drop" + ); + assert!( + request.options.into_bytes().try_into_mut().is_ok(), + "the only handle to the options block" + ); + } + /// The produce extractor keeps the caller's context, so the write is fenced /// with the incarnation the caller saw rather than the current one. #[test]
diff --git a/core/server/src/http/reads.rs b/core/server/src/http/reads.rs index 81eb672..9a2de98 100644 --- a/core/server/src/http/reads.rs +++ b/core/server/src/http/reads.rs
@@ -594,10 +594,10 @@ use iggy_binary_protocol::primitives::partition_assignment::CreatedPartitionAssignment; use iggy_binary_protocol::requests::messages::SendMessagesHeader; use iggy_binary_protocol::requests::streams::{ - CreateStreamRequest, DeleteStreamRequest, UpdateStreamRequest, + CreateStreamRequest, DeleteStreamRequest, UpdateStreamResolvedRequest, }; use iggy_binary_protocol::requests::topics::{ - CreateTopicRequest, CreateTopicWithAssignmentsRequest, UpdateTopicRequest, + CreateTopicRequest, CreateTopicWithAssignmentsRequest, UpdateTopicResolvedRequest, }; use iggy_binary_protocol::requests::users::{ CreateUserRequest, DeleteUserRequest, UpdatePermissionsRequest, @@ -838,8 +838,8 @@ (true, true, None), (false, true, None), (false, true, Some(Operation::DeleteStream)), - (false, true, Some(Operation::UpdateStream)), - (false, true, Some(Operation::UpdateTopic)), + (false, true, Some(Operation::UpdateStreamResolved)), + (false, true, Some(Operation::UpdateTopicResolved)), ] { let (sender, shard_inbox, _shard_replies) = shard_channel(0, 1, 1); let state = read_state_on(Some(sender), None); @@ -885,6 +885,7 @@ context: metadata.context(0), fragments: Vec::new().into(), current_offset: 0, + encryptor: None, } } PartitionRead::ConsumerOffsetOnPrimary { .. } => { @@ -923,17 +924,19 @@ stream_id: WireIdentifier::numeric(0), } .to_bytes(), - Operation::UpdateStream => UpdateStreamRequest { + Operation::UpdateStreamResolved => UpdateStreamResolvedRequest { stream_id: WireIdentifier::numeric(0), - name: WireName::new("renamed").unwrap(), - options: WireOptions::empty(), + name: Some(WireName::new("renamed").unwrap()), + explicit_options: WireOptions::empty(), + derived_options: WireOptions::empty(), } .to_bytes(), - Operation::UpdateTopic => UpdateTopicRequest { + Operation::UpdateTopicResolved => UpdateTopicResolvedRequest { stream_id: WireIdentifier::numeric(0), topic_id: WireIdentifier::numeric(0), - name: WireName::new("renamed").unwrap(), - options: WireOptions::empty(), + name: Some(WireName::new("renamed").unwrap()), + explicit_options: WireOptions::empty(), + derived_options: WireOptions::empty(), } .to_bytes(), _ => unreachable!(), @@ -1107,6 +1110,7 @@ context: metadata.context(0), fragments: Vec::new().into(), current_offset: 0, + encryptor: None, }, PartitionRead::ConsumerOffsetOnPrimary { .. } => PartitionReadReply::ConsumerOffset { stored: Some(0), @@ -1228,11 +1232,12 @@ commit( &mut inner, - &UpdateTopicRequest { + &UpdateTopicResolvedRequest { stream_id: WireIdentifier::named("events").unwrap(), topic_id: WireIdentifier::named("orders").unwrap(), - name: WireName::new("renamed").unwrap(), - options: WireOptions::empty(), + name: Some(WireName::new("renamed").unwrap()), + explicit_options: WireOptions::empty(), + derived_options: WireOptions::empty(), }, ); let replacement_topic = create_topic(&mut inner, "events", "orders", 2); @@ -1243,10 +1248,11 @@ if rename_stream { commit( &mut inner, - &UpdateStreamRequest { + &UpdateStreamResolvedRequest { stream_id: WireIdentifier::named("events").unwrap(), - name: WireName::new("renamed-stream").unwrap(), - options: WireOptions::empty(), + name: Some(WireName::new("renamed-stream").unwrap()), + explicit_options: WireOptions::empty(), + derived_options: WireOptions::empty(), }, ); let replacement_stream = create_stream(&mut inner, "events"); @@ -1629,7 +1635,7 @@ validate_checksum: config.partition.validate_checksum, segment_size: IggyByteSize::from(iggy_common::DEFAULT_SEGMENT_SIZE), preallocate_segments: iggy_common::DEFAULT_PREALLOCATE_SEGMENTS, - encryptor: None, + key_context: Arc::default(), path_layout: PartitionPathLayout::default(), }, );
diff --git a/core/server/src/http/submit.rs b/core/server/src/http/submit.rs index 0171f35..4a5ee9d 100644 --- a/core/server/src/http/submit.rs +++ b/core/server/src/http/submit.rs
@@ -170,15 +170,19 @@ /// [`submit_committed`] then forgets the entry so the caller's retry /// re-registers cleanly. /// -/// Both shard-0 request rewrites run here before consensus, mirroring the TCP +/// The [`http_chain`] stages run here before consensus, mirroring the TCP /// dispatch path: the PAT rewrite mints a raw token and replicates only its hash -/// (`CreatePersonalAccessToken`), and the user-password rewrite hashes the new +/// (`CreatePersonalAccessToken`), the user-password rewrite hashes the new /// password and, for `ChangePassword`, strips the current one and verifies it -/// against the stored hash (`CreateUser` / `ChangePassword`). Both are no-ops -/// for every other operation, so plaintext secrets never enter consensus on any -/// write. A third resolution handles `DeleteSegments`, which is not itself a -/// consensus op: it is rewritten to the metadata `TruncatePartition` that commits -/// the trim (see [`resolve_delete_segments_truncate`]), so the truncate rides +/// against the stored hash (`CreateUser` / `ChangePassword`), the static bounds +/// admit every options block through the catalog, and the secret options +/// rewrite replaces a stream key with its wrap set (`CreateStream`). The +/// header this gateway builds leaves `request_checksum` zero, so no digest of +/// the original body rides the prepare either, and no plaintext secret from +/// an HTTP write enters consensus. A further resolution handles +/// `DeleteSegments`, which is not itself a consensus op: it is rewritten to the +/// metadata `TruncatePartition` that commits the trim (see +/// [`resolve_delete_segments_truncate`]), so the truncate rides /// this session's gate id. A rejection (a malformed body or an unresolved /// delete-segments namespace) still spends the id like any other exit. A failed /// current-password check is not a rejection here: the op still commits with an
diff --git a/core/server/src/lib.rs b/core/server/src/lib.rs index 225410e..a08d83f 100644 --- a/core/server/src/lib.rs +++ b/core/server/src/lib.rs
@@ -54,6 +54,8 @@ pub(crate) mod rewrite; pub(crate) mod session_manager; pub mod shell; +pub(crate) mod stream_key_rewrapper; +pub(crate) mod stream_keys; pub(crate) mod users; pub(crate) mod wire;
diff --git a/core/server/src/partition_helpers.rs b/core/server/src/partition_helpers.rs index 46d2088..a08665b 100644 --- a/core/server/src/partition_helpers.rs +++ b/core/server/src/partition_helpers.rs
@@ -2463,7 +2463,7 @@ validate_checksum: true, segment_size: IggyByteSize::from(1_048_576_u64), preallocate_segments: false, - encryptor: None, + key_context: Arc::default(), path_layout: PartitionPathLayout { streams_root: config.get_streams_path(), },
diff --git a/core/server/src/partition_reconciler.rs b/core/server/src/partition_reconciler.rs index 501fac5..d460832 100644 --- a/core/server/src/partition_reconciler.rs +++ b/core/server/src/partition_reconciler.rs
@@ -166,7 +166,9 @@ AckLevel, Command, Operation, PrepareHeader, ReplyHeader, RoutedRequestHeader, WireConsumer, WireEncode, WireIdentifier, }; -use iggy_common::{ConsumerKind, IggyTimestamp}; +use iggy_common::{ + ConsumerKind, IggyError, IggyTimestamp, StreamEncryption, check_critical_options, +}; use journal::superblock::SuperblockStore; use journal::{Journal, JournalHandle}; use message_bus::AUTO_COMMIT_CLIENT_ID; @@ -177,7 +179,7 @@ RETIRED_PARTITION_OP, }; use metadata::stm::stream::{Partition, StatsRegistry}; -use partitions::{delete_partitions_from_disk, read_created_revision}; +use partitions::{StreamKeyIdentity, delete_partitions_from_disk, read_created_revision}; use server_common::Message; use server_common::sharding::{IggyNamespace, ShardId}; use shard::MetadataSubmit; @@ -258,6 +260,9 @@ /// change. stats_registry: Arc<StatsRegistry>, failure_state: RefCell<AHashMap<(IggyNamespace, FailureCause), FailureRecord>>, + /// Owned namespaces boot left unbuilt, whose files a delete committed + /// before any build must still remove. + unbuilt_owned: RefCell<AHashSet<IggyNamespace>>, /// `Streams::revision` observed at the end of the last pass that fully /// converged. Paired with `last_pass_noop` for the fast-skip in /// [`reconcile_once`] (no O(N) scan when nothing changed). @@ -295,6 +300,7 @@ replica_count, stats_registry, failure_state: RefCell::new(AHashMap::new()), + unbuilt_owned: RefCell::new(AHashSet::new()), last_revision: Cell::new(None), last_pass_noop: Cell::new(false), group_offset_cleanup: GroupOffsetCleanup::default(), @@ -303,6 +309,48 @@ } } + /// Record the owned namespaces boot did not build. + pub fn mark_unbuilt(&self, namespaces: impl IntoIterator<Item = IggyNamespace>) { + self.unbuilt_owned.borrow_mut().extend(namespaces); + } + + /// Owned namespaces that never built and left the committed target. Boot + /// records its own, and a failed build keeps its `Add` record until the + /// removals pass, which runs before stale records are pruned. + fn unbuilt_ghosts(&self, target_set: &AHashSet<IggyNamespace>) -> Vec<IggyNamespace> { + let partitions = self.shard.plane.partitions(); + let mut ghosts = Vec::new(); + self.unbuilt_owned.borrow_mut().retain(|ns| { + if partitions.contains(ns) { + return false; + } + if target_set.contains(ns) { + return true; + } + ghosts.push(*ns); + false + }); + ghosts.extend( + self.failure_state + .borrow() + .keys() + .filter(|(ns, cause)| *cause == FailureCause::Add && !target_set.contains(ns)) + .map(|(ns, _)| *ns), + ); + ghosts.retain(|ns| !partitions.contains(ns) && !partitions.is_tombstoned(ns)); + ghosts.sort_unstable(); + ghosts.dedup(); + ghosts + } + + /// `true` when a recorded failure is due for its retry. + fn has_retry_due(&self, now: Instant) -> bool { + self.failure_state + .borrow() + .values() + .any(|record| record.next_retry_at <= now) + } + fn is_backed_off(&self, ns: IggyNamespace, cause: FailureCause, now: Instant) -> bool { let state = self.failure_state.borrow(); if state.is_empty() { @@ -429,6 +477,8 @@ routed: usize, removed_local: usize, removed_routed: usize, + /// Logged only. A pass that skipped only backed-off namespaces converged + /// until a retry comes due, which the fast-skip checks on its own. backoff_skipped: usize, /// Stale incarnations (slab-key reuse) torn down for rebuild. stale: usize, @@ -466,7 +516,6 @@ + self.routed + self.removed_local + self.removed_routed - + self.backoff_skipped + self.stale + self.cg_offsets_submitted + self.cg_offsets_completed @@ -507,8 +556,8 @@ // Fast-skip: committed partition set unchanged since the last // fully-converged pass and no backoff retry due, so the O(N) diff is // pure waste. Safe because reconcile is level-triggered: the next - // partition-shaping commit bumps `revision`, a pending retry keeps - // `failure_state` non-empty, and a pass that found work it could not + // partition-shaping commit bumps `revision`, a retry coming due forces + // the pass, and a pass that found work it could not // finish (a deferred rebuild, an incomplete trim) leaves // `last_pass_noop` false; any of the three forces the next pass. // @@ -521,7 +570,7 @@ // namespace's legitimate convergence window. if ctx.last_revision.get() == Some(revision) && ctx.last_pass_noop.get() - && ctx.failure_state.borrow().is_empty() + && !ctx.has_retry_due(Instant::now()) && ctx.group_offset_cleanup.completed.get() == 0 && !ctx.shard.has_parked_partition_frames() && ctx.last_group_offset_reconcile_epoch.get() == group_offset_epoch @@ -546,6 +595,7 @@ reconcile_parked_frames(ctx, &mut counters); reconcile_group_offsets(&ctx.shard, &ctx.group_offset_cleanup, &mut counters); reconcile_segment_truncations(ctx, &mut counters); + reconcile_stream_keys(ctx); report_lifecycles(ctx).await; let local_set: AHashSet<IggyNamespace> = @@ -589,6 +639,23 @@ true } +/// Forget the keys of deleted stream incarnations, so a reused stream id can +/// never meet a previous stream's key. +fn reconcile_stream_keys(ctx: &ReconcilerCtx) { + let partitions = ctx.shard.plane.partitions(); + if !partitions.has_stream_keys() { + return; + } + let () = ctx.shard.plane.metadata().mux_stm.streams().read(|inner| { + partitions.retain_stream_keys(|identity| { + inner + .items + .get(identity.stream_id) + .is_some_and(|stream| stream.created_op == identity.created_op) + }); + }); +} + /// Runs on every wake, apart from the full pass. A fence applies, or its /// primary changes, without a `Streams::revision` bump. Each report commit /// also wakes every shard, so a full pass per report grows a delete @@ -755,11 +822,54 @@ // Resolved only for namespaces actually built, not once per committed // partition every pass. A stream, topic or partition that vanished // between the target snapshot and this read defers to the next pass. - let Some((partition_stats, topic_runtime, partition_metadata)) = - fetch_partition_build_inputs(ctx, ns) + let Some(PartitionBuildInputs { + stats: partition_stats, + runtime: topic_runtime, + metadata: partition_metadata, + stream_key, + encryption, + }) = fetch_partition_build_inputs(ctx, ns) else { continue; }; + let topic_runtime = match topic_runtime { + Ok(topic_runtime) => topic_runtime, + Err(error) => { + ctx.record_failure(ns, FailureCause::Add, now); + ctx.shard.metrics().record_partition_reconcile_failure(); + error!( + shard = shard_id, + stream_id = ns.stream_id(), + topic_id = ns.topic_id(), + partition_id = ns.partition_id(), + %error, + "Partition of topic {} cannot be built, its options hold critical state this node cannot read", + ns.topic_id() + ); + continue; + } + }; + // An encrypted partition is built only with its own stream's key. A + // key that does not open here leaves the partition unbuilt, never + // served as plaintext or under another key. + let encryptor = match partitions.resolve_stream_encryptor(stream_key, &encryption) { + Ok(encryptor) => encryptor, + Err(error) => { + ctx.record_failure(ns, FailureCause::Add, now); + ctx.shard.metrics().record_partition_reconcile_failure(); + error!( + shard = shard_id, + stream_id = ns.stream_id(), + topic_id = ns.topic_id(), + partition_id = ns.partition_id(), + kek_fingerprints = %partitions.key_context().kek_fingerprints().join(","), + %error, + "Partition of stream {} cannot be built, its key does not open on this node", + ns.stream_id() + ); + continue; + } + }; // A directory already on disk is a prior life of this namespace. The // usual shape: this replica applied the create before a crash, but its @@ -836,7 +946,8 @@ .map(Some) }; match built { - Ok(Some(partition)) => { + Ok(Some(mut partition)) => { + partition.set_encryptor(encryptor); ctx.shard.enqueue_reconcile_op(ReconcileOp::InsertOwned { namespace: ns, partition: Box::new(partition), @@ -988,6 +1099,13 @@ tear_down_owned_partition(ctx, ns, counters).await; } + // A namespace deferred for its stream key or left by a failed build has + // files on disk but no partition and no tombstone, so neither loop above + // reaches it once a delete commits. + for ns in ctx.unbuilt_ghosts(target_set) { + tear_down_owned_partition(ctx, ns, counters).await; + } + // Skip namespaces still locally owned (disk-delete-failed ghosts): // pruning their shards_table row would strand peer routing. let still_owned: AHashSet<IggyNamespace> = partitions.namespaces().copied().collect(); @@ -1531,6 +1649,17 @@ .remove_partitions(ns.stream_id(), ns.topic_id(), &[ns.partition_id()]); } +/// What building one partition reads from committed metadata. +struct PartitionBuildInputs { + stats: Arc<iggy_common::PartitionStats>, + /// `UnsupportedCriticalOption` when the topic holds critical state this + /// build cannot read. + runtime: Result<iggy_common::TopicRuntimeOptions, IggyError>, + metadata: Partition, + stream_key: StreamKeyIdentity, + encryption: StreamEncryption, +} + /// Everything one namespace's build needs out of committed metadata: the shared /// `Arc<PartitionStats>`, the topic's runtime options, and the committed /// [`Partition`] record the disk loader reads `created_at` / `created_revision` @@ -1549,11 +1678,7 @@ fn fetch_partition_build_inputs( ctx: &ReconcilerCtx, ns: IggyNamespace, -) -> Option<( - Arc<iggy_common::PartitionStats>, - iggy_common::TopicRuntimeOptions, - Partition, -)> { +) -> Option<PartitionBuildInputs> { ctx.shard.plane.metadata().mux_stm.streams().read(|inner| { let stream = inner.items.get(ns.stream_id())?; let topic = stream.topics.get(ns.topic_id())?; @@ -1564,16 +1689,22 @@ .clone(); // Get-or-create in the shared registry so the owning shard's counters // are the same `Arc` every shard's `get_topic` reply reads. - Some(( - inner.stats_registry.partition( + Some(PartitionBuildInputs { + stats: inner.stats_registry.partition( ns.stream_id(), ns.topic_id(), &partition, topic.stats.clone(), ), - iggy_common::TopicRuntimeOptions::from_resource_options(&topic.options), - partition, - )) + runtime: check_critical_options(&topic.options) + .map(|()| iggy_common::TopicRuntimeOptions::from_resource_options(&topic.options)), + metadata: partition, + stream_key: StreamKeyIdentity { + stream_id: stream.id, + created_op: stream.created_op, + }, + encryption: StreamEncryption::from(&stream.options), + }) }) } @@ -1895,6 +2026,7 @@ reconcile_partition_lifecycles, reconcile_pending_revocations, reconcile_retired_lifecycles, }; + use bytes::Bytes; use configs::server::ServerConfig; use consensus::{MetadataHandle, PartitionsHandle, Sequencer, StateTransferStage}; use iggy_binary_protocol::codec::{WireDecode, WireEncode}; @@ -1906,7 +2038,9 @@ CreatePartitionsRequest, CreatePartitionsWithAssignmentsRequest, RetireConsumerGroupOwnersRequest, }; - use iggy_binary_protocol::requests::streams::{CreateStreamRequest, DeleteStreamRequest}; + use iggy_binary_protocol::requests::streams::{ + CreateStreamRequest, DeleteStreamRequest, RewrapStreamKeyEntry, RewrapStreamKeysRequest, + }; use iggy_binary_protocol::requests::topics::{ CreateTopicRequest, CreateTopicWithAssignmentsRequest, DeleteTopicRequest, }; @@ -1914,6 +2048,11 @@ AckLevel, Command, Operation, PrepareHeader, RepairRangeReplyHeader, RoutedRequestHeader, WireConsumer, WireIdentifier, WireOptions, }; + use iggy_common::wire_conversions::resource_options_to_wire; + use iggy_common::{ + HeaderKey, HeaderValue, OptionKey, OptionValue, OptionsProvenance, ResourceOptions, + ServerKeyContext, StreamEncryptionKey, StreamKeyOrigin, WrapSet, critical_keys_marker, + }; use message_bus::IggyMessageBus; use metadata::IggyMetadata; use metadata::MuxStateMachine; @@ -1932,6 +2071,7 @@ use shard::{IggyShard, PartitionConsensusConfig, ReconcileOp, ShardIdentity}; use std::cell::Cell; use std::mem::size_of; + use std::path::Path; use std::rc::Rc; use std::sync::Arc; use std::sync::atomic::Ordering; @@ -1948,6 +2088,8 @@ >; const CLUSTER_ID: u128 = 1; + const CONFIG_KEY: &str = "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE="; + const OTHER_CONFIG_KEY: &str = "AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA="; /// Sanity test that ensures the `()` channel can coalesce wakes /// without blocking the producer when the consumer hasn't drained @@ -2098,6 +2240,46 @@ .expect("CreateStream apply succeeds"); } + /// [`seed_stream`] for an encrypted stream, in the committed form the edge + /// writes once it has wrapped the stream's key. + fn seed_encrypted_stream(mux: &TestMux, op: u64, name: &str, set: &WrapSet) { + let options = ResourceOptions::from([ + ( + HeaderKey::from(OptionKey::EncryptionKeyWrapped), + OptionValue::explicit(HeaderValue::try_from(set.as_ref()).expect("set fits")), + ), + ( + HeaderKey::from(OptionKey::CriticalKeys), + OptionValue::explicit( + critical_keys_marker(&[OptionKey::EncryptionKeyWrapped]).expect("marker"), + ), + ), + ]); + let req = CreateStreamRequest { + name: WireName::new(name).expect("test stream name fits WireName"), + options: resource_options_to_wire(&options, OptionsProvenance::All) + .expect("options encode"), + }; + mux.update(build_prepare(op, Operation::CreateStream, &req)) + .expect("CreateStream apply succeeds"); + } + + fn key_context(config_key: &str) -> Arc<ServerKeyContext> { + Arc::new(ServerKeyContext::from_config(config_key, &[], true).expect("valid config key")) + } + + fn wrap_stream_key(context: &ServerKeyContext, seed: u8) -> WrapSet { + let key = StreamEncryptionKey::try_from([seed; 32].as_slice()).expect("32 bytes"); + WrapSet::from(context.wrap(&key, StreamKeyOrigin::Server).expect("wraps")) + } + + fn served_encrypted(shard: &TestShard, ns: IggyNamespace) -> Option<bool> { + shard + .plane + .partitions() + .with_partition(&ns, |partition| partition.encryptor().is_some()) + } + /// Drive a `CreateTopicWithAssignments` commit. fn seed_topic( mux: &TestMux, @@ -2386,6 +2568,16 @@ /// Uses `IggyShard::without_inbox` so no inter-shard pump runs; the /// reconciler can be driven directly by `reconcile_once`. fn build_test_shard(shard_id: u16, config: &ServerConfig, mux: TestMux) -> Rc<TestShard> { + build_test_shard_with_keys(shard_id, config, mux, Arc::default()) + } + + /// [`build_test_shard`] on a node holding `key_context`. + fn build_test_shard_with_keys( + shard_id: u16, + config: &ServerConfig, + mux: TestMux, + key_context: Arc<ServerKeyContext>, + ) -> Rc<TestShard> { let bus = Rc::new(IggyMessageBus::with_config(shard_id, config)); let metadata: IggyMetadata< consensus::VsrConsensus<Rc<IggyMessageBus>>, @@ -2401,7 +2593,7 @@ validate_checksum: true, segment_size: iggy_common::IggyByteSize::from(iggy_common::DEFAULT_SEGMENT_SIZE), preallocate_segments: false, - encryptor: None, + key_context, // The reconciler reads segment and offset paths off this layout, // so it must root where `test_config` rooted the server's own. path_layout: PartitionPathLayout { @@ -2637,6 +2829,211 @@ ); } + #[compio::test] + async fn given_topic_with_unknown_critical_option_when_reconciling_should_not_build_its_partition() + { + let tmp = TempDir::new().expect("tempdir for system path"); + let config = test_config(&tmp); + let mux = TestMux::default(); + seed_stream(&mux, 1, "stream-a"); + let unreadable = ResourceOptions::from([ + ( + HeaderKey::from(OptionKey::CriticalKeys), + OptionValue::explicit(HeaderValue::try_from("a_future_key").expect("marker fits")), + ), + ( + HeaderKey::try_from("a_future_key").expect("key fits"), + OptionValue::explicit(HeaderValue::from(7u64)), + ), + ]); + let request = CreateTopicWithAssignmentsRequest { + request: CreateTopicRequest { + stream_id: WireIdentifier::numeric(0), + partitions_count: 1, + name: WireName::new("topic-a").expect("test topic name fits WireName"), + options: WireOptions::empty(), + }, + derived_options: resource_options_to_wire(&unreadable, OptionsProvenance::All) + .expect("options encode"), + partitions: vec![assignment(0, 1)], + }; + mux.update(build_prepare( + 2, + Operation::CreateTopicWithAssignments, + &request, + )) + .expect("CreateTopicWithAssignments apply succeeds"); + let shard = build_test_shard(0, &config, mux); + let ctx = make_ctx(Rc::clone(&shard), 1, Rc::new(config)); + + reconcile_pass(&ctx).await; + + assert!( + !shard + .plane + .partitions() + .contains(&IggyNamespace::new(0, 0, 0)), + "a topic this build cannot read must not be served" + ); + } + + #[compio::test] + async fn given_stream_key_that_does_not_open_when_reconciling_should_not_build_its_partition() { + let stream_context = key_context(CONFIG_KEY); + let ns = IggyNamespace::new(0, 0, 0); + for (node_context, opens) in [ + (Arc::clone(&stream_context), true), + (key_context(OTHER_CONFIG_KEY), false), + ] { + let tmp = TempDir::new().expect("tempdir for system path"); + let config = test_config(&tmp); + let partition_dir = config.get_partition_path(0, 0, 0); + let mux = TestMux::default(); + seed_encrypted_stream(&mux, 1, "stream-a", &wrap_stream_key(&stream_context, 1)); + seed_topic(&mux, 2, 0, "topic-a", vec![assignment(0, 1)]); + let shard = build_test_shard_with_keys(0, &config, mux, node_context); + let ctx = make_ctx(Rc::clone(&shard), 1, Rc::new(config)); + + reconcile_pass(&ctx).await; + + assert_eq!( + served_encrypted(&shard, ns), + opens.then_some(true), + "a partition is served only under its own stream's key" + ); + assert_eq!( + ctx.failure_state + .borrow() + .contains_key(&(ns, FailureCause::Add)), + !opens + ); + assert_eq!(Path::new(&partition_dir).exists(), opens); + } + } + + #[compio::test] + async fn given_only_backed_off_add_when_reconciling_should_skip_until_retry_is_due() { + let tmp = TempDir::new().expect("tempdir for system path"); + let config = test_config(&tmp); + let stream_context = key_context(CONFIG_KEY); + let node_context = key_context(OTHER_CONFIG_KEY); + let mux = TestMux::default(); + seed_encrypted_stream(&mux, 1, "stream-a", &wrap_stream_key(&stream_context, 1)); + seed_topic(&mux, 2, 0, "topic-a", vec![assignment(0, 1)]); + let shard = build_test_shard_with_keys(0, &config, mux, node_context); + let ctx = make_ctx(shard, 1, Rc::new(config)); + + assert!(reconcile_once(&ctx).await); + assert!(!ctx.failure_state.borrow().is_empty()); + assert!(!reconcile_once(&ctx).await); + + for record in ctx.failure_state.borrow_mut().values_mut() { + record.next_retry_at = Instant::now(); + } + assert!(reconcile_once(&ctx).await); + } + + #[compio::test] + async fn given_unbuilt_encrypted_partition_when_its_stream_is_deleted_should_remove_its_files() + { + for boot_deferred in [false, true] { + let tmp = TempDir::new().expect("tempdir for system path"); + let config = test_config(&tmp); + let partition_dir = config.get_partition_path(0, 0, 0); + let stream_context = key_context(CONFIG_KEY); + let mux = TestMux::default(); + seed_encrypted_stream(&mux, 1, "stream-a", &wrap_stream_key(&stream_context, 1)); + seed_topic(&mux, 2, 0, "topic-a", vec![assignment(0, 1)]); + let shard = build_test_shard_with_keys(0, &config, mux, key_context(OTHER_CONFIG_KEY)); + let ctx = make_ctx(Rc::clone(&shard), 1, Rc::new(config)); + let ns = IggyNamespace::new(0, 0, 0); + std::fs::create_dir_all(&partition_dir).expect("prior partition files"); + if boot_deferred { + ctx.mark_unbuilt([ns]); + } else { + reconcile_pass(&ctx).await; + assert!( + ctx.failure_state + .borrow() + .contains_key(&(ns, FailureCause::Add)) + ); + } + + seed_delete_stream(&shard.plane.metadata().mux_stm, 3, 0); + reconcile_pass(&ctx).await; + + assert!( + !Path::new(&partition_dir).exists(), + "boot deferred {boot_deferred}: a deleted partition that never built keeps no files" + ); + assert!(!shard.plane.partitions().is_tombstoned(&ns)); + } + } + + #[compio::test] + async fn given_rewrap_committed_after_a_failed_build_when_reconciling_should_build_under_the_new_wrap() + { + let tmp = TempDir::new().expect("tempdir for system path"); + let config = test_config(&tmp); + let old_context = key_context(CONFIG_KEY); + let new_context = key_context(OTHER_CONFIG_KEY); + let stored = wrap_stream_key(&old_context, 1); + let mux = TestMux::default(); + seed_encrypted_stream(&mux, 1, "stream-a", &stored); + seed_topic(&mux, 2, 0, "topic-a", vec![assignment(0, 1)]); + let shard = build_test_shard_with_keys(0, &config, mux, Arc::clone(&new_context)); + let ctx = make_ctx(Rc::clone(&shard), 1, Rc::new(config)); + let ns = IggyNamespace::new(0, 0, 0); + + reconcile_pass(&ctx).await; + assert_eq!(served_encrypted(&shard, ns), None); + + let mux = &shard.plane.metadata().mux_stm; + let created_op = mux + .streams() + .read(|inner| inner.items.get(0).map(|stream| stream.created_op)) + .expect("stream exists"); + let rewrap = RewrapStreamKeysRequest { + entries: vec![RewrapStreamKeyEntry { + stream_id: 0, + created_op, + key_fingerprint: stored + .key_fingerprint_bytes() + .try_into() + .expect("8-byte fingerprint"), + wrap_set: Bytes::copy_from_slice(wrap_stream_key(&new_context, 1).as_ref()), + }], + }; + mux.update(build_prepare(3, Operation::RewrapStreamKeys, &rewrap)) + .expect("RewrapStreamKeys apply succeeds"); + for record in ctx.failure_state.borrow_mut().values_mut() { + record.next_retry_at = Instant::now(); + } + reconcile_pass(&ctx).await; + + assert_eq!(served_encrypted(&shard, ns), Some(true)); + } + + #[compio::test] + async fn given_deleted_encrypted_stream_when_reconciling_should_forget_its_key() { + let tmp = TempDir::new().expect("tempdir for system path"); + let config = test_config(&tmp); + let context = key_context(CONFIG_KEY); + let mux = TestMux::default(); + seed_encrypted_stream(&mux, 1, "stream-a", &wrap_stream_key(&context, 1)); + seed_topic(&mux, 2, 0, "topic-a", vec![assignment(0, 1)]); + let shard = build_test_shard_with_keys(0, &config, mux, context); + let ctx = make_ctx(Rc::clone(&shard), 1, Rc::new(config)); + + reconcile_pass(&ctx).await; + assert!(shard.plane.partitions().has_stream_keys()); + + seed_delete_stream(&shard.plane.metadata().mux_stm, 3, 0); + reconcile_pass(&ctx).await; + + assert!(!shard.plane.partitions().has_stream_keys()); + } + /// A delete and a recreate of the same ids can commit while a pass runs, /// so the pass holds an older epoch than the committed partition. The /// directory of the committed incarnation must survive that deferral, @@ -2652,8 +3049,9 @@ let shard = build_test_shard(0, &config, mux); let ctx = make_ctx(Rc::clone(&shard), 1, Rc::new(config)); let ns = IggyNamespace::new(0, 0, 0); - let (_, _, partition) = - fetch_partition_build_inputs(&ctx, ns).expect("committed namespace resolves"); + let partition = fetch_partition_build_inputs(&ctx, ns) + .expect("committed namespace resolves") + .metadata; let snapshot_epoch = partition .created_revision .checked_sub(1) @@ -2749,8 +3147,9 @@ reconcile_once(&ctx).await; ctx.shard.apply_reconcile_ops(); - let (stats, ..) = - fetch_partition_build_inputs(&ctx, ns).expect("materialised namespace has stats"); + let stats = fetch_partition_build_inputs(&ctx, ns) + .expect("materialised namespace has stats") + .stats; stats.increment_size_bytes(512); seed_delete_topic(&ctx.shard.plane.metadata().mux_stm, 3, 0, 0); @@ -2790,8 +3189,9 @@ reconcile_once(&ctx).await; ctx.shard.apply_reconcile_ops(); - let (stats, _, partition) = + let inputs = fetch_partition_build_inputs(&ctx, ns).expect("materialised namespace has stats"); + let (stats, partition) = (inputs.stats, inputs.metadata); let topic_stats = stats.parent(); seed_delete_topic(&ctx.shard.plane.metadata().mux_stm, 3, 0, 0); @@ -2879,8 +3279,9 @@ // `ensure_initial_segment` plants exactly one segment per build and // folds it into the namespace's shared stats, so this counter is the // observable that separates them. - let (stats, ..) = - fetch_partition_build_inputs(&ctx, ns).expect("materialised namespace has stats"); + let stats = fetch_partition_build_inputs(&ctx, ns) + .expect("materialised namespace has stats") + .stats; assert_eq!( stats.segments_count_inconsistent(), 1, @@ -2917,8 +3318,9 @@ let ctx = make_ctx(Rc::clone(&shard), 1, Rc::new(config.clone())); let ns = IggyNamespace::new(0, 0, 0); - let (stats, ..) = - fetch_partition_build_inputs(&ctx, ns).expect("committed namespace has stats"); + let stats = fetch_partition_build_inputs(&ctx, ns) + .expect("committed namespace has stats") + .stats; let live = build_partition_fresh( &config, ctx.shard.plane.partitions().config(),
diff --git a/core/server/src/reply_frame.rs b/core/server/src/reply_frame.rs index 2d3f3ed..e15fa93 100644 --- a/core/server/src/reply_frame.rs +++ b/core/server/src/reply_frame.rs
@@ -577,13 +577,10 @@ #[cfg(test)] mod tests { use super::*; + use crate::dispatch::test_support::stored_batch_record; use iggy_binary_protocol::{Operation, WireDecode}; use iggy_common::Aes256GcmEncryptor; - use server_common::send_messages::{ - BatchHeader, COMMAND_HEADER_SIZE, IggyMessage, IggyMessageHeader, IggyMessages, - PREPARE_SPLIT_POINT, SendMessagesOwned, encrypt_batch_request, frozen_batch_header, - }; - use server_common::sharding::IggyNamespace; + use server_common::send_messages::{BatchHeader, COMMAND_HEADER_SIZE, frozen_batch_header}; fn pat_request_header() -> RoutedRequestHeader { let zeroed = [0u8; std::mem::size_of::<RoutedRequestHeader>()]; @@ -867,52 +864,12 @@ )); } - /// A stored record encrypted the way the primary encrypts at ingestion. - fn encrypted_record(encryptor: &EncryptorKind) -> Frozen<MESSAGE_ALIGN> { - let namespace = IggyNamespace::new(1, 1, 3); - let mut messages = IggyMessages::with_capacity(2); - for (id, payload) in [(7u128, &b"first-payload"[..]), (8, &b"second-payload"[..])] { - messages.push(IggyMessage { - header: IggyMessageHeader { - id, - origin_timestamp: 1_000, - ..Default::default() - }, - payload: Bytes::copy_from_slice(payload), - user_headers: None, - }); - } - let owned = SendMessagesOwned::from_messages(namespace, &messages).expect("build batch"); - let header_size = std::mem::size_of::<RoutedRequestHeader>(); - let total = header_size + owned.header.total_size(); - let mut buffer = Owned::<MESSAGE_ALIGN>::zeroed(total); - { - let header: &mut RoutedRequestHeader = - bytemuck::checked::try_from_bytes_mut(&mut buffer.as_mut_slice()[..header_size]) - .expect("zeroed bytes form a valid RoutedRequestHeader"); - header.command = Command::Request; - header.operation = Operation::SendMessages; - header.client = 1; - header.session = 1; - header.request = 1; - header.size = u32::try_from(total).expect("size fits u32"); - } - let bytes = buffer.as_mut_slice(); - owned - .header - .encode_into(&mut bytes[header_size..header_size + COMMAND_HEADER_SIZE]); - bytes[PREPARE_SPLIT_POINT..].copy_from_slice(&owned.blob); - let canonical = Message::try_from(buffer).expect("request message is valid"); - let encrypted = encrypt_batch_request(canonical, encryptor).expect("encrypt batch"); - let record = &encrypted.as_slice()[header_size..encrypted.header().size as usize]; - Owned::<MESSAGE_ALIGN>::copy_from_slice(record).into() - } - #[test] fn polled_reply_encrypted_records_take_the_flattening_path() { let encryptor = EncryptorKind::Aes256Gcm(Aes256GcmEncryptor::new(&[7u8; 32]).expect("valid 32B key")); - let fragments = PollFragments::from_iter([Fragment::whole(encrypted_record(&encryptor))]); + let fragments = + PollFragments::from_iter([Fragment::whole(stored_batch_record(Some(&encryptor)))]); let expected = flattened_reply(fragments.clone(), Some(&encryptor)); let reply = vectored_reply(fragments, Some(&encryptor)).expect("decrypting reply"); assert_eq!(reply.fragments().len(), 1);
diff --git a/core/server/src/responses.rs b/core/server/src/responses.rs index e4ddca4..7a8daca 100644 --- a/core/server/src/responses.rs +++ b/core/server/src/responses.rs
@@ -59,30 +59,98 @@ GetStreamResponse, TopicHeader as StreamTopicHeader, }; use iggy_binary_protocol::responses::streams::get_streams::GetStreamsResponse; +use iggy_binary_protocol::responses::system::DescribeOptionsResponse; use iggy_binary_protocol::responses::system::get_cluster_metadata::{ ClusterMetadataResponse, ClusterNodeResponse, }; use iggy_binary_protocol::responses::system::get_stats::StatsResponse; -use iggy_binary_protocol::responses::system::{DescribeOptionsResponse, OptionDescriptor}; use iggy_binary_protocol::responses::topics::get_topic::{GetTopicResponse, PartitionResponse}; use iggy_binary_protocol::responses::topics::get_topics::GetTopicsResponse; use iggy_binary_protocol::responses::users::get_user::UserDetailsResponse; use iggy_binary_protocol::responses::users::get_users::GetUsersResponse; use iggy_binary_protocol::responses::users::user_response::UserResponse; use iggy_binary_protocol::{ - ReplyHeader, RoutedRequestHeader, WireDecode, WireEncode, WireIdentifier, WireName, + ReplyHeader, RoutedRequestHeader, WireDecode, WireEncode, WireIdentifier, WireName, WireOptions, }; -use iggy_common::wire_conversions::{resource_options_to_wire, resource_options_to_wire_split}; -use iggy_common::{HeaderKind, IggyError, IggyTimestamp, OptionsProvenance, topic_option_keys}; +use iggy_common::wire_conversions::{describe_options_response, resource_options_to_wire_split}; +use iggy_common::{ + IggyError, IggyTimestamp, OptionsScope, ResourceOptions, UserStatus, public_options, +}; use journal::superblock::SuperblockStore; use journal::{Journal, JournalHandle}; use metadata::impls::metadata::StreamsFrontend; +use metadata::stm::stream::{Stream, StreamsInner, Topic}; +use metadata::stm::user::User; use server_common::Message; use shard::ConnectedClientInfo; use std::cell::RefCell; use std::net::IpAddr; use std::rc::Rc; use std::sync::Arc; +use tracing::warn; + +/// The stored fields a stream or topic read returns. Reads copy them under the +/// metadata read guard and project the options after it, because the writer +/// publishing the next apply waits for every reader to leave. +struct StoredEntry { + id: usize, + created_at: IggyTimestamp, + /// Topics of a stream, partitions of a topic. + children_count: usize, + size_bytes: u64, + messages_count: u64, + name: Arc<str>, + options: ResourceOptions, +} + +/// The stored fields a user read returns, copied like [`StoredEntry`]. +struct StoredUser { + id: u32, + created_at: IggyTimestamp, + status: UserStatus, + username: Arc<str>, + options: ResourceOptions, +} + +impl From<&Stream> for StoredEntry { + fn from(stream: &Stream) -> Self { + Self { + id: stream.id, + created_at: stream.created_at, + children_count: stream.topics.len(), + size_bytes: stream.stats.size_bytes_inconsistent(), + messages_count: stream.stats.messages_count_inconsistent(), + name: Arc::clone(&stream.name), + options: stream.options.clone(), + } + } +} + +impl From<&Topic> for StoredEntry { + fn from(topic: &Topic) -> Self { + Self { + id: topic.id, + created_at: topic.created_at, + children_count: topic.partitions.len(), + size_bytes: topic.stats.size_bytes_inconsistent(), + messages_count: topic.stats.messages_count_inconsistent(), + name: Arc::clone(&topic.name), + options: topic.options.clone(), + } + } +} + +impl From<&User> for StoredUser { + fn from(user: &User) -> Self { + Self { + id: user.id, + created_at: user.created_at, + status: user.status, + username: Arc::clone(&user.username), + options: user.options.clone(), + } + } +} /// Build the `get_me` reply for the requesting connection. Identity /// (`user_id`, transport kind, peer address) comes from the per-shard @@ -560,23 +628,29 @@ S: 'static, SB: SuperblockStore + 'static, { - shard.plane.metadata().mux_stm.streams().read(|streams| { - let Some(stream_id) = streams.resolve_stream_id(stream_id) else { - return Ok(None); - }; - let stream = streams - .items - .get(stream_id) - .ok_or(IggyError::InvalidIdentifier)?; - Ok(Some(GetStreamResponse { - stream: stream_response(stream)?, - topics: stream - .topics - .iter() - .map(|(_, topic)| topic_header(topic)) - .collect::<Result<Vec<_>, _>>()?, - })) - }) + let stored = + shard + .plane + .metadata() + .mux_stm + .streams() + .read(|streams| -> Result<_, IggyError> { + let Some(stream_id) = streams.resolve_stream_id(stream_id) else { + return Ok(None); + }; + let stream = streams + .items + .get(stream_id) + .ok_or(IggyError::InvalidIdentifier)?; + Ok(Some((StoredEntry::from(stream), stored_topics(stream)))) + })?; + let Some((stream, topics)) = stored else { + return Ok(None); + }; + Ok(Some(GetStreamResponse { + stream: stream_response(&stream)?, + topics: listed_topics(stream.id, &topics)?, + })) } fn build_get_streams_response<B, MJ, S, SB>( @@ -589,155 +663,120 @@ S: 'static, SB: SuperblockStore + 'static, { - shard.plane.metadata().mux_stm.streams().read(|streams| { - streams - .items - .iter() - .map(|(_, stream)| stream_response(stream)) - .collect::<Result<Vec<_>, _>>() - .map(|streams| GetStreamsResponse { streams }) - }) + let streams = shard + .plane + .metadata() + .mux_stm + .streams() + .read(stored_streams); + listed_streams(&streams).map(|streams| GetStreamsResponse { streams }) } -/// Every key `CreateTopic` accepts, with the kind, default and bounds of each. -/// -/// Split out of [`build_describe_options_response`] so the descriptions have room -/// to state the bounds each value is checked against: this catalog is the only -/// place an operator learns them. -/// -/// Every default is a build constant: these knobs stopped being config-derived -/// when the `[system.*]` keys became topic options, so the catalog reads them -/// straight from `iggy_common`. -#[allow(clippy::too_many_lines)] -fn topic_option_descriptors() -> Result<Vec<OptionDescriptor>, IggyError> { - Ok(vec![ - OptionDescriptor { - key: WireName::new(topic_option_keys::COMPRESSION_ALGORITHM) - .map_err(|_| IggyError::InvalidFormat)?, - kind: HeaderKind::String.as_code(), - default_value: Bytes::from_static(b"none"), - description: "Compression algorithm (none, gzip); stored and reported, not applied to messages".to_string(), - }, - OptionDescriptor { - key: WireName::new(topic_option_keys::MESSAGE_EXPIRY) - .map_err(|_| IggyError::InvalidFormat)?, - kind: HeaderKind::Uint64.as_code(), - default_value: Bytes::copy_from_slice( - &iggy_common::DEFAULT_MESSAGE_EXPIRY.to_le_bytes(), - ), - description: "Message expiry in microseconds, or a humantime string \ - (e.g. 7 days)" - .to_string(), - }, - OptionDescriptor { - key: WireName::new(topic_option_keys::MAX_TOPIC_SIZE) - .map_err(|_| IggyError::InvalidFormat)?, - kind: HeaderKind::Uint64.as_code(), - default_value: Bytes::copy_from_slice( - &iggy_common::DEFAULT_MAX_TOPIC_SIZE.to_le_bytes(), - ), - description: "Topic-wide sealed-segment size cap, split across all partitions, in bytes \ - or a byte-size string (e.g. 1 GiB); finite values must be at least the segment size" - .to_string(), - }, - OptionDescriptor { - key: WireName::new(topic_option_keys::SEGMENT_SIZE) - .map_err(|_| IggyError::InvalidFormat)?, - kind: HeaderKind::Uint64.as_code(), - default_value: Bytes::copy_from_slice(&iggy_common::DEFAULT_SEGMENT_SIZE.to_le_bytes()), - description: format!( - "Segment size in bytes, or a byte-size string (e.g. 128 MiB); a 512-byte \ - multiple within {}..={}", - iggy_common::MIN_TOPIC_SEGMENT_SIZE, - iggy_common::MAX_TOPIC_SEGMENT_SIZE - ), - }, - OptionDescriptor { - key: WireName::new(topic_option_keys::DURABILITY).map_err(|_| IggyError::InvalidFormat)?, - kind: HeaderKind::String.as_code(), - default_value: Bytes::from_static(b"replicated"), - description: "Message completion: replicated or persisted. Independently defaults to replicated. A singleton quorum has one copy. In replicated groups, persisted messages use WAL references to segment bodies, retaining their inodes by hard link until WAL reclamation. The full body size still counts against partition.wal_bytes_max.".to_string(), - }, - OptionDescriptor { - key: WireName::new(topic_option_keys::CONSUMER_OFFSET_DURABILITY).map_err(|_| IggyError::InvalidFormat)?, - kind: HeaderKind::String.as_code(), - default_value: Bytes::from_static(b"replicated"), - description: "Explicit offset completion: replicated or persisted. Independently defaults to replicated. Poll auto-commit remains asynchronous. In replicated groups, persisted offsets also enable WAL references to segment bodies, retaining their inodes by hard link until reclamation, even with replicated message durability. Full body sizes count against partition.wal_bytes_max.".to_string(), - }, - OptionDescriptor { - key: WireName::new(topic_option_keys::PARTITION_RESIZE_POLICY) - .map_err(|_| IggyError::InvalidFormat)?, - kind: HeaderKind::String.as_code(), - default_value: Bytes::from_static(b"mutable"), - description: "Immutable after creation: mutable permits partition addition/deletion without a bounded cutover or per-key ordering across layouts; fixed rejects both. Retention and topic deletion remain destructive.".to_string(), - }, - OptionDescriptor { - key: WireName::new(topic_option_keys::MESSAGES_REQUIRED_TO_SAVE) - .map_err(|_| IggyError::InvalidFormat)?, - kind: HeaderKind::Uint32.as_code(), - default_value: Bytes::copy_from_slice( - &iggy_common::DEFAULT_MESSAGES_REQUIRED_TO_SAVE.to_le_bytes(), - ), - description: format!( - "Ordinary message-count flush trigger; 1..={}. Required persistence, \ - capacity pressure, and lifecycle operations can flush earlier", - iggy_common::MAX_MESSAGES_REQUIRED_TO_SAVE - ), - }, - OptionDescriptor { - key: WireName::new(topic_option_keys::SIZE_OF_MESSAGES_REQUIRED_TO_SAVE) - .map_err(|_| IggyError::InvalidFormat)?, - kind: HeaderKind::Uint64.as_code(), - default_value: Bytes::copy_from_slice( - &iggy_common::DEFAULT_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE.to_le_bytes(), - ), - description: format!( - "Flush the journal once it holds this many bytes, or a byte-size \ - string; whichever threshold trips first flushes. At most {}", - iggy_common::MAX_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE - ), - }, - OptionDescriptor { - key: WireName::new(topic_option_keys::PREALLOCATE_SEGMENTS) - .map_err(|_| IggyError::InvalidFormat)?, - kind: HeaderKind::Bool.as_code(), - default_value: Bytes::copy_from_slice(&[u8::from( - iggy_common::DEFAULT_PREALLOCATE_SEGMENTS, - )]), - description: format!( - "Request segment_size bytes of disk reservation when each segment opens. \ - Unsupported or failed reservations fall back to ordinary allocation \ - with a warning. Reservation runs inline on the owning shard. \ - segment_size * partitions_count is capped at {} bytes per create", - iggy_common::MAX_PREALLOCATED_TOPIC_BYTES - ), - }, - ]) +fn stored_streams(streams: &StreamsInner) -> Vec<StoredEntry> { + streams + .items + .iter() + .map(|(_, stream)| StoredEntry::from(stream)) + .collect() +} + +fn stored_topics(stream: &Stream) -> Vec<StoredEntry> { + stream + .topics + .iter() + .map(|(_, topic)| StoredEntry::from(topic)) + .collect() +} + +/// Every stream a list read serves, see [`servable`]. +fn listed_streams(streams: &[StoredEntry]) -> Result<Vec<StreamResponse>, IggyError> { + streams + .iter() + .filter_map(|stream| { + servable( + stream_response(stream), + OptionsScope::Stream, + stream.id, + None, + ) + .transpose() + }) + .collect() +} + +/// Every topic of stream `stream_id` a list read serves, see [`servable`]. +fn listed_topics( + stream_id: usize, + topics: &[StoredEntry], +) -> Result<Vec<StreamTopicHeader>, IggyError> { + topics + .iter() + .filter_map(|topic| { + servable( + topic_header(topic), + OptionsScope::Topic, + topic.id, + Some(stream_id), + ) + .transpose() + }) + .collect() +} + +/// A list entry, or `None` for a resource whose stored critical state this +/// build cannot read. That resource still fails its own read, but it must not +/// fail the list every other resource is served in. +fn servable<T>( + entry: Result<T, IggyError>, + scope: OptionsScope, + resource_id: usize, + stream_id: Option<usize>, +) -> Result<Option<T>, IggyError> { + match entry { + Ok(entry) => Ok(Some(entry)), + Err(IggyError::UnsupportedCriticalOption(key)) => { + warn!( + scope = %scope, + resource_id, + stream_id, + "Leaving {scope} {resource_id} out of the list, its critical option {key} is not readable by this build." + ); + Ok(None) + } + Err(error) => Err(error), + } } /// Serve the option catalog for one resource scope. -/// -/// Streams and users have no catalog keys yet, so their scopes return empty -/// (every key is rejected at create until one lands). fn build_describe_options_response(body: &[u8]) -> Result<DescribeOptionsResponse, IggyError> { let request = DescribeOptionsRequest::decode_from(body).map_err(|_| IggyError::InvalidCommand)?; - let entries = match request.scope { - OPTIONS_SCOPE_TOPIC => topic_option_descriptors()?, - OPTIONS_SCOPE_STREAM | OPTIONS_SCOPE_USER => Vec::new(), + let scope = match request.scope { + OPTIONS_SCOPE_TOPIC | OPTIONS_SCOPE_STREAM | OPTIONS_SCOPE_USER => { + OptionsScope::from_code(request.scope)? + } _ => return Err(IggyError::InvalidCommand), }; - Ok(DescribeOptionsResponse { entries }) + describe_options_response(scope) } -#[allow(clippy::cast_possible_truncation)] -fn user_response(user: &metadata::stm::user::User) -> Result<UserResponse, IggyError> { +/// The explicit and derived blocks a read returns for `stored`. +fn public_blocks( + scope: OptionsScope, + stored: &ResourceOptions, +) -> Result<(WireOptions, WireOptions), IggyError> { + resource_options_to_wire_split(&public_options(scope, stored)?) +} + +fn user_response(user: &StoredUser) -> Result<UserResponse, IggyError> { + let (options, derived_options) = public_blocks(OptionsScope::User, &user.options)?; Ok(UserResponse { id: user.id, created_at: user.created_at.as_micros(), status: user.status.as_code(), username: WireName::new(user.username.as_ref()).map_err(|_| IggyError::InvalidFormat)?, - options: resource_options_to_wire(&user.options, OptionsProvenance::Explicit)?, + options, + derived_options, }) } @@ -751,14 +790,26 @@ S: 'static, SB: SuperblockStore + 'static, { - shard.plane.metadata().mux_stm.users().read(|users| { + let users = shard.plane.metadata().mux_stm.users().read(|users| { users .items .iter() - .map(|(_, user)| user_response(user)) - .collect::<Result<Vec<_>, _>>() - .map(|users| GetUsersResponse { users }) - }) + .map(|(_, user)| StoredUser::from(user)) + .collect::<Vec<_>>() + }); + users + .iter() + .filter_map(|user| { + servable( + user_response(user), + OptionsScope::User, + user.id as usize, + None, + ) + .transpose() + }) + .collect::<Result<Vec<_>, _>>() + .map(|users| GetUsersResponse { users }) } fn build_get_user_response<B, MJ, S, SB>( @@ -772,19 +823,27 @@ S: 'static, SB: SuperblockStore + 'static, { - shard.plane.metadata().mux_stm.users().read(|users| { - let Some(id) = users.resolve_user_id(user_id) else { - return Ok(None); - }; - let user = users.items.get(id).ok_or(IggyError::InvalidIdentifier)?; - Ok(Some(UserDetailsResponse { - user: user_response(user)?, - permissions: user - .permissions - .as_ref() - .map(|p| iggy_common::wire_conversions::permissions_to_wire(p)), - })) - }) + let stored = shard + .plane + .metadata() + .mux_stm + .users() + .read(|users| -> Result<_, IggyError> { + let Some(id) = users.resolve_user_id(user_id) else { + return Ok(None); + }; + let user = users.items.get(id).ok_or(IggyError::InvalidIdentifier)?; + Ok(Some((StoredUser::from(user), user.permissions.clone()))) + })?; + let Some((user, permissions)) = stored else { + return Ok(None); + }; + Ok(Some(UserDetailsResponse { + user: user_response(&user)?, + permissions: permissions + .as_deref() + .map(iggy_common::wire_conversions::permissions_to_wire), + })) } fn personal_access_tokens_response( @@ -816,38 +875,49 @@ S: 'static, SB: SuperblockStore + 'static, { - shard.plane.metadata().mux_stm.streams().read(|streams| { - let Some(stream_id) = streams.resolve_stream_id(stream_id) else { - return Ok(None); - }; - let Some(topic_id) = streams.resolve_topic_id(stream_id, topic_id) else { - return Ok(None); - }; - let stream = streams - .items - .get(stream_id) - .ok_or(IggyError::InvalidIdentifier)?; - let topic = stream - .topics - .get(topic_id) - .ok_or(IggyError::InvalidIdentifier)?; - Ok(Some(GetTopicResponse { - topic: topic_header(topic)?, - partitions: topic - .partitions - .iter() - .map(|partition| { - partition_response( - streams, - stream_id, - topic_id, - partition, - shard.plane.metadata().applied_frontier().get(), - ) - }) - .collect::<Result<Vec<_>, _>>()?, - })) - }) + let stored = + shard + .plane + .metadata() + .mux_stm + .streams() + .read(|streams| -> Result<_, IggyError> { + let Some(stream_id) = streams.resolve_stream_id(stream_id) else { + return Ok(None); + }; + let Some(topic_id) = streams.resolve_topic_id(stream_id, topic_id) else { + return Ok(None); + }; + let stream = streams + .items + .get(stream_id) + .ok_or(IggyError::InvalidIdentifier)?; + let topic = stream + .topics + .get(topic_id) + .ok_or(IggyError::InvalidIdentifier)?; + let partitions = topic + .partitions + .iter() + .map(|partition| { + partition_response( + streams, + stream_id, + topic_id, + partition, + shard.plane.metadata().applied_frontier().get(), + ) + }) + .collect::<Result<Vec<_>, _>>()?; + Ok(Some((StoredEntry::from(topic), partitions))) + })?; + let Some((topic, partitions)) = stored else { + return Ok(None); + }; + Ok(Some(GetTopicResponse { + topic: topic_header(&topic)?, + partitions, + })) } fn build_get_topics_response<B, MJ, S, SB>( @@ -861,52 +931,51 @@ S: 'static, SB: SuperblockStore + 'static, { - shard.plane.metadata().mux_stm.streams().read(|streams| { - // Legacy parity: a missing stream lists as empty, not StreamNotFound. - let Some(resolved_stream) = streams.resolve_stream_id(stream_id) else { - return Ok(GetTopicsResponse { topics: Vec::new() }); - }; - let stream = streams - .items - .get(resolved_stream) - .ok_or(IggyError::InvalidIdentifier)?; - stream - .topics - .iter() - .map(|(_, topic)| topic_header(topic)) - .collect::<Result<Vec<_>, _>>() - .map(|topics| GetTopicsResponse { topics }) - }) + let stored = + shard + .plane + .metadata() + .mux_stm + .streams() + .read(|streams| -> Result<_, IggyError> { + // Legacy parity: a missing stream lists as empty, not StreamNotFound. + let Some(resolved_stream) = streams.resolve_stream_id(stream_id) else { + return Ok(None); + }; + let stream = streams + .items + .get(resolved_stream) + .ok_or(IggyError::InvalidIdentifier)?; + Ok(Some((stream.id, stored_topics(stream)))) + })?; + let Some((stream_id, topics)) = stored else { + return Ok(GetTopicsResponse { topics: Vec::new() }); + }; + listed_topics(stream_id, &topics).map(|topics| GetTopicsResponse { topics }) } -fn stream_response(stream: &metadata::stm::stream::Stream) -> Result<StreamResponse, IggyError> { +fn stream_response(stream: &StoredEntry) -> Result<StreamResponse, IggyError> { + let (options, derived_options) = public_blocks(OptionsScope::Stream, &stream.options)?; Ok(StreamResponse { id: usize_to_u32(stream.id)?, created_at: stream.created_at.as_micros(), - topics_count: usize_to_u32(stream.topics.len())?, - size_bytes: stream.stats.size_bytes_inconsistent(), - messages_count: stream.stats.messages_count_inconsistent(), + topics_count: usize_to_u32(stream.children_count)?, + size_bytes: stream.size_bytes, + messages_count: stream.messages_count, name: WireName::new(stream.name.as_ref()).map_err(|_| IggyError::InvalidFormat)?, - options: resource_options_to_wire(&stream.options, OptionsProvenance::Explicit)?, + options, + derived_options, }) } -/// Stored `message_expiry` and `max_topic_size` echo verbatim, `ServerDefault` -/// as the wire sentinel (0), matching legacy: create admission resolves the -/// sentinels against server config before replication, so a stored sentinel -/// came from an update and must read back as `ServerDefault`, not as the node -/// default frozen at read time. -fn topic_header(topic: &metadata::stm::stream::Topic) -> Result<StreamTopicHeader, IggyError> { - let (options, derived_options) = resource_options_to_wire_split(&topic.options)?; +fn topic_header(topic: &StoredEntry) -> Result<StreamTopicHeader, IggyError> { + let (options, derived_options) = public_blocks(OptionsScope::Topic, &topic.options)?; Ok(StreamTopicHeader { id: usize_to_u32(topic.id)?, created_at: topic.created_at.as_micros(), - partitions_count: usize_to_u32(topic.partitions.len())?, - message_expiry: u64::from(topic.message_expiry), - compression_algorithm: topic.compression_algorithm.as_code(), - max_topic_size: topic.max_topic_size.as_bytes_u64(), - size_bytes: topic.stats.size_bytes_inconsistent(), - messages_count: topic.stats.messages_count_inconsistent(), + partitions_count: usize_to_u32(topic.children_count)?, + size_bytes: topic.size_bytes, + messages_count: topic.messages_count, name: WireName::new(topic.name.as_ref()).map_err(|_| IggyError::InvalidFormat)?, options, derived_options, @@ -1013,6 +1082,57 @@ #[cfg(test)] mod tests { use super::*; + use iggy_common::{HeaderKey, HeaderValue, OptionKey, OptionValue}; + use std::str::FromStr; + + #[test] + fn given_one_unreadable_stream_and_topic_when_listing_should_serve_the_rest() { + let unreadable = || { + ResourceOptions::from([ + ( + HeaderKey::from(OptionKey::CriticalKeys), + OptionValue::explicit(HeaderValue::try_from("future_key").unwrap()), + ), + ( + HeaderKey::from_str("future_key").unwrap(), + OptionValue::explicit(HeaderValue::from(true)), + ), + ]) + }; + let created_at = IggyTimestamp::from(1u64); + let mut streams = StreamsInner::new(); + let mut readable = Stream::new(Arc::from("readable"), created_at); + for (name, options) in [("kept", ResourceOptions::new()), ("hidden", unreadable())] { + let mut topic = Topic::new(Arc::from(name), created_at, readable.stats.clone()); + topic.set_options(options); + readable.topics.insert(topic); + } + let mut hidden = Stream::new(Arc::from("hidden"), created_at); + hidden.options = unreadable(); + let readable_slot = streams.items.insert(readable); + let hidden_slot = streams.items.insert(hidden); + + let names = |names: Vec<&WireName>| -> Vec<String> { + names.iter().map(|name| name.as_str().to_owned()).collect() + }; + let listed = listed_streams(&stored_streams(&streams)).expect("the stream list builds"); + assert_eq!( + names(listed.iter().map(|stream| &stream.name).collect()), + ["readable"] + ); + let readable = streams.items.get(readable_slot).expect("stream stored"); + let topics = + listed_topics(readable.id, &stored_topics(readable)).expect("the topic list builds"); + assert_eq!( + names(topics.iter().map(|topic| &topic.name).collect()), + ["kept"] + ); + let hidden = streams.items.get(hidden_slot).expect("stream stored"); + assert!(matches!( + stream_response(&StoredEntry::from(hidden)), + Err(IggyError::UnsupportedCriticalOption(_)) + )); + } #[test] fn personal_access_tokens_response_preserves_order_and_encodes_never_as_zero() { @@ -1086,10 +1206,10 @@ id: 0, name: Arc::from("topic"), created_at, - message_expiry: iggy_common::IggyExpiry::NeverExpire, - compression_algorithm: iggy_common::CompressionAlgorithm::None, - max_topic_size: iggy_common::MaxTopicSize::Unlimited, options: iggy_common::ResourceOptions::default(), + resolved: iggy_common::TopicResolvedOptions::from( + &iggy_common::ResourceOptions::default(), + ), stats: topic_stats.clone(), partitions: vec![ Partition::new(0, 1, created_at, 0, 0), @@ -1138,56 +1258,32 @@ } #[test] - fn topic_header_echoes_stored_size_and_expiry_verbatim() { - use iggy_common::{ - CompressionAlgorithm, IggyDuration, IggyExpiry, MaxTopicSize, ResourceOptions, - StreamStats, TopicStats, - }; - use std::sync::atomic::AtomicUsize; + fn given_topic_options_when_projecting_header_should_split_blocks_by_provenance() { + use iggy_common::wire_conversions::resource_options_from_wire; + use iggy_common::{HeaderKey, HeaderValue, OptionKey, OptionValue, StreamStats}; - let parent = Arc::new(StreamStats::default()); - let topic_with = |max_topic_size, message_expiry| metadata::stm::stream::Topic { - id: 0, - name: Arc::from("topic"), - created_at: IggyTimestamp::from(1u64), - message_expiry, - compression_algorithm: CompressionAlgorithm::None, - max_topic_size, - options: ResourceOptions::default(), - stats: Arc::new(TopicStats::new(parent.clone())), - partitions: Vec::new(), - round_robin_counter: Arc::new(AtomicUsize::new(0)), - consumer_groups: ahash::AHashMap::default(), - consumer_group_index: ahash::AHashMap::default(), - next_consumer_group_id: 0, - consumer_group_catalog_op: 0, - }; + let mut options = ResourceOptions::new(); + options.insert( + HeaderKey::from(OptionKey::MessageExpiry), + OptionValue::explicit(HeaderValue::from(5_000_000u64)), + ); + options.insert( + HeaderKey::from(OptionKey::MaxTopicSize), + OptionValue::derived(HeaderValue::from(u64::MAX)), + ); + let mut topic = metadata::stm::stream::Topic::new( + Arc::from("topic"), + IggyTimestamp::from(1u64), + Arc::new(StreamStats::default()), + ); + topic.set_options(options.clone()); - // Stored `ServerDefault` sentinels echo the wire sentinel (0) - // verbatim, so an update to `ServerDefault` reads back as - // `ServerDefault` instead of the node default frozen at read time. - let sentinel = topic_header(&topic_with( - MaxTopicSize::ServerDefault, - IggyExpiry::ServerDefault, - )) - .expect("topic header builds"); - assert_eq!(sentinel.max_topic_size, 0); - assert_eq!(sentinel.message_expiry, 0); + let header = topic_header(&StoredEntry::from(&topic)).expect("topic header builds"); - // Explicit values round-trip unchanged. - let custom = topic_header(&topic_with( - MaxTopicSize::from(1024u64), - IggyExpiry::ExpireDuration(IggyDuration::from(5_000_000u64)), - )) - .expect("topic header builds"); - assert_eq!(custom.max_topic_size, 1024); - assert_eq!(custom.message_expiry, 5_000_000); - let unlimited = topic_header(&topic_with( - MaxTopicSize::Unlimited, - IggyExpiry::NeverExpire, - )) - .expect("topic header builds"); - assert_eq!(unlimited.max_topic_size, u64::MAX); - assert_eq!(unlimited.message_expiry, u64::MAX); + let mut explicit = resource_options_from_wire(&header.options, true).unwrap(); + explicit.extend(resource_options_from_wire(&header.derived_options, false).unwrap()); + assert_eq!(explicit, options); + assert_eq!((&header.options).into_iter().count(), 1); + assert_eq!((&header.derived_options).into_iter().count(), 1); } }
diff --git a/core/server/src/rewrite.rs b/core/server/src/rewrite.rs index 4172861..ab5e614 100644 --- a/core/server/src/rewrite.rs +++ b/core/server/src/rewrite.rs
@@ -39,28 +39,36 @@ use crate::segment_cleaner::UNENFORCEABLE_TOPIC_SIZE_WARN; use crate::session_manager::SessionManager; use crate::shell::{ShellBus, ShellShard}; +use crate::stream_keys::{ + Bounded, defers_stream_keys, erase_stream_create, maybe_rewrite_secret_options_request, +}; use crate::users::maybe_rewrite_user_password_request; -use crate::wire::request_body; -use consensus::MetadataHandle; +use crate::wire::{request_body, rewrite_request_body}; +use consensus::{MetadataHandle, PartitionsHandle}; use iggy_binary_protocol::requests::partitions::{ CreatePartitionsRequest, DeletePartitionsRequest, }; -use iggy_binary_protocol::requests::streams::{CreateStreamRequest, UpdateStreamRequest}; -use iggy_binary_protocol::requests::topics::{CreateTopicRequest, UpdateTopicRequest}; -use iggy_binary_protocol::requests::users::{CreateUserRequest, UpdateUserRequest}; +use iggy_binary_protocol::requests::streams::CreateStreamRequest; +use iggy_binary_protocol::requests::system::{ResourceTarget, UpdateResourceRequest}; +use iggy_binary_protocol::requests::topics::CreateTopicRequest; +use iggy_binary_protocol::requests::users::CreateUserRequest; use iggy_binary_protocol::{ MAX_PARTITIONS_PER_REQUEST, Operation, PrepareHeader, RoutedRequestHeader, WireDecode, - WireIdentifier, WireOptions, + WireEncode, WireIdentifier, WireOptions, }; +use iggy_common::defaults::DEFAULT_ROOT_USER_ID; +use iggy_common::wire_conversions::resource_options_to_wire; use iggy_common::{ - IggyByteSize, IggyError, MaxTopicSize, TopicCreateOptions, UPDATABLE_STREAM_OPTION_KEYS, - UPDATABLE_TOPIC_OPTION_KEYS, UPDATABLE_USER_OPTION_KEYS, validate_preallocated_topic_bytes, + AdmittedOptions, IggyByteSize, IggyError, MaxTopicSize, OptionAdmissionContext, OptionColumns, + OptionOperation, OptionsProvenance, OptionsScope, ResourceOptions, TopicCreateOptions, + TopicResolvedOptions, UserStatus, admit, derived_defaults, validate_preallocated_topic_bytes, validate_topic_segment_size, }; use journal::superblock::SuperblockStore; use journal::{Journal, JournalHandle}; use metadata::impls::metadata::StreamsFrontend; use metadata::stm::stream::Streams; +use metadata::stm::user::Users; use server_common::Message; use std::cell::RefCell; use std::rc::Rc; @@ -72,6 +80,7 @@ PersonalAccessToken, UserPassword, StaticBounds, + SecretOptions, ConsumerGroup, } @@ -84,6 +93,7 @@ Self::PersonalAccessToken => "personal_access_token", Self::UserPassword => "user_password", Self::StaticBounds => "static_bounds", + Self::SecretOptions => "secret_options", Self::ConsumerGroup => "consumer_group", } } @@ -98,13 +108,15 @@ /// The TCP funnel's pre-consensus rewrite chain, in order: the PAT rewrite /// (resolving the acting user from the transport `sessions` binding), the -/// password rewrite, the static bounds gate, then the consumer-group -/// Join/Leave identity rewrite ([`crate::consumer_group`]). Mirrors -/// [`http_chain`] plus the bounds step, which the binary wire needs because -/// it has no `command.validate()` layer, and the group step, which only the -/// binary wire serves. Returns the rewritten request and the raw PAT token -/// the funnel substitutes into the committed reply; a rejection names the -/// failing stage for the deny log line. +/// password rewrite, the static bounds gate (which also replaces a user +/// create's options with the admitted canonical block and hands a stream +/// create on decoded), the secret options rewrite that builds the stream +/// create's body with its key wrapped, then the consumer-group Join/Leave +/// identity rewrite ([`crate::consumer_group`]). +/// Mirrors [`http_chain`] plus the group step, which only the binary wire +/// serves. Returns the rewritten request and the raw PAT token the funnel +/// substitutes into the committed reply. A rejection names the failing stage +/// for the deny log line. pub fn tcp_chain<B, MJ, S, SB>( shard: &Rc<ShellShard<B, MJ, S, SB>>, sessions: &Rc<RefCell<SessionManager>>, @@ -135,10 +147,24 @@ stage: RewriteStage::UserPassword, error, })?; - static_bounds(shard, &request).map_err(|error| RewriteDeny { + let user_id = sessions.borrow().get_user_id(transport_client_id); + let bounded = static_bounds(shard, user_id, request).map_err(|error| RewriteDeny { stage: RewriteStage::StaticBounds, error, })?; + let key_context = shard.plane.partitions().key_context(); + let request = maybe_rewrite_secret_options_request(&key_context, bounded, || { + shard + .plane + .metadata() + .mux_stm + .streams() + .read(|streams| defers_stream_keys(streams, &key_context)) + }) + .map_err(|error| RewriteDeny { + stage: RewriteStage::SecretOptions, + error, + })?; let request = maybe_rewrite_consumer_group_request(request).map_err(|error| RewriteDeny { stage: RewriteStage::ConsumerGroup, error, @@ -147,12 +173,13 @@ } /// The HTTP submit's pre-consensus rewrite chain: the PAT rewrite for the -/// already-authenticated `user_id`, then the password rewrite. Mirrors +/// already-authenticated `user_id`, the password rewrite, the same static +/// bounds as [`tcp_chain`], so both transports admit options through one +/// catalog gate and answer the same request with the same error, then the +/// secret options rewrite. Mirrors /// [`tcp_chain`] minus the session lookup (the HTTP listener authenticates -/// against its own session table and resolves the acting user itself), -/// minus the static bounds step (HTTP enforces the same bounds in its -/// handlers via `command.validate()` plus the validators below), and minus -/// the group step, because HTTP serves no join or leave. Returns the +/// against its own session table and resolves the acting user itself) and +/// minus the group step, because HTTP serves no join or leave. Returns the /// rewritten request and the raw PAT token the caller substitutes into the /// committed reply. pub fn http_chain<B, MJ, S, SB>( @@ -169,6 +196,16 @@ { let (request, raw_token) = rewrite_pat_request_for_user(user_id, request)?; let request = maybe_rewrite_user_password_request(shard, request)?; + let bounded = static_bounds(shard, Some(user_id), request)?; + let key_context = shard.plane.partitions().key_context(); + let request = maybe_rewrite_secret_options_request(&key_context, bounded, || { + shard + .plane + .metadata() + .mux_stm + .streams() + .read(|streams| defers_stream_keys(streams, &key_context)) + })?; Ok((request, raw_token)) } @@ -314,30 +351,21 @@ ); } -/// Reject option keys outside the resource's catalog, pre-consensus. Unknown -/// keys are rejected rather than skipped: a silently ignored knob would hand -/// the client server defaults without it ever learning. Streams and users -/// have no catalog keys yet, so `known` is empty for both until one lands. -pub fn validate_option_keys(options: &WireOptions, known: &[&str]) -> Result<(), IggyError> { - for entry in options { - // Wire validation already enforced UTF-8 string keys. - let key = String::from_utf8_lossy(entry.key); - if !known.contains(&key.as_ref()) { - return Err(IggyError::UnsupportedOptionKey(key.into_owned())); - } - } - Ok(()) -} - /// Static bounds run pre-consensus so a rejected request burns no -/// replicated log entry; HTTP covers the same bounds via -/// `command.validate()`. A body that fails to decode denies typed too -/// (`InvalidCommand`), instead of riding consensus just to fail there. -#[allow(clippy::too_many_lines)] +/// replicated log entry. Both chains run them, so every option block passes +/// the same catalog admission whatever the transport. A body that fails to +/// decode denies typed too (`InvalidCommand`), instead of riding consensus +/// just to fail there. A user create leaves with the admitted canonical +/// options, so apply stores one kind per key whatever the client sent. A +/// stream create leaves decoded with its admitted options, so the secret +/// stage builds its body without encoding a client key again, and a refused +/// one has its copies of the key erased. `user_id` is the acting user, `None` +/// when the transport has not bound one. fn static_bounds<B, MJ, S, SB>( shard: &Rc<ShellShard<B, MJ, S, SB>>, - request: &Message<RoutedRequestHeader>, -) -> Result<(), IggyError> + user_id: Option<u32>, + request: Message<RoutedRequestHeader>, +) -> Result<Bounded, IggyError> where B: ShellBus, MJ: JournalHandle + 'static, @@ -346,31 +374,33 @@ SB: SuperblockStore + 'static, { match request.header().operation { - Operation::CreateTopic => CreateTopicRequest::decode_from(request_body(request)) + Operation::CreateTopic => CreateTopicRequest::decode_from(request_body(&request)) .map_err(|_| IggyError::InvalidCommand) .and_then(|create_topic| { - // `parse` doubles as the catalog gate: an unknown key or a - // malformed value denies typed here, pre-consensus. - let options = TopicCreateOptions::parse(&create_topic.options)?; - if let Some(segment_size) = options.segment_size { + let admitted = admit_create( + OptionsScope::Topic, + &create_topic.options, + OptionColumns { + name: Some(create_topic.name.as_str().to_owned()), + ..OptionColumns::default() + }, + )?; + if let Some(segment_size) = + TopicCreateOptions::from_resource_options(&admitted.explicit).segment_size + { validate_topic_segment_size( segment_size.as_bytes_u64(), iggy_common::MAX_TOPIC_SEGMENT_SIZE, )?; } - let segment_size = options.segment_size.map_or_else( - || iggy_common::DEFAULT_SEGMENT_SIZE, - |segment_size| segment_size.as_bytes_u64(), - ); - if options - .preallocate_segments - .unwrap_or(iggy_common::DEFAULT_PREALLOCATE_SEGMENTS) - { + let mut stored = derived_defaults(OptionsScope::Topic, &admitted.explicit)?; + stored.extend(admitted.explicit); + let resolved = TopicResolvedOptions::from(&stored); + let segment_size = resolved.runtime.effective_segment_size().as_bytes_u64(); + if resolved.runtime.preallocate_segments == Some(true) { validate_preallocated_topic_bytes(segment_size, create_topic.partitions_count)?; } - let max_topic_size = options - .max_topic_size - .unwrap_or(MaxTopicSize::ServerDefault); + let max_topic_size = resolved.max_topic_size; validate_topic_bounds(create_topic.partitions_count, max_topic_size, segment_size)?; warn_unenforceable_topic_size( max_topic_size, @@ -379,8 +409,8 @@ create_topic.partitions_count, ); Ok(()) - }), - Operation::CreatePartitions => CreatePartitionsRequest::decode_from(request_body(request)) + })?, + Operation::CreatePartitions => CreatePartitionsRequest::decode_from(request_body(&request)) .map_err(|_| IggyError::InvalidCommand) .and_then(|create_partitions| { validate_partitions_change_count(create_partitions.partitions_count)?; @@ -393,70 +423,778 @@ create_partitions.partitions_count, ); Ok(()) - }), - Operation::DeletePartitions => DeletePartitionsRequest::decode_from(request_body(request)) + })?, + Operation::DeletePartitions => DeletePartitionsRequest::decode_from(request_body(&request)) .map_err(|_| IggyError::InvalidCommand) .and_then(|delete_partitions| { validate_partitions_change_count(delete_partitions.partitions_count) - }), - // Only the updatable subset: the create-time knobs are pushed to - // partitions when the topic is built and nothing re-pushes them, so - // accepting one here would store a value no partition ever sees. - Operation::UpdateTopic => UpdateTopicRequest::decode_from(request_body(request)) + })?, + Operation::UpdateResource => UpdateResourceRequest::decode_from(request_body(&request)) .map_err(|_| IggyError::InvalidCommand) - .and_then(|update_topic| { - validate_option_keys(&update_topic.options, UPDATABLE_TOPIC_OPTION_KEYS)?; - let options = TopicCreateOptions::parse(&update_topic.options)?; - let Some(max_topic_size) = options.max_topic_size else { - return Ok(()); - }; - // An update can lower the cap below one segment just as a - // create can, and the stored map would then report a size the - // topic can never enforce. The floor is this topic's own - // segment size, since that key is create-only. - let metadata = shard.plane.metadata(); - let streams = metadata.mux_stm.streams(); - let segment_size = streams - .topic_segment_size(&update_topic.stream_id, &update_topic.topic_id) - .map_or_else( - || iggy_common::DEFAULT_SEGMENT_SIZE, - |segment_size| segment_size.as_bytes_u64(), - ); - validate_topic_size_floor(max_topic_size, segment_size)?; - let partitions_count = streams - .topic_partitions_count(&update_topic.stream_id, &update_topic.topic_id) - .unwrap_or(0); - warn_unenforceable_topic_size( - max_topic_size, - segment_size, - shard.bus_max_message_size(), - u32::try_from(partitions_count).unwrap_or(u32::MAX), - ); - Ok(()) - }), - Operation::UpdateStream => UpdateStreamRequest::decode_from(request_body(request)) - .map_err(|_| IggyError::InvalidCommand) - .and_then(|update_stream| { - validate_option_keys(&update_stream.options, UPDATABLE_STREAM_OPTION_KEYS) - }), - Operation::UpdateUser => UpdateUserRequest::decode_from(request_body(request)) - .map_err(|_| IggyError::InvalidCommand) - .and_then(|update_user| { - validate_option_keys(&update_user.options, UPDATABLE_USER_OPTION_KEYS) - }), - Operation::CreateStream => CreateStreamRequest::decode_from(request_body(request)) - .map_err(|_| IggyError::InvalidCommand) - .and_then(|create_stream| validate_option_keys(&create_stream.options, &[])), - Operation::CreateUser => CreateUserRequest::decode_from(request_body(request)) - .map_err(|_| IggyError::InvalidCommand) - .and_then(|create_user| validate_option_keys(&create_user.options, &[])), - _ => Ok(()), + .and_then(|patch| patch_bounds(shard, user_id, &patch))?, + Operation::CreateStream => return bound_stream_create(request), + Operation::CreateUser => { + let mut create_user = CreateUserRequest::decode_from(request_body(&request)) + .map_err(|_| IggyError::InvalidCommand)?; + let admitted = admit_create( + OptionsScope::User, + &create_user.options, + OptionColumns { + username: Some(create_user.username.as_str().to_owned()), + ..OptionColumns::default() + }, + )?; + UserStatus::from_code(create_user.status).map_err(|_| IggyError::InvalidUserStatus)?; + if let Some(options) = canonical_block(&create_user.options, &admitted)? { + create_user.options = options; + return rewrite_request_body(&request, &create_user.to_bytes()) + .map(Bounded::Request); + } + } + _ => {} } + Ok(Bounded::Request(request)) +} + +/// Decode and admit a stream create, handed on decoded with its admitted +/// options. A refused one has its copies of a client key erased. +fn bound_stream_create(request: Message<RoutedRequestHeader>) -> Result<Bounded, IggyError> { + let Ok(create) = CreateStreamRequest::decode_from(request_body(&request)) else { + erase_stream_create(request, None); + return Err(IggyError::InvalidCommand); + }; + let admitted = admit_create( + OptionsScope::Stream, + &create.options, + OptionColumns { + name: Some(create.name.as_str().to_owned()), + ..OptionColumns::default() + }, + ); + match admitted { + Ok(admitted) => Ok(Bounded::StreamCreate { + request, + create, + options: admitted.explicit, + }), + Err(error) => { + erase_stream_create(request, Some(create)); + Err(error) + } + } +} + +/// The admitted canonical block when it differs from what the client sent. +fn canonical_block( + client: &WireOptions, + admitted: &AdmittedOptions, +) -> Result<Option<WireOptions>, IggyError> { + let canonical = resource_options_to_wire(&admitted.explicit, OptionsProvenance::All)?; + Ok((canonical.as_bytes() != client.as_bytes()).then_some(canonical)) +} + +/// Catalog admission for a generic patch, plus the floor a topic size cap +/// keeps against the topic's own segment size. +/// +/// The capacity check and the floor read the target's stored state, so they +/// run only for a caller this node lets update the target. Any other caller +/// is admitted on the request alone and refused by apply, so no answer here +/// tells it what the target stores. +fn patch_bounds<B, MJ, S, SB>( + shard: &Rc<ShellShard<B, MJ, S, SB>>, + user_id: Option<u32>, + patch: &UpdateResourceRequest, +) -> Result<(), IggyError> +where + B: ShellBus, + MJ: JournalHandle + 'static, + MJ::Target: Journal<Entry = Message<PrepareHeader>, Header = PrepareHeader>, + S: 'static, + SB: SuperblockStore + 'static, +{ + let metadata = shard.plane.metadata(); + let streams = metadata.mux_stm.streams(); + let users = metadata.mux_stm.users(); + let authorized_user_id = + user_id.filter(|user_id| may_update(streams, users, *user_id, &patch.target)); + let current = authorized_user_id.and_then(|_| stored_options(streams, users, &patch.target)); + let stored_state_visible = current.is_some(); + let admitted = admit_patch(patch, current)?; + if let (Some(actor_id), ResourceTarget::User { user_id: target_id }) = + (authorized_user_id, &patch.target) + { + let target_is_root = users + .read(|inner| inner.resolve_user_id(target_id) == Some(DEFAULT_ROOT_USER_ID as usize)); + if target_is_root + && (admitted.columns.username.is_some() || admitted.columns.status.is_some()) + && (actor_id != DEFAULT_ROOT_USER_ID + || admitted.columns.status == Some(UserStatus::Inactive)) + { + return Err(IggyError::CannotModifyRootUser); + } + } + let ResourceTarget::Topic { + stream_id, + topic_id, + } = &patch.target + else { + return Ok(()); + }; + let Some(max_topic_size) = TopicCreateOptions::from_resource_options(&admitted.explicit) + .max_topic_size + .filter(|_| stored_state_visible) + else { + return Ok(()); + }; + // An update can lower the cap below one segment just as a create can, and + // the stored map would then report a size the topic can never enforce. The + // floor is this topic's own segment size, since that key is create-only. + let segment_size = streams.topic_segment_size(stream_id, topic_id).map_or_else( + || iggy_common::DEFAULT_SEGMENT_SIZE, + |segment_size| segment_size.as_bytes_u64(), + ); + validate_topic_size_floor(max_topic_size, segment_size)?; + let partitions_count = streams + .topic_partitions_count(stream_id, topic_id) + .unwrap_or(0); + warn_unenforceable_topic_size( + max_topic_size, + segment_size, + shard.bus_max_message_size(), + u32::try_from(partitions_count).unwrap_or(u32::MAX), + ); + Ok(()) +} + +/// Catalog admission for a create's options block, with the create's own +/// fixed fields as columns so the block cannot shadow them. +fn admit_create( + scope: OptionsScope, + options: &WireOptions, + create_columns: OptionColumns, +) -> Result<AdmittedOptions, IggyError> { + admit( + scope, + OptionOperation::Create, + options, + &[], + &OptionAdmissionContext { + current: None, + create_columns, + }, + ) +} + +/// Whether this node's applied permissions let `user_id` update `target`, +/// the rule apply runs again in committed order. `false` for a target this +/// node cannot resolve, which apply reports. +fn may_update(streams: &Streams, users: &Users, user_id: u32, target: &ResourceTarget) -> bool { + let decision = match target { + ResourceTarget::Stream { stream_id } => { + streams.resolve_stream_id(stream_id).map(|stream_id| { + users.authorize(|permissioner| permissioner.update_stream(user_id, stream_id)) + }) + } + ResourceTarget::Topic { + stream_id, + topic_id, + } => streams + .resolve_topic_ids(stream_id, topic_id) + .map(|(stream_id, topic_id)| { + users.authorize(|permissioner| { + permissioner.update_topic(user_id, stream_id, topic_id) + }) + }), + ResourceTarget::User { .. } => { + Some(users.authorize(|permissioner| permissioner.update_user(user_id))) + } + }; + matches!(decision, Some(Ok(()))) +} + +/// The options the patch target stores, `None` for a target that does not +/// exist, which apply reports. +fn stored_options( + streams: &Streams, + users: &Users, + target: &ResourceTarget, +) -> Option<ResourceOptions> { + match target { + ResourceTarget::Stream { stream_id } => streams.read(|inner| { + let stream = inner.resolve_stream_id(stream_id)?; + Some(inner.items.get(stream)?.options.clone()) + }), + ResourceTarget::Topic { + stream_id, + topic_id, + } => streams.read(|inner| { + let stream = inner.resolve_stream_id(stream_id)?; + let topic = inner.resolve_topic_id(stream, topic_id)?; + Some(inner.items.get(stream)?.topics.get(topic)?.options.clone()) + }), + ResourceTarget::User { user_id } => users.read(|inner| { + let user = inner.resolve_user_id(user_id)?; + Some(inner.items.get(user)?.options.clone()) + }), + } +} + +/// Catalog admission for a generic patch against the target's stored +/// options, with the same rules the metadata primary applies again when it +/// resolves the patch. +fn admit_patch( + patch: &UpdateResourceRequest, + current: Option<ResourceOptions>, +) -> Result<AdmittedOptions, IggyError> { + let reset: Vec<String> = patch + .reset + .iter() + .map(|name| name.as_str().to_owned()) + .collect(); + admit( + OptionsScope::from_code(patch.target.scope_code())?, + OptionOperation::Update, + &patch.options, + &reset, + &OptionAdmissionContext { + current, + create_columns: OptionColumns::default(), + }, + ) } #[cfg(test)] mod tests { use super::*; + use crate::dispatch::test_support::{ + SpyBus, TestShard, create_one_partition, create_reader_request, prepare_message, + register_reply, request_message, test_shard, test_shard_with_key_context, + }; + use bytes::Bytes; + use iggy_binary_protocol::requests::system::WireResetKeys; + use iggy_binary_protocol::requests::topics::CreateTopicWithAssignmentsRequest; + use iggy_binary_protocol::{MAX_OPTIONS, WireName}; + use iggy_common::defaults::DEFAULT_ROOT_USER_ID; + use iggy_common::wire_conversions::resource_options_from_wire; + use iggy_common::{ + HeaderKey, HeaderValue, IggyDuration, IggyExpiry, OptionKey, OptionValue, OptionsUpdate, + ServerKeyContext, StreamCreateOptions, StreamEncryptionKey, StreamKeyOrigin, + TopicUpdateOptions, UserUpdateOptions, stream_wrap_set, text, + }; + use message_bus::installer::conn_info::ClientTransportKind; + use metadata::stm::StateMachine as _; + use std::collections::BTreeMap; + use std::str::FromStr; + use std::sync::Arc; + use std::time::Instant; + + const CONFIG_KEY: &str = "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE="; + + #[test] + #[allow(clippy::too_many_lines)] + fn given_patch_constraints_when_using_either_chain_should_return_same_error() { + let bus = SpyBus::default(); + let shard = Rc::new(test_shard(&bus, 0, 1, 1)); + create_one_partition(&shard); + let sessions = root_sessions(&shard, 1); + let topic = || ResourceTarget::Topic { + stream_id: WireIdentifier::numeric(0), + topic_id: WireIdentifier::numeric(0), + }; + let stream = || ResourceTarget::Stream { + stream_id: WireIdentifier::numeric(0), + }; + let raw_topic_patch = |key: OptionKey, value: &str| { + let patch = TopicUpdateOptions { + raw: BTreeMap::from([(key.to_string(), value.to_owned())]), + ..TopicUpdateOptions::default() + } + .to_patch() + .unwrap(); + UpdateResourceRequest { + target: topic(), + reset: patch.reset, + options: patch.set, + } + .to_bytes() + }; + let short_username = UserUpdateOptions { + raw: BTreeMap::from([(OptionKey::Username.to_string(), "ab".to_owned())]), + ..UserUpdateOptions::default() + } + .to_patch() + .unwrap(); + let mut name_column = ResourceOptions::new(); + name_column.insert( + HeaderKey::from(OptionKey::Name), + OptionValue::explicit(HeaderValue::try_from("shadow").unwrap()), + ); + let json_patch = |target: ResourceTarget, body: &str| { + let patch = serde_json::from_str::<OptionsUpdate>(body) + .unwrap() + .to_patch() + .unwrap(); + UpdateResourceRequest { + target, + reset: patch.reset, + options: patch.set, + } + .to_bytes() + }; + let user = || ResourceTarget::User { + user_id: WireIdentifier::numeric(0), + }; + let cases: Vec<(Operation, Bytes, IggyError)> = vec![ + ( + Operation::UpdateResource, + json_patch(topic(), r#"{"name": 42}"#), + IggyError::InvalidTopicName, + ), + ( + Operation::UpdateResource, + json_patch(user(), r#"{"status": true}"#), + IggyError::InvalidUserStatus, + ), + ( + Operation::UpdateResource, + raw_topic_patch(OptionKey::SegmentSize, "64 MiB"), + IggyError::OptionNotUpdatable(OptionKey::SegmentSize.to_string()), + ), + ( + Operation::UpdateResource, + raw_topic_patch(OptionKey::MaxTopicSize, "1 MiB"), + IggyError::InvalidTopicSize( + MaxTopicSize::from(1024 * 1024u64), + IggyByteSize::from(iggy_common::DEFAULT_SEGMENT_SIZE), + ), + ), + ( + Operation::UpdateResource, + UpdateResourceRequest { + target: stream(), + reset: WireResetKeys::try_from(vec![WireName::new("name").unwrap()]).unwrap(), + options: WireOptions::empty(), + } + .to_bytes(), + IggyError::OptionNotResettable(OptionKey::Name.to_string()), + ), + ( + Operation::UpdateResource, + UpdateResourceRequest { + target: stream(), + reset: WireResetKeys::default(), + options: resource_options_to_wire( + &ResourceOptions::from([( + HeaderKey::from(OptionKey::EncryptionKeySource), + OptionValue::explicit(HeaderValue::try_from("stream").unwrap()), + )]), + OptionsProvenance::All, + ) + .unwrap(), + } + .to_bytes(), + IggyError::ReadOnlyOption(OptionKey::EncryptionKeySource.to_string()), + ), + ( + Operation::UpdateResource, + UpdateResourceRequest { + target: ResourceTarget::User { + user_id: WireIdentifier::numeric(0), + }, + reset: short_username.reset, + options: short_username.set, + } + .to_bytes(), + IggyError::InvalidUsername, + ), + ( + Operation::CreateStream, + CreateStreamRequest { + name: WireName::new("orders").unwrap(), + options: resource_options_to_wire(&name_column, OptionsProvenance::All) + .unwrap(), + } + .to_bytes(), + IggyError::InvalidOptionValue(OptionKey::Name.to_string()), + ), + ( + Operation::CreateStream, + CreateStreamRequest { + name: WireName::new("orders").unwrap(), + options: StreamCreateOptions { + encryption_key: Some( + StreamEncryptionKey::try_from([7u8; 32].as_slice()).unwrap(), + ), + ..StreamCreateOptions::default() + } + .to_wire() + .unwrap(), + } + .to_bytes(), + IggyError::StreamEncryptionUnavailable, + ), + ]; + + for (operation, body, expected) in cases { + let tcp = tcp_chain( + &shard, + &sessions, + 1, + request_message(operation, 1, 1, 1, &body), + ) + .map(|_| ()) + .map_err(|deny| deny.error); + let http = http_chain( + &shard, + DEFAULT_ROOT_USER_ID, + request_message(operation, 1, 1, 1, &body), + ) + .map(|_| ()); + assert_eq!(tcp, Err(expected.clone()), "binary chain, {operation:?}"); + assert_eq!(http, Err(expected), "HTTP chain, {operation:?}"); + } + } + + #[test] + fn given_caller_without_update_permission_when_patching_topic_below_its_segment_should_leave_the_refusal_to_apply() + { + const READER: u32 = 1; + let bus = SpyBus::default(); + let shard = Rc::new(test_shard(&bus, 0, 1, 1)); + create_one_partition(&shard); + let created = shard + .plane + .metadata() + .mux_stm + .update(prepare_message( + Operation::CreateUser, + 1, + 3, + &create_reader_request().to_bytes(), + )) + .expect("the reader is created"); + assert_eq!(created.code, 0); + let patch = TopicUpdateOptions { + raw: BTreeMap::from([(OptionKey::MaxTopicSize.to_string(), "1 MiB".to_owned())]), + ..TopicUpdateOptions::default() + } + .to_patch() + .expect("the patch encodes"); + let body = UpdateResourceRequest { + target: ResourceTarget::Topic { + stream_id: WireIdentifier::numeric(0), + topic_id: WireIdentifier::numeric(0), + }, + reset: patch.reset, + options: patch.set, + } + .to_bytes(); + let chain = |user_id: u32| { + http_chain( + &shard, + user_id, + request_message(Operation::UpdateResource, 1, 1, 1, &body), + ) + .map(|_| ()) + }; + + assert_eq!( + chain(READER), + Ok(()), + "a caller apply refuses must not learn the topic's segment size here" + ); + assert_eq!( + chain(DEFAULT_ROOT_USER_ID), + Err(IggyError::InvalidTopicSize( + MaxTopicSize::from(1024 * 1024u64), + IggyByteSize::from(iggy_common::DEFAULT_SEGMENT_SIZE), + )) + ); + } + + #[test] + fn given_root_deactivation_when_admitting_patch_should_refuse_before_consensus() { + let bus = SpyBus::default(); + let shard = Rc::new(test_shard(&bus, 0, 1, 1)); + shard + .plane + .metadata() + .mux_stm + .users() + .ensure_root_user("iggy", "hash"); + let patch = UserUpdateOptions { + status: Some(UserStatus::Inactive), + ..UserUpdateOptions::default() + } + .to_patch() + .expect("status patch encodes"); + let body = UpdateResourceRequest { + target: ResourceTarget::User { + user_id: WireIdentifier::numeric(DEFAULT_ROOT_USER_ID), + }, + reset: patch.reset, + options: patch.set, + } + .to_bytes(); + + let result = http_chain( + &shard, + DEFAULT_ROOT_USER_ID, + request_message(Operation::UpdateResource, 1, 1, 1, &body), + ); + assert!( + matches!(result, Err(IggyError::CannotModifyRootUser)), + "actual: {:?}", + result.err() + ); + assert!(bus.replica_sends.borrow().is_empty()); + } + + #[test] + fn given_manage_users_caller_when_patching_root_should_refuse_rename_and_deactivation() { + let bus = SpyBus::default(); + let shard = Rc::new(test_shard(&bus, 0, 1, 1)); + let metadata = shard.plane.metadata(); + metadata.mux_stm.users().ensure_root_user("iggy", "hash"); + let mut manager = create_reader_request(); + manager.username = WireName::new("manager").expect("valid name"); + manager + .permissions + .as_mut() + .expect("permissions") + .global + .manage_users = true; + let created = metadata + .mux_stm + .update(prepare_message( + Operation::CreateUser, + 1, + 3, + &manager.to_bytes(), + )) + .expect("create manager"); + assert_eq!(created.code, 0); + + for update in [ + UserUpdateOptions { + username: Some("root-renamed".to_owned()), + ..UserUpdateOptions::default() + }, + UserUpdateOptions { + status: Some(UserStatus::Inactive), + ..UserUpdateOptions::default() + }, + ] { + let patch = update.to_patch().expect("patch encodes"); + let body = UpdateResourceRequest { + target: ResourceTarget::User { + user_id: WireIdentifier::numeric(DEFAULT_ROOT_USER_ID), + }, + reset: patch.reset, + options: patch.set, + } + .to_bytes(); + let result = http_chain( + &shard, + 1, + request_message(Operation::UpdateResource, 1, 1, 1, &body), + ); + assert!(matches!(result, Err(IggyError::CannotModifyRootUser))); + } + assert!(bus.replica_sends.borrow().is_empty()); + } + + #[test] + fn given_out_of_range_status_when_creating_user_with_either_chain_should_refuse_invalid_status() + { + let bus = SpyBus::default(); + let shard = Rc::new(test_shard(&bus, 0, 1, 1)); + let sessions = Rc::new(RefCell::new(SessionManager::new())); + let body = CreateUserRequest { + username: WireName::new("status-user").unwrap(), + password: "secret123".to_owned(), + status: 9, + permissions: None, + options: WireOptions::empty(), + } + .to_bytes(); + + let tcp = tcp_chain( + &shard, + &sessions, + 1, + request_message(Operation::CreateUser, 1, 1, 1, &body), + ) + .map(|_| ()) + .map_err(|deny| deny.error); + let http = http_chain( + &shard, + 0, + request_message(Operation::CreateUser, 1, 1, 1, &body), + ) + .map(|_| ()); + + assert_eq!(tcp, Err(IggyError::InvalidUserStatus), "binary chain"); + assert_eq!(http, Err(IggyError::InvalidUserStatus), "HTTP chain"); + } + + #[test] + fn given_text_stream_key_when_bounding_create_should_encode_no_request_with_the_key() { + let bus = SpyBus::default(); + let shard = Rc::new(test_shard(&bus, 0, 1, 1)); + let key = StreamEncryptionKey::try_from([7u8; 32].as_slice()).expect("32 bytes"); + let body = CreateStreamRequest { + name: WireName::new("orders").expect("a name"), + options: text_key_block(&key), + } + .to_bytes(); + + let sent = request_message(Operation::CreateStream, 1, 1, 1, &body); + + let bounded = static_bounds(&shard, Some(DEFAULT_ROOT_USER_ID), sent.clone()) + .expect("the create admits"); + + let Bounded::StreamCreate { + request, options, .. + } = bounded + else { + panic!("a stream create goes on decoded"); + }; + assert_eq!(request.as_slice(), sent.as_slice(), "no request is rebuilt"); + assert_eq!( + options.get(&HeaderKey::from(OptionKey::EncryptionKey)), + Some(&OptionValue::explicit( + HeaderValue::try_from(key.as_bytes().as_slice()).expect("a raw value") + )), + "the admitted key is canonical" + ); + } + + #[test] + fn given_stream_key_in_either_form_when_running_either_chain_should_forward_only_its_wrap_set() + { + let bus = SpyBus::default(); + let context = + Arc::new(ServerKeyContext::from_config(CONFIG_KEY, &[], true).expect("a valid key")); + let shard = Rc::new(test_shard_with_key_context( + &bus, + 0, + 1, + 1, + Arc::clone(&context), + )); + let sessions = root_sessions(&shard, 1); + let key = StreamEncryptionKey::try_from([7u8; 32].as_slice()).expect("32 bytes"); + let raw_block = StreamCreateOptions { + encryption_key: Some(key.clone()), + ..StreamCreateOptions::default() + } + .to_wire() + .expect("encodes"); + + for options in [raw_block, text_key_block(&key)] { + let body = CreateStreamRequest { + name: WireName::new("orders").expect("a name"), + options, + } + .to_bytes(); + let (tcp, _) = tcp_chain( + &shard, + &sessions, + 1, + request_message(Operation::CreateStream, 1, 1, 1, &body), + ) + .map_err(|deny| deny.error) + .expect("binary chain"); + let (http, _) = http_chain( + &shard, + DEFAULT_ROOT_USER_ID, + request_message(Operation::CreateStream, 1, 1, 1, &body), + ) + .expect("HTTP chain"); + + for forwarded in [tcp, http] { + let forwarded = request_body(&forwarded); + assert!(!contains(forwarded, key.as_bytes()), "raw key travels on"); + assert!( + !contains(forwarded, text::as_base64(key.as_bytes()).as_bytes()), + "base64 key travels on" + ); + let create = CreateStreamRequest::decode_from(forwarded).expect("decodes"); + let stored = resource_options_from_wire(&create.options, true).expect("a block"); + let set = stream_wrap_set(&stored) + .expect("readable") + .expect("a wrap set"); + assert_eq!(set.origin(), StreamKeyOrigin::Client); + assert_eq!(context.open(&set).expect("opens"), key); + } + } + } + + #[test] + fn given_stored_map_at_capacity_when_patching_a_new_key_should_refuse_at_the_edge() { + let bus = SpyBus::default(); + let shard = Rc::new(test_shard(&bus, 0, 1, 1)); + create_one_partition(&shard); + let full: ResourceOptions = (0..MAX_OPTIONS) + .map(|index| { + ( + HeaderKey::from_str(&format!("future_{index:04}")).unwrap(), + OptionValue::explicit(HeaderValue::from(true)), + ) + }) + .collect(); + shard + .plane + .metadata() + .mux_stm + .update(prepare_message( + Operation::CreateTopicWithAssignments, + 1, + 3, + &CreateTopicWithAssignmentsRequest { + request: CreateTopicRequest { + stream_id: WireIdentifier::numeric(0), + partitions_count: 0, + name: WireName::new("full").unwrap(), + options: resource_options_to_wire(&full, OptionsProvenance::All).unwrap(), + }, + derived_options: WireOptions::empty(), + partitions: Vec::new(), + } + .to_bytes(), + )) + .unwrap(); + let patch = TopicUpdateOptions { + message_expiry: Some(IggyExpiry::ExpireDuration(IggyDuration::from( + 60_000_000u64, + ))), + ..TopicUpdateOptions::default() + } + .to_patch() + .unwrap(); + let body = UpdateResourceRequest { + target: ResourceTarget::Topic { + stream_id: WireIdentifier::numeric(0), + topic_id: WireIdentifier::numeric(1), + }, + reset: patch.reset, + options: patch.set, + } + .to_bytes(); + + let bounded = static_bounds( + &shard, + Some(DEFAULT_ROOT_USER_ID), + request_message(Operation::UpdateResource, 1, 1, 1, &body), + ) + .map(|_| ()); + + assert!( + matches!(bounded, Err(IggyError::OptionsBlockTooLarge(_))), + "{bounded:?}" + ); + } #[test] fn create_topic_bounds_deny_pre_consensus() { @@ -543,4 +1281,64 @@ "the cap still applies" ); } + + fn contains(haystack: &[u8], needle: &[u8]) -> bool { + haystack + .windows(needle.len()) + .any(|window| window == needle) + } + + fn text_key_block(key: &StreamEncryptionKey) -> WireOptions { + resource_options_to_wire( + &ResourceOptions::from([( + HeaderKey::from(OptionKey::EncryptionKey), + OptionValue::explicit( + HeaderValue::try_from(text::as_base64(key.as_bytes()).as_str()) + .expect("a text value"), + ), + )]), + OptionsProvenance::All, + ) + .expect("encodes") + } + + /// A session table whose `transport` connection is bound to root. + fn root_sessions(shard: &TestShard, transport: u128) -> Rc<RefCell<SessionManager>> { + const CLIENT: u128 = 1; + const SESSION: u64 = 1; + let attachment = { + let mut client_table = shard.plane.metadata().client_table.borrow_mut(); + client_table + .commit_register( + CLIENT, + DEFAULT_ROOT_USER_ID, + [0x5a; 32], + register_reply(CLIENT, SESSION), + ) + .expect("root registers"); + client_table + .attach_session(CLIENT, SESSION, DEFAULT_ROOT_USER_ID) + .expect("root attaches") + }; + let sessions = Rc::new(RefCell::new(SessionManager::new())); + sessions.borrow_mut().ensure_connection( + transport, + "127.0.0.1:34567".parse().expect("a socket address"), + ClientTransportKind::Tcp, + Instant::now(), + ); + sessions + .borrow_mut() + .bind_authenticated_connection( + transport, + CLIENT, + SESSION, + DEFAULT_ROOT_USER_ID, + attachment, + SESSION, + Instant::now(), + ) + .expect("root binds"); + sessions + } }
diff --git a/core/server/src/server_error.rs b/core/server/src/server_error.rs index c0571ac..f66415a 100644 --- a/core/server/src/server_error.rs +++ b/core/server/src/server_error.rs
@@ -43,6 +43,25 @@ }, #[error(transparent)] Iggy(Box<iggy_common::IggyError>), + #[error( + "stream {stream_id} is encrypted under server keys with fingerprints [{stored_kek_fingerprints}], this node holds [{node_kek_fingerprints}]: {source}. Configure the key the stream was wrapped with as the encryption key or as a previous key" + )] + StreamKeyUnavailable { + stream_id: usize, + stored_kek_fingerprints: String, + node_kek_fingerprints: String, + #[source] + source: Box<iggy_common::IggyError>, + }, + #[error( + "topic {topic_id} of stream {stream_id} holds critical options this build cannot read: {source}. Run a build that understands them" + )] + TopicOptionsUnavailable { + stream_id: usize, + topic_id: usize, + #[source] + source: Box<iggy_common::IggyError>, + }, #[error("failed to load server config")] Config(#[source] configs::ConfigurationError), #[error("failed to allocate shards from sharding.cpu_allocation")]
diff --git a/core/server/src/stream_key_rewrapper.rs b/core/server/src/stream_key_rewrapper.rs new file mode 100644 index 0000000..eee3f9e --- /dev/null +++ b/core/server/src/stream_key_rewrapper.rs
@@ -0,0 +1,673 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::shell::{ServerShard, ShellBus, ShellShard}; +use bytes::Bytes; +use consensus::MetadataHandle; +use iggy_binary_protocol::PrepareHeader; +use iggy_binary_protocol::requests::streams::{RewrapStreamKeyEntry, RewrapStreamKeysRequest}; +use iggy_common::{ + HeaderKey, IggyError, MAX_WRAP_SET_ENVELOPES, OptionKey, RewrapPlanError, ServerKeyContext, + stream_wrap_set, +}; +use journal::superblock::SuperblockStore; +use journal::{Journal, JournalHandle}; +use metadata::impls::metadata::StreamsFrontend; +use metadata::stm::stream::{Stream, StreamsInner}; +use server_common::Message; +use shard::Receiver; +use std::collections::BTreeSet; +use std::rc::Rc; +use std::time::Duration; +use tracing::{info, trace, warn}; + +/// How often the metadata primary checks whether a sweep is due. A tick with +/// an unchanged view and applied op does nothing else. +pub const REWRAP_INTERVAL: Duration = Duration::from_secs(5); +/// Upper bound on streams rewrapped by one replicated operation, so one +/// rotation never monopolizes the metadata pipeline. +const MAX_ENTRIES_PER_OPERATION: usize = 64; + +/// What the rewrapper keeps between ticks. +#[derive(Debug, Default)] +struct Rewrapper { + /// Metadata view and last applied op of the last finished sweep. + swept: Option<(u32, u64)>, + /// Stream incarnations already reported as stuck, so each warns once. + stuck: BTreeSet<(usize, u64)>, + /// Whether the last finished sweep found every set settled. + settled: bool, +} + +/// Outcome of one tick. +#[derive(Debug, PartialEq, Eq)] +enum Sweep { + Stopped, + NotPrimary, + Unchanged, + Interrupted, + Finished, +} + +/// A stream that needs a rewrap this node cannot make. +#[derive(Debug)] +struct StuckStream { + stream_id: usize, + created_op: u64, + kek_fingerprints: String, + error: RewrapPlanError, +} + +/// Run the rewrapper until `stop` fires, planning with `context`, this node's +/// keys plus the retired fingerprints. Spawned only when that context is +/// rotating. Only the caught-up metadata primary proposes, so a rotation is +/// replicated once and every replica stores the same wrap sets. A node never +/// rewrites them locally. +pub async fn run_stream_key_rewrapper( + shard: Rc<ServerShard>, + context: ServerKeyContext, + stop: Receiver<()>, +) { + trace!(shard = shard.id, "stream key rewrapper started"); + let mut rewrapper = Rewrapper::default(); + loop { + if compio::time::timeout(REWRAP_INTERVAL, stop.recv()) + .await + .is_ok() + || rewrapper.sweep(&shard, &context, &stop).await == Sweep::Stopped + { + break; + } + } + trace!(shard = shard.id, "stream key rewrapper exited"); +} + +impl Rewrapper { + /// Scan every stream once when this node is the caught-up primary and + /// the metadata moved since the last finished sweep, then propose the + /// unsettled ones in bounded operations. + async fn sweep<B, MJ, S, SB>( + &mut self, + shard: &Rc<ShellShard<B, MJ, S, SB>>, + context: &ServerKeyContext, + stop: &Receiver<()>, + ) -> Sweep + where + B: ShellBus, + MJ: JournalHandle + 'static, + MJ::Target: Journal<Entry = Message<PrepareHeader>, Header = PrepareHeader>, + S: 'static, + SB: SuperblockStore + 'static, + { + if stop.try_recv().is_ok() { + return Sweep::Stopped; + } + let metadata = shard.plane.metadata(); + let Some(consensus) = metadata + .consensus + .as_ref() + .filter(|_| metadata.is_caught_up_primary()) + else { + self.swept = None; + return Sweep::NotPrimary; + }; + // Read with the scan below and no await between, so the mark names + // exactly the state the scan saw. + let mark = (consensus.view(), consensus.commit_min()); + if self.swept == Some(mark) { + return Sweep::Unchanged; + } + let pending = metadata + .mux_stm + .streams() + .read(|streams| unsettled_streams(streams, context)); + let mut rewrapped = 0; + let mut stuck = BTreeSet::new(); + for ids in pending.chunks(MAX_ENTRIES_PER_OPERATION) { + if stop.try_recv().is_ok() { + return Sweep::Stopped; + } + let (entries, failures) = metadata + .mux_stm + .streams() + .read(|streams| planned_entries(streams, context, ids)); + for failure in failures { + let incarnation = (failure.stream_id, failure.created_op); + if !self.stuck.contains(&incarnation) { + match &failure.error { + RewrapPlanError::WrapSetFull => warn!( + stream_id = failure.stream_id, + stored_kek_fingerprints = %failure.kek_fingerprints, + node_kek_fingerprints = %context.kek_fingerprints().join(","), + "Stream {} wrap set is full with {MAX_WRAP_SET_ENVELOPES} wraps, none under the current server key or a retired one, so add one of its fingerprints to retired_kek_fingerprints", + failure.stream_id + ), + RewrapPlanError::Key(error) => warn!( + stream_id = failure.stream_id, + stored_kek_fingerprints = %failure.kek_fingerprints, + node_kek_fingerprints = %context.kek_fingerprints().join(","), + error = %error, + "Stream {} needs a key rewrap this node cannot make: {error}", + failure.stream_id + ), + } + } + stuck.insert(incarnation); + } + if entries.is_empty() { + continue; + } + let planned = entries.len(); + match metadata + .submit_rewrap_stream_keys_in_process(RewrapStreamKeysRequest { entries }) + .await + { + Ok(commit) => { + rewrapped += planned; + info!( + streams = planned, + commit, + key_context = ?context, + "Rewrapped {planned} stream keys: added the current server key's wrap where missing and dropped retired ones" + ); + } + Err(error) => { + warn!( + ?error, + "Stream key rewrap not committed, retrying on the next tick" + ); + self.stuck.extend(stuck); + return Sweep::Interrupted; + } + } + } + let settled = rewrapped == 0 && stuck.is_empty(); + if !settled { + info!( + rewrapped, + stuck = stuck.len(), + "Stream key sweep rewrapped {rewrapped} streams, {} still need a rewrap this node cannot make", + stuck.len() + ); + } else if !self.settled { + info!( + kek_fingerprints = %context.kek_fingerprints().join(","), + "Every encrypted stream holds a wrap under the current server key and none under a retired one" + ); + } + self.swept = Some(mark); + self.stuck = stuck; + self.settled = settled; + Sweep::Finished + } +} + +/// Ids of the encrypted streams whose stored set is not what this node's +/// keys call for, in one pass that allocates only the result. +fn unsettled_streams(streams: &StreamsInner, context: &ServerKeyContext) -> Vec<usize> { + let wrapped_key = HeaderKey::from(OptionKey::EncryptionKeyWrapped); + streams + .items + .iter() + .filter(|(_, stream)| { + stream + .options + .get(&wrapped_key) + .is_some_and(|option| !context.is_settled(option.value.as_bytes())) + }) + .map(|(id, _)| id) + .collect() +} + +/// One operation's entries for the streams `ids` names, planned against +/// their current state, and the streams among them this node cannot rewrap. +/// A stream deleted or settled since the scan is skipped. +fn planned_entries( + streams: &StreamsInner, + context: &ServerKeyContext, + ids: &[usize], +) -> (Vec<RewrapStreamKeyEntry>, Vec<StuckStream>) { + let mut entries = Vec::with_capacity(ids.len()); + let mut stuck = Vec::new(); + for stream in ids.iter().filter_map(|id| streams.items.get(*id)) { + match planned_entry(stream, context) { + Ok(Some(entry)) => entries.push(entry), + Ok(None) => {} + Err(error) => stuck.push(StuckStream { + stream_id: stream.id, + created_op: stream.created_op, + kek_fingerprints: stream_wrap_set(&stream.options) + .ok() + .flatten() + .map(|set| set.kek_fingerprints().join(",")) + .unwrap_or_default(), + error, + }), + } + } + (entries, stuck) +} + +fn planned_entry( + stream: &Stream, + context: &ServerKeyContext, +) -> Result<Option<RewrapStreamKeyEntry>, RewrapPlanError> { + let Some(stored) = stream_wrap_set(&stream.options)? else { + return Ok(None); + }; + let Some(target) = context.target_set(&stored)? else { + return Ok(None); + }; + Ok(Some(RewrapStreamKeyEntry { + stream_id: u32::try_from(stream.id).map_err(|_| IggyError::InvalidStreamId)?, + created_op: stream.created_op, + key_fingerprint: stored.key_fingerprint_bytes().try_into().map_err(|_| { + IggyError::UnsupportedCriticalOption(OptionKey::EncryptionKeyWrapped.to_string()) + })?, + wrap_set: Bytes::copy_from_slice(target.as_ref()), + })) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::dispatch::test_support::{ + SpyBus, TestShard, prepare_message, test_shard_with_key_context, + }; + use consensus::{Consensus as _, Plane as _, PlaneKind, Sequencer}; + use iggy_binary_protocol::requests::streams::CreateStreamRequest; + use iggy_binary_protocol::{Operation, PrepareOkHeader, WireEncode, WireName}; + use iggy_common::wire_conversions::resource_options_to_wire; + use iggy_common::{ + HeaderValue, IggyTimestamp, OptionValue, OptionsProvenance, ResourceOptions, + StreamEncryptionKey, StreamKeyOrigin, WrapSet, critical_keys_marker, + }; + use journal::prepare_journal::PrepareJournal; + use metadata::stm::{StateHandler, StateMachine}; + use std::sync::Arc; + use tempfile::TempDir; + + const OLD_KEY: &str = "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE="; + const NEW_KEY: &str = "AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA="; + const STRANGER_KEY: &str = "ICEiIyQlJicoKSorLC0uLzAxMjM0NTY3ODk6Ozw9Pj8="; + + #[compio::test] + async fn given_backup_or_lagging_node_when_ticking_should_not_propose() { + let rotated = context(NEW_KEY, &[OLD_KEY]); + let (_stop_sender, stop) = shard::channel(1); + for (replica, replica_count, outcome) in + [(1, 3, Sweep::NotPrimary), (0, 1, Sweep::Finished)] + { + let bus = SpyBus::default(); + let shard = Rc::new(test_shard_with_key_context( + &bus, + replica, + replica_count, + 1, + Arc::clone(&rotated), + )); + // A backup holds a stale set and must leave it to the primary. + // The sole replica is the primary, but its set already matches. + let stored_under = if replica_count == 1 { NEW_KEY } else { OLD_KEY }; + seed_stream(&shard, "seeded", &context(stored_under, &[])); + let consensus = shard.plane.metadata().consensus.as_ref().unwrap(); + let sequence = consensus.sequencer().current_sequence(); + + assert_eq!( + Rewrapper::default().sweep(&shard, &rotated, &stop).await, + outcome + ); + + assert_eq!( + consensus.sequencer().current_sequence(), + sequence, + "replica {replica} of {replica_count} must not propose" + ); + assert!(bus.replica_sends.borrow().is_empty()); + } + } + + #[compio::test] + async fn given_unchanged_metadata_when_sweeping_again_should_skip_the_scan() { + let bus = SpyBus::default(); + let rotated = context(NEW_KEY, &[OLD_KEY]); + let shard = Rc::new(test_shard_with_key_context( + &bus, + 0, + 1, + 1, + Arc::clone(&rotated), + )); + seed_stream(&shard, "seeded", &context(STRANGER_KEY, &[])); + let (_stop_sender, stop) = shard::channel(1); + let mut rewrapper = Rewrapper::default(); + + assert_eq!( + rewrapper.sweep(&shard, &rotated, &stop).await, + Sweep::Finished + ); + assert_eq!(rewrapper.stuck.len(), 1, "the stranger's set is reported"); + assert!(!rewrapper.settled); + assert_eq!( + rewrapper.sweep(&shard, &rotated, &stop).await, + Sweep::Unchanged + ); + assert_eq!(rewrapper.stuck.len(), 1); + } + + #[compio::test] + async fn given_settled_primary_when_a_stale_stream_commits_should_rescan_and_rewrap_it() { + let directory = TempDir::new().expect("metadata journal directory"); + let rotated = context(NEW_KEY, &[OLD_KEY]); + let shard = journal_backed_primary(&directory, &rotated).await; + let (_stop_sender, stop) = shard::channel(1); + let mut rewrapper = Rewrapper::default(); + assert_eq!( + rewrapper.sweep(&shard, &rotated, &stop).await, + Sweep::Finished + ); + assert!(rewrapper.settled); + assert_eq!( + rewrapper.sweep(&shard, &rotated, &stop).await, + Sweep::Unchanged + ); + + // An edge still on the old config wraps a new stream under the old key only. + commit_stream(&shard, "created-mid-rotation", &context(OLD_KEY, &[])).await; + let consensus = shard.plane.metadata().consensus.as_ref().expect("primary"); + let before = consensus.commit_min(); + + assert_eq!( + rewrapper.sweep(&shard, &rotated, &stop).await, + Sweep::Finished + ); + assert_eq!(consensus.commit_min(), before + 1, "one rewrap commits"); + assert_eq!(unsettled(&shard, &rotated), Vec::<usize>::new()); + } + + #[compio::test] + async fn given_more_stale_streams_than_one_operation_when_sweeping_should_commit_every_chunk() { + let directory = TempDir::new().expect("metadata journal directory"); + let rotated = context(NEW_KEY, &[OLD_KEY]); + let shard = journal_backed_primary(&directory, &rotated).await; + let old = context(OLD_KEY, &[]); + let stale_count = 2 * MAX_ENTRIES_PER_OPERATION + 8; + for index in 0..stale_count { + seed_stream(&shard, &format!("stale-{index}"), &old); + } + let (_stop_sender, stop) = shard::channel(1); + let consensus = shard.plane.metadata().consensus.as_ref().expect("primary"); + let before = consensus.commit_min(); + + assert_eq!( + Rewrapper::default().sweep(&shard, &rotated, &stop).await, + Sweep::Finished + ); + + assert_eq!( + consensus.commit_min() - before, + stale_count.div_ceil(MAX_ENTRIES_PER_OPERATION) as u64 + ); + assert_eq!(unsettled(&shard, &rotated), Vec::<usize>::new()); + } + + #[test] + fn given_more_stale_streams_than_one_operation_when_sweeping_should_plan_each_from_one_scan() { + let old = context(OLD_KEY, &[]); + let rotated = context(NEW_KEY, &[OLD_KEY]); + let mut streams = StreamsInner::new(); + let stale_count = 2 * MAX_ENTRIES_PER_OPERATION + 8; + for index in 0..stale_count { + create_stream(&mut streams, &format!("stale-{index}"), &old, 1); + create_stream(&mut streams, &format!("settled-{index}"), &rotated, 2); + } + + let pending = unsettled_streams(&streams, &rotated); + + assert_eq!(pending.len(), stale_count); + let mut planned = Vec::new(); + for ids in pending.chunks(MAX_ENTRIES_PER_OPERATION) { + let (entries, stuck) = planned_entries(&streams, &rotated, ids); + assert!(entries.len() <= MAX_ENTRIES_PER_OPERATION); + assert!(stuck.is_empty()); + planned.extend(entries); + } + let mut planned_ids: Vec<u32> = planned.iter().map(|entry| entry.stream_id).collect(); + planned_ids.dedup(); + assert_eq!(planned_ids.len(), stale_count, "each stale stream once"); + for entry in &planned { + let stream = streams.items.get(entry.stream_id as usize).expect("exists"); + assert!(stream.name.starts_with("stale-")); + let set = WrapSet::try_from(entry.wrap_set.as_ref()).expect("a valid set"); + assert_eq!(set.envelopes().len(), 2); + assert_eq!(set.key_fingerprint_bytes(), entry.key_fingerprint); + assert!(rotated.is_settled(set.as_ref())); + } + } + + #[test] + fn given_old_and_current_sets_when_planning_should_entry_only_the_stale_one() { + let mut streams = StreamsInner::new(); + create_stream(&mut streams, "old", &context(OLD_KEY, &[]), 1); + create_stream(&mut streams, "current", &context(NEW_KEY, &[]), 2); + let rotated = context(NEW_KEY, &[OLD_KEY]); + + let pending = unsettled_streams(&streams, &rotated); + let (entries, stuck) = planned_entries(&streams, &rotated, &pending); + + assert_eq!(pending, vec![0]); + assert!(stuck.is_empty()); + assert_eq!(entries.len(), 1); + let entry = &entries[0]; + assert_eq!(entry.stream_id, 0); + let set = WrapSet::try_from(entry.wrap_set.as_ref()).expect("a valid set"); + assert_eq!(set.envelopes().len(), 2); + assert_eq!(set.key_fingerprint_bytes(), entry.key_fingerprint); + } + + #[test] + fn given_sets_this_node_cannot_rewrap_when_planning_should_report_them() { + let mut streams = StreamsInner::new(); + create_stream(&mut streams, "stranger", &context(STRANGER_KEY, &[]), 1); + create_stream(&mut streams, "old", &context(OLD_KEY, &[]), 2); + let rotated = context(NEW_KEY, &[OLD_KEY]); + + let pending = unsettled_streams(&streams, &rotated); + let (entries, stuck) = planned_entries(&streams, &rotated, &pending); + + assert_eq!(pending, vec![0, 1]); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].stream_id, 1); + assert_eq!(stuck.len(), 1); + assert_eq!(stuck[0].stream_id, 0); + assert!(matches!( + stuck[0].error, + RewrapPlanError::Key(IggyError::CannotDecryptData) + )); + assert_eq!( + stuck[0].kek_fingerprints, + context(STRANGER_KEY, &[]).kek_fingerprint().expect("a key") + ); + } + + #[test] + fn given_retired_kek_when_planning_should_drop_its_wraps_and_keep_unknown_ones() { + let rotated = context(NEW_KEY, &[OLD_KEY]); + let old_fingerprint = context(OLD_KEY, &[]).kek_fingerprint().expect("a key"); + let phase_two = Arc::new( + ServerKeyContext::from_config(NEW_KEY, &[], true) + .and_then(|context| context.with_retired_kek_fingerprints(&[old_fingerprint])) + .expect("valid keys"), + ); + let mut streams = StreamsInner::new(); + create_stream(&mut streams, "both", &rotated, 1); + let stranger_and_new = WrapSet::new(vec![ + context(STRANGER_KEY, &[]) + .wrap(&seeded_key(2), StreamKeyOrigin::Server) + .expect("wraps"), + phase_two + .wrap(&seeded_key(2), StreamKeyOrigin::Server) + .expect("wraps"), + ]) + .expect("two wraps"); + insert_stream(&mut streams, "unknown-kept", &stranger_and_new); + + let pending = unsettled_streams(&streams, &phase_two); + let (entries, stuck) = planned_entries(&streams, &phase_two, &pending); + + assert_eq!(pending, vec![0]); + assert!(stuck.is_empty()); + let set = WrapSet::try_from(entries[0].wrap_set.as_ref()).expect("a valid set"); + assert_eq!(set.kek_fingerprints(), phase_two.kek_fingerprints()); + } + + fn context(key: &str, previous: &[&str]) -> Arc<ServerKeyContext> { + let previous: Vec<String> = previous.iter().map(|key| (*key).to_owned()).collect(); + Arc::new(ServerKeyContext::from_config(key, &previous, true).expect("valid keys")) + } + + fn seeded_key(seed: u8) -> StreamEncryptionKey { + StreamEncryptionKey::try_from([seed; 32].as_slice()).expect("32 bytes") + } + + fn create_stream( + streams: &mut StreamsInner, + name: &str, + wrapping: &ServerKeyContext, + seed: u8, + ) { + let set = wrapping + .wrap_set(&seeded_key(seed), StreamKeyOrigin::Server) + .expect("wraps"); + insert_stream(streams, name, &set); + } + + fn insert_stream(streams: &mut StreamsInner, name: &str, set: &WrapSet) { + let mut options = ResourceOptions::new(); + options.insert( + HeaderKey::from(OptionKey::EncryptionKeyWrapped), + OptionValue::explicit(HeaderValue::try_from(set.as_ref()).expect("fits")), + ); + options.insert( + HeaderKey::from(OptionKey::CriticalKeys), + OptionValue::explicit( + critical_keys_marker(&[OptionKey::EncryptionKeyWrapped]).expect("marker"), + ), + ); + let request = CreateStreamRequest { + name: WireName::new(name).expect("valid name"), + options: resource_options_to_wire(&options, OptionsProvenance::All).expect("encodes"), + }; + let reply = StateHandler::apply(&request, streams, IggyTimestamp::now()); + assert_eq!(reply.code, 0); + } + + async fn journal_backed_primary( + directory: &TempDir, + keys: &Arc<ServerKeyContext>, + ) -> Rc<TestShard> { + let mut shard = test_shard_with_key_context(&SpyBus::default(), 0, 1, 1, Arc::clone(keys)); + shard.plane.metadata_mut().journal = Some( + PrepareJournal::open(&directory.path().join("journal.wal"), 0) + .await + .expect("opens the metadata journal"), + ); + Rc::new(shard) + } + + /// Commits a create through the primary's replicate path, as the create an + /// edge rewrote reaches it. + async fn commit_stream(shard: &TestShard, name: &str, wrapping: &ServerKeyContext) { + let metadata = shard.plane.metadata(); + let consensus = metadata.consensus.as_ref().expect("primary"); + let prepare = consensus::seal_prepare_checksum( + prepare_message( + Operation::CreateStream, + 0, + 1, + &create_stream_body(name, wrapping), + ) + .transmute_header(|old, header: &mut PrepareHeader| { + *header = old; + header.session = 0; + }), + ); + consensus.pipeline_message(PlaneKind::Metadata, &prepare); + metadata.on_replicate(prepare).await; + let mut loopback = Vec::new(); + consensus.drain_loopback_into(&mut loopback); + for ack in loopback { + metadata + .on_ack( + ack.try_into_typed::<PrepareOkHeader>() + .expect("loopback holds self PrepareOks"), + ) + .await; + } + } + + fn unsettled(shard: &TestShard, keys: &ServerKeyContext) -> Vec<usize> { + shard + .plane + .metadata() + .mux_stm + .streams() + .read(|streams| unsettled_streams(streams, keys)) + } + + fn seed_stream<B, MJ, S, SB>( + shard: &Rc<ShellShard<B, MJ, S, SB>>, + name: &str, + wrapping: &ServerKeyContext, + ) where + B: ShellBus, + MJ: JournalHandle + 'static, + MJ::Target: Journal<Entry = Message<PrepareHeader>, Header = PrepareHeader>, + S: 'static, + SB: SuperblockStore + 'static, + { + shard + .plane + .metadata() + .mux_stm + .update(prepare_message( + Operation::CreateStream, + 1, + 1, + &create_stream_body(name, wrapping), + )) + .expect("applies"); + } + + fn create_stream_body(name: &str, wrapping: &ServerKeyContext) -> Bytes { + let set = WrapSet::from( + wrapping + .wrap(&seeded_key(5), StreamKeyOrigin::Client) + .expect("wraps"), + ); + let mut options = ResourceOptions::new(); + options.insert( + HeaderKey::from(OptionKey::EncryptionKeyWrapped), + OptionValue::explicit(HeaderValue::try_from(set.as_ref()).expect("fits")), + ); + CreateStreamRequest { + name: WireName::new(name).expect("valid name"), + options: resource_options_to_wire(&options, OptionsProvenance::All).expect("encodes"), + } + .to_bytes() + } +}
diff --git a/core/server/src/stream_keys.rs b/core/server/src/stream_keys.rs new file mode 100644 index 0000000..e27cddd --- /dev/null +++ b/core/server/src/stream_keys.rs
@@ -0,0 +1,450 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::wire::rewrite_request_body; +use bytes::Bytes; +use iggy_binary_protocol::requests::streams::CreateStreamRequest; +use iggy_binary_protocol::{RoutedRequestHeader, WireEncode}; +use iggy_common::wire_conversions::resource_options_to_wire; +use iggy_common::{ + HeaderKey, HeaderValue, IggyError, OptionKey, OptionValue, OptionsProvenance, ResourceOptions, + ServerKeyContext, StreamEncryptionKey, StreamKeyOrigin, critical_keys_marker, erase_buffer, +}; +use metadata::stm::stream::StreamsInner; +use secrecy::zeroize::Zeroize; +use server_common::Message; + +/// A request past static bounds. A `CreateStream` keeps its decoded body and +/// admitted options for [`maybe_rewrite_secret_options_request`], so a client +/// key is decoded once and never encoded into a request in the clear again. +pub enum Bounded { + Request(Message<RoutedRequestHeader>), + StreamCreate { + request: Message<RoutedRequestHeader>, + create: CreateStreamRequest, + /// The admitted canonical options, a client key among them. + options: ResourceOptions, + }, +} + +/// Wrap a `CreateStream`'s key before the request leaves this node: the +/// client's key when it sent one, or a key generated here when it did not and +/// this node encrypts new streams. The key is wrapped under every configured +/// server key, so the stream also opens on nodes a rotation has not reached. +/// TCP requests carry only the wrap set and the critical marker onward, so no +/// plaintext key reaches a replica, the log or a snapshot. An HTTP follower +/// relays the raw body to the primary, which wraps the key there. The body is +/// built from the admitted options, and the copies of a client key this node +/// decoded are erased whatever the outcome. Runs after catalog admission, +/// which refuses a client-sent wrapped key. +/// +/// # Errors +/// +/// `StreamEncryptionUnavailable` for a client key on a node without a server +/// key, or for any key while `deferring` reports a committed stream key this +/// node cannot open, `CannotEncryptData` when generating or wrapping fails. +pub fn maybe_rewrite_secret_options_request( + context: &ServerKeyContext, + bounded: Bounded, + deferring: impl FnOnce() -> bool, +) -> Result<Message<RoutedRequestHeader>, IggyError> { + let (request, create, options) = match bounded { + Bounded::Request(request) => return Ok(request), + Bounded::StreamCreate { + request, + create, + options, + } => (request, create, options), + }; + match sealed_body(context, &create, options, deferring) { + Ok(None) => Ok(request), + Ok(Some(body)) => { + let rewritten = rewrite_request_body(&request, &body); + erase_stream_create(request, Some(create)); + rewritten + } + Err(error) => { + erase_stream_create(request, Some(create)); + Err(error) + } + } +} + +/// Zero a create's copies of a client key once it is rewritten or refused: +/// the decoded options block, when there is one, and the request body. +pub fn erase_stream_create( + mut request: Message<RoutedRequestHeader>, + create: Option<CreateStreamRequest>, +) { + if let Some(create) = create { + erase_buffer(create.options.into_bytes()); + } + request.as_mut_slice()[size_of::<RoutedRequestHeader>()..].zeroize(); +} + +/// The create body apply stores: the admitted options with the stream key +/// wrapped in place of a client key, or `None` when the client's own body +/// already is it, which holds no key. +fn sealed_body( + context: &ServerKeyContext, + create: &CreateStreamRequest, + mut options: ResourceOptions, + deferring: impl FnOnce() -> bool, +) -> Result<Option<Bytes>, IggyError> { + let client_key = match options.remove(&HeaderKey::from(OptionKey::EncryptionKey)) { + Some(plaintext) => { + let key = StreamEncryptionKey::try_from(plaintext.value.as_bytes()); + plaintext.erase(); + Some(key?) + } + None => None, + }; + let key = match client_key { + Some(_) if !context.accepts_stream_keys() => { + return Err(IggyError::StreamEncryptionUnavailable); + } + Some(key) => Some((key, StreamKeyOrigin::Client)), + None if context.generates_stream_keys() => { + Some((StreamEncryptionKey::generate()?, StreamKeyOrigin::Server)) + } + None => None, + }; + if let Some((key, origin)) = key { + // Keys that leave a committed stream unopenable may be keys no peer + // holds, and a stream wrapped only under them opens nowhere else. + if deferring() { + return Err(IggyError::StreamEncryptionUnavailable); + } + let set = context.wrap_set(&key, origin)?; + options.insert( + HeaderKey::from(OptionKey::EncryptionKeyWrapped), + OptionValue::explicit(HeaderValue::try_from(set.as_ref())?), + ); + options.insert( + HeaderKey::from(OptionKey::CriticalKeys), + OptionValue::explicit(critical_keys_marker(&[OptionKey::EncryptionKeyWrapped])?), + ); + } + let block = resource_options_to_wire(&options, OptionsProvenance::All)?; + if block.as_bytes() == create.options.as_bytes() { + return Ok(None); + } + Ok(Some( + CreateStreamRequest { + name: create.name.clone(), + options: block, + } + .to_bytes(), + )) +} + +/// Whether a committed stream holds no wrap under this node's keys, so this +/// node defers serving it. One fingerprint scan over every stream, run only +/// for a create that wraps a key. +#[must_use] +pub fn defers_stream_keys(streams: &StreamsInner, context: &ServerKeyContext) -> bool { + let wrapped_key = HeaderKey::from(OptionKey::EncryptionKeyWrapped); + streams.items.iter().any(|(_, stream)| { + stream + .options + .get(&wrapped_key) + .is_some_and(|option| !context.holds_wrap_for(option.value.as_bytes())) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::dispatch::test_support::request_message; + use crate::wire::request_body; + use iggy_binary_protocol::{Operation, WireDecode, WireName, WireOptions}; + use iggy_common::wire_conversions::resource_options_from_wire; + use iggy_common::{ + AdmittedOptions, IggyTimestamp, OptionAdmissionContext, OptionColumns, OptionOperation, + OptionsScope, StreamCreateOptions, WrapSet, admit, stream_wrap_set, text, + }; + use metadata::stm::StateHandler; + + const CONFIG_KEY: &str = "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE="; + const NEW_CONFIG_KEY: &str = "AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA="; + + #[test] + fn given_custom_key_on_binary_backup_when_rewriting_should_forward_only_wrapped_bytes() { + let context = context(true); + let key = StreamEncryptionKey::try_from([7u8; 32].as_slice()).unwrap(); + for options in [ + StreamCreateOptions { + encryption_key: Some(key.clone()), + ..StreamCreateOptions::default() + } + .to_wire() + .unwrap(), + raw_key_block(&text::as_base64(key.as_bytes())), + ] { + let rewritten = rewrite(&context, options).expect("rewrites"); + let body = request_body(&rewritten).to_vec(); + assert!(!contains(&body, key.as_bytes()), "raw key travels on"); + assert!( + !contains(&body, text::as_base64(key.as_bytes()).as_bytes()), + "base64 key travels on" + ); + let set = stored_set(&rewritten).expect("a wrap set"); + assert_eq!(set.envelopes().len(), 1); + assert_eq!(set.origin(), StreamKeyOrigin::Client); + assert_eq!(context.open(&set).unwrap(), key); + } + } + + #[test] + fn given_keyless_create_with_encryption_enabled_when_rewriting_should_attach_server_origin_wrap_set() + { + let context = context(true); + let first = + stored_set(&rewrite(&context, WireOptions::empty()).unwrap()).expect("a generated key"); + let second = + stored_set(&rewrite(&context, WireOptions::empty()).unwrap()).expect("a generated key"); + assert_eq!(first.origin(), StreamKeyOrigin::Server); + assert!(context.open(&first).is_ok()); + assert_ne!(first.key_fingerprint(), second.key_fingerprint()); + + let disabled = self::context(false); + let request = create_request(WireOptions::empty()); + let passed = + maybe_rewrite_secret_options_request(&disabled, bounded(request.clone()), || false) + .expect("passes through"); + assert_eq!(passed.as_slice(), request.as_slice()); + } + + #[test] + fn given_decoded_create_when_erasing_should_hold_the_only_handle_to_its_options() { + let request = create_request(raw_key_block(&text::as_base64(&[5u8; 32]))); + let create = CreateStreamRequest::decode_from(request_body(&request)).expect("decodes"); + assert!( + create.options.into_bytes().is_unique(), + "the decoded block shares the request buffer, so erasing it would zero nothing" + ); + } + + #[test] + fn given_rotated_node_when_wrapping_a_create_should_open_on_nodes_holding_only_the_old_key() { + let rotated = ServerKeyContext::from_config(NEW_CONFIG_KEY, &[CONFIG_KEY.to_owned()], true) + .expect("valid keys"); + let old_only = context(true); + let new_only = ServerKeyContext::from_config(NEW_CONFIG_KEY, &[], true).expect("valid key"); + let key = StreamEncryptionKey::try_from([3u8; 32].as_slice()).expect("32 bytes"); + for options in [ + WireOptions::empty(), + StreamCreateOptions { + encryption_key: Some(key), + ..StreamCreateOptions::default() + } + .to_wire() + .expect("encodes"), + ] { + let set = stored_set(&rewrite(&rotated, options).expect("rewrites")).expect("a set"); + let opened = rotated.open(&set).expect("opens on the rotated node"); + assert_eq!(old_only.open(&set).expect("opens on an old node"), opened); + assert_eq!(new_only.open(&set).expect("opens after phase 2"), opened); + } + let single = stored_set(&rewrite(&new_only, WireOptions::empty()).expect("rewrites")) + .expect("a set"); + assert_eq!(single.envelopes().len(), 1); + } + + #[test] + fn given_client_internal_key_when_admitting_should_reject_before_wrapping() { + let context = context(true); + let forged = WrapSet::from( + context + .wrap( + &StreamEncryptionKey::try_from([1u8; 32].as_slice()).unwrap(), + StreamKeyOrigin::Client, + ) + .unwrap(), + ); + for (key, value) in [ + ( + OptionKey::EncryptionKeyWrapped, + HeaderValue::try_from(forged.as_ref()).unwrap(), + ), + ( + OptionKey::CriticalKeys, + critical_keys_marker(&[OptionKey::EncryptionKeyWrapped]).unwrap(), + ), + ] { + let mut options = ResourceOptions::new(); + options.insert(HeaderKey::from(key), OptionValue::explicit(value)); + let block = resource_options_to_wire(&options, OptionsProvenance::All).unwrap(); + assert_eq!( + admit_create(&block).unwrap_err().as_code(), + IggyError::ReadOnlyOption(key.to_string()).as_code() + ); + } + let valid = StreamCreateOptions { + encryption_key: Some(StreamEncryptionKey::try_from([2u8; 32].as_slice()).unwrap()), + ..StreamCreateOptions::default() + } + .to_wire() + .unwrap(); + assert!(admit_create(&valid).is_ok()); + assert!(rewrite(&context, valid.clone()).is_ok()); + assert_eq!( + rewrite(&ServerKeyContext::disabled(), valid) + .unwrap_err() + .as_code(), + IggyError::StreamEncryptionUnavailable.as_code() + ); + } + + #[test] + fn given_committed_stream_this_node_cannot_open_when_wrapping_a_create_should_refuse() { + let context = context(true); + let stranger = ServerKeyContext::from_config(NEW_CONFIG_KEY, &[], true).expect("valid key"); + let rotated = ServerKeyContext::from_config(NEW_CONFIG_KEY, &[CONFIG_KEY.to_owned()], true) + .expect("valid keys"); + let mut streams = StreamsInner::new(); + insert_wrapped_stream(&mut streams, "opens-here", &context); + assert!(!defers_stream_keys(&streams, &context)); + insert_wrapped_stream(&mut streams, "stranger", &stranger); + assert!(defers_stream_keys(&streams, &context)); + assert!(!defers_stream_keys(&streams, &rotated)); + + let client_key = StreamCreateOptions { + encryption_key: Some(StreamEncryptionKey::try_from([4u8; 32].as_slice()).unwrap()), + ..StreamCreateOptions::default() + } + .to_wire() + .unwrap(); + for options in [WireOptions::empty(), client_key] { + let refusal = maybe_rewrite_secret_options_request( + &context, + bounded(create_request(options)), + || defers_stream_keys(&streams, &context), + ) + .expect_err("a deferring node must not wrap a new stream key"); + assert_eq!( + refusal.as_code(), + IggyError::StreamEncryptionUnavailable.as_code() + ); + } + let plaintext = self::context(false); + let request = create_request(WireOptions::empty()); + let passed = + maybe_rewrite_secret_options_request(&plaintext, bounded(request.clone()), || { + defers_stream_keys(&streams, &plaintext) + }) + .expect("a create that wraps nothing passes"); + assert_eq!(passed.as_slice(), request.as_slice()); + } + + fn context(enabled: bool) -> ServerKeyContext { + ServerKeyContext::from_config(CONFIG_KEY, &[], enabled).unwrap() + } + + fn create_request(options: WireOptions) -> Message<RoutedRequestHeader> { + let body = CreateStreamRequest { + name: WireName::new("orders").unwrap(), + options, + } + .to_bytes(); + request_message(Operation::CreateStream, 1, 1, 1, &body) + } + + fn rewrite( + context: &ServerKeyContext, + options: WireOptions, + ) -> Result<Message<RoutedRequestHeader>, IggyError> { + maybe_rewrite_secret_options_request(context, bounded(create_request(options)), || false) + } + + /// `request` past static bounds, admitted the way both chains admit it. + fn bounded(request: Message<RoutedRequestHeader>) -> Bounded { + let create = CreateStreamRequest::decode_from(request_body(&request)).expect("decodes"); + let admitted = admit_create(&create.options).expect("admits"); + Bounded::StreamCreate { + request, + create, + options: admitted.explicit, + } + } + + fn stored_set(request: &Message<RoutedRequestHeader>) -> Option<WrapSet> { + let create = CreateStreamRequest::decode_from(request_body(request)).unwrap(); + let options = resource_options_from_wire(&create.options, true).unwrap(); + assert!(!options.contains_key(&HeaderKey::from(OptionKey::EncryptionKey))); + stream_wrap_set(&options).unwrap() + } + + fn insert_wrapped_stream(streams: &mut StreamsInner, name: &str, wrapping: &ServerKeyContext) { + let set = wrapping + .wrap_set( + &StreamEncryptionKey::try_from([9u8; 32].as_slice()).unwrap(), + StreamKeyOrigin::Server, + ) + .unwrap(); + let mut options = ResourceOptions::new(); + options.insert( + HeaderKey::from(OptionKey::EncryptionKeyWrapped), + OptionValue::explicit(HeaderValue::try_from(set.as_ref()).unwrap()), + ); + options.insert( + HeaderKey::from(OptionKey::CriticalKeys), + OptionValue::explicit( + critical_keys_marker(&[OptionKey::EncryptionKeyWrapped]).unwrap(), + ), + ); + let request = CreateStreamRequest { + name: WireName::new(name).unwrap(), + options: resource_options_to_wire(&options, OptionsProvenance::All).unwrap(), + }; + assert_eq!( + StateHandler::apply(&request, streams, IggyTimestamp::now()).code, + 0 + ); + } + + fn raw_key_block(base64: &str) -> WireOptions { + let mut options = ResourceOptions::new(); + options.insert( + HeaderKey::from(OptionKey::EncryptionKey), + OptionValue::explicit(HeaderValue::try_from(base64).unwrap()), + ); + resource_options_to_wire(&options, OptionsProvenance::All).unwrap() + } + + fn admit_create(block: &WireOptions) -> Result<AdmittedOptions, IggyError> { + admit( + OptionsScope::Stream, + OptionOperation::Create, + block, + &[], + &OptionAdmissionContext { + current: None, + create_columns: OptionColumns { + name: Some("orders".to_owned()), + ..OptionColumns::default() + }, + }, + ) + } + + fn contains(haystack: &[u8], needle: &[u8]) -> bool { + haystack + .windows(needle.len()) + .any(|window| window == needle) + } +}
diff --git a/core/server/src/users.rs b/core/server/src/users.rs index 3b146cb..bf07bab 100644 --- a/core/server/src/users.rs +++ b/core/server/src/users.rs
@@ -28,11 +28,12 @@ //! //! `ChangePassword` also verifies the caller-supplied `current_password` //! against the target's stored hash before the op replicates (the legacy -//! server does this for self- and admin-initiated changes alike), and it -//! always empties that field, so no plaintext credential ever rides the -//! replicated prepare/WAL. A mismatch is committed as a rejecting no-op +//! server does this for self- and admin-initiated changes alike), pins the +//! target to the id of the user it verified, and always empties that field, +//! so no plaintext credential ever rides the replicated prepare/WAL. A +//! mismatch or an unresolved target is committed as a rejecting no-op //! (signalled by an empty new password) rather than denied pre-consensus, so -//! the caller's request sequence stays contiguous; see +//! the caller's request sequence stays contiguous. See //! `verify_and_rewrite_change_password`. use crate::shell::{ShellBus, ShellShard}; @@ -41,8 +42,8 @@ use consensus::MetadataHandle; use iggy_binary_protocol::codec::{WireDecode, WireEncode}; use iggy_binary_protocol::requests::users::{ChangePasswordRequest, CreateUserRequest}; -use iggy_binary_protocol::{Operation, PrepareHeader, RoutedRequestHeader}; -use iggy_common::IggyError; +use iggy_binary_protocol::{Operation, PrepareHeader, RoutedRequestHeader, WireIdentifier}; +use iggy_common::{IggyError, UserId}; use journal::superblock::SuperblockStore; use journal::{Journal, JournalHandle}; use metadata::impls::metadata::StreamsFrontend; @@ -53,10 +54,10 @@ /// /// `CreateUser` hashes the new password. `ChangePassword` strips the current /// password unconditionally (so no plaintext enters consensus) and verifies it -/// against the target's stored hash when the target resolves; a mismatch is -/// signalled to the replicated apply as a committed `InvalidCredentials` -/// rejection (see [`verify_and_rewrite_change_password`]). Every other operation -/// passes through unchanged. Returns [`IggyError::InvalidCommand`] only on an +/// against the target's stored hash. A mismatch or an unresolved target is +/// signalled to the replicated apply as a committed rejection (see +/// [`verify_and_rewrite_change_password`]). Every other operation passes +/// through unchanged. Returns [`IggyError::InvalidCommand`] only on an /// undecodable password body. pub fn maybe_rewrite_user_password_request<B, MJ, S, SB>( shard: &Rc<ShellShard<B, MJ, S, SB>>, @@ -81,13 +82,18 @@ Operation::ChangePassword => { let mut wire = ChangePasswordRequest::decode_from(body).map_err(|_| IggyError::InvalidCommand)?; - let stored_hash = shard + let target = shard .plane .metadata() .mux_stm .users() - .read(|users| users.password_hash_of(&wire.user_id)); - verify_and_rewrite_change_password(&mut wire, stored_hash.as_deref()) + .read(|users| users.resolve_password_hash(&wire.user_id)); + verify_and_rewrite_change_password( + &mut wire, + target + .as_ref() + .map(|(user_id, stored_hash)| (*user_id, stored_hash.as_ref())), + ) } _ => return Ok(request), }; @@ -97,30 +103,36 @@ /// Strip the current password, then re-encode the body for replication. /// -/// When the target resolves (`stored_hash` is `Some`) the supplied -/// `current_password` is verified against it first. A mismatch does NOT deny -/// pre-consensus: that would consume the client's request id without recording -/// it in the replicated `ClientTable`, so a retry of that id would re-execute -/// instead of deduping. Instead the new password is emptied, which signals the -/// committed apply to reject with `InvalidCredentials` (a committed no-op that -/// advances the watermark). +/// `target` is the id and stored hash of the user `wire.user_id` resolves to +/// on this node, `None` when it does not resolve. The supplied +/// `current_password` is verified against that hash, and `wire.user_id` is +/// pinned to that id. Apply resolves the identifier again, so without the pin +/// a name rebound before apply (root renamed to the verified name) would +/// point the change at a user whose password was never checked. /// -/// Either way the current password is stripped and the accepted new one hashed, -/// so no plaintext credential ever enters consensus. An unresolved target keeps -/// its hashed new password and the replicated apply rejects it with -/// `UserNotFound`. ACCEPTED RESIDUAL: if the target is created in the narrow -/// window between this rewrite and the apply, the current-password check is -/// skipped for that one op; RBAC still gates cross-user changes. Concurrent -/// same-user `ChangePassword` ops each verify against the hash read here, so -/// the last to commit wins over a now-stale predecessor; there is no -/// compare-and-swap, which is benign and inherent to distributed submission. +/// A mismatch or an unresolved target does NOT deny pre-consensus: that would +/// consume the client's request id without recording it in the replicated +/// `ClientTable`, so a retry of that id would re-execute instead of deduping. +/// Instead the new password is emptied, which signals the committed apply to +/// reject (a committed no-op that advances the watermark). Apply answers +/// `UserNotFound` when the target is still missing and `InvalidCredentials` +/// otherwise, also for a target created or renamed into place before apply. +/// +/// Either way the current password is stripped and an accepted new one hashed, +/// so no plaintext credential ever enters consensus. Concurrent same-user +/// `ChangePassword` ops each verify against the hash read here, so the last to +/// commit wins over a now-stale predecessor. There is no compare-and-swap, +/// which is benign and inherent to distributed submission. fn verify_and_rewrite_change_password( wire: &mut ChangePasswordRequest, - stored_hash: Option<&str>, + target: Option<(UserId, &str)>, ) -> Bytes { - let credentials_ok = match stored_hash { - Some(stored_hash) => crypto::verify_password(&wire.current_password, stored_hash), - None => true, + let credentials_ok = match target { + Some((user_id, stored_hash)) => { + wire.user_id = WireIdentifier::numeric(user_id); + crypto::verify_password(&wire.current_password, stored_hash) + } + None => false, }; wire.current_password = String::new(); wire.new_password = if credentials_ok { @@ -138,25 +150,33 @@ #[cfg(test)] mod tests { use super::*; - use iggy_binary_protocol::WireIdentifier; + use crate::dispatch::test_support::{ + FIRST_BOOT, SpyBus, TestShard, prepare_message, request_message, test_shard, + }; + use iggy_binary_protocol::requests::users::UpdateUserResolvedRequest; + use iggy_binary_protocol::{WireGlobalPermissions, WireName, WireOptions, WirePermissions}; + use iggy_common::UserStatus; + use iggy_common::defaults::DEFAULT_ROOT_USER_ID; + use metadata::stm::authz::GatedApply; + use metadata::stm::result::{ApplyReply, ChangePasswordResult}; + use std::mem::size_of; + use std::sync::Arc; const CURRENT_PASSWORD: &str = "current-password"; const NEW_PASSWORD: &str = "brand-new-password"; - - fn change_password_request() -> ChangePasswordRequest { - ChangePasswordRequest { - user_id: WireIdentifier::numeric(7), - current_password: CURRENT_PASSWORD.to_owned(), - new_password: NEW_PASSWORD.to_owned(), - } - } + const ROOT_USERNAME: &str = "iggy"; + const ROOT_PASSWORD: &str = "root-password"; + const ADMIN_USERNAME: &str = "mallory"; + const ADMIN_PASSWORD: &str = "mallory-password"; + const UNUSED_USERNAME: &str = "unused-name"; + const VSR_CLIENT_ID: u128 = 1; #[test] fn given_wrong_current_password_when_rewrite_should_signal_reject() { let stored_hash = crypto::hash_password("some-other-password"); let mut wire = change_password_request(); - let body = verify_and_rewrite_change_password(&mut wire, Some(&stored_hash)); + let body = verify_and_rewrite_change_password(&mut wire, Some((7, &stored_hash))); let decoded = ChangePasswordRequest::decode_from(&body).expect("re-encoded body round-trips"); @@ -177,7 +197,7 @@ let stored_hash = crypto::hash_password(CURRENT_PASSWORD); let mut wire = change_password_request(); - let body = verify_and_rewrite_change_password(&mut wire, Some(&stored_hash)); + let body = verify_and_rewrite_change_password(&mut wire, Some((7, &stored_hash))); let decoded = ChangePasswordRequest::decode_from(&body).expect("re-encoded body round-trips"); @@ -196,11 +216,9 @@ } #[test] - fn given_unresolved_target_when_rewrite_should_hash_and_strip_without_verify() { + fn given_unresolved_target_when_rewrite_should_signal_reject() { let mut wire = change_password_request(); - // No stored hash: nothing to verify against, but the plaintext must - // still never reach consensus, so the body is rewritten regardless. let body = verify_and_rewrite_change_password(&mut wire, None); let decoded = @@ -210,8 +228,173 @@ "current password must be stripped even for an unresolved target" ); assert!( - crypto::verify_password(NEW_PASSWORD, &decoded.new_password), - "new password must still be hashed for an unresolved target" + decoded.new_password.is_empty(), + "an unverified target must signal reject via an empty new password" + ); + } + + #[test] + fn given_root_renamed_to_target_after_rewrite_when_committed_should_keep_root_password() { + let shard = credentials_shard(); + let admin = create_admin(&shard); + let change = edge_rewritten_change_password(&shard, UNUSED_USERNAME, CURRENT_PASSWORD); + + let rename = commit( + &shard, + DEFAULT_ROOT_USER_ID, + Operation::UpdateUserResolved, + &rename_body(DEFAULT_ROOT_USER_ID, UNUSED_USERNAME), + ); + assert_eq!(rename.code, 0, "root may rename itself"); + let reply = commit(&shard, admin, Operation::ChangePassword, &change); + + assert_eq!( + reply.code, + u32::from(ChangePasswordResult::InvalidCredentials) + ); + assert_password(&shard, DEFAULT_ROOT_USER_ID, ROOT_PASSWORD); + } + + #[test] + fn given_names_swapped_after_verify_when_committed_should_change_only_verified_user() { + let shard = credentials_shard(); + let admin = create_admin(&shard); + let change = edge_rewritten_change_password(&shard, ADMIN_USERNAME, ADMIN_PASSWORD); + + for (user_id, username) in [ + (admin, "mallory-old"), + (DEFAULT_ROOT_USER_ID, ADMIN_USERNAME), + ] { + let rename = commit( + &shard, + user_id, + Operation::UpdateUserResolved, + &rename_body(user_id, username), + ); + assert_eq!(rename.code, 0, "a user may rename itself to {username}"); + } + let reply = commit(&shard, admin, Operation::ChangePassword, &change); + + assert_eq!(reply.code, 0, "the verified user changes its own password"); + assert_password(&shard, DEFAULT_ROOT_USER_ID, ROOT_PASSWORD); + assert_password(&shard, admin, NEW_PASSWORD); + } + + fn change_password_request() -> ChangePasswordRequest { + ChangePasswordRequest { + user_id: WireIdentifier::numeric(7), + current_password: CURRENT_PASSWORD.to_owned(), + new_password: NEW_PASSWORD.to_owned(), + } + } + + fn credentials_shard() -> Rc<TestShard> { + let shard = Rc::new(test_shard(&SpyBus::default(), 0, 1, FIRST_BOOT)); + shard + .plane + .metadata() + .mux_stm + .users() + .ensure_root_user(ROOT_USERNAME, &crypto::hash_password(ROOT_PASSWORD)); + shard + } + + /// Creates a user holding `manage_users` alone and returns its id. + fn create_admin(shard: &Rc<TestShard>) -> u32 { + let body = CreateUserRequest { + username: WireName::new(ADMIN_USERNAME).expect("valid username"), + password: crypto::hash_password(ADMIN_PASSWORD), + status: UserStatus::Active.as_code(), + permissions: Some(WirePermissions { + global: WireGlobalPermissions { + manage_servers: false, + read_servers: false, + manage_users: true, + read_users: false, + manage_streams: false, + read_streams: false, + manage_topics: false, + read_topics: false, + poll_messages: false, + send_messages: false, + }, + streams: Vec::new(), + }), + options: WireOptions::empty(), + } + .to_bytes(); + let created = commit(shard, DEFAULT_ROOT_USER_ID, Operation::CreateUser, &body); + assert_eq!(created.code, 0, "root creates the admin"); + shard + .plane + .metadata() + .mux_stm + .users() + .read(|users| users.user_by_name(ADMIN_USERNAME).map(|user| user.id)) + .expect("admin exists") + } + + /// The body the edge replicates for a `ChangePassword` it rewrites against + /// the state it has applied so far. + fn edge_rewritten_change_password( + shard: &Rc<TestShard>, + target: &str, + current_password: &str, + ) -> Bytes { + let body = ChangePasswordRequest { + user_id: WireIdentifier::named(target).expect("valid target name"), + current_password: current_password.to_owned(), + new_password: NEW_PASSWORD.to_owned(), + } + .to_bytes(); + let request = request_message(Operation::ChangePassword, VSR_CLIENT_ID, 1, 1, &body); + let rewritten = + maybe_rewrite_user_password_request(shard, request).expect("well-formed body rewrites"); + Bytes::copy_from_slice(request_body(&rewritten)) + } + + fn rename_body(user_id: u32, username: &str) -> Bytes { + UpdateUserResolvedRequest { + user_id: WireIdentifier::numeric(user_id), + username: Some(WireName::new(username).expect("valid username")), + status: None, + explicit_options: WireOptions::empty(), + derived_options: WireOptions::empty(), + } + .to_bytes() + } + + /// Commits `body` as `caller` through the in-apply RBAC gate. + fn commit(shard: &Rc<TestShard>, caller: u32, operation: Operation, body: &[u8]) -> ApplyReply { + let mut prepare = prepare_message(operation, VSR_CLIENT_ID, 1, body); + bytemuck::checked::from_bytes_mut::<PrepareHeader>( + &mut prepare.as_mut_slice()[..size_of::<PrepareHeader>()], + ) + .user_id = caller; + shard + .plane + .metadata() + .mux_stm + .gated_update(consensus::seal_prepare_checksum(prepare)) + .expect("committed body applies") + } + + fn assert_password(shard: &Rc<TestShard>, user_id: u32, password: &str) { + let stored_hash = shard + .plane + .metadata() + .mux_stm + .users() + .read(|users| { + users + .items + .get(user_id as usize) + .map(|user| Arc::clone(&user.password_hash)) + }) + .expect("user exists"); + assert!( + crypto::verify_password(password, &stored_hash), + "user {user_id} must hold the expected password" ); } }
diff --git a/core/server/src/wire.rs b/core/server/src/wire.rs index ecaea97..9df6d39 100644 --- a/core/server/src/wire.rs +++ b/core/server/src/wire.rs
@@ -30,23 +30,6 @@ &request.as_slice()[std::mem::size_of::<RoutedRequestHeader>()..request.header().size as usize] } -/// Check a client's `request_checksum` against the body it stamps. -/// -/// Must run BEFORE any body rewrite: PAT / password / consumer-group paths -/// substitute server-chosen bytes. Zero means unstamped and skips this check; -/// retry identity is validated separately. -/// -/// # Errors -/// [`IggyError::InvalidFormat`] when the stamp disagrees with the body. -pub fn verify_request_checksum(request: &Message<RoutedRequestHeader>) -> Result<(), IggyError> { - let stamped = request.header().request_checksum; - if stamped == 0 || u128::from(iggy_common::calculate_checksum(request_body(request))) == stamped - { - return Ok(()); - } - Err(IggyError::InvalidFormat) -} - /// Map the transport kind to the legacy wire discriminant /// (`1=TCP, 2=QUIC, 4=WebSocket`); TLS variants report their base /// transport. `ClientTransportKind` is `#[non_exhaustive]`, so any other @@ -87,7 +70,7 @@ // downstream. Clear rather than recompute; carrying them forward is a stale claim. header.checksum = 0; header.checksum_body = 0; - // Preserve the original validated client stamp in the prepare and receipt; + // Preserve the original client request identifier in the prepare and receipt; // the substituted body's integrity is sealed separately. rewritten.as_mut_slice()[std::mem::size_of::<RoutedRequestHeader>()..].copy_from_slice(body); Ok(rewritten) @@ -123,8 +106,8 @@ #[test] fn given_a_body_rewrite_should_keep_the_client_stamp_and_clear_the_stale_seals() { // The secret-bearing wire body is swapped for the hash-carrying replicated - // one. `request_checksum` describes what the client sent and admission has - // already checked it, so it must survive; the other two describe the body + // one. `request_checksum` identifies the logical client request, so it + // must survive; the other two describe the body // that just went away. let original = request(b"plaintext-secret", 0x1234); let rewritten = rewrite_request_body(&original, &Bytes::from_static(b"argon2-hash"))
diff --git a/core/shard/src/coordinator.rs b/core/shard/src/coordinator.rs index d1d6e72..c9e8c0e 100644 --- a/core/shard/src/coordinator.rs +++ b/core/shard/src/coordinator.rs
@@ -687,7 +687,7 @@ validate_checksum: true, segment_size, preallocate_segments: false, - encryptor: None, + key_context: Arc::default(), path_layout: PartitionPathLayout::default(), }, );
diff --git a/core/shard/src/lib.rs b/core/shard/src/lib.rs index 68c47fb..7bae459 100644 --- a/core/shard/src/lib.rs +++ b/core/shard/src/lib.rs
@@ -58,7 +58,7 @@ #[cfg(feature = "simulator")] use iggy_common::PartitionStats; use iggy_common::variadic; -use iggy_common::{ConsumerKind, IggyError, IggyExpiry, IggyTimestamp}; +use iggy_common::{ConsumerKind, EncryptorKind, IggyError, IggyExpiry, IggyTimestamp}; use journal::superblock::{PingPongSuperblock, SuperblockStore}; use journal::{Journal, JournalHandle}; use message_bus::client_listener::RequestHandler; @@ -84,7 +84,6 @@ use std::collections::{BTreeMap, HashMap, VecDeque}; use std::future::Future; use std::rc::Rc; -#[cfg(feature = "simulator")] use std::sync::Arc; use std::time::Duration; @@ -486,6 +485,8 @@ context: iggy_binary_protocol::primitives::partition_history::PartitionContext, fragments: PollFragments, current_offset: u64, + /// The owning partition's encryptor, captured with the fragments. + encryptor: Option<Arc<EncryptorKind>>, }, ConsumerOffset { stored: Option<u64>, @@ -1941,6 +1942,8 @@ count: senders.len(), })?; let nonce_seed = boot_nonce(metadata.consensus.as_ref()); + // Metadata admits a stream's wrap set with the keys its partitions open. + let metadata = metadata.with_stream_key_context(partitions.key_context()); let plane = MuxPlane::new(variadic!(metadata, partitions)); let ShardIdentity { id, name } = identity; let poll_completions = @@ -2545,6 +2548,8 @@ let (_reply_tx, reply_inbox) = channel(1); let metrics = crate::metrics::ShardMetrics::for_shard(); let nonce_seed = boot_nonce(metadata.consensus.as_ref()); + // Metadata admits a stream's wrap set with the keys its partitions open. + let metadata = metadata.with_stream_key_context(partitions.key_context()); let plane = MuxPlane::new(variadic!(metadata, partitions)); let ShardIdentity { id, name } = identity; let host: Rc<dyn ShardHost> = Rc::new(NoopHost); @@ -15408,7 +15413,7 @@ validate_checksum: true, segment_size, preallocate_segments: false, - encryptor: None, + key_context: Arc::default(), path_layout: PartitionPathLayout::default(), }, );
diff --git a/core/shard/src/partition_io.rs b/core/shard/src/partition_io.rs index 30fc68d..dc1bbf3 100644 --- a/core/shard/src/partition_io.rs +++ b/core/shard/src/partition_io.rs
@@ -7742,7 +7742,7 @@ validate_checksum: true, segment_size: IggyByteSize::from(SEGMENT_BYTES), preallocate_segments: false, - encryptor: None, + key_context: Arc::default(), path_layout: PartitionPathLayout::default(), }; let partitions = IggyPartitions::new(shard_id, config);
diff --git a/core/shard/src/poll.rs b/core/shard/src/poll.rs index 57455f7..1005449 100644 --- a/core/shard/src/poll.rs +++ b/core/shard/src/poll.rs
@@ -517,6 +517,7 @@ fragments, current_offset, replication, + encryptor, }) => { // The owner has admitted progress, but the offset may still be // queued. Release the reply before waiting for replication. @@ -525,6 +526,7 @@ context, fragments, current_offset, + encryptor, }); if let Some(replication) = replication { partitions
diff --git a/core/shard/src/poll/completion.rs b/core/shard/src/poll/completion.rs index 40d20d3..0fc4eeb 100644 --- a/core/shard/src/poll/completion.rs +++ b/core/shard/src/poll/completion.rs
@@ -477,7 +477,7 @@ validate_checksum: true, segment_size: IggyByteSize::from(1_048_576_u64), preallocate_segments: false, - encryptor: None, + key_context: Arc::default(), path_layout: PartitionPathLayout::default(), }, );
diff --git a/core/shard/src/poll/test_support.rs b/core/shard/src/poll/test_support.rs index f226b8b..78faabf 100644 --- a/core/shard/src/poll/test_support.rs +++ b/core/shard/src/poll/test_support.rs
@@ -44,7 +44,7 @@ validate_checksum: true, segment_size, preallocate_segments: false, - encryptor: None, + key_context: Arc::default(), path_layout: PartitionPathLayout::default(), } }
diff --git a/core/shard/src/primary_rebalance/tests.rs b/core/shard/src/primary_rebalance/tests.rs index f301572..2d78f4e 100644 --- a/core/shard/src/primary_rebalance/tests.rs +++ b/core/shard/src/primary_rebalance/tests.rs
@@ -143,7 +143,7 @@ validate_checksum: true, segment_size, preallocate_segments: false, - encryptor: None, + key_context: Arc::default(), path_layout: PartitionPathLayout::default(), }, );
diff --git a/core/simulator/src/client.rs b/core/simulator/src/client.rs index 10b7cf3..6c12e6f 100644 --- a/core/simulator/src/client.rs +++ b/core/simulator/src/client.rs
@@ -38,20 +38,26 @@ }; use iggy_binary_protocol::requests::segments::DeleteSegmentsRequest; use iggy_binary_protocol::requests::streams::{ - CreateStreamRequest, DeleteStreamRequest, GetStreamRequest, UpdateStreamRequest, + CreateStreamRequest, DeleteStreamRequest, GetStreamRequest, WRAPPED_STREAM_KEY_LENGTH, }; -use iggy_binary_protocol::requests::system::{BindSessionRequest, SessionIdentity}; -use iggy_binary_protocol::requests::topics::{ - CreateTopicRequest, DeleteTopicRequest, UpdateTopicRequest, +use iggy_binary_protocol::requests::system::{ + BindSessionRequest, ResourceTarget, SessionIdentity, UpdateResourceRequest, }; +use iggy_binary_protocol::requests::topics::{CreateTopicRequest, DeleteTopicRequest}; use iggy_binary_protocol::requests::users::login_register::BindSecret; use iggy_binary_protocol::requests::users::{ ChangePasswordRequest, CreateUserRequest, DeleteUserRequest, LoginRegisterRequest, - UpdatePermissionsRequest, UpdateUserRequest, + UpdatePermissionsRequest, }; use iggy_binary_protocol::{ AckLevel, ClientVersionInfo, IGGY_PROTOCOL_VERSION, Operation, RoutedRequestHeader, WireEncode, - WireIdentifier, WireName, WireOptions, WirePartitioning, WirePollingStrategy, + WireIdentifier, WireName, WireOptions, WirePartitioning, WirePermissions, WirePollingStrategy, +}; +use iggy_common::wire_conversions::resource_options_to_wire; +use iggy_common::{ + HeaderKey, HeaderValue, OptionKey, OptionValue, OptionsPatch, OptionsProvenance, + ResourceOptions, StreamUpdateOptions, TopicUpdateOptions, UserStatus, UserUpdateOptions, + critical_keys_marker, }; use metadata::stm::user::{ CreatePersonalAccessTokenRequest, DeletePersonalAccessTokenRequest, ROOT_USER_CREATED_AT_MICROS, @@ -62,6 +68,30 @@ use std::cell::{Cell, RefCell}; use std::collections::BTreeMap; +/// Key-encryption key [`SIM_STREAM_WRAP_SET`] is sealed under. +pub const SIM_STREAM_KEK: &str = "c2ltdWxhdG9yLXN0cmVhbS1rZXktZW5jcnlwdGlvbi0="; +/// Key-encryption key [`SIM_ROTATED_WRAP_SET`] is sealed under. +pub const SIM_ROTATED_KEK: &str = "c2ltdWxhdG9yLXJvdGF0ZWQta2V5LWVuY3J5cHRpb24="; +/// Client stream key both fixed wrap sets hold. +pub const SIM_STREAM_KEY: [u8; 32] = [0x51; 32]; +/// [`SIM_STREAM_KEY`] wrapped under [`SIM_STREAM_KEK`]. Precomputed, since a +/// fresh wrap draws its nonce from the OS generator and breaks seeded replay. +pub const SIM_STREAM_WRAP_SET: [u8; WRAPPED_STREAM_KEY_LENGTH] = [ + 0x01, 0x01, 0xd5, 0x40, 0xc9, 0x77, 0x57, 0xd4, 0x39, 0x46, 0xec, 0x07, 0x1d, 0xb1, 0x3d, 0xfb, + 0x78, 0x80, 0x18, 0xfc, 0xae, 0x2b, 0xb8, 0x25, 0x96, 0x8a, 0xd7, 0x9d, 0x65, 0xbc, 0x41, 0x08, + 0xd6, 0x1d, 0xed, 0x5e, 0x85, 0x01, 0x54, 0xcd, 0x8d, 0x17, 0x34, 0x16, 0x9f, 0x19, 0x6e, 0xc6, + 0xf7, 0xdc, 0x23, 0xcb, 0x1d, 0x9e, 0x04, 0x7a, 0xc7, 0xa6, 0xbf, 0x8b, 0xf2, 0x81, 0x42, 0x32, + 0xa9, 0x58, 0xd0, 0x29, 0x44, 0x35, 0x31, 0x19, 0x21, 0x47, 0x46, 0xe2, 0x94, 0xc5, +]; +/// [`SIM_STREAM_KEY`] wrapped under [`SIM_ROTATED_KEK`], the set a rewrap stores. +pub const SIM_ROTATED_WRAP_SET: [u8; WRAPPED_STREAM_KEY_LENGTH] = [ + 0x01, 0x01, 0x28, 0x05, 0x68, 0xfc, 0x9e, 0xaa, 0x81, 0x5f, 0x96, 0x5f, 0x30, 0x69, 0xfd, 0x3f, + 0x3c, 0x50, 0xd8, 0xf5, 0x48, 0xdb, 0x49, 0xb4, 0x0e, 0x11, 0xbb, 0xd0, 0x66, 0x1e, 0xb3, 0xff, + 0x95, 0xe5, 0x24, 0x1e, 0x4a, 0xb0, 0x69, 0x04, 0xee, 0x40, 0x43, 0xe5, 0x36, 0x81, 0x09, 0xf1, + 0x16, 0x43, 0xa3, 0x11, 0xf3, 0x53, 0xcd, 0x76, 0x8d, 0x4a, 0xff, 0x95, 0xc4, 0xbd, 0xc8, 0x84, + 0x41, 0x84, 0xf2, 0x31, 0x01, 0x37, 0x31, 0x19, 0x21, 0x47, 0x46, 0xe2, 0x94, 0xc5, +]; + // TODO: Proper client which implements the full client SDK API pub struct SimClient { client_id: u128, @@ -345,6 +375,38 @@ self.build_request(Operation::CreateStream, &payload) } + /// A `CreateStream` already in the replicated form an edge produces: the + /// fixed [`SIM_STREAM_WRAP_SET`] and the critical marker naming it. The + /// dispatch shell refuses a client-sent wrap set, so only the raw path + /// commits one. + /// + /// # Panics + /// Panics if the stream name is not a valid wire name. + pub fn create_keyed_stream(&self, name: &str) -> Message<RoutedRequestHeader> { + let options = ResourceOptions::from([ + ( + HeaderKey::from(OptionKey::EncryptionKeyWrapped), + OptionValue::explicit( + HeaderValue::try_from(SIM_STREAM_WRAP_SET.as_slice()) + .expect("a wrap set fits a header value"), + ), + ), + ( + HeaderKey::from(OptionKey::CriticalKeys), + OptionValue::explicit( + critical_keys_marker(&[OptionKey::EncryptionKeyWrapped]) + .expect("the wrapped key is a critical option"), + ), + ), + ]); + let wire = CreateStreamRequest { + name: WireName::new(name).expect("stream name must be valid"), + options: resource_options_to_wire(&options, OptionsProvenance::All) + .expect("the keyed options fit a block"), + }; + self.build_request(Operation::CreateStream, &wire.to_bytes()) + } + /// # Panics /// Panics if the stream name cannot be converted to a `WireIdentifier`. pub fn delete_stream(&self, name: &str) -> Message<RoutedRequestHeader> { @@ -360,12 +422,18 @@ /// Panics if the new name or the existing stream name is not a valid /// `WireName`. pub fn update_stream(&self, stream: &str, new_name: &str) -> Message<RoutedRequestHeader> { - let wire = UpdateStreamRequest { - stream_id: WireIdentifier::named(stream).expect("stream name must be valid"), - name: WireName::new(new_name).expect("stream name must be valid"), - options: WireOptions::empty(), - }; - self.build_request(Operation::UpdateStream, &wire.to_bytes()) + let patch = StreamUpdateOptions { + name: Some(new_name.to_owned()), + ..StreamUpdateOptions::default() + } + .to_patch() + .expect("stream name must be valid"); + self.update_resource( + ResourceTarget::Stream { + stream_id: WireIdentifier::named(stream).expect("stream name must be valid"), + }, + patch, + ) } /// # Panics @@ -392,20 +460,22 @@ } /// # Panics - /// Panics if `stream`, `topic`, or `new_name` is not a valid `WireName`. + /// Panics if `stream` or `topic` is not a valid `WireName`, or `options` + /// fail the client-side patch checks. pub fn update_topic( &self, stream: &str, topic: &str, - new_name: &str, + options: &TopicUpdateOptions, ) -> Message<RoutedRequestHeader> { - let wire = UpdateTopicRequest { - stream_id: WireIdentifier::named(stream).expect("stream name must be valid"), - topic_id: WireIdentifier::named(topic).expect("topic name must be valid"), - name: WireName::new(new_name).expect("topic name must be valid"), - options: WireOptions::empty(), - }; - self.build_request(Operation::UpdateTopic, &wire.to_bytes()) + let patch = options.to_patch().expect("topic patch must be valid"); + self.update_resource( + ResourceTarget::Topic { + stream_id: WireIdentifier::named(stream).expect("stream name must be valid"), + topic_id: WireIdentifier::named(topic).expect("topic name must be valid"), + }, + patch, + ) } /// # Panics @@ -540,11 +610,23 @@ password: &str, status: u8, ) -> Message<RoutedRequestHeader> { + self.create_user_with_permissions(username, password, status, None) + } + + /// # Panics + /// Panics if `username` is not a valid `WireName`. + pub fn create_user_with_permissions( + &self, + username: &str, + password: &str, + status: u8, + permissions: Option<WirePermissions>, + ) -> Message<RoutedRequestHeader> { let wire = CreateUserRequest { username: WireName::new(username).expect("username must be valid"), password: password.to_string(), status, - permissions: None, + permissions, options: WireOptions::empty(), }; self.build_request(Operation::CreateUser, &wire.to_bytes()) @@ -557,15 +639,34 @@ &self, user: &str, new_username: Option<&str>, - status: Option<u8>, + status: Option<UserStatus>, ) -> Message<RoutedRequestHeader> { - let wire = UpdateUserRequest { - user_id: WireIdentifier::named(user).expect("username must be valid"), - username: new_username.map(|n| WireName::new(n).expect("username must be valid")), + let patch = UserUpdateOptions { + username: new_username.map(ToOwned::to_owned), status, - options: WireOptions::empty(), + ..UserUpdateOptions::default() + } + .to_patch() + .expect("username must be valid"); + self.update_resource( + ResourceTarget::User { + user_id: WireIdentifier::named(user).expect("username must be valid"), + }, + patch, + ) + } + + fn update_resource( + &self, + target: ResourceTarget, + patch: OptionsPatch, + ) -> Message<RoutedRequestHeader> { + let wire = UpdateResourceRequest { + target, + reset: patch.reset, + options: patch.set, }; - self.build_request(Operation::UpdateUser, &wire.to_bytes()) + self.build_request(Operation::UpdateResource, &wire.to_bytes()) } /// # Panics @@ -884,7 +985,7 @@ let mut header = self.header(operation, group.inner(), total_size); self.stamp_partition_context(&mut header); - header.request_checksum = u128::from(iggy_common::calculate_checksum(payload)); + header.request_checksum = 0; let header_bytes = bytemuck::bytes_of(&header); let mut buffer = Vec::with_capacity(total_size); @@ -903,7 +1004,9 @@ // ops go through `build_request_with_namespace`), and metadata // requests carry the metadata consensus group on the wire. let mut header = self.header(operation, METADATA_GROUP, total_size); - header.request_checksum = u128::from(iggy_common::calculate_checksum(payload)); + // The simulator keeps request bytes deterministic for seeded replay. + // Sequence identity must never expose a hash of a secret-bearing body. + header.request_checksum = u128::from(header.request) + 1; let header_bytes = bytemuck::bytes_of(&header); let mut buffer = Vec::with_capacity(total_size); @@ -964,6 +1067,21 @@ #[cfg(test)] mod tests { use super::*; + use iggy_common::{ServerKeyContext, StreamEncryptionKey, StreamKeyOrigin, WrapSet}; + + #[test] + fn given_fixed_wrap_sets_when_opened_under_their_keks_should_hold_the_same_client_key() { + let key = StreamEncryptionKey::try_from(SIM_STREAM_KEY.as_slice()).expect("32 bytes"); + for (kek, bytes) in [ + (SIM_STREAM_KEK, SIM_STREAM_WRAP_SET), + (SIM_ROTATED_KEK, SIM_ROTATED_WRAP_SET), + ] { + let context = ServerKeyContext::from_config(kek, &[], false).expect("valid kek"); + let set = WrapSet::try_from(bytes.as_slice()).expect("a valid wrap set"); + assert_eq!(set.origin(), StreamKeyOrigin::Client); + assert_eq!(context.open(&set).expect("opens under its kek"), key); + } + } // A partition send consumes a request id like any other replicated op, so // an interleaved run numbers both planes from one sequence and the metadata
diff --git a/core/simulator/src/lib.rs b/core/simulator/src/lib.rs index 80c6351..3c02208 100644 --- a/core/simulator/src/lib.rs +++ b/core/simulator/src/lib.rs
@@ -749,6 +749,25 @@ /// If no login reply arrives within `SETUP_TOTAL_STEPS`, or it carries no /// session. pub fn shell_login_via(&mut self, client: &SimClient, target: u8) { + self.shell_login_as( + client, + target, + replica::SHELL_ROOT_USERNAME, + replica::SHELL_ROOT_PASSWORD, + ); + } + + /// [`Self::shell_login_via`] with the credentials of a user the run created. + /// + /// # Panics + /// As [`Self::shell_login_via`]. + pub fn shell_login_as( + &mut self, + client: &SimClient, + target: u8, + username: &str, + password: &str, + ) { // Connection metadata on every replica, as `install_client_fd` does in // production. `ensure_transport_connection` reads it to admit the connection // into the SessionManager, which the login's bind @@ -765,9 +784,7 @@ // From here this client talks the real client protocol. See // `SimClient::shell_wire`. client.set_shell_wire(); - let msg = client - .login(replica::SHELL_ROOT_USERNAME, replica::SHELL_ROOT_PASSWORD) - .into_generic(); + let msg = client.login(username, password).into_generic(); // `build_reply_with_body` maps the session field to `op`. let session = self .await_setup_reply( @@ -2156,7 +2173,7 @@ cluster, is_commit_for, prepare_for, sharded_cluster, }; use super::*; - use crate::client::SimClient; + use crate::client::{SIM_ROTATED_WRAP_SET, SIM_STREAM_WRAP_SET, SimClient}; use crate::workload::apply_sim_commands; use bytes::Bytes; use consensus::{ @@ -2164,19 +2181,21 @@ }; use futures::FutureExt; use iggy_binary_protocol::requests::partitions::InstallConsumerGroupOwnerRequest; + use iggy_binary_protocol::requests::streams::{RewrapStreamKeyEntry, RewrapStreamKeysRequest}; use iggy_binary_protocol::requests::users::CreateUserRequest; use iggy_binary_protocol::{ AckLevel, ConsensusHeader, Operation, PrepareOkHeader, RoutedRequestHeader, StartViewHeader, WireDecode, WireEncode, WireGlobalPermissions, WireIdentifier, WireName, WireOptions, WirePermissions, }; - use iggy_common::{ConsumerKind, IggyError}; + use iggy_common::{ConsumerKind, HeaderKey, IggyError, OptionKey, WrapSet}; use journal::{Journal, JournalHandle}; use message_bus::lifecycle::Shutdown; use message_bus::{AUTO_COMMIT_CLIENT_ID, ConnectionInstaller, MessageBus}; use metadata::stm::stream::Streams; use server_common::MessageBag; use server_common::sharding::{IggyNamespace, LIST_CLIENTS_GATHER_TIMEOUT, METADATA_GROUP}; + use std::cell::Cell; use std::sync::atomic::{AtomicU64, Ordering}; struct RootInitialization { @@ -3020,7 +3039,7 @@ const SHARDS_PER_REPLICA: u16 = 2; let (mut sim, client) = sharded_cluster(0x5245_4352, SHARDS_PER_REPLICA); sim.register_client_with_primary(&client); - complete_metadata_op(&mut sim, &client, client.create_stream(STREAM)); + complete_metadata_op(&mut sim, &client, client.create_keyed_stream(STREAM)); complete_metadata_op(&mut sim, &client, client.create_topic(STREAM, TOPIC, 1)); let (namespace, deleted) = incarnation(&sim); assert_ne!( @@ -4980,6 +4999,170 @@ } } + /// A keyed create commits its wrap set on every replica, then a rewrap + /// replaces it there while an entry naming another incarnation is skipped. + #[test] + fn given_keyed_stream_when_rewrapped_should_store_the_new_set_on_every_replica() { + const PROGRESS_STEPS: usize = 4_000; + const STREAM: &str = "keyed"; + let (mut sim, client) = cluster(0x5245_5752); + sim.register_client_with_primary(&client); + complete_metadata_op(&mut sim, &client, client.create_keyed_stream(STREAM)); + let (stream_id, created_op) = keyed_stream(&sim, 0, STREAM); + for replica in 0..sim.replica_count { + assert_eq!( + keyed_stream_wrap_set(&sim, replica, STREAM), + Some(SIM_STREAM_WRAP_SET.to_vec()) + ); + } + + let key_fingerprint = WrapSet::try_from(SIM_STREAM_WRAP_SET.as_slice()) + .expect("a valid wrap set") + .key_fingerprint_bytes() + .try_into() + .expect("an eight byte fingerprint"); + let request = RewrapStreamKeysRequest { + entries: vec![ + RewrapStreamKeyEntry { + stream_id, + created_op, + key_fingerprint, + wrap_set: Bytes::from_static(&SIM_ROTATED_WRAP_SET), + }, + RewrapStreamKeyEntry { + stream_id, + created_op: created_op + 1, + key_fingerprint, + wrap_set: Bytes::from_static(&SIM_STREAM_WRAP_SET), + }, + ], + }; + let primary = usize::from(sim.metadata_primary_index().expect("a metadata primary")); + let shard = Rc::clone(&sim.replicas[primary].shards[0]); + let committed = Rc::new(Cell::new(None)); + let outcome = Rc::clone(&committed); + let task = sim.executor.spawn(async move { + let result = shard + .plane + .metadata() + .submit_rewrap_stream_keys_in_process(request) + .await; + outcome.set(Some(result.is_ok())); + }); + sim.replicas[primary].pump_tasks.push(task); + assert!( + (0..PROGRESS_STEPS).any(|_| { + sim.step(); + committed.get().is_some() + && (0..sim.replica_count).all(|replica| { + keyed_stream_wrap_set(&sim, replica, STREAM) + == Some(SIM_ROTATED_WRAP_SET.to_vec()) + }) + }), + "every replica must store the rewrapped set, and keep it past the stale entry" + ); + assert_eq!(committed.get(), Some(true)); + } + + /// A user deactivating itself ends its own session on commit. Every replica + /// keeps that session's receipt, which a client table snapshot admits after + /// the session end only for a self-deactivation. + #[test] + fn given_shell_user_when_deactivating_itself_should_end_its_session_and_keep_the_receipt() { + const PROGRESS_STEPS: usize = 4_000; + const ROOT_CLIENT: u128 = 1; + const USER_CLIENT: u128 = 2; + const USERNAME: &str = "self-deactivating"; + const PASSWORD: &str = "self-deactivating-password"; + server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { + enabled: false, + size: iggy_common::IggyByteSize::from(0u64), + bucket_capacity: 1, + }); + let mut sim = Simulator::with_shards_shell( + 3, + 1, + [ROOT_CLIENT, USER_CLIENT].into_iter(), + PacketSimulatorOptions { + node_count: 3, + client_count: 2, + seed: 0x4445_4143, + ..Default::default() + }, + ); + let root = SimClient::new(ROOT_CLIENT); + sim.shell_login(&root); + let permissions = WirePermissions { + global: WireGlobalPermissions { + manage_servers: false, + read_servers: false, + manage_users: true, + read_users: true, + manage_streams: false, + read_streams: false, + manage_topics: false, + read_topics: false, + poll_messages: false, + send_messages: false, + }, + streams: Vec::new(), + }; + let created = submit_and_wait_for_reply( + &mut sim, + ROOT_CLIENT, + 0, + root.create_user_with_permissions( + USERNAME, + PASSWORD, + iggy_common::UserStatus::Active.as_code(), + Some(permissions), + ), + ); + assert_eq!(metadata::stm::result::result_code(created.body()), Some(0)); + + let user = SimClient::new(USER_CLIENT); + sim.shell_login_as(&user, 0, USERNAME, PASSWORD); + let deactivated = submit_and_wait_for_reply( + &mut sim, + USER_CLIENT, + 0, + user.update_user(USERNAME, None, Some(iggy_common::UserStatus::Inactive)), + ); + assert_eq!(deactivated.header().status, 0); + assert_eq!( + metadata::stm::result::result_code(deactivated.body()), + Some(0) + ); + + assert!( + (0..PROGRESS_STEPS).any(|_| { + sim.step(); + sim.replicas.iter().all(|replica| { + let table = replica.shards[0].plane.metadata().client_table.borrow(); + table.get_epoch(USER_CLIENT).is_none() + && table + .ended_sessions() + .any(|session| session.client_id == USER_CLIENT) + }) + }), + "every replica must end the deactivated user's session" + ); + for replica in &sim.replicas { + let table = replica.shards[0].plane.metadata().client_table.borrow(); + assert_eq!( + table + .get_reply(USER_CLIENT) + .expect("the ended session keeps its receipt") + .header() + .operation, + Operation::UpdateUserResolved + ); + assert!( + consensus::client_table::ClientTable::from_snapshot(table.to_snapshot()).is_ok() + ); + } + } + pub fn submit_and_wait_for_reply( sim: &mut Simulator, client_id: u128, @@ -5000,10 +5183,51 @@ panic!("request {request_id} did not receive a reply"); } + /// `(stream id, created op)` of `name` on `replica`. + fn keyed_stream(sim: &Simulator, replica: u8, name: &str) -> (u32, u64) { + let streams = sim.replicas[usize::from(replica)].shards[0] + .plane + .metadata() + .mux_stm + .streams(); + let stream_id = streams + .resolve_stream_id(&WireIdentifier::named(name).expect("valid name")) + .expect("the stream is committed"); + let created_op = streams.read(|inner| { + inner + .items + .get(stream_id) + .expect("the stream is committed") + .created_op + }); + ( + u32::try_from(stream_id).expect("stream id fits u32"), + created_op, + ) + } + + /// The wrap set `replica` stores for stream `name`. + fn keyed_stream_wrap_set(sim: &Simulator, replica: u8, name: &str) -> Option<Vec<u8>> { + let streams = sim.replicas[usize::from(replica)].shards[0] + .plane + .metadata() + .mux_stm + .streams(); + let stream_id = streams.resolve_stream_id(&WireIdentifier::named(name).ok()?)?; + streams.read(|inner| { + inner.items.get(stream_id).and_then(|stream| { + stream + .options + .get(&HeaderKey::from(OptionKey::EncryptionKeyWrapped)) + .map(|wrapped| wrapped.value.as_bytes().to_vec()) + }) + }) + } + /// Submit a metadata op to replica 0 and step until every live replica applied /// it and finished its lifecycle. Deletes reply before their partition fences /// install. - fn complete_metadata_op( + pub fn complete_metadata_op( sim: &mut Simulator, client: &SimClient, request: Message<RoutedRequestHeader>,
diff --git a/core/simulator/src/replica.rs b/core/simulator/src/replica.rs index 438ca2a..f9ae0b4 100644 --- a/core/simulator/src/replica.rs +++ b/core/simulator/src/replica.rs
@@ -16,6 +16,7 @@ // under the License. use crate::bus::{SharedSimOutbox, SimOutbox}; +use crate::client::SIM_STREAM_KEK; use crate::deps::SimSuperblock; use crate::deps::{MemStorage, SimJournal, SimMuxStateMachine, SimSnapshot}; use configs::server::PersonalAccessTokenConfig; @@ -24,8 +25,8 @@ ClientTable, ConsensusClock, HistoryRecovery, LocalPipeline, Sequencer, VsrConsensus, VsrState, }; use futures::FutureExt; -use iggy_common::IggyByteSize; use iggy_common::variadic; +use iggy_common::{IggyByteSize, ServerKeyContext}; use journal::Journal; use message_bus::MessageBus; use metadata::impls::metadata::IggySnapshot; @@ -438,7 +439,12 @@ validate_checksum: true, segment_size: IggyByteSize::from(iggy_common::DEFAULT_SEGMENT_SIZE), preallocate_segments: false, - encryptor: None, + // Opens the fixed wrap set keyed creates carry, so metadata admits them. + // Partitions stay plaintext: the simulator never resolves an encryptor. + key_context: Arc::new( + ServerKeyContext::from_config(SIM_STREAM_KEK, &[], false) + .expect("the simulator key-encryption key is valid"), + ), path_layout: PartitionPathLayout::default(), };
diff --git a/core/simulator/src/storage/bookmark_recovery.rs b/core/simulator/src/storage/bookmark_recovery.rs index cfe91cf..d7d9135 100644 --- a/core/simulator/src/storage/bookmark_recovery.rs +++ b/core/simulator/src/storage/bookmark_recovery.rs
@@ -378,7 +378,7 @@ validate_checksum: true, segment_size: IggyByteSize::from(1024 * 1024), preallocate_segments: false, - encryptor: None, + key_context: Arc::default(), path_layout: PartitionPathLayout::default(), } }
diff --git a/core/simulator/src/workload/actions.rs b/core/simulator/src/workload/actions.rs index 64c6bd0..696b660 100644 --- a/core/simulator/src/workload/actions.rs +++ b/core/simulator/src/workload/actions.rs
@@ -53,6 +53,7 @@ LeaveFixtureGroup, StoreStaleOwnerOffset, PollMessages, + CreateKeyedStream, } // Lock the discriminants of the first three variants. Inserting a new variant before
diff --git a/core/simulator/src/workload/ops/create_keyed_stream.rs b/core/simulator/src/workload/ops/create_keyed_stream.rs new file mode 100644 index 0000000..6d1bd0b --- /dev/null +++ b/core/simulator/src/workload/ops/create_keyed_stream.rs
@@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! `CreateStream` carrying a wrapped client stream key. +//! +//! Targets `Ok` with a fresh name, or `NameAlreadyExists` by reusing a live +//! stream name. The raw path commits the fixed wrap set, and the dispatch shell +//! refuses it as a client-sent wrapped key. + +use iggy_binary_protocol::RoutedRequestHeader; +use rand_xoshiro::Xoshiro256PlusPlus; +use server_common::Message; + +use crate::client::SimClient; +use crate::workload::effect::Effect; +use crate::workload::options::WorkloadOptions; +use crate::workload::shadow::Shadow; + +pub use metadata::stm::result::CreateStreamResult as Outcome; + +#[derive(Debug, Clone)] +pub struct Input { + pub name: String, +} + +pub const OUTCOMES: &[Outcome] = &[Outcome::Ok, Outcome::NameAlreadyExists]; + +pub fn sample( + shadow: &mut Shadow, + outcome: Outcome, + prng: &mut Xoshiro256PlusPlus, + _options: &WorkloadOptions, +) -> Option<Input> { + match outcome { + Outcome::Ok => Some(Input { + name: shadow.fresh_stream_name(), + }), + Outcome::NameAlreadyExists => Some(Input { + name: shadow.pick_stream_name(prng)?, + }), + // Not targeted, as for `create_stream`: the fixed wrap set is always a + // valid option, and the slab ceiling is out of a run's reach. + Outcome::InvalidOptionValue | Outcome::TooManyStreams => None, + } +} + +#[must_use] +pub fn build_message(client: &SimClient, input: &Input) -> Message<RoutedRequestHeader> { + client.create_keyed_stream(&input.name) +} + +/// Decode committed `code` into this op's outcome. +/// +/// # Panics +/// On an undeclared code. `on_reply` rejects unrecognized codes first. +#[must_use] +pub const fn classify_reply(code: u32) -> Outcome { + Outcome::from_u32(code).expect("on_reply rejects unrecognized result codes before classify") +} + +#[must_use] +pub fn predicted_effect(input: &Input, outcome: Outcome) -> Effect { + match outcome { + Outcome::Ok => Effect::AddStream { + name: input.name.clone(), + }, + Outcome::NameAlreadyExists | Outcome::InvalidOptionValue | Outcome::TooManyStreams => { + Effect::None + } + } +}
diff --git a/core/simulator/src/workload/ops/mod.rs b/core/simulator/src/workload/ops/mod.rs index c6639d8..a5eb37d 100644 --- a/core/simulator/src/workload/ops/mod.rs +++ b/core/simulator/src/workload/ops/mod.rs
@@ -31,6 +31,7 @@ pub mod change_password; pub mod create_consumer_group; +pub mod create_keyed_stream; pub mod create_partitions; pub mod create_personal_access_token; pub mod create_stream; @@ -217,4 +218,5 @@ LeaveFixtureGroup => leave_fixture_group, StoreStaleOwnerOffset => store_stale_owner_offset, PollMessages => poll_messages, + CreateKeyedStream => create_keyed_stream, }
diff --git a/core/simulator/src/workload/ops/update_stream.rs b/core/simulator/src/workload/ops/update_stream.rs index c85bccf..9410823 100644 --- a/core/simulator/src/workload/ops/update_stream.rs +++ b/core/simulator/src/workload/ops/update_stream.rs
@@ -59,7 +59,7 @@ Outcome::NameAlreadyExists => { unreachable!("update_stream does not target NameAlreadyExists") } - Outcome::InvalidOptionValue => { + Outcome::InvalidOptionValue | Outcome::OptionsBlockTooLarge => { unreachable!("the simulator only sends an empty update options block") } }
diff --git a/core/simulator/src/workload/ops/update_topic.rs b/core/simulator/src/workload/ops/update_topic.rs index fcded2d..e6cf766 100644 --- a/core/simulator/src/workload/ops/update_topic.rs +++ b/core/simulator/src/workload/ops/update_topic.rs
@@ -17,11 +17,17 @@ //! `UpdateTopic` op. //! -//! Targets `Ok` (rename live topic to fresh name), `StreamNotFound` (fabricated -//! parent stream), or `TopicNotFound` (live stream, fabricated topic). +//! Targets `Ok` (rename live topic to fresh name, sometimes also setting or +//! resetting the updatable options), `StreamNotFound` (fabricated parent +//! stream), or `TopicNotFound` (live stream, fabricated topic). //! `NameAlreadyExists` not targeted. +use std::collections::BTreeSet; +use std::time::Duration; + use iggy_binary_protocol::RoutedRequestHeader; +use iggy_common::{IggyDuration, IggyExpiry, OptionKey, TopicUpdateOptions}; +use rand::RngExt; use rand_xoshiro::Xoshiro256PlusPlus; use server_common::Message; @@ -37,6 +43,10 @@ pub stream: String, pub topic: String, pub new_name: String, + /// Set beside the rename so replicas must agree on the canonical value. + pub message_expiry: Option<IggyExpiry>, + /// Reset beside the rename so replicas must agree on the default resolved at prepare. + pub reset: BTreeSet<OptionKey>, } pub const OUTCOMES: &[Outcome] = &[Outcome::Ok, Outcome::StreamNotFound, Outcome::TopicNotFound]; @@ -51,16 +61,32 @@ Outcome::Ok => { let (stream, topic) = shadow.pick_topic_pair(prng)?; let new_name = shadow.fresh_name("topic"); + let (message_expiry, reset) = match prng.random_range(0..3u8) { + 0 => (None, BTreeSet::new()), + 1 => { + let minutes = 1 + prng.random_range(0..60u64); + let expiry = IggyDuration::from(Duration::from_secs(minutes * 60)); + (Some(IggyExpiry::ExpireDuration(expiry)), BTreeSet::new()) + } + _ => ( + None, + BTreeSet::from([OptionKey::MessageExpiry, OptionKey::MaxTopicSize]), + ), + }; Some(Input { stream, topic, new_name, + message_expiry, + reset, }) } Outcome::StreamNotFound => Some(Input { stream: shadow.fabricate_absent_name("stream"), topic: shadow.fabricate_absent_name("topic"), new_name: shadow.fresh_name("topic"), + message_expiry: None, + reset: BTreeSet::new(), }), Outcome::TopicNotFound => { let stream = shadow.pick_stream_name(prng)?; @@ -68,20 +94,34 @@ stream, topic: shadow.fabricate_absent_name("topic"), new_name: shadow.fresh_name("topic"), + message_expiry: None, + reset: BTreeSet::new(), }) } Outcome::NameAlreadyExists => { unreachable!("update_topic does not target NameAlreadyExists") } - Outcome::InvalidOptionValue | Outcome::UnsupportedOptionKey => { - unreachable!("the simulator only sends an empty update options block") + Outcome::InvalidOptionValue | Outcome::OptionsBlockTooLarge => { + unreachable!("the simulator only sends valid option values") + } + Outcome::InvalidTopicSize => { + unreachable!("the simulator never sets an explicit max topic size") } } } #[must_use] pub fn build_message(client: &SimClient, input: &Input) -> Message<RoutedRequestHeader> { - client.update_topic(&input.stream, &input.topic, &input.new_name) + client.update_topic( + &input.stream, + &input.topic, + &TopicUpdateOptions { + name: Some(input.new_name.clone()), + message_expiry: input.message_expiry, + reset: input.reset.clone(), + ..TopicUpdateOptions::default() + }, + ) } /// Decode committed `code` into this op's outcome.
diff --git a/core/simulator/src/workload/ops/update_user.rs b/core/simulator/src/workload/ops/update_user.rs index b76cbf8..e83b49b 100644 --- a/core/simulator/src/workload/ops/update_user.rs +++ b/core/simulator/src/workload/ops/update_user.rs
@@ -15,10 +15,13 @@ // specific language governing permissions and limitations // under the License. -//! `UpdateUser` op. Targets `Ok` (rename live user to fresh name) or -//! `UserNotFound` (fabricated user). `UsernameAlreadyExists` not targeted. +//! `UpdateUser` op. Targets `Ok` (rename live user to fresh name and set it +//! active or inactive) or `UserNotFound` (fabricated user). +//! `UsernameAlreadyExists` not targeted. use iggy_binary_protocol::RoutedRequestHeader; +use iggy_common::UserStatus; +use rand::RngExt; use rand_xoshiro::Xoshiro256PlusPlus; use server_common::Message; @@ -33,7 +36,7 @@ pub struct Input { pub user: String, pub new_username: Option<String>, - pub status: Option<u8>, + pub status: Option<UserStatus>, /// Current password from shadow; carried via `Effect::RenameUser` so the /// shadow re-keys `passwords` under `new_username`. pub current_password: String, @@ -51,17 +54,22 @@ Outcome::Ok => { let user = shadow.pick_user_name(prng)?; let current_password = shadow.password_for(&user)?.to_string(); + let status = if prng.random_bool(0.5) { + UserStatus::Inactive + } else { + UserStatus::Active + }; Some(Input { user, new_username: Some(shadow.fresh_name("user")), - status: Some(1), + status: Some(status), current_password, }) } Outcome::UserNotFound => Some(Input { user: shadow.fabricate_absent_name("user"), new_username: Some(shadow.fresh_name("user")), - status: Some(1), + status: Some(UserStatus::Active), current_password: String::new(), }), Outcome::UsernameAlreadyExists => { @@ -70,9 +78,15 @@ Outcome::InvalidUsername => { unreachable!("update_user does not target InvalidUsername") } - Outcome::InvalidOptionValue => { + Outcome::InvalidOptionValue | Outcome::OptionsBlockTooLarge => { unreachable!("the simulator only sends an empty update options block") } + Outcome::InvalidUserStatus => { + unreachable!("update_user only sends a valid status") + } + Outcome::CannotModifyRootUser => { + unreachable!("the simulator only selects non-root users") + } } } @@ -104,6 +118,36 @@ Outcome::UserNotFound | Outcome::UsernameAlreadyExists | Outcome::InvalidUsername - | Outcome::InvalidOptionValue => Effect::None, + | Outcome::InvalidUserStatus + | Outcome::InvalidOptionValue + | Outcome::OptionsBlockTooLarge + | Outcome::CannotModifyRootUser => Effect::None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::workload::ids::IdPermutation; + use rand_xoshiro::rand_core::SeedableRng; + use server_common::sharding::IggyNamespace; + + const SEED: u64 = 0x5553_4552; + + #[test] + fn given_live_users_when_sampling_updates_should_draw_both_statuses() { + let namespaces = vec![IggyNamespace::new(1, 1, 0)]; + let options = WorkloadOptions::new(SEED, 3, namespaces.clone()); + let mut shadow = Shadow::new(namespaces, IdPermutation::Identity); + let mut prng = Xoshiro256PlusPlus::seed_from_u64(SEED); + let mut statuses = Vec::new(); + for _ in 0..32 { + let name = shadow.fresh_name("user"); + shadow.apply(Effect::AddUser { name }); + let input = sample(&mut shadow, Outcome::Ok, &mut prng, &options).expect("a live user"); + statuses.push(input.status); + } + assert!(statuses.contains(&Some(UserStatus::Active))); + assert!(statuses.contains(&Some(UserStatus::Inactive))); } }
diff --git a/core/simulator/src/workload/options.rs b/core/simulator/src/workload/options.rs index ba901b2..a1f58cc 100644 --- a/core/simulator/src/workload/options.rs +++ b/core/simulator/src/workload/options.rs
@@ -126,7 +126,8 @@ #[must_use] pub fn metadata_only() -> Self { Self::new(&[ - (Action::CreateStream, 12), + (Action::CreateStream, 9), + (Action::CreateKeyedStream, 3), (Action::UpdateStream, 10), (Action::DeleteStream, 6), (Action::CreateTopic, 12),
diff --git a/core/simulator/src/workload/oracle.rs b/core/simulator/src/workload/oracle.rs index f302e96..1ef4242 100644 --- a/core/simulator/src/workload/oracle.rs +++ b/core/simulator/src/workload/oracle.rs
@@ -52,7 +52,7 @@ use iggy_binary_protocol::primitives::partition_history::ConsumerGroupOwner; use iggy_binary_protocol::requests::partitions::RetireConsumerGroupOwnersRequest; use iggy_binary_protocol::{Operation, WireDecode}; -use iggy_common::ConsumerKind; +use iggy_common::{ConsumerKind, ResourceOptions, UserStatus}; use metadata::impls::metadata::StreamsFrontend; use server_common::sharding::IggyNamespace; use std::collections::{BTreeMap, BTreeSet}; @@ -79,6 +79,9 @@ /// crashes leave behind. pub(crate) const PENDING_REVOCATION_TIMEOUTS_MAX: u64 = 2; +/// Everything [`assert_committed_metadata_agrees`] compares for one replica. +type ReplicaState = (CommittedMetadata, CommittedSettings); + /// Committed metadata entity sets read from one replica. Stored sorted so /// equality is independent of slab / hashmap iteration order, which is not /// stable across replicas. @@ -115,6 +118,17 @@ } } +/// Committed state the name-keyed shadow does not predict, compared only across +/// replicas: every value was resolved before its prepare committed. +#[derive(Debug, PartialEq)] +struct CommittedSettings { + /// Created op and options per stream name. + streams: BTreeMap<String, (u64, ResourceOptions)>, + topics: BTreeMap<(String, String), ResourceOptions>, + /// Status and options per username. + users: BTreeMap<String, (UserStatus, ResourceOptions)>, +} + /// Drain the system after the active workload, then settle. /// /// Stops submitting new requests and steps until no client request is @@ -780,7 +794,7 @@ /// consensus, so they agree only while the seed runs at the same point on every /// replica, and a divergence there is the slab-order bug. fn assert_committed_metadata_agrees(sim: &Simulator, live: &[usize], seed: u64) -> usize { - let mut by_commit: BTreeMap<u64, (usize, CommittedMetadata)> = BTreeMap::new(); + let mut by_commit: BTreeMap<u64, (usize, ReplicaState)> = BTreeMap::new(); let mut compared = 0; for &replica_idx in live { let Some(consensus) = sim.replicas[replica_idx].shards[0] @@ -791,7 +805,7 @@ else { continue; }; - let committed = read_committed_metadata(&sim.replicas[replica_idx].shards[0]); + let committed = read_replica_state(&sim.replicas[replica_idx].shards[0]); match by_commit.get(&consensus.commit_min()) { Some((owner, canonical)) => { assert_eq!( @@ -1065,6 +1079,13 @@ /// Read one replica's committed metadata. `read` enters the committed (left) /// buffer of the left-right state machine, so uncommitted writes are invisible. +fn read_replica_state(replica: &Replica) -> ReplicaState { + ( + read_committed_metadata(replica), + read_committed_settings(replica), + ) +} + fn read_committed_metadata(replica: &Replica) -> CommittedMetadata { let stm = &replica.plane.metadata().mux_stm; @@ -1106,6 +1127,42 @@ } } +fn read_committed_settings(replica: &Replica) -> CommittedSettings { + let stm = &replica.plane.metadata().mux_stm; + let (streams, topics) = stm.streams().read(|inner| { + let mut streams = BTreeMap::new(); + let mut topics = BTreeMap::new(); + for (_, stream) in &inner.items { + let stream_name = stream.name.to_string(); + for (_, topic) in &stream.topics { + topics.insert( + (stream_name.clone(), topic.name.to_string()), + topic.options.clone(), + ); + } + streams.insert(stream_name, (stream.created_op, stream.options.clone())); + } + (streams, topics) + }); + let users = stm.users().read(|inner| { + inner + .items + .iter() + .map(|(_, user)| { + ( + user.username.to_string(), + (user.status, user.options.clone()), + ) + }) + .collect() + }); + CommittedSettings { + streams, + topics, + users, + } +} + /// Project the shadow's predicted entity sets into the comparable shape. All /// shadow names carry [`WORKLOAD_PREFIX`], so this lines up with /// [`CommittedMetadata::workload_owned`]. @@ -1122,15 +1179,18 @@ mod tests { use super::*; use crate::packet::PacketSimulatorOptions; - use crate::tests::commit_sends_everywhere; + use crate::partition_repair_driver_tests::cluster; + use crate::tests::{commit_sends_everywhere, complete_metadata_op}; use crate::workload::actions::Action; use crate::workload::effect::Effect; use crate::workload::options::{ActionWeights, WorkloadOptions}; use crate::workload::{create_fixture_groups, run_group_lifecycle}; use iggy_binary_protocol::batch::{BATCH_HEADER_SIZE, BATCH_MESSAGE_HEADER_SIZE}; use iggy_binary_protocol::{HEADER_SIZE, Operation}; + use iggy_common::{IggyDuration, IggyExpiry, TopicUpdateOptions}; use journal::Journal; use server_common::iobuf::Owned; + use std::time::Duration; const SEED: u64 = 0x000F_F5E7; const CLIENT: u128 = 1; @@ -1178,6 +1238,58 @@ } #[test] + fn given_committed_setting_changes_when_reading_replica_state_should_see_each() { + const STREAM: &str = "wl-settings-stream"; + const TOPIC: &str = "wl-settings-topic"; + const USER: &str = "wl-settings-user"; + let (mut sim, client) = cluster(SEED); + sim.register_client_with_primary(&client); + let state = |sim: &Simulator| read_replica_state(&sim.replicas[0].shards[0]); + complete_metadata_op(&mut sim, &client, client.create_stream(STREAM)); + complete_metadata_op(&mut sim, &client, client.create_topic(STREAM, TOPIC, 1)); + complete_metadata_op( + &mut sim, + &client, + client.create_user(USER, "pw-settings", UserStatus::Active.as_code()), + ); + let created = state(&sim); + + let expiry = IggyDuration::from(Duration::from_secs(60)); + let options = TopicUpdateOptions { + message_expiry: Some(IggyExpiry::ExpireDuration(expiry)), + ..TopicUpdateOptions::default() + }; + complete_metadata_op( + &mut sim, + &client, + client.update_topic(STREAM, TOPIC, &options), + ); + let expiry_set = state(&sim); + assert_ne!( + created, expiry_set, + "a topic option change must be compared" + ); + + let deactivate = client.update_user(USER, None, Some(UserStatus::Inactive)); + complete_metadata_op(&mut sim, &client, deactivate); + let deactivated = state(&sim); + assert_ne!( + expiry_set, deactivated, + "a user status change must be compared" + ); + + complete_metadata_op(&mut sim, &client, client.delete_topic(STREAM, TOPIC)); + let first_incarnation = state(&sim); + complete_metadata_op(&mut sim, &client, client.delete_stream(STREAM)); + complete_metadata_op(&mut sim, &client, client.create_stream(STREAM)); + assert_ne!( + first_incarnation, + state(&sim), + "a recreated stream of the same name must be compared" + ); + } + + #[test] #[should_panic(expected = "different consumer offsets at committed prefix")] fn given_a_backup_offset_divergence_when_quiesced_should_fail_the_oracle() { let (sim, workload) = offset_fixture();
diff --git a/core/tools/src/data-seeder/seeder.rs b/core/tools/src/data-seeder/seeder.rs index 14b26e3..e7f4c28 100644 --- a/core/tools/src/data-seeder/seeder.rs +++ b/core/tools/src/data-seeder/seeder.rs
@@ -33,13 +33,19 @@ async fn create_streams(client: &IggyClient) -> Result<Vec<(String, u32)>, IggyError> { let mut streams = Vec::new(); - let prod_stream = client.create_stream(PROD_STREAM_NAME).await?; + let prod_stream = client + .create_stream(PROD_STREAM_NAME, &StreamCreateOptions::default()) + .await?; streams.push((PROD_STREAM_NAME.to_string(), prod_stream.id)); - let test_stream = client.create_stream(TEST_STREAM_NAME).await?; + let test_stream = client + .create_stream(TEST_STREAM_NAME, &StreamCreateOptions::default()) + .await?; streams.push((TEST_STREAM_NAME.to_string(), test_stream.id)); - let dev_stream = client.create_stream(DEV_STREAM_NAME).await?; + let dev_stream = client + .create_stream(DEV_STREAM_NAME, &StreamCreateOptions::default()) + .await?; streams.push((DEV_STREAM_NAME.to_string(), dev_stream.id)); Ok(streams)
diff --git a/examples/csharp/src/TcpTls/Iggy_SDK.Examples.TcpTls.Producer/Program.cs b/examples/csharp/src/TcpTls/Iggy_SDK.Examples.TcpTls.Producer/Program.cs index 4033326..6acdfcc 100644 --- a/examples/csharp/src/TcpTls/Iggy_SDK.Examples.TcpTls.Producer/Program.cs +++ b/examples/csharp/src/TcpTls/Iggy_SDK.Examples.TcpTls.Producer/Program.cs
@@ -28,6 +28,7 @@ using System.Text; using Apache.Iggy; using Apache.Iggy.Configuration; +using Apache.Iggy.Contracts; using Apache.Iggy.Enums; using Apache.Iggy.Exceptions; using Apache.Iggy.Factory; @@ -85,7 +86,7 @@ Identifier.String(streamName), topicName, 1, - CompressionAlgorithm.None + new TopicOptions { CompressionAlgorithm = CompressionAlgorithm.None } ); logger.LogInformation("Topic was created."); }
diff --git a/examples/go/getting-started/producer/main.go b/examples/go/getting-started/producer/main.go index 74bd366..2af157c 100644 --- a/examples/go/getting-started/producer/main.go +++ b/examples/go/getting-started/producer/main.go
@@ -70,20 +70,19 @@ } func initSystem(ctx context.Context, client iggcon.Client) { - if _, err := client.CreateStream(ctx, "sample-stream"); err != nil { + if _, err := client.CreateStream(ctx, "sample-stream", nil); err != nil { log.Printf("WARN: Stream already exists or error: %v", err) } log.Println("Stream was created.") streamIdentifier, _ := iggcon.NewIdentifier(StreamId) + partitions := uint32(1) + expiry := iggcon.IggyExpiryNeverExpire if _, err := client.CreateTopic( ctx, streamIdentifier, "sample-topic", - 1, - iggcon.CompressionAlgorithmNone, - iggcon.IggyExpiryNeverExpire, - 0); err != nil { + &iggcon.TopicOptions{PartitionsCount: &partitions, MessageExpiry: &expiry}); err != nil { log.Printf("WARN: Topic already exists and will not be created again or error: %v", err) } log.Println("Topic was created.")
diff --git a/examples/java/src/main/java/org/apache/iggy/examples/async/AsyncProducer.java b/examples/java/src/main/java/org/apache/iggy/examples/async/AsyncProducer.java index 1cf7b82..ca38e15 100644 --- a/examples/java/src/main/java/org/apache/iggy/examples/async/AsyncProducer.java +++ b/examples/java/src/main/java/org/apache/iggy/examples/async/AsyncProducer.java
@@ -25,11 +25,9 @@ import org.apache.iggy.identifier.TopicId; import org.apache.iggy.message.Message; import org.apache.iggy.message.Partitioning; -import org.apache.iggy.topic.CompressionAlgorithm; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.math.BigInteger; import java.util.ArrayList; import java.util.List; import java.util.UUID; @@ -135,13 +133,7 @@ } log.info("Creating topic: {}", TOPIC_NAME); return client.topics() - .createTopic( - StreamId.of(STREAM_NAME), - 1L, - CompressionAlgorithm.None, - BigInteger.ZERO, - BigInteger.ZERO, - TOPIC_NAME) + .createTopic(StreamId.of(STREAM_NAME), TOPIC_NAME, 1L) .thenAccept(created -> log.info("Topic created: {}", created.name())); }); }
diff --git a/examples/java/src/main/java/org/apache/iggy/examples/gettingstarted/producer/GettingStartedProducer.java b/examples/java/src/main/java/org/apache/iggy/examples/gettingstarted/producer/GettingStartedProducer.java index 8f3899e..1776b82 100644 --- a/examples/java/src/main/java/org/apache/iggy/examples/gettingstarted/producer/GettingStartedProducer.java +++ b/examples/java/src/main/java/org/apache/iggy/examples/gettingstarted/producer/GettingStartedProducer.java
@@ -25,12 +25,10 @@ import org.apache.iggy.message.Message; import org.apache.iggy.message.Partitioning; import org.apache.iggy.stream.StreamDetails; -import org.apache.iggy.topic.CompressionAlgorithm; import org.apache.iggy.topic.TopicDetails; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.math.BigInteger; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -117,8 +115,7 @@ log.warn("Topic already exists and will not be created again."); return; } - client.topics() - .createTopic(STREAM_ID, 1L, CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, TOPIC_NAME); + client.topics().createTopic(STREAM_ID, TOPIC_NAME, 1L); log.info("Topic {} was created.", TOPIC_NAME); } }
diff --git a/examples/java/src/main/java/org/apache/iggy/examples/messageenvelope/producer/MessageEnvelopeProducer.java b/examples/java/src/main/java/org/apache/iggy/examples/messageenvelope/producer/MessageEnvelopeProducer.java index 7ef9c30..bd2d9a7 100644 --- a/examples/java/src/main/java/org/apache/iggy/examples/messageenvelope/producer/MessageEnvelopeProducer.java +++ b/examples/java/src/main/java/org/apache/iggy/examples/messageenvelope/producer/MessageEnvelopeProducer.java
@@ -29,7 +29,6 @@ import org.apache.iggy.message.MessageId; import org.apache.iggy.message.Partitioning; import org.apache.iggy.stream.StreamDetails; -import org.apache.iggy.topic.CompressionAlgorithm; import org.apache.iggy.topic.TopicDetails; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -76,9 +75,7 @@ if (topic.isPresent()) { log.warn("Topic already exists and will not be created again."); } else { - client.topics() - .createTopic( - STREAM_ID, 1L, CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, TOPIC_NAME); + client.topics().createTopic(STREAM_ID, TOPIC_NAME, 1L); log.info("Topic {} was created.", TOPIC_NAME); }
diff --git a/examples/java/src/main/java/org/apache/iggy/examples/messageheaders/producer/MessageHeadersProducer.java b/examples/java/src/main/java/org/apache/iggy/examples/messageheaders/producer/MessageHeadersProducer.java index e56abd5..adc5336 100644 --- a/examples/java/src/main/java/org/apache/iggy/examples/messageheaders/producer/MessageHeadersProducer.java +++ b/examples/java/src/main/java/org/apache/iggy/examples/messageheaders/producer/MessageHeadersProducer.java
@@ -29,12 +29,10 @@ import org.apache.iggy.message.Message; import org.apache.iggy.message.Partitioning; import org.apache.iggy.stream.StreamDetails; -import org.apache.iggy.topic.CompressionAlgorithm; import org.apache.iggy.topic.TopicDetails; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.math.BigInteger; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -77,9 +75,7 @@ if (topic.isPresent()) { log.warn("Topic already exists and will not be created again."); } else { - client.topics() - .createTopic( - STREAM_ID, 1L, CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, TOPIC_NAME); + client.topics().createTopic(STREAM_ID, TOPIC_NAME, 1L); log.info("Topic {} was created.", TOPIC_NAME); }
diff --git a/examples/java/src/main/java/org/apache/iggy/examples/multitenant/producer/MultiTenantProducer.java b/examples/java/src/main/java/org/apache/iggy/examples/multitenant/producer/MultiTenantProducer.java index 9ed1e82..d111c27 100644 --- a/examples/java/src/main/java/org/apache/iggy/examples/multitenant/producer/MultiTenantProducer.java +++ b/examples/java/src/main/java/org/apache/iggy/examples/multitenant/producer/MultiTenantProducer.java
@@ -26,7 +26,6 @@ import org.apache.iggy.message.Message; import org.apache.iggy.message.Partitioning; import org.apache.iggy.stream.StreamDetails; -import org.apache.iggy.topic.CompressionAlgorithm; import org.apache.iggy.topic.TopicDetails; import org.apache.iggy.user.GlobalPermissions; import org.apache.iggy.user.Permissions; @@ -37,7 +36,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.math.BigInteger; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -254,14 +252,7 @@ return; } - client.topics() - .createTopic( - streamId, - (long) partitionsCount, - CompressionAlgorithm.None, - BigInteger.ZERO, - BigInteger.ZERO, - topicId.getName()); + client.topics().createTopic(streamId, topicId.getName(), (long) partitionsCount); log.info("Created topic {} for stream {}", topicId.getName(), streamId.getName()); }
diff --git a/examples/java/src/main/java/org/apache/iggy/examples/sinkdataproducer/SinkDataProducer.java b/examples/java/src/main/java/org/apache/iggy/examples/sinkdataproducer/SinkDataProducer.java index bd92783..23707a4 100644 --- a/examples/java/src/main/java/org/apache/iggy/examples/sinkdataproducer/SinkDataProducer.java +++ b/examples/java/src/main/java/org/apache/iggy/examples/sinkdataproducer/SinkDataProducer.java
@@ -24,7 +24,6 @@ import org.apache.iggy.identifier.TopicId; import org.apache.iggy.message.Message; import org.apache.iggy.message.Partitioning; -import org.apache.iggy.topic.CompressionAlgorithm; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import tools.jackson.core.JacksonException; @@ -110,14 +109,7 @@ client.topics() .getTopic(streamId, topicId) .ifPresentOrElse(existing -> log.info("Topic {} already exists.", topicId.getName()), () -> { - client.topics() - .createTopic( - streamId, - 1L, - CompressionAlgorithm.None, - java.math.BigInteger.ZERO, - java.math.BigInteger.ZERO, - topicId.getName()); + client.topics().createTopic(streamId, topicId.getName(), 1L); log.info("Created topic {}.", topicId.getName()); }); }
diff --git a/examples/java/src/main/java/org/apache/iggy/examples/streambuilder/StreamBasic.java b/examples/java/src/main/java/org/apache/iggy/examples/streambuilder/StreamBasic.java index 3ad8e7e..20f0178 100644 --- a/examples/java/src/main/java/org/apache/iggy/examples/streambuilder/StreamBasic.java +++ b/examples/java/src/main/java/org/apache/iggy/examples/streambuilder/StreamBasic.java
@@ -28,7 +28,6 @@ import org.apache.iggy.message.PolledMessages; import org.apache.iggy.message.PollingStrategy; import org.apache.iggy.stream.StreamDetails; -import org.apache.iggy.topic.CompressionAlgorithm; import org.apache.iggy.topic.TopicDetails; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -126,9 +125,7 @@ if (topic.isPresent()) { log.info("Topic {} already exists.", TOPIC_NAME); } else { - client.topics() - .createTopic( - STREAM_ID, 1L, CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, TOPIC_NAME); + client.topics().createTopic(STREAM_ID, TOPIC_NAME, 1L); log.info("Topic {} was created.", TOPIC_NAME); } }
diff --git a/examples/java/src/main/java/org/apache/iggy/examples/tcptls/producer/TcpTlsProducer.java b/examples/java/src/main/java/org/apache/iggy/examples/tcptls/producer/TcpTlsProducer.java index 5dd18a8..7e555b0 100644 --- a/examples/java/src/main/java/org/apache/iggy/examples/tcptls/producer/TcpTlsProducer.java +++ b/examples/java/src/main/java/org/apache/iggy/examples/tcptls/producer/TcpTlsProducer.java
@@ -25,12 +25,10 @@ import org.apache.iggy.message.Message; import org.apache.iggy.message.Partitioning; import org.apache.iggy.stream.StreamDetails; -import org.apache.iggy.topic.CompressionAlgorithm; import org.apache.iggy.topic.TopicDetails; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.math.BigInteger; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -138,8 +136,7 @@ log.warn("Topic already exists and will not be created again."); return; } - client.topics() - .createTopic(STREAM_ID, 1L, CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, TOPIC_NAME); + client.topics().createTopic(STREAM_ID, TOPIC_NAME, 1L); log.info("Topic {} was created.", TOPIC_NAME); } }
diff --git a/examples/java/src/test/java/org/apache/iggy/examples/MultiTenantAccessTest.java b/examples/java/src/test/java/org/apache/iggy/examples/MultiTenantAccessTest.java index 7d2130b..92f2bf6 100644 --- a/examples/java/src/test/java/org/apache/iggy/examples/MultiTenantAccessTest.java +++ b/examples/java/src/test/java/org/apache/iggy/examples/MultiTenantAccessTest.java
@@ -29,7 +29,6 @@ import org.apache.iggy.exception.IggyErrorCode; import org.apache.iggy.identifier.StreamId; import org.apache.iggy.stream.StreamDetails; -import org.apache.iggy.topic.CompressionAlgorithm; import org.apache.iggy.topic.TopicDetails; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -113,22 +112,19 @@ } } if (service == StreamsClient.class) { - return Optional.of( - new StreamDetails(0L, BigInteger.ZERO, stream, "0 B", BigInteger.ZERO, 0L, List.of())); + return Optional.of(new StreamDetails( + 0L, + BigInteger.ZERO, + stream, + "0 B", + BigInteger.ZERO, + 0L, + List.of(), + Map.of(), + Map.of())); } return Optional.of(new TopicDetails( - 0L, - BigInteger.ZERO, - "events", - "0 B", - BigInteger.ZERO, - CompressionAlgorithm.None, - BigInteger.ZERO, - BigInteger.ZERO, - 0L, - List.of(), - Map.of(), - Map.of())); + 0L, BigInteger.ZERO, "events", "0 B", BigInteger.ZERO, 0L, List.of(), Map.of(), Map.of())); })); }
diff --git a/examples/node/README.md b/examples/node/README.md index 8966a6c..c9ae8bd 100644 --- a/examples/node/README.md +++ b/examples/node/README.md
@@ -100,7 +100,7 @@ npm run test:stream-builder ``` -Uses the standard client API to send and consume three messages, then deletes the resources it created. +Uses the standard client API to send and consume three messages, then deletes the resources it created. It also logs who chose the new stream's at-rest key: `server` when the server runs with encryption enabled, `none` for a plaintext stream. ### Sink Data Producer
diff --git a/examples/node/package-lock.json b/examples/node/package-lock.json index e12f301..b6ac323 100644 --- a/examples/node/package-lock.json +++ b/examples/node/package-lock.json
@@ -23,21 +23,20 @@ }, "../../foreign/node": { "name": "apache-iggy", - "version": "0.10.1-edge.1", + "version": "0.10.1-edge.2", "license": "Apache-2.0", "dependencies": { "@node-rs/xxhash": "1.7.8", "debug": "4.4.3", - "generic-pool": "3.9.0", "uuidv7": "1.2.1" }, "devDependencies": { "@commitlint/cli": "21.2.3", "@commitlint/config-conventional": "21.2.3", - "@cucumber/cucumber": "13.2.1", + "@cucumber/cucumber": "13.3.0", "@swc-node/register": "1.12.1", "@types/debug": "4.1.13", - "@types/node": "26.6.3", + "@types/node": "26.6.4", "c8": "^12.0.0", "husky": "9.1.7", "typescript": "6.0.3",
diff --git a/examples/node/src/stream-builder/example.ts b/examples/node/src/stream-builder/example.ts index c9943c1..75d6684 100644 --- a/examples/node/src/stream-builder/example.ts +++ b/examples/node/src/stream-builder/example.ts
@@ -61,7 +61,7 @@ compressionAlgorithm: 1, }); - log(`Stream created: ${stream.id}`); + log(`Stream created: ${stream.id}, encryption key source: ${stream.encryptionKeySource}`); log(`Topic created: ${topic.id}`); return { client, stream, topic };
diff --git a/examples/python/basic/producer.py b/examples/python/basic/producer.py index 13a4ae3..188fa90 100644 --- a/examples/python/basic/producer.py +++ b/examples/python/basic/producer.py
@@ -71,11 +71,7 @@ logger.info(f"Creating topic {TOPIC_NAME} in stream {STREAM_NAME}") topic: TopicDetails | None = await client.get_topic(STREAM_NAME, TOPIC_NAME) if topic is None: - await client.create_topic( - stream=STREAM_NAME, - partitions_count=1, - name=TOPIC_NAME, - ) + await client.create_topic(stream=STREAM_NAME, name=TOPIC_NAME) logger.info("Topic was created successfully.") else: logger.warning(f"Topic {topic.name} already exists with ID {topic.id}")
diff --git a/examples/python/getting-started/producer.py b/examples/python/getting-started/producer.py index 2cde76e..d5bdc3c 100755 --- a/examples/python/getting-started/producer.py +++ b/examples/python/getting-started/producer.py
@@ -181,11 +181,7 @@ logger.info(f"Creating topic {TOPIC_NAME} in stream {STREAM_NAME}") topic: TopicDetails | None = await client.get_topic(STREAM_NAME, TOPIC_NAME) if topic is None: - await client.create_topic( - stream=STREAM_NAME, - partitions_count=1, - name=TOPIC_NAME, - ) + await client.create_topic(stream=STREAM_NAME, name=TOPIC_NAME) logger.info("Topic was created successfully.") else: logger.warning(f"Topic {topic.name} already exists with ID {topic.id}")
diff --git a/examples/python/high-level/producer.py b/examples/python/high-level/producer.py index a23156c..a5a45942 100644 --- a/examples/python/high-level/producer.py +++ b/examples/python/high-level/producer.py
@@ -20,7 +20,13 @@ from datetime import timedelta from typing import NamedTuple -from apache_iggy import DirectProducerConfig, IggyClient, Partitioning, SendMessage +from apache_iggy import ( + DirectProducerConfig, + IggyClient, + Partitioning, + SendMessage, + TopicOptions, +) from loguru import logger STREAM_NAME = "high-level-stream" @@ -54,7 +60,7 @@ await client.create_topic( stream=SEND_TO_STREAM_NAME, name=SEND_TO_TOPIC_NAME, - partitions_count=2, + options=TopicOptions(partitions_count=2), )
diff --git a/examples/python/message-headers/common.py b/examples/python/message-headers/common.py index addaa66..95609b9 100644 --- a/examples/python/message-headers/common.py +++ b/examples/python/message-headers/common.py
@@ -129,11 +129,7 @@ logger.info(f"Creating topic {TOPIC_NAME} in stream {STREAM_NAME}") topic: TopicDetails | None = await client.get_topic(STREAM_NAME, TOPIC_NAME) if topic is None: - await client.create_topic( - stream=STREAM_NAME, - partitions_count=1, - name=TOPIC_NAME, - ) + await client.create_topic(stream=STREAM_NAME, name=TOPIC_NAME) logger.info("Topic was created successfully.") else: logger.warning(f"Topic {topic.name} already exists with ID {topic.id}")
diff --git a/examples/python/uv.lock b/examples/python/uv.lock index c8ec57f..0a8c64f 100644 --- a/examples/python/uv.lock +++ b/examples/python/uv.lock
@@ -8,7 +8,7 @@ [[package]] name = "apache-iggy" -version = "0.9.1.dev1" +version = "0.9.1.dev2" source = { directory = "../../foreign/python" } [package.metadata]
diff --git a/examples/rust/src/getting-started/producer/main.rs b/examples/rust/src/getting-started/producer/main.rs index 6e76ef7..93bf0e0 100644 --- a/examples/rust/src/getting-started/producer/main.rs +++ b/examples/rust/src/getting-started/producer/main.rs
@@ -54,7 +54,10 @@ } async fn init_system(client: &IggyClient) -> (u32, u32) { - let stream = match client.create_stream(STREAM_NAME).await { + let stream = match client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + { Ok(stream) => { info!("Stream was created."); stream
diff --git a/examples/rust/src/message-headers/message-compression/producer/main.rs b/examples/rust/src/message-headers/message-compression/producer/main.rs index e23b61a..52c2d6f 100644 --- a/examples/rust/src/message-headers/message-compression/producer/main.rs +++ b/examples/rust/src/message-headers/message-compression/producer/main.rs
@@ -35,7 +35,7 @@ // Create a Stream. client - .create_stream(STREAM_NAME) + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) .await .expect("Stream was NOT created! Remove /local_data or start a fresh server with the --fresh flag to run this example.");
diff --git a/examples/rust/src/multi-tenant/consumer/main.rs b/examples/rust/src/multi-tenant/consumer/main.rs index 8cfad37..25c5491 100644 --- a/examples/rust/src/multi-tenant/consumer/main.rs +++ b/examples/rust/src/multi-tenant/consumer/main.rs
@@ -221,7 +221,13 @@ ..Default::default() }; let user = client - .create_user(username, PASSWORD, UserStatus::Active, Some(permissions)) + .create_user( + username, + PASSWORD, + UserStatus::Active, + Some(permissions), + &UserCreateOptions::default(), + ) .await?; info!( "Created user: {username} with ID: {}, with permissions for topics: {:?} in stream: {stream_name}",
diff --git a/examples/rust/src/multi-tenant/producer/main.rs b/examples/rust/src/multi-tenant/producer/main.rs index d748f77..834115e 100644 --- a/examples/rust/src/multi-tenant/producer/main.rs +++ b/examples/rust/src/multi-tenant/producer/main.rs
@@ -326,7 +326,9 @@ username: &str, client: &IggyClient, ) -> Result<(), IggyError> { - let stream = client.create_stream(stream_name).await?; + let stream = client + .create_stream(stream_name, &StreamCreateOptions::default()) + .await?; info!("Created stream: {stream_name} with ID: {}", stream.id); let mut streams_permissions = BTreeMap::new(); streams_permissions.insert( @@ -342,7 +344,13 @@ ..Default::default() }; let user = client - .create_user(username, PASSWORD, UserStatus::Active, Some(permissions)) + .create_user( + username, + PASSWORD, + UserStatus::Active, + Some(permissions), + &UserCreateOptions::default(), + ) .await?; info!( "Created user: {username} with ID: {}, with permissions for stream: {stream_name}",
diff --git a/examples/rust/src/shared/system.rs b/examples/rust/src/shared/system.rs index c90d200..63d0d55 100644 --- a/examples/rust/src/shared/system.rs +++ b/examples/rust/src/shared/system.rs
@@ -81,7 +81,9 @@ } info!("Stream does not exist, creating..."); - client.create_stream(&args.stream_id).await?; + client + .create_stream(&args.stream_id, &StreamCreateOptions::default()) + .await?; let compression_algorithm = CompressionAlgorithm::from_code(args.compression_algorithm)?; client .create_topic(
diff --git a/examples/rust/src/tcp-tls/producer/main.rs b/examples/rust/src/tcp-tls/producer/main.rs index b8a0e69..15d0b69 100644 --- a/examples/rust/src/tcp-tls/producer/main.rs +++ b/examples/rust/src/tcp-tls/producer/main.rs
@@ -75,7 +75,10 @@ } async fn init_system(client: &IggyClient) -> (u32, u32) { - let stream = match client.create_stream(STREAM_NAME).await { + let stream = match client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + { Ok(stream) => { info!("Stream was created."); stream
diff --git a/foreign/cpp/BUILD.bazel b/foreign/cpp/BUILD.bazel index 417d7d7..b1ca2a1 100644 --- a/foreign/cpp/BUILD.bazel +++ b/foreign/cpp/BUILD.bazel
@@ -196,6 +196,7 @@ "tests/bdd/background_steps.cpp", "tests/bdd/messaging_steps.cpp", "tests/bdd/raw_command_steps.cpp", + "tests/bdd/resource_options_steps.cpp", "tests/bdd/session_contract_steps.cpp", "tests/bdd/world.hpp", ], @@ -239,9 +240,13 @@ "-fexec-charset=UTF-8", ], }), + data = [ + "tests/fixtures/resource_options.json", + ], deps = [ ":iggy-cpp", "@googletest//:gtest_main", + "@nlohmann_json//:json", ], ) @@ -264,11 +269,15 @@ "-fexec-charset=UTF-8", ], }), + data = [ + "tests/fixtures/resource_options.json", + ], tags = [ "e2e", ], deps = [ ":iggy-cpp", "@googletest//:gtest_main", + "@nlohmann_json//:json", ], )
diff --git a/foreign/cpp/Cargo.toml b/foreign/cpp/Cargo.toml index d55f8b1..c213672 100644 --- a/foreign/cpp/Cargo.toml +++ b/foreign/cpp/Cargo.toml
@@ -17,8 +17,9 @@ [package] name = "iggy-cpp" -version = "0.1.3" +version = "0.1.4" edition = "2024" +license = "Apache-2.0" [package.metadata.cargo-machete] ignored = ["cxx-build"] @@ -32,6 +33,8 @@ iggy = { path = "../../core/sdk" } iggy_binary_protocol = { path = "../../core/binary_protocol" } iggy_common = { path = "../../core/common" } +serde = { version = "1.0.229", features = ["derive"] } +serde_json = "1.0.154" # Explicitly enable the runtime + I/O drivers required by `Runtime::enable_all()` in lib.rs. # Listing the features here insulates this crate from upstream SDK feature changes. tokio = { version = "1.53.3", features = ["rt-multi-thread", "macros", "time", "net", "io-util"] }
diff --git a/foreign/cpp/MODULE.bazel b/foreign/cpp/MODULE.bazel index bb870f6..20faa4d 100644 --- a/foreign/cpp/MODULE.bazel +++ b/foreign/cpp/MODULE.bazel
@@ -17,7 +17,7 @@ module( name = "iggy_cpp", - version = "0.1.3", + version = "0.1.4", ) bazel_dep(name = "rules_cc", version = "0.2.26") @@ -26,6 +26,7 @@ bazel_dep(name = "cucumber-cpp", version = "0.9.0") bazel_dep(name = "rules_rust", version = "0.74.0") bazel_dep(name = "abseil-cpp", version = "20260526.0") +bazel_dep(name = "nlohmann_json", version = "3.12.0.bcr.1", dev_dependency = True) iggy_source = use_repo_rule("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository") iggy_source(
diff --git a/foreign/cpp/README.md b/foreign/cpp/README.md index e98848e..5c89d98 100644 --- a/foreign/cpp/README.md +++ b/foreign/cpp/README.md
@@ -25,6 +25,8 @@ bazel test //:e2e ``` +The E2E suite expects a local server on the default ports with encryption enabled, because the resource options cases create streams with their own keys and expect a server-generated key for keyless streams. Start it with `IGGY_ENCRYPTION_ENABLED=true` and `IGGY_ENCRYPTION_KEY` set to a base64 encoded 32-byte key, for example `IGGY_ENCRYPTION_ENABLED=true IGGY_ENCRYPTION_KEY=$(head -c 32 /dev/urandom | base64) cargo run --bin iggy-server -- --with-default-root-credentials`. + Receipt access follows the server's [session and revocation policy](../../core/server/README.md#sessions-and-retry-receipts). An unresolved operation cannot recover its receipt after its original session ends, even after a new login or account reactivation. A fresh send can duplicate @@ -36,6 +38,29 @@ this delay is at most `cluster.commit_broadcast_interval` (500 ms by default). See [metadata freshness and counters](../../core/server/README.md#metadata-freshness-and-counters). +## Resource options + +Streams, topics and users take typed create and update options. Unset settings use the server default on create and keep their stored value on update. `SetRawEntries()` and `RawReset()` name keys newer than this SDK build and leave the decision to the server. `DescribeOptions()` serves each scope's catalog. + +```cpp +const std::string key_base64 = LoadStreamKey(); +const auto stream = client.CreateStream("payments", iggy::StreamCreateOptions().SetEncryptionKeyBase64(key_base64)); +const bool same_key = stream.EncryptionKeyFingerprint() == iggy::EncryptionKeyFingerprint(key_base64); + +const auto stream_id = iggy::Identifier::Numeric(stream.Id()); +const auto topic = client.CreateTopic(stream_id, "orders", + iggy::TopicCreateOptions() + .SetPartitionsCount(3) + .SetDurability(iggy::Durability::Persisted) + .SetPartitionResizePolicy(iggy::PartitionResizePolicy::Fixed)); +client.UpdateTopic(stream_id, iggy::Identifier::Numeric(topic.Id()), + iggy::TopicUpdateOptions() + .SetMessageExpiry(iggy::Expiry::Duration(3600000000)) + .Reset(iggy::TopicOptionKey::MaxTopicSize)); +``` + +A stream key must be 32 random bytes, never derived from a password, because reads expose its fingerprint. It travels inside the create request, so protecting it in transit with TLS is the caller's job. The server stores it wrapped and never returns it. Reads report only its source and fingerprints. Without a key, a server with encryption enabled generates one for the stream. An invalid key is refused before anything is sent. + Binary mutations retain an uncertain operation until it is resolved or abandoned. `IggyException::OperationId()` carries an exact `std::uint64_t` for `operation_uncertain`, `lane_blocked`, and `operation_not_retained`.
diff --git a/foreign/cpp/include/iggy.hpp b/foreign/cpp/include/iggy.hpp index 4c00beb..a8f0d11 100644 --- a/foreign/cpp/include/iggy.hpp +++ b/foreign/cpp/include/iggy.hpp
@@ -31,6 +31,7 @@ #include <map> #include <memory> #include <optional> +#include <set> #include <stdexcept> #include <string> #include <string_view> @@ -131,6 +132,8 @@ } friend class IggyBlockingClient; + friend std::string EncryptionKeyFingerprint(const std::vector<std::uint8_t> &key); + friend std::string EncryptionKeyFingerprint(const std::string &key_base64); std::optional<std::uint32_t> code_; // Shared, so that copying the exception cannot throw. @@ -1104,15 +1107,16 @@ }; /** - * @brief Options recorded for a stream or topic. + * @brief Options recorded for a stream, topic, or user. * - * Explicit() contains values supplied when the resource was created. Derived() - * contains values resolved from the server configuration at that time. Derived - * values describe the resource's creation settings and can differ when the - * resource is recreated with a different server configuration. + * Explicit() contains values a client supplied at creation or in a later + * update. Derived() contains values the server filled in: defaults resolved + * at admission, values restored by a reset, and the read-only stream + * encryption projections. Key material is never returned. * - * This is a response-only model returned by Options(). Use TopicCreateOptions - * to configure a new topic. Stream creation currently accepts only a name. + * This is a response-only model returned by Options(). Use + * TopicCreateOptions, StreamCreateOptions, or UserCreateOptions to configure a + * new resource. */ class ResourceOptions final { public: @@ -1123,10 +1127,8 @@ [[nodiscard]] const std::map<std::string, HeaderField> &Explicit() const noexcept { return explicit_; } /** - * @brief Returns entries derived from configured defaults at admission. + * @brief Returns entries the server filled in. * @return Derived entries as map from option name to typed value. - * @note Stream responses currently expose explicit entries only, so this - * collection is empty for Stream and StreamDetails. */ [[nodiscard]] const std::map<std::string, HeaderField> &Derived() const noexcept { return derived_; } @@ -1185,7 +1187,7 @@ [[nodiscard]] const std::string &Username() const noexcept { return username_; } /** - * @brief Returns explicit user creation options. + * @brief Returns user options and their provenance. * @return Options owned by this value. */ [[nodiscard]] const ResourceOptions &Options() const noexcept { return options_; } @@ -1254,7 +1256,7 @@ [[nodiscard]] const std::optional<::iggy::Permissions> &Permissions() const noexcept { return permissions_; } /** - * @brief Returns explicit user creation options. + * @brief Returns user options and their provenance. * @return Options owned by this value. */ [[nodiscard]] const ResourceOptions &Options() const noexcept { return options_; } @@ -1328,19 +1330,21 @@ [[nodiscard]] std::uint64_t SizeBytes() const noexcept { return size_bytes_; } /** - * @brief Returns the server-encoded message retention value. + * @brief Returns the message retention resolved from the topic options. * @return Retention value in microseconds or a protocol sentinel. */ [[nodiscard]] std::uint64_t MessageExpiry() const noexcept { return message_expiry_; } /** - * @brief Returns the server-selected storage compression algorithm. + * @brief Returns the storage compression algorithm resolved from the topic + * options. * @return Algorithm name owned by this value. */ [[nodiscard]] const std::string &CompressionAlgorithm() const noexcept { return compression_algorithm_; } /** - * @brief Returns the configured maximum retained topic size. + * @brief Returns the maximum retained topic size resolved from the topic + * options. * @return Maximum size in bytes. */ [[nodiscard]] std::uint64_t MaxTopicSize() const noexcept { return max_topic_size_; } @@ -1578,19 +1582,20 @@ [[nodiscard]] std::uint64_t SizeBytes() const noexcept { return size_bytes_; } /** - * @brief Returns the server-encoded message retention value. + * @brief Returns the message retention resolved from the topic options. * @return Retention value in microseconds or a protocol sentinel. */ [[nodiscard]] std::uint64_t MessageExpiry() const noexcept { return message_expiry_; } /** - * @brief Returns the storage compression algorithm selected for this topic. + * @brief Returns the storage compression algorithm resolved from the topic + * options. * @return Algorithm name owned by this value. */ [[nodiscard]] const std::string &CompressionAlgorithm() const noexcept { return compression_algorithm_; } /** - * @brief Returns the maximum retained size configured for this topic. + * @brief Returns the maximum retained size resolved from the topic options. * @return Maximum size in bytes. */ [[nodiscard]] std::uint64_t MaxTopicSize() const noexcept { return max_topic_size_; } @@ -1664,6 +1669,30 @@ }; /** + * @brief Who chose a stream's at-rest data key. + * + * Reads report the source and fingerprints of a stream's key, never the key + * itself. + */ +enum class StreamEncryptionSource : std::uint8_t { + None = 0, ///< The stream is stored as plaintext for life. + Stream = 1, ///< The client supplied the key at creation. + Server = 2, ///< The server generated the key at creation. +}; + +constexpr std::string_view to_string(const StreamEncryptionSource source) { + switch (source) { + case StreamEncryptionSource::None: + return "none"; + case StreamEncryptionSource::Stream: + return "stream"; + case StreamEncryptionSource::Server: + return "server"; + } + throw std::invalid_argument("Unknown stream encryption source"); +} + +/** * @brief Snapshot of one stream's metadata and aggregate statistics. * * CreateStream() and GetStream() return this value. @@ -1732,12 +1761,46 @@ [[nodiscard]] const std::vector<Topic> &Topics() const noexcept { return topics_; } /** - * @brief Returns explicit stream creation options. + * @brief Returns stream options and their provenance. + * + * Derived() carries the read-only encryption projections. Key material is + * never returned. * @return Options owned by this value. - * @note The current bridge does not return derived stream options. */ [[nodiscard]] const ResourceOptions &Options() const noexcept { return options_; } + /** + * @brief Returns who chose the stream's at-rest key. + * @return Key source. A plaintext stream reports + * StreamEncryptionSource::None. `std::nullopt` means the read + * carried no source entry or one this SDK build does not know. + */ + [[nodiscard]] std::optional<StreamEncryptionSource> EncryptionKeySource() const noexcept { + return encryption_key_source_; + } + + /** + * @brief Returns the fingerprint of the stream's key. + * + * The value is 16 lowercase hex characters. Compare it with + * EncryptionKeyFingerprint() of a key you hold to check which key the + * stream uses. + * @return Fingerprint, or `std::nullopt` for a plaintext stream. + */ + [[nodiscard]] const std::optional<std::string> &EncryptionKeyFingerprint() const noexcept { + return encryption_key_fingerprint_; + } + + /** + * @brief Returns the fingerprints of the server keys that wrap the + * stream's key. + * @return One fingerprint outside a server key rotation, empty for a + * plaintext stream. + */ + [[nodiscard]] const std::vector<std::string> &EncryptionKekFingerprints() const noexcept { + return encryption_kek_fingerprints_; + } + private: StreamDetails(std::uint32_t id, std::uint64_t created_at, @@ -1746,7 +1809,10 @@ std::uint64_t messages_count, std::uint32_t topics_count, std::vector<Topic> topics, - ResourceOptions options) + ResourceOptions options, + std::optional<StreamEncryptionSource> encryption_key_source, + std::optional<std::string> encryption_key_fingerprint, + std::vector<std::string> encryption_kek_fingerprints) : id_(id), created_at_(created_at), name_(std::move(name)), @@ -1754,7 +1820,10 @@ messages_count_(messages_count), topics_count_(topics_count), topics_(std::move(topics)), - options_(std::move(options)) {} + options_(std::move(options)), + encryption_key_source_(encryption_key_source), + encryption_key_fingerprint_(std::move(encryption_key_fingerprint)), + encryption_kek_fingerprints_(std::move(encryption_kek_fingerprints)) {} static StreamDetails FromFfi(ffi::StreamDetails stream); @@ -1768,6 +1837,9 @@ std::uint32_t topics_count_; std::vector<Topic> topics_; ResourceOptions options_; + std::optional<StreamEncryptionSource> encryption_key_source_; + std::optional<std::string> encryption_key_fingerprint_; + std::vector<std::string> encryption_kek_fingerprints_; }; /** @@ -1821,12 +1893,46 @@ [[nodiscard]] std::uint32_t TopicsCount() const noexcept { return topics_count_; } /** - * @brief Returns explicit stream creation options. + * @brief Returns stream options and their provenance. + * + * Derived() carries the read-only encryption projections. Key material is + * never returned. * @return Options owned by this value. - * @note The current bridge does not return derived stream options. */ [[nodiscard]] const ResourceOptions &Options() const noexcept { return options_; } + /** + * @brief Returns who chose the stream's at-rest key. + * @return Key source. A plaintext stream reports + * StreamEncryptionSource::None. `std::nullopt` means the read + * carried no source entry or one this SDK build does not know. + */ + [[nodiscard]] std::optional<StreamEncryptionSource> EncryptionKeySource() const noexcept { + return encryption_key_source_; + } + + /** + * @brief Returns the fingerprint of the stream's key. + * + * The value is 16 lowercase hex characters. Compare it with + * EncryptionKeyFingerprint() of a key you hold to check which key the + * stream uses. + * @return Fingerprint, or `std::nullopt` for a plaintext stream. + */ + [[nodiscard]] const std::optional<std::string> &EncryptionKeyFingerprint() const noexcept { + return encryption_key_fingerprint_; + } + + /** + * @brief Returns the fingerprints of the server keys that wrap the + * stream's key. + * @return One fingerprint outside a server key rotation, empty for a + * plaintext stream. + */ + [[nodiscard]] const std::vector<std::string> &EncryptionKekFingerprints() const noexcept { + return encryption_kek_fingerprints_; + } + private: Stream(std::uint32_t id, std::uint64_t created_at, @@ -1834,14 +1940,20 @@ std::uint64_t size_bytes, std::uint64_t messages_count, std::uint32_t topics_count, - ResourceOptions options) + ResourceOptions options, + std::optional<StreamEncryptionSource> encryption_key_source, + std::optional<std::string> encryption_key_fingerprint, + std::vector<std::string> encryption_kek_fingerprints) : id_(id), created_at_(created_at), name_(std::move(name)), size_bytes_(size_bytes), messages_count_(messages_count), topics_count_(topics_count), - options_(std::move(options)) {} + options_(std::move(options)), + encryption_key_source_(encryption_key_source), + encryption_key_fingerprint_(std::move(encryption_key_fingerprint)), + encryption_kek_fingerprints_(std::move(encryption_kek_fingerprints)) {} static Stream FromFfi(ffi::Stream stream); @@ -1854,6 +1966,9 @@ std::uint64_t messages_count_; std::uint32_t topics_count_; ResourceOptions options_; + std::optional<StreamEncryptionSource> encryption_key_source_; + std::optional<std::string> encryption_key_fingerprint_; + std::vector<std::string> encryption_kek_fingerprints_; }; /** @@ -2768,15 +2883,128 @@ } /** + * @brief Whether a topic's partition set may change after creation. + */ +enum class PartitionResizePolicy : std::uint8_t { + Mutable, ///< Partitions may be created and deleted later. + Fixed, ///< The partition set is fixed at topic creation. +}; + +constexpr std::string_view to_string(const PartitionResizePolicy policy) { + switch (policy) { + case PartitionResizePolicy::Mutable: + return "mutable"; + case PartitionResizePolicy::Fixed: + return "fixed"; + } + throw std::invalid_argument("Unknown partition resize policy"); +} + +/** + * @brief Option keys of the stream scope, in catalog order. + * + * Name is updatable. EncryptionKey is a create-only secret. The fingerprint + * and source keys are read-only projections. No stream key is resettable. + */ +enum class StreamOptionKey : std::uint8_t { + Name, + EncryptionKey, + EncryptionKeySource, + EncryptionKeyFingerprint, + EncryptionKekFingerprint, +}; + +constexpr std::string_view to_string(const StreamOptionKey key) { + switch (key) { + case StreamOptionKey::Name: + return "name"; + case StreamOptionKey::EncryptionKey: + return "encryption_key"; + case StreamOptionKey::EncryptionKeySource: + return "encryption_key_source"; + case StreamOptionKey::EncryptionKeyFingerprint: + return "encryption_key_fingerprint"; + case StreamOptionKey::EncryptionKekFingerprint: + return "encryption_kek_fingerprint"; + } + throw std::invalid_argument("Unknown stream option key"); +} + +/** + * @brief Option keys of the topic scope, in catalog order. + * + * Name, MessageExpiry, and MaxTopicSize are updatable, and MessageExpiry and + * MaxTopicSize are also resettable. Every other key is create-only. + */ +enum class TopicOptionKey : std::uint8_t { + Name, + CompressionAlgorithm, + MessageExpiry, + MaxTopicSize, + SegmentSize, + Durability, + ConsumerOffsetDurability, + PartitionResizePolicy, + MessagesRequiredToSave, + SizeOfMessagesRequiredToSave, + PreallocateSegments, +}; + +constexpr std::string_view to_string(const TopicOptionKey key) { + switch (key) { + case TopicOptionKey::Name: + return "name"; + case TopicOptionKey::CompressionAlgorithm: + return "compression_algorithm"; + case TopicOptionKey::MessageExpiry: + return "message_expiry"; + case TopicOptionKey::MaxTopicSize: + return "max_topic_size"; + case TopicOptionKey::SegmentSize: + return "segment_size"; + case TopicOptionKey::Durability: + return "durability"; + case TopicOptionKey::ConsumerOffsetDurability: + return "consumer_offset_durability"; + case TopicOptionKey::PartitionResizePolicy: + return "partition_resize_policy"; + case TopicOptionKey::MessagesRequiredToSave: + return "messages_required_to_save"; + case TopicOptionKey::SizeOfMessagesRequiredToSave: + return "size_of_messages_required_to_save"; + case TopicOptionKey::PreallocateSegments: + return "preallocate_segments"; + } + throw std::invalid_argument("Unknown topic option key"); +} + +/** + * @brief Option keys of the user scope, in catalog order. + * + * Both keys are updatable. Neither is resettable. + */ +enum class UserOptionKey : std::uint8_t { + Username, + Status, +}; + +constexpr std::string_view to_string(const UserOptionKey key) { + switch (key) { + case UserOptionKey::Username: + return "username"; + case UserOptionKey::Status: + return "status"; + } + throw std::invalid_argument("Unknown user option key"); +} + +/** * @brief Options for creating a topic. * * Use the typed setters to configure supported topic settings. Leave a setting * unset to use the server default. Use SetRawEntries() for supported options * that do not yet have a typed setter. When both specify the same option, the * typed setting takes precedence. - * - * For example, the entry `partition_resize_policy` set to `fixed` makes the - * server reject later partition creation and deletion on this topic. */ class TopicCreateOptions final { public: @@ -2919,6 +3147,26 @@ } /** + * @brief Returns whether the topic's partition set may change later. + * @return Configured policy, or `std::nullopt` to use the server default + * (`mutable`). + */ + [[nodiscard]] std::optional<::iggy::PartitionResizePolicy> PartitionResizePolicy() const noexcept { + return partition_resize_policy_; + } + + /** + * @brief Sets whether the topic's partition set may change later. + * @param policy PartitionResizePolicy::Fixed makes the server reject later + * partition creation and deletion on this topic. + * @return Reference to this options object. + */ + TopicCreateOptions &SetPartitionResizePolicy(::iggy::PartitionResizePolicy policy) noexcept { + partition_resize_policy_ = policy; + return *this; + } + + /** * @brief Returns the message-count threshold for flushing the journal. * @return Configured threshold, or `std::nullopt` to use the server default. */ @@ -3028,6 +3276,7 @@ std::optional<std::uint64_t> segment_size_; std::optional<::iggy::Durability> durability_; std::optional<::iggy::Durability> consumer_offset_durability_; + std::optional<::iggy::PartitionResizePolicy> partition_resize_policy_; std::optional<std::uint32_t> messages_required_to_save_; std::optional<std::uint64_t> size_of_messages_required_to_save_; std::optional<bool> preallocate_segments_; @@ -3040,33 +3289,33 @@ * @brief Options for updating a topic. * * Use this class to change a topic's mutable settings. Leave a setting unset - * to retain its current value. Topic creation settings, such as the partition - * count and segment size, cannot be changed after the topic is created. + * to retain its current value, or name it in Reset() to return it to the + * server default. Topic creation settings, such as the compression + * algorithm, partition count, and segment size, cannot be changed after the + * topic is created. * * Use the typed setters for supported settings. SetRawEntries() can configure * other supported mutable settings. When both configure the same setting, the - * typed setting takes precedence. + * typed setting takes precedence. An update must change at least one setting, + * and must not both set and reset the same one. */ class TopicUpdateOptions final { public: TopicUpdateOptions() = default; /** - * @brief Returns the requested storage compression update. - * @return Compression algorithm to apply, or `std::nullopt` when this - * update leaves compression unchanged. + * @brief Returns the requested topic name. + * @return New name, or `std::nullopt` when this update keeps the name. */ - [[nodiscard]] const std::optional<::iggy::CompressionAlgorithm> &CompressionAlgorithm() const noexcept { - return compression_algorithm_; - } + [[nodiscard]] const std::optional<std::string> &Name() const noexcept { return name_; } /** - * @brief Sets the storage compression algorithm. - * @param compression_algorithm Compression algorithm to apply. + * @brief Renames the topic. + * @param name New topic name, unique within its stream. * @return Reference to this options object. */ - TopicUpdateOptions &SetCompressionAlgorithm(::iggy::CompressionAlgorithm compression_algorithm) { - compression_algorithm_ = std::move(compression_algorithm); + TopicUpdateOptions &SetName(std::string name) { + name_ = std::move(name); return *this; } @@ -3080,7 +3329,8 @@ /** * @brief Sets the message retention policy. * @param message_expiry Expiry policy to apply. Expiry::ServerDefault() - * leaves the current policy unchanged. + * leaves the current policy unchanged. Use + * Reset(TopicOptionKey::MessageExpiry) to return to the default. * @return Reference to this options object. */ TopicUpdateOptions &SetMessageExpiry(::iggy::Expiry message_expiry) { @@ -3102,7 +3352,8 @@ /** * @brief Sets the maximum retained topic size. * @param max_topic_size Maximum size to retain. MaxTopicSize::ServerDefault() - * leaves the current limit unchanged. + * leaves the current limit unchanged. Use + * Reset(TopicOptionKey::MaxTopicSize) to return to the default. * @return Reference to this options object. */ TopicUpdateOptions &SetMaxTopicSize(::iggy::MaxTopicSize max_topic_size) { @@ -3152,11 +3403,147 @@ return *this; } + /** + * @brief Returns the settings this update resets to their defaults. + * @return Keys to reset, ordered by catalog position. + */ + [[nodiscard]] const std::set<TopicOptionKey> &ResetKeys() const noexcept { return reset_; } + + /** + * @brief Resets a setting to its server default. + * @param key Setting to reset. Only MessageExpiry and MaxTopicSize are + * resettable. UpdateTopic() refuses any other key before sending + * with the error the server would return. + * @return Reference to this options object. + */ + TopicUpdateOptions &Reset(TopicOptionKey key) { + reset_.insert(key); + return *this; + } + + /** + * @brief Returns the settings this update resets by name. + * @return Names to reset, for keys newer than this SDK build. + */ + [[nodiscard]] const std::set<std::string> &RawResetKeys() const noexcept { return raw_reset_; } + + /** + * @brief Resets a setting by name to its server default. + * + * Use this for keys newer than this SDK build. The server decides whether + * the key exists and may be reset. + * @param key Setting name to reset. + * @return Reference to this options object. + */ + TopicUpdateOptions &RawReset(std::string key) { + raw_reset_.insert(std::move(key)); + return *this; + } + private: - std::optional<::iggy::CompressionAlgorithm> compression_algorithm_; + std::optional<std::string> name_; std::optional<::iggy::Expiry> message_expiry_; std::optional<::iggy::MaxTopicSize> max_topic_size_; std::map<std::string, std::string> raw_; + std::set<TopicOptionKey> reset_; + std::set<std::string> raw_reset_; + + friend class IggyBlockingClient; +}; + +/** + * @brief Options for creating a stream. + * + * A stream may carry its own 32-byte data encryption key. The key must be 32 + * random bytes, never derived from a password, because reads expose its + * fingerprint. It travels inside the create request over whatever transport + * the client uses, so protecting it in transit with TLS is the caller's + * responsibility. The server stores the key wrapped and never returns it: + * reads report only its source and fingerprints. Without a key, a server with + * encryption enabled generates one for the stream. + * + * An invalid key is refused by CreateStream() before anything is sent. This + * type never prints or logs the key. + */ +class StreamCreateOptions final { + public: + StreamCreateOptions() = default; + + /** + * @brief Returns whether this create carries an encryption key. + * @return `true` after SetEncryptionKey() or SetEncryptionKeyBase64(). + */ + [[nodiscard]] bool HasEncryptionKey() const noexcept { + return encryption_key_.has_value() || encryption_key_base64_.has_value(); + } + + /** + * @brief Sets the stream's data encryption key from raw bytes. + * @param key Exactly 32 random bytes. CreateStream() refuses any other + * length with invalid_encryption_key. + * @return Reference to this options object. + */ + StreamCreateOptions &SetEncryptionKey(const std::vector<std::uint8_t> &key) { + encryption_key_ = key; + encryption_key_base64_.reset(); + return *this; + } + + /** + * @brief Sets the stream's data encryption key from its base64 text. + * @param key Canonical standard padded base64 of exactly 32 bytes. + * CreateStream() refuses whitespace, the URL-safe alphabet, missing + * padding, and noncanonical trailing bits with + * invalid_encryption_key. + * @return Reference to this options object. + */ + StreamCreateOptions &SetEncryptionKeyBase64(std::string key) { + encryption_key_base64_ = std::move(key); + encryption_key_.reset(); + return *this; + } + + /** + * @brief Returns additional stream settings as key-value pairs. + * @return Ordered map of setting names and values. + * @note A dedicated setter takes precedence when it configures the same + * setting. + */ + [[nodiscard]] const std::map<std::string, std::string> &RawEntries() const noexcept { return raw_; } + + /** + * @brief Adds or replaces additional stream settings. + * + * Use this for keys newer than this SDK build. Unsupported names and + * invalid values are rejected when the stream is created. + * @param entries Setting names and values to add. + * @return Reference to this options object. + */ + StreamCreateOptions &SetRawEntries(const std::map<std::string, std::string> &entries) { + for (const auto &entry : entries) { + raw_.insert_or_assign(entry.first, entry.second); + } + return *this; + } + /** + * @brief Adds or replaces additional stream settings. + * @param entries Setting names and values to move into this options object. + * @return Reference to this options object. + * @see SetRawEntries(const std::map<std::string, std::string>&) + */ + StreamCreateOptions &SetRawEntries(std::map<std::string, std::string> &&entries) { + while (!entries.empty()) { + auto node = entries.extract(entries.begin()); + raw_.erase(node.key()); + raw_.insert(std::move(node)); + } + return *this; + } + + private: + std::optional<std::vector<std::uint8_t>> encryption_key_; + std::optional<std::string> encryption_key_base64_; + std::map<std::string, std::string> raw_; friend class IggyBlockingClient; }; @@ -3164,27 +3551,45 @@ /** * @brief Options for updating a stream. * - * Use this class to supply stream settings to UpdateStream(). Currently, Iggy - * does not support updating stream settings, so the server rejects every - * supplied setting. The raw entries are retained for compatibility with future - * server versions that add mutable stream settings. + * Leave a setting unset to retain its current value. The name is the only + * updatable stream setting, and no stream setting is resettable, so Reset() + * exists to refuse a reset before anything is sent, with the error the server + * would return. RawReset() names keys newer than this SDK build and leaves the + * decision to the server. An update must change at least one setting. */ class StreamUpdateOptions final { public: StreamUpdateOptions() = default; /** + * @brief Returns the requested stream name. + * @return New name, or `std::nullopt` when this update keeps the name. + */ + [[nodiscard]] const std::optional<std::string> &Name() const noexcept { return name_; } + + /** + * @brief Renames the stream. + * @param name New unique stream name. + * @return Reference to this options object. + */ + StreamUpdateOptions &SetName(std::string name) { + name_ = std::move(name); + return *this; + } + + /** * @brief Returns the requested stream settings as key-value pairs. * @return Ordered map of setting names and values. - * @note The server currently rejects all stream settings. */ [[nodiscard]] const std::map<std::string, std::string> &RawEntries() const noexcept { return raw_; } /** * @brief Adds or replaces requested stream settings. + * + * Use this for updatable keys newer than this SDK build. Unsupported or + * immutable settings are rejected when the stream is updated. * @param entries Setting names and values to add. * @return Reference to this options object. - * @note The server currently rejects all stream settings. */ StreamUpdateOptions &SetRawEntries(const std::map<std::string, std::string> &entries) { for (const auto &entry : entries) { @@ -3197,7 +3602,6 @@ * @param entries Setting names and values to move into this options object. * @return Reference to this options object. * @see SetRawEntries(const std::map<std::string, std::string>&) - * @note The server currently rejects all stream settings. */ StreamUpdateOptions &SetRawEntries(std::map<std::string, std::string> &&entries) { while (!entries.empty()) { @@ -3208,6 +3612,96 @@ return *this; } + /** + * @brief Returns the settings this update resets to their defaults. + * @return Keys to reset, ordered by catalog position. + */ + [[nodiscard]] const std::set<StreamOptionKey> &ResetKeys() const noexcept { return reset_; } + + /** + * @brief Requests a reset of a stream setting. + * @param key Setting to reset. No stream key is resettable in this + * release, so UpdateStream() refuses every reset before sending. + * @return Reference to this options object. + */ + StreamUpdateOptions &Reset(StreamOptionKey key) { + reset_.insert(key); + return *this; + } + + /** + * @brief Returns the settings this update resets by name. + * @return Names to reset, for keys newer than this SDK build. + */ + [[nodiscard]] const std::set<std::string> &RawResetKeys() const noexcept { return raw_reset_; } + + /** + * @brief Resets a setting by name to its server default. + * + * Use this for keys newer than this SDK build. The server decides whether + * the key exists and may be reset. + * @param key Setting name to reset. + * @return Reference to this options object. + */ + StreamUpdateOptions &RawReset(std::string key) { + raw_reset_.insert(std::move(key)); + return *this; + } + + private: + std::optional<std::string> name_; + std::map<std::string, std::string> raw_; + std::set<StreamOptionKey> reset_; + std::set<std::string> raw_reset_; + + friend class IggyBlockingClient; +}; + +/** + * @brief Options for creating a user. + * + * No typed user setting exists yet. The type exists so a future setting is + * a new setter rather than a signature change. + */ +class UserCreateOptions final { + public: + UserCreateOptions() = default; + + /** + * @brief Returns additional user settings as key-value pairs. + * @return Ordered map of setting names and values. + */ + [[nodiscard]] const std::map<std::string, std::string> &RawEntries() const noexcept { return raw_; } + + /** + * @brief Adds or replaces additional user settings. + * + * Use this for keys newer than this SDK build. Unsupported names and + * invalid values are rejected when the user is created. + * @param entries Setting names and values to add. + * @return Reference to this options object. + */ + UserCreateOptions &SetRawEntries(const std::map<std::string, std::string> &entries) { + for (const auto &entry : entries) { + raw_.insert_or_assign(entry.first, entry.second); + } + return *this; + } + /** + * @brief Adds or replaces additional user settings. + * @param entries Setting names and values to move into this options object. + * @return Reference to this options object. + * @see SetRawEntries(const std::map<std::string, std::string>&) + */ + UserCreateOptions &SetRawEntries(std::map<std::string, std::string> &&entries) { + while (!entries.empty()) { + auto node = entries.extract(entries.begin()); + raw_.erase(node.key()); + raw_.insert(std::move(node)); + } + return *this; + } + private: std::map<std::string, std::string> raw_; @@ -3217,11 +3711,11 @@ /** * @brief Options for updating a user. * - * Use this class to supply user settings to UpdateUser(). Updating a user - * patches only the supplied settings; omitted settings remain unchanged. - * Currently, Iggy does not support updating user settings, so the server - * rejects every supplied setting. The raw entries are retained for - * compatibility with future server versions that add mutable user settings. + * Updating a user patches only the supplied settings. Omitted settings remain + * unchanged. No user setting is resettable, so Reset() exists to refuse a + * reset before anything is sent, with the error the server would return. + * RawReset() names keys newer than this SDK build and leaves the decision to + * the server. An update must change at least one setting. */ class UserUpdateOptions final { public: @@ -3229,17 +3723,53 @@ UserUpdateOptions() = default; /** + * @brief Returns the requested user name. + * @return New name, or `std::nullopt` when this update keeps the name. + */ + [[nodiscard]] const std::optional<std::string> &Username() const noexcept { return username_; } + + /** + * @brief Renames the user. + * + * Renaming the authenticated user also updates the credentials the client + * uses when it reconnects. + * @param username New unique user name, 3 to 50 bytes. + * @return Reference to this options object. + */ + UserUpdateOptions &SetUsername(std::string username) { + username_ = std::move(username); + return *this; + } + + /** + * @brief Returns the requested authentication status. + * @return New status, or `std::nullopt` when this update keeps the status. + */ + [[nodiscard]] std::optional<UserStatus> Status() const noexcept { return status_; } + + /** + * @brief Sets the user's authentication status. + * @param status New status. Deactivating a user ends its sessions. + * @return Reference to this options object. + */ + UserUpdateOptions &SetStatus(UserStatus status) noexcept { + status_ = status; + return *this; + } + + /** * @brief Returns the requested user settings as key-value pairs. * @return Ordered map of setting names and values. - * @note The server currently rejects all user settings. */ [[nodiscard]] const std::map<std::string, std::string> &RawEntries() const noexcept { return raw_; } /** * @brief Adds or replaces requested user settings. + * + * Use this for updatable keys newer than this SDK build. Unsupported or + * immutable settings are rejected when the user is updated. * @param entries Setting names and values to add. * @return Reference to this options object. - * @note The server currently rejects all user settings. */ UserUpdateOptions &SetRawEntries(const std::map<std::string, std::string> &entries) { for (const auto &entry : entries) { @@ -3252,7 +3782,6 @@ * @param entries Setting names and values to move into this options object. * @return Reference to this options object. * @see SetRawEntries(const std::map<std::string, std::string>&) - * @note The server currently rejects all user settings. */ UserUpdateOptions &SetRawEntries(std::map<std::string, std::string> &&entries) { while (!entries.empty()) { @@ -3263,13 +3792,184 @@ return *this; } + /** + * @brief Returns the settings this update resets to their defaults. + * @return Keys to reset, ordered by catalog position. + */ + [[nodiscard]] const std::set<UserOptionKey> &ResetKeys() const noexcept { return reset_; } + + /** + * @brief Requests a reset of a user setting. + * @param key Setting to reset. No user key is resettable in this release, + * so UpdateUser() refuses every reset before sending. + * @return Reference to this options object. + */ + UserUpdateOptions &Reset(UserOptionKey key) { + reset_.insert(key); + return *this; + } + + /** + * @brief Returns the settings this update resets by name. + * @return Names to reset, for keys newer than this SDK build. + */ + [[nodiscard]] const std::set<std::string> &RawResetKeys() const noexcept { return raw_reset_; } + + /** + * @brief Resets a setting by name to its server default. + * + * Use this for keys newer than this SDK build. The server decides whether + * the key exists and may be reset. + * @param key Setting name to reset. + * @return Reference to this options object. + */ + UserUpdateOptions &RawReset(std::string key) { + raw_reset_.insert(std::move(key)); + return *this; + } + private: + std::optional<std::string> username_; + std::optional<UserStatus> status_; std::map<std::string, std::string> raw_; + std::set<UserOptionKey> reset_; + std::set<std::string> raw_reset_; friend class IggyBlockingClient; }; /** + * @brief Resource kind whose option catalog DescribeOptions() serves. + */ +enum class OptionsScope : std::uint8_t { + Topic = 1, + Stream = 2, + User = 3, +}; + +constexpr std::string_view to_string(const OptionsScope scope) { + switch (scope) { + case OptionsScope::Topic: + return "topic"; + case OptionsScope::Stream: + return "stream"; + case OptionsScope::User: + return "user"; + } + throw std::invalid_argument("Unknown options scope"); +} + +/** + * @brief One key of a resource's option catalog. + * + * DescribeOptions() returns these values. The flags say what a create or an + * update accepts for the key: binary transports report a refused key by error + * code only, so this is the discovery surface for valid keys. + */ +class OptionSpec final { + public: + /** + * @brief Returns the option key. + * @return Key name owned by this value. + */ + [[nodiscard]] const std::string &Key() const noexcept { return key_; } + + /** + * @brief Returns the kind the value is encoded under. + * @return Header kind of the value and of DefaultValue(). + */ + [[nodiscard]] HeaderKind Kind() const noexcept { return kind_; } + + /** + * @brief Returns the key's default in Kind()'s encoding. + * @return Encoded default, empty when the key has no default. + */ + [[nodiscard]] const std::vector<std::uint8_t> &DefaultValue() const noexcept { return default_value_; } + + /** + * @brief Returns the human-readable description. + * @return Description owned by this value. + */ + [[nodiscard]] const std::string &Description() const noexcept { return description_; } + + /** @brief Returns whether an update may set the key. */ + [[nodiscard]] bool Updatable() const noexcept { return updatable_; } + + /** @brief Returns whether an update may reset the key to its default. */ + [[nodiscard]] bool Resettable() const noexcept { return resettable_; } + + /** @brief Returns whether the key is write-only and never returned. */ + [[nodiscard]] bool Secret() const noexcept { return secret_; } + + /** @brief Returns whether the server computes the key on reads only. */ + [[nodiscard]] bool ReadOnly() const noexcept { return read_only_; } + + /** @brief Returns whether a node that does not understand the key refuses to serve the resource. */ + [[nodiscard]] bool Critical() const noexcept { return critical_; } + + private: + OptionSpec(std::string key, + HeaderKind kind, + std::vector<std::uint8_t> default_value, + std::string description, + bool updatable, + bool resettable, + bool secret, + bool read_only, + bool critical) + : key_(std::move(key)), + kind_(kind), + default_value_(std::move(default_value)), + description_(std::move(description)), + updatable_(updatable), + resettable_(resettable), + secret_(secret), + read_only_(read_only), + critical_(critical) {} + + static OptionSpec FromFfi(ffi::OptionSpec spec); + + friend class IggyBlockingClient; + + std::string key_; + HeaderKind kind_; + std::vector<std::uint8_t> default_value_; + std::string description_; + bool updatable_; + bool resettable_; + bool secret_; + bool read_only_; + bool critical_; +}; + +/** + * @brief Computes the fingerprint of a stream encryption key. + * + * The fingerprint is 16 lowercase hex characters, the same value reads + * report through StreamDetails::EncryptionKeyFingerprint(). It is computed + * locally and identifies a key without revealing it. + * + * @param key Exactly 32 raw key bytes. + * @return Key fingerprint. + * @throws IggyException with Kind() invalid_encryption_key when @p key is not + * 32 bytes long. The message never contains key material. + */ +std::string EncryptionKeyFingerprint(const std::vector<std::uint8_t> &key); + +/** + * @brief Computes the fingerprint of a stream encryption key given as base64. + * + * Takes the same form as StreamCreateOptions::SetEncryptionKeyBase64(). + * + * @param key_base64 Canonical standard padded base64 of exactly 32 bytes. + * @return Key fingerprint. + * @throws IggyException with Kind() invalid_encryption_key when @p key_base64 + * is not canonical or does not decode to 32 bytes. The message never + * contains key material. + */ +std::string EncryptionKeyFingerprint(const std::string &key_base64); + +/** * @brief Starting position for polling messages. * * @note The strategy kind and value are passed across the Rust FFI as a pair. @@ -3341,6 +4041,7 @@ * * - Login(), Logout(), ResolveOperation(), and AbandonOperation(). * - CreateUser(), UpdateUser(), and DeleteUser(). + * - EncryptionKeyFingerprint() and invalid keys passed to CreateStream(). * - CreateStream(), UpdateStream(), and DeleteStream(). * - CreateTopic(), UpdateTopic(), and DeleteTopic(). * - CreatePartitions(), DeletePartitions(), and DeleteSegments(). @@ -3620,16 +4321,18 @@ * @param password Initial user password. * @param status Initial authentication status. * @param permissions Optional permission assignment. + * @param options User creation options. * @return Details of the newly created user. - * @throws IggyException if the client is unavailable or unauthenticated; - * the name, password, status, or permissions are invalid; the name - * is already in use; the caller lacks manage-users permission; or - * the request fails. + * @throws IggyException if the client is unavailable or unauthenticated, + * the name, password, status, permissions, or an option are + * invalid, the name is already in use, the caller lacks + * manage-users permission, or the request fails. */ UserInfoDetails CreateUser(std::string username, std::string password, UserStatus status, - const std::optional<Permissions> &permissions = std::nullopt); + const std::optional<Permissions> &permissions = std::nullopt, + const UserCreateOptions &options = {}); /** * @brief Deletes a user account. @@ -3650,33 +4353,26 @@ /** * @brief Changes a user's name, status, and mutable settings. * - * A supplied name must contain between 3 and 50 bytes and remain unique. - * Setting the status to inactive prevents subsequent authentication. - * Passing `std::nullopt` for the name or status leaves that field - * unchanged. The supplied UserUpdateOptions changes only the settings it - * contains; settings left unset retain their current values. User settings - * are currently not mutable, so the options object must be empty. Passing - * `std::nullopt` for both fields and an empty options object is accepted as - * a no-op. + * The supplied UserUpdateOptions changes only the settings it contains. + * Settings left unset retain their current values. A supplied name must + * contain between 3 and 50 bytes and remain unique. Setting the status to + * inactive prevents subsequent authentication. An update that changes + * nothing, sets and resets the same setting, or resets a typed user + * setting is refused before anything is sent. Renaming the authenticated user also + * updates the credentials the client uses when it reconnects. * * A failed or unknown transport outcome can leave the update committed. * Retrieve the user before retrying with different values. * * @param user User to update, addressed by numeric ID or name. - * @param username New unique name, or `std::nullopt` to retain the name. - * @param status New authentication status, or `std::nullopt` to retain the - * status. - * @param options User update options. - * @throws IggyException if the client is unavailable or unauthenticated; - * an identifier, name, status, or option is invalid; an option is - * unsupported; the user does not exist; the name is already in - * use; the caller lacks manage-users permission; or the request - * fails. + * @param options Settings to change. + * @throws IggyException if the client is unavailable or unauthenticated, + * an identifier, name, status, or option is invalid, an option is + * unsupported or not resettable, the user does not exist, the name + * is already in use, the caller lacks manage-users permission, or + * the request fails. */ - void UpdateUser(const Identifier &user, - std::optional<std::string> username, - std::optional<UserStatus> status, - const UserUpdateOptions &options = {}); + void UpdateUser(const Identifier &user, const UserUpdateOptions &options); /** * @brief Creates a top-level stream in the cluster metadata. @@ -3685,29 +4381,41 @@ * partitions, or messages. Its name must be unique, non-empty, and no more * than 255 UTF-8 bytes. * + * StreamCreateOptions may carry the stream's own encryption key. The key + * travels in the request, the server stores it wrapped and never returns + * it, and the result reports only its source and fingerprints. Without a + * key, a server with encryption enabled generates one for the stream. An + * invalid key is refused before anything is sent. + * * A transport failure after submission can leave the stream created. Look * it up by name before retrying or choosing another name. * * @param name Unique stream name. + * @param options Stream creation options. * @return Details of the newly created, topic-less stream. - * @throws IggyException if the client is unavailable or unauthenticated; - * the name is invalid or already in use; the caller lacks - * stream-management permission; or the request fails. + * @throws IggyException if the client is unavailable or unauthenticated, + * the name, the key, or an option is invalid, the name is already + * in use, the caller lacks stream-management permission, or the + * request fails. */ - StreamDetails CreateStream(std::string name); + StreamDetails CreateStream(std::string name, const StreamCreateOptions &options = {}); /** - * @brief Renames a stream. + * @brief Renames a stream or changes its mutable settings. * - * @param stream Stream to rename, addressed by numeric ID or name. - * @param name New unique stream name. - * @param options Stream update options (currently no updatable keys; `raw` - * carries forward-compatible keys, each rejected until catalogued). + * The supplied StreamUpdateOptions changes only the settings it contains. + * Renaming keeps the stream's ID, encryption key, and data. An update that + * changes nothing or resets any stream setting is refused before anything + * is sent. + * + * @param stream Stream to update, addressed by numeric ID or name. + * @param options Settings to change. * @throws IggyException if the client is unavailable, the caller lacks - * stream-management permission, either value is invalid, the stream - * does not exist, the name is already taken, or the request fails. + * stream-management permission, a value is invalid, a setting is + * not updatable or resettable, the stream does not exist, the name + * is already taken, or the request fails. */ - void UpdateStream(const Identifier &stream, std::string name, const StreamUpdateOptions &options = {}); + void UpdateStream(const Identifier &stream, const StreamUpdateOptions &options); /** * @brief Lists stream summaries visible to the authenticated user. @@ -3768,27 +4476,27 @@ TopicDetails CreateTopic(const Identifier &stream, std::string name, const TopicCreateOptions &options = {}); /** - * @brief Renames a topic and updates its mutable configuration. + * @brief Renames a topic, changes its mutable settings, or resets them. * * The supplied TopicUpdateOptions changes only the settings it contains; - * settings left unset retain their current values. Topic creation settings, - * such as the partition count and segment size, cannot be changed after the - * topic is created. Use SetRawEntries() for supported mutable settings - * without a dedicated setter. + * settings left unset retain their current values and their provenance. + * Topic creation settings, such as the compression algorithm, partition + * count, and segment size, cannot be changed after the topic is created. + * Use SetRawEntries() for supported mutable settings without a dedicated + * setter. An update that changes nothing, sets and resets the same + * setting, or resets a setting that has no default is refused before + * anything is sent. * * @param stream Parent stream, addressed by numeric ID or name. * @param topic Topic to update, addressed by numeric ID or name. - * @param name New unique topic name within @p stream. - * @param options Topic update options. - * @throws IggyException if the client is unavailable or unauthenticated; - * an identifier, name, setting, or option is invalid; the stream or - * topic does not exist; the caller lacks permission; or the server - * rejects or cannot commit the write. + * @param options Settings to change or reset. + * @throws IggyException if the client is unavailable or unauthenticated, + * an identifier, name, setting, or option is invalid, a setting is + * not updatable or resettable, the stream or topic does not exist, + * the caller lacks permission, or the server rejects or cannot + * commit the write. */ - void UpdateTopic(const Identifier &stream, - const Identifier &topic, - std::string name, - const TopicUpdateOptions &options = {}); + void UpdateTopic(const Identifier &stream, const Identifier &topic, const TopicUpdateOptions &options); /** * @brief Lists topic summaries in a stream. @@ -4178,6 +4886,20 @@ std::vector<ClientInfo> GetClients(); /** + * @brief Returns the option catalog of one resource scope. + * + * Each entry names a key with its kind, default, description, and whether + * an update may set or reset it, whether it is secret, read-only, or + * critical. Internal keys are not listed. + * + * @param scope Resource scope to describe. + * @return Catalog entries in catalog order. + * @throws IggyException if the client is unavailable or unauthenticated, + * or the request fails. + */ + std::vector<OptionSpec> DescribeOptions(OptionsScope scope); + + /** * @brief Retrieves server process, storage, and resource statistics. * * The returned fields are an observed snapshot and can change immediately.
diff --git a/foreign/cpp/src/client.cpp b/foreign/cpp/src/client.cpp index 6099443..7be218a 100644 --- a/foreign/cpp/src/client.cpp +++ b/foreign/cpp/src/client.cpp
@@ -43,12 +43,37 @@ return ffi_options; } +template <typename Key> +rust::Vec<rust::String> ToFfiResetKeys(const std::set<Key> &keys) { + rust::Vec<rust::String> ffi_keys; + ffi_keys.reserve(keys.size()); + for (const auto key : keys) { + const auto name = to_string(key); + ffi_keys.push_back(rust::String(name.data(), name.size())); + } + return ffi_keys; +} + +rust::Vec<rust::String> ToFfiRawResetKeys(const std::set<std::string> &keys) { + rust::Vec<rust::String> ffi_keys; + ffi_keys.reserve(keys.size()); + for (const auto &key : keys) { + ffi_keys.push_back(rust::String(key)); + } + return ffi_keys; +} + +rust::Vec<std::uint8_t> ToFfiBytes(const std::vector<std::uint8_t> &bytes) { + rust::Vec<std::uint8_t> ffi_bytes; + ffi_bytes.reserve(bytes.size()); + for (const auto byte : bytes) { + ffi_bytes.push_back(byte); + } + return ffi_bytes; +} + template <typename FfiOptions, typename Options> void SetMutableTopicOptions(FfiOptions &ffi_options, const Options &options) { - if (const auto &value = options.CompressionAlgorithm()) { - ffi_options.has_compression_algorithm = true; - ffi_options.compression_algorithm = std::string(value->Value()); - } if (const auto &value = options.MessageExpiry()) { ffi_options.has_message_expiry = true; ffi_options.message_expiry_kind = std::string(value->Kind()); @@ -138,12 +163,16 @@ UserInfoDetails IggyBlockingClient::CreateUser(std::string username, std::string password, UserStatus status, - const std::optional<Permissions> &permissions) { - return RethrowAsIggyException([this, &username, &password, status, &permissions](ffi::ErrorInfo &error_info) { + const std::optional<Permissions> &permissions, + const UserCreateOptions &options) { + return RethrowAsIggyException([this, &username, &password, status, &permissions, + &options](ffi::ErrorInfo &error_info) { auto ffi_permissions = permissions ? permissions->ToFfi() : ffi::Permissions{}; + ffi::UserCreateOptions ffi_options{}; + ffi_options.raw_options = ToFfiRawOptions(options.RawEntries()); return UserInfoDetails::FromFfi(Handle()->create_user(username, password, static_cast<ffi::UserStatus>(status), permissions.has_value(), std::move(ffi_permissions), - error_info)); + std::move(ffi_options), error_info)); }); } @@ -152,28 +181,45 @@ [this, &user](ffi::ErrorInfo &error_info) { Handle()->delete_user(user.ToFfi(), error_info); }); } -void IggyBlockingClient::UpdateUser(const Identifier &user, - std::optional<std::string> username, - std::optional<UserStatus> status, - const UserUpdateOptions &options) { - RethrowAsIggyException([this, &user, &username, status, &options](ffi::ErrorInfo &error_info) { - auto ffi_options = ToFfiRawOptions(options.RawEntries()); - Handle()->update_user(user.ToFfi(), username.has_value(), username.value_or(""), status.has_value(), - static_cast<ffi::UserStatus>(status.value_or(UserStatus::Active)), std::move(ffi_options), - error_info); +void IggyBlockingClient::UpdateUser(const Identifier &user, const UserUpdateOptions &options) { + RethrowAsIggyException([this, &user, &options](ffi::ErrorInfo &error_info) { + ffi::UserUpdateOptions ffi_options{}; + ffi_options.has_username = options.Username().has_value(); + ffi_options.username = options.Username().value_or(""); + ffi_options.has_status = options.Status().has_value(); + ffi_options.status = static_cast<ffi::UserStatus>(options.Status().value_or(UserStatus::Active)); + ffi_options.reset = ToFfiResetKeys(options.ResetKeys()); + ffi_options.raw_reset = ToFfiRawResetKeys(options.RawResetKeys()); + ffi_options.raw_options = ToFfiRawOptions(options.RawEntries()); + Handle()->update_user(user.ToFfi(), std::move(ffi_options), error_info); }); } -StreamDetails IggyBlockingClient::CreateStream(std::string name) { - return RethrowAsIggyException([this, &name](ffi::ErrorInfo &error_info) { - return StreamDetails::FromFfi(Handle()->create_stream(name, error_info)); +StreamDetails IggyBlockingClient::CreateStream(std::string name, const StreamCreateOptions &options) { + return RethrowAsIggyException([this, &name, &options](ffi::ErrorInfo &error_info) { + ffi::StreamCreateOptions ffi_options{}; + if (options.encryption_key_) { + ffi_options.has_encryption_key = true; + ffi_options.encryption_key = ToFfiBytes(*options.encryption_key_); + } + if (options.encryption_key_base64_) { + ffi_options.has_encryption_key_base64 = true; + ffi_options.encryption_key_base64 = *options.encryption_key_base64_; + } + ffi_options.raw_options = ToFfiRawOptions(options.RawEntries()); + return StreamDetails::FromFfi(Handle()->create_stream(name, std::move(ffi_options), error_info)); }); } -void IggyBlockingClient::UpdateStream(const Identifier &stream, std::string name, const StreamUpdateOptions &options) { - RethrowAsIggyException([this, &stream, &name, &options](ffi::ErrorInfo &error_info) { - auto ffi_options = ToFfiRawOptions(options.RawEntries()); - Handle()->update_stream(stream.ToFfi(), name, std::move(ffi_options), error_info); +void IggyBlockingClient::UpdateStream(const Identifier &stream, const StreamUpdateOptions &options) { + RethrowAsIggyException([this, &stream, &options](ffi::ErrorInfo &error_info) { + ffi::StreamUpdateOptions ffi_options{}; + ffi_options.has_name = options.Name().has_value(); + ffi_options.name = options.Name().value_or(""); + ffi_options.reset = ToFfiResetKeys(options.ResetKeys()); + ffi_options.raw_reset = ToFfiRawResetKeys(options.RawResetKeys()); + ffi_options.raw_options = ToFfiRawOptions(options.RawEntries()); + Handle()->update_stream(stream.ToFfi(), std::move(ffi_options), error_info); }); } @@ -205,6 +251,10 @@ return RethrowAsIggyException([this, &stream, &name, &options](ffi::ErrorInfo &error_info) { ffi::TopicCreateOptions ffi_options{}; SetMutableTopicOptions(ffi_options, options); + if (const auto &value = options.CompressionAlgorithm()) { + ffi_options.has_compression_algorithm = true; + ffi_options.compression_algorithm = std::string(value->Value()); + } if (auto value = options.PartitionsCount()) { ffi_options.has_partitions_count = true; ffi_options.partitions_count = *value; @@ -233,6 +283,10 @@ ffi_options.has_consumer_offset_durability = false; ffi_options.consumer_offset_durability = ""; } + if (auto value = options.PartitionResizePolicy()) { + ffi_options.has_partition_resize_policy = true; + ffi_options.partition_resize_policy = std::string(iggy::to_string(*value)); + } if (auto value = options.MessagesRequiredToSave()) { ffi_options.has_messages_required_to_save = true; ffi_options.messages_required_to_save = *value; @@ -260,13 +314,16 @@ void IggyBlockingClient::UpdateTopic(const Identifier &stream, const Identifier &topic, - std::string name, const TopicUpdateOptions &options) { - RethrowAsIggyException([this, &stream, &topic, &name, &options](ffi::ErrorInfo &error_info) { + RethrowAsIggyException([this, &stream, &topic, &options](ffi::ErrorInfo &error_info) { ffi::TopicUpdateOptions ffi_options{}; SetMutableTopicOptions(ffi_options, options); + ffi_options.has_name = options.Name().has_value(); + ffi_options.name = options.Name().value_or(""); + ffi_options.reset = ToFfiResetKeys(options.ResetKeys()); + ffi_options.raw_reset = ToFfiRawResetKeys(options.RawResetKeys()); - Handle()->update_topic(stream.ToFfi(), topic.ToFfi(), name, std::move(ffi_options), error_info); + Handle()->update_topic(stream.ToFfi(), topic.ToFfi(), std::move(ffi_options), error_info); }); } @@ -450,6 +507,19 @@ }); } +std::vector<OptionSpec> IggyBlockingClient::DescribeOptions(OptionsScope scope) { + return RethrowAsIggyException([this, scope] { + const auto scope_name = to_string(scope); + std::vector<OptionSpec> specs; + auto ffi_specs = Handle()->describe_options(rust::String(scope_name.data(), scope_name.size())); + specs.reserve(ffi_specs.size()); + for (auto &spec : ffi_specs) { + specs.push_back(OptionSpec::FromFfi(std::move(spec))); + } + return specs; + }); +} + Stats IggyBlockingClient::GetStats() { return RethrowAsIggyException([this] { return Stats::FromFfi(Handle()->get_stats()); }); } @@ -476,6 +546,28 @@ ffi::delete_client(client); } +std::string EncryptionKeyFingerprint(const std::vector<std::uint8_t> &key) { + ffi::ErrorInfo error_info{}; + try { + const auto fingerprint = + ffi::encryption_key_fingerprint(rust::Slice<const std::uint8_t>(key.data(), key.size()), error_info); + return std::string(fingerprint.data(), fingerprint.size()); + } catch (const std::exception &error) { + throw IggyException(error.what(), error_info); + } +} + +std::string EncryptionKeyFingerprint(const std::string &key_base64) { + ffi::ErrorInfo error_info{}; + try { + const auto fingerprint = + ffi::encryption_key_fingerprint_base64(rust::Str(key_base64.data(), key_base64.size()), error_info); + return std::string(fingerprint.data(), fingerprint.size()); + } catch (const std::exception &error) { + throw IggyException(error.what(), error_info); + } +} + IggyBlockingClient::Builder::Builder() = default; IggyBlockingClient::Builder &IggyBlockingClient::Builder::WithServerAddress(std::string server_address) {
diff --git a/foreign/cpp/src/client.rs b/foreign/cpp/src/client.rs index 21aceed..b9a62e9 100644 --- a/foreign/cpp/src/client.rs +++ b/foreign/cpp/src/client.rs
@@ -25,12 +25,17 @@ NonZeroIggyDuration as RustNonZeroIggyDuration, OptionsScope as RustOptionsScope, PartitionClient, PartitionContext, Partitioning, Permissions as RustPermissions, PollingStrategy, SegmentClient, SnapshotCompression as RustSnapshotCompression, StreamClient, - StreamUpdateOptions, SystemClient as RustSystemClient, + StreamCreateOptions as RustStreamCreateOptions, StreamEncryptionKey, + StreamUpdateOptions as RustStreamUpdateOptions, SystemClient as RustSystemClient, SystemSnapshotType as RustSystemSnapshotType, TopicClient, TopicCreateOptions as RustTopicCreateOptions, TopicUpdateOptions as RustTopicUpdateOptions, - UserClient, UserStatus as RustUserStatus, UserUpdateOptions, + UserClient, UserCreateOptions as RustUserCreateOptions, UserStatus as RustUserStatus, + UserUpdateOptions as RustUserUpdateOptions, }; -use iggy_common::{Credentials as RustCredentials, IggyError, PendingOperationsConfig}; +use iggy_common::{ + Credentials as RustCredentials, IggyError, PendingOperationsConfig, stream_key_fingerprint, +}; +use serde::Deserialize; use std::collections::HashSet; use std::convert::TryFrom; use std::num::NonZeroUsize; @@ -78,10 +83,24 @@ }) } +/// The `id` of the JSON error body an HTTP refusal carries. +#[derive(Deserialize)] +struct HttpErrorBody { + id: u32, +} + impl ffi::ErrorInfo { fn record(&mut self, error: &IggyError) { self.code = error.as_code(); self.kind = error.as_string().to_owned(); + // HTTP reports an option refusal only in its body, binary transports by the error itself. + if let IggyError::HttpResponseError(_, body) = error + && let Ok(body) = serde_json::from_str::<HttpErrorBody>(body) + && is_option_error(&IggyError::from_code(body.id)) + { + self.code = body.id; + self.kind = IggyError::from_code_as_string(body.id).to_owned(); + } let operation = error.operation_id(); self.has_operation_id = operation.is_some(); self.operation_id = operation.unwrap_or_default(); @@ -298,13 +317,24 @@ pub fn create_stream( &self, stream_name: String, + options: ffi::StreamCreateOptions, error_info: &mut ffi::ErrorInfo, ) -> Result<ffi::StreamDetails, String> { *error_info = ffi::ErrorInfo::default(); + let encryption_key = options + .encryption_key() + .inspect_err(|error| error_info.record(error)) + .map_err(|error| format!("Could not create stream '{stream_name}': {error}"))?; + let create_options = RustStreamCreateOptions { + encryption_key, + raw: ffi_options_to_raw(options.raw_options) + .map_err(|error| format!("Could not create stream '{stream_name}': {error}"))?, + }; + RUNTIME.block_on(async { let stream_details = self .inner - .create_stream(&stream_name) + .create_stream(&stream_name, &create_options) .await .inspect_err(|error| error_info.record(error)) .map_err(|error| format!("Could not create stream '{stream_name}': {error}"))?; @@ -315,28 +345,21 @@ pub fn update_stream( &self, stream_id: ffi::Identifier, - stream_name: String, - options: Vec<ffi::HeaderEntry>, + options: ffi::StreamUpdateOptions, error_info: &mut ffi::ErrorInfo, ) -> Result<(), String> { *error_info = ffi::ErrorInfo::default(); let rust_stream_id = RustIdentifier::try_from(stream_id) - .map_err(|error| format!("Could not update stream '{stream_name}': {error}"))?; - let update_options = StreamUpdateOptions { - raw: ffi_options_to_raw(options) - .map_err(|error| format!("Could not update stream '{stream_name}': {error}"))?, - }; + .map_err(|error| format!("Could not update stream: {error}"))?; + let update_options = RustStreamUpdateOptions::try_from(options) + .map_err(|error| format!("Could not update stream '{rust_stream_id}': {error}"))?; RUNTIME.block_on(async { self.inner - .update_stream(&rust_stream_id, &stream_name, &update_options) + .update_stream(&rust_stream_id, &update_options) .await .inspect_err(|error| error_info.record(error)) - .map_err(|error| { - format!( - "Could not update stream '{rust_stream_id}' to '{stream_name}': {error}" - ) - })?; + .map_err(|error| format!("Could not update stream '{rust_stream_id}': {error}"))?; Ok(()) }) } @@ -628,21 +651,25 @@ &self, stream_id: ffi::Identifier, topic_id: ffi::Identifier, - topic_name: String, options: ffi::TopicUpdateOptions, error_info: &mut ffi::ErrorInfo, ) -> Result<(), String> { *error_info = ffi::ErrorInfo::default(); - let rust_stream_id = RustIdentifier::try_from(stream_id) - .map_err(|error| format!("Could not update topic '{topic_name}': {error}"))?; - let rust_topic_id = RustIdentifier::try_from(topic_id) - .map_err(|error| format!("Could not update topic '{topic_name}': {error}"))?; - let update_options = RustTopicUpdateOptions::try_from(options) - .map_err(|error| format!("Could not update topic '{topic_name}': {error}"))?; + let rust_stream_id = RustIdentifier::try_from(stream_id).map_err(|error| { + format!("Could not update topic: invalid stream identifier: {error}") + })?; + let rust_topic_id = RustIdentifier::try_from(topic_id).map_err(|error| { + format!("Could not update topic: invalid topic identifier: {error}") + })?; + let update_options = RustTopicUpdateOptions::try_from(options).map_err(|error| { + format!( + "Could not update topic '{rust_topic_id}' on stream '{rust_stream_id}': {error}" + ) + })?; RUNTIME.block_on(async { self.inner - .update_topic(&rust_stream_id, &rust_topic_id, &topic_name, &update_options) + .update_topic(&rust_stream_id, &rust_topic_id, &update_options) .await .inspect_err(|error| error_info.record(error)) .map_err(|error| { @@ -1286,6 +1313,7 @@ }) } + #[allow(clippy::too_many_arguments)] pub fn create_user( &self, username: String, @@ -1293,6 +1321,7 @@ status: ffi::UserStatus, has_permissions: bool, permissions: ffi::Permissions, + options: ffi::UserCreateOptions, error_info: &mut ffi::ErrorInfo, ) -> Result<ffi::UserInfoDetails, String> { *error_info = ffi::ErrorInfo::default(); @@ -1302,11 +1331,19 @@ .then(|| RustPermissions::try_from(permissions)) .transpose() .map_err(|error| format!("Could not create user '{username}': {error}"))?; + let create_options = RustUserCreateOptions::try_from(options) + .map_err(|error| format!("Could not create user '{username}': {error}"))?; RUNTIME.block_on(async { let user = self .inner - .create_user(&username, &password, rust_status, rust_permissions) + .create_user( + &username, + &password, + rust_status, + rust_permissions, + &create_options, + ) .await .inspect_err(|error| error_info.record(error)) .map_err(|error| format!("Could not create user '{username}': {error}"))?; @@ -1333,36 +1370,21 @@ }) } - #[allow(clippy::too_many_arguments)] pub fn update_user( &self, user_id: ffi::Identifier, - has_username: bool, - username: String, - has_status: bool, - status: ffi::UserStatus, - options: Vec<ffi::HeaderEntry>, + options: ffi::UserUpdateOptions, error_info: &mut ffi::ErrorInfo, ) -> Result<(), String> { *error_info = ffi::ErrorInfo::default(); let rust_user_id = RustIdentifier::try_from(user_id) .map_err(|error| format!("Could not update user: invalid user identifier: {error}"))?; - let rust_status = has_status - .then(|| RustUserStatus::try_from(status)) - .transpose() + let rust_options = RustUserUpdateOptions::try_from(options) .map_err(|error| format!("Could not update user '{rust_user_id}': {error}"))?; - let raw = ffi_options_to_raw(options) - .map_err(|error| format!("Could not update user '{rust_user_id}': {error}"))?; - let rust_options = UserUpdateOptions { raw }; RUNTIME.block_on(async { self.inner - .update_user( - &rust_user_id, - has_username.then_some(username.as_str()), - rust_status, - &rust_options, - ) + .update_user(&rust_user_id, &rust_options) .await .inspect_err(|error| error_info.record(error)) .map_err(|error| format!("Could not update user '{rust_user_id}': {error}"))?; @@ -1443,6 +1465,44 @@ } } +/// Fingerprint of a 32-byte stream key. Any other length is refused with +/// `InvalidEncryptionKey`. +pub fn encryption_key_fingerprint( + key: &[u8], + error_info: &mut ffi::ErrorInfo, +) -> Result<String, String> { + *error_info = ffi::ErrorInfo::default(); + stream_key_fingerprint(key) + .inspect_err(|error| error_info.record(error)) + .map_err(|error| format!("Could not compute the stream key fingerprint: {error}")) +} + +pub fn encryption_key_fingerprint_base64( + key: &str, + error_info: &mut ffi::ErrorInfo, +) -> Result<String, String> { + *error_info = ffi::ErrorInfo::default(); + StreamEncryptionKey::from_str(key) + .map(|key| key.fingerprint()) + .inspect_err(|error| error_info.record(error)) + .map_err(|error| format!("Could not compute the stream key fingerprint: {error}")) +} + +fn is_option_error(error: &IggyError) -> bool { + matches!( + error, + IggyError::UnsupportedOptionKey(_) + | IggyError::InvalidOptionValue(_) + | IggyError::OptionsBlockTooLarge(_) + | IggyError::OptionNotUpdatable(_) + | IggyError::OptionNotResettable(_) + | IggyError::ReadOnlyOption(_) + | IggyError::StreamEncryptionUnavailable + | IggyError::UnsupportedCriticalOption(_) + | IggyError::CannotModifyRootUser + ) +} + pub unsafe fn delete_connection(client: *mut Client) { if !client.is_null() { unsafe { @@ -1529,6 +1589,37 @@ } #[test] + fn given_http_option_refusal_when_recording_should_report_the_server_error_code() { + let mut error_info = ffi::ErrorInfo::default(); + error_info.record(&IggyError::HttpResponseError( + 400, + r#"{"id":4105,"code":"read_only_option","reason":"Option is read-only: critical_keys","field":null}"# + .to_owned(), + )); + assert_eq!( + error_info.code, + IggyError::ReadOnlyOption(String::new()).as_code() + ); + assert_eq!(error_info.kind, "read_only_option"); + + let mut root = ffi::ErrorInfo::default(); + root.record(&IggyError::HttpResponseError( + 400, + r#"{"id":4108,"code":"cannot_modify_root_user","reason":"Cannot modify root user","field":null}"# + .to_owned(), + )); + assert_eq!(root.code, IggyError::CannotModifyRootUser.as_code()); + assert_eq!(root.kind, "cannot_modify_root_user"); + + let mut other = ffi::ErrorInfo::default(); + other.record(&IggyError::HttpResponseError( + 400, + r#"{"id":1013,"reason":"invalid"}"#.to_owned(), + )); + assert_eq!(other.kind, "http_response_error"); + } + + #[test] fn should_reject_an_unknown_polling_strategy() { assert_eq!( polling_strategy("latest", 0, Some(CONTEXT))
diff --git a/foreign/cpp/src/lib.rs b/foreign/cpp/src/lib.rs index e6e8fee..055a8cc 100644 --- a/foreign/cpp/src/lib.rs +++ b/foreign/cpp/src/lib.rs
@@ -22,7 +22,10 @@ mod producer; mod type_conversion; -use client::{Client, delete_connection as delete_client, from_connection_string, new_connection}; +use client::{ + Client, delete_connection as delete_client, encryption_key_fingerprint, + encryption_key_fingerprint_base64, from_connection_string, new_connection, +}; use consumer::Consumer; use messages::make_message; use producer::Producer; @@ -117,9 +120,27 @@ size_bytes: u64, messages_count: u64, topics_count: u32, - /// Creation options. Streams have no catalog keys yet, so this is - /// empty until one lands. + /// Options the creating client set explicitly. Key material is never + /// returned. options: Vec<HeaderEntry>, + /// Options the server filled in, including the read-only encryption + /// projections. + derived_options: Vec<HeaderEntry>, + has_encryption_key_source: bool, + encryption_key_source: StreamEncryptionSource, + has_encryption_key_fingerprint: bool, + encryption_key_fingerprint: String, + /// One per server key that wraps the stream's key, empty for a + /// plaintext stream. + encryption_kek_fingerprints: Vec<String>, + } + + /// Who chose a stream's at-rest data key. + #[repr(u8)] + enum StreamEncryptionSource { + None = 0, + Stream = 1, + Server = 2, } #[repr(u8)] @@ -151,13 +172,13 @@ value: HeaderField, } - /// One key a resource's create command accepts, as served by + /// One key of a resource's option catalog, as served by /// `describe_options`. /// - /// This is the discovery surface for the keys `create_topic` takes. A key - /// outside the server catalog is refused at create, and the binary - /// transports carry back only an error code, so nothing in the rejection - /// names the keys that would have worked. + /// This is the discovery surface for the keys a create or an update takes. + /// A key outside the server catalog is refused, and the binary transports + /// carry back only an error code, so nothing in the rejection names the + /// keys that would have worked. struct OptionSpec { key: String, /// Wire kind code the value is encoded under, the same encoding @@ -166,6 +187,15 @@ /// The default in `kind`'s encoding. Empty when the key has no default. default_value: Vec<u8>, description: String, + updatable: bool, + resettable: bool, + /// Write-only: accepted at create, never returned. + secret: bool, + /// Computed by the server and returned on reads, never accepted. + read_only: bool, + /// A node that does not understand the key refuses to serve the + /// resource. + critical: bool, } struct IggyMessageToSend { @@ -234,6 +264,13 @@ topics: Vec<Topic>, /// See [`Stream::options`]. options: Vec<HeaderEntry>, + /// See [`Stream::derived_options`]. + derived_options: Vec<HeaderEntry>, + has_encryption_key_source: bool, + encryption_key_source: StreamEncryptionSource, + has_encryption_key_fingerprint: bool, + encryption_key_fingerprint: String, + encryption_kek_fingerprints: Vec<String>, } struct ConsumerGroupMember { @@ -414,6 +451,7 @@ status: UserStatus, username: String, options: Vec<HeaderEntry>, + derived_options: Vec<HeaderEntry>, } struct UserInfoDetails { @@ -424,6 +462,7 @@ has_permissions: bool, permissions: Permissions, options: Vec<HeaderEntry>, + derived_options: Vec<HeaderEntry>, } struct LoginInfo { @@ -485,17 +524,56 @@ size_of_messages_required_to_save: u64, has_preallocate_segments: bool, preallocate_segments: bool, + has_partition_resize_policy: bool, + partition_resize_policy: String, raw_options: Vec<HeaderEntry>, } + /// Reset keys travel as catalog names and are validated against the + /// scope's catalog before anything is sent. Raw reset names are left to + /// the server. struct TopicUpdateOptions { - has_compression_algorithm: bool, - compression_algorithm: String, + has_name: bool, + name: String, has_message_expiry: bool, message_expiry_kind: String, message_expiry_value: u64, has_max_topic_size: bool, max_topic_size: String, + reset: Vec<String>, + raw_reset: Vec<String>, + raw_options: Vec<HeaderEntry>, + } + + /// The key is either 32 raw bytes or their canonical padded base64, never + /// both. An invalid key is refused before anything is sent. + struct StreamCreateOptions { + has_encryption_key: bool, + encryption_key: Vec<u8>, + has_encryption_key_base64: bool, + encryption_key_base64: String, + raw_options: Vec<HeaderEntry>, + } + + struct StreamUpdateOptions { + has_name: bool, + name: String, + reset: Vec<String>, + raw_reset: Vec<String>, + raw_options: Vec<HeaderEntry>, + } + + struct UserCreateOptions { + raw_options: Vec<HeaderEntry>, + } + + struct UserUpdateOptions { + has_username: bool, + username: String, + has_status: bool, + status: UserStatus, + reset: Vec<String>, + raw_reset: Vec<String>, raw_options: Vec<HeaderEntry>, } @@ -525,13 +603,13 @@ fn create_stream( self: &Client, stream_name: String, + options: StreamCreateOptions, error_info: &mut ErrorInfo, ) -> Result<StreamDetails>; fn update_stream( self: &Client, stream_id: Identifier, - stream_name: String, - options: Vec<HeaderEntry>, + options: StreamUpdateOptions, error_info: &mut ErrorInfo, ) -> Result<()>; fn get_streams(self: &Client) -> Result<Vec<Stream>>; @@ -558,7 +636,6 @@ self: &Client, stream_id: Identifier, topic_id: Identifier, - topic_name: String, options: TopicUpdateOptions, error_info: &mut ErrorInfo, ) -> Result<()>; @@ -723,6 +800,13 @@ /// "user". A scope with no keys yet answers with an empty vector, which /// is an empty catalog rather than a failure. fn describe_options(self: &Client, scope: String) -> Result<Vec<OptionSpec>>; + /// Fingerprint of a 32-byte stream key, computed locally. + fn encryption_key_fingerprint(key: &[u8], error_info: &mut ErrorInfo) -> Result<String>; + /// Fingerprint of a stream key given as canonical padded base64. + fn encryption_key_fingerprint_base64( + key: &str, + error_info: &mut ErrorInfo, + ) -> Result<String>; fn ping(self: &Client) -> Result<()>; fn heartbeat_interval(self: &Client) -> u64; fn snapshot( @@ -750,6 +834,7 @@ ) -> Result<()>; fn get_user(self: &Client, user_id: Identifier) -> Result<UserInfoDetails>; fn get_users(self: &Client) -> Result<Vec<UserInfo>>; + #[allow(clippy::too_many_arguments)] fn create_user( self: &Client, username: String, @@ -757,6 +842,7 @@ status: UserStatus, has_permissions: bool, permissions: Permissions, + options: UserCreateOptions, error_info: &mut ErrorInfo, ) -> Result<UserInfoDetails>; fn delete_user( @@ -764,15 +850,10 @@ user_id: Identifier, error_info: &mut ErrorInfo, ) -> Result<()>; - #[allow(clippy::too_many_arguments)] fn update_user( self: &Client, user_id: Identifier, - has_username: bool, - username: String, - has_status: bool, - status: UserStatus, - options: Vec<HeaderEntry>, + options: UserUpdateOptions, error_info: &mut ErrorInfo, ) -> Result<()>; fn update_permissions(
diff --git a/foreign/cpp/src/type_conversion.rs b/foreign/cpp/src/type_conversion.rs index eb77780..e712eb3 100644 --- a/foreign/cpp/src/type_conversion.rs +++ b/foreign/cpp/src/type_conversion.rs
@@ -15,21 +15,28 @@ // specific language governing permissions and limitations // under the License. -use std::{collections::BTreeMap, str::FromStr}; +use std::{ + collections::{BTreeMap, BTreeSet}, + str::FromStr, +}; use bytes::Bytes; use iggy::prelude::{ CompressionAlgorithm as RustCompressionAlgorithm, ConsumerGroupDetails as RustConsumerGroupDetails, Durability as RustDurability, IdKind, Identifier as RustIdentifier, IggyByteSize as RustIggyByteSize, IggyExpiry as RustIggyExpiry, - IggyMessage as RustIggyMessage, MaxTopicSize as RustMaxTopicSize, OptionSpec as RustOptionSpec, - Partition as RustPartition, PartitionContext as RustPartitionContext, + IggyMessage as RustIggyMessage, MaxTopicSize as RustMaxTopicSize, OptionKey, + OptionSpec as RustOptionSpec, Partition as RustPartition, + PartitionContext as RustPartitionContext, PartitionResizePolicy as RustPartitionResizePolicy, PolledMessages as RustPolledMessages, SendMessagesConfirmationResponse as RustSendMessagesConfirmationResponse, SendMessagesResponse as RustSendMessagesResponse, Stream as RustStream, - StreamDetails as RustStreamDetails, Topic as RustTopic, + StreamDetails as RustStreamDetails, StreamEncryptionKey, + StreamEncryptionSource as RustStreamEncryptionSource, + StreamUpdateOptions as RustStreamUpdateOptions, Topic as RustTopic, TopicCreateOptions as RustTopicCreateOptions, TopicDetails as RustTopicDetails, - TopicUpdateOptions as RustTopicUpdateOptions, Validatable, + TopicUpdateOptions as RustTopicUpdateOptions, UserCreateOptions as RustUserCreateOptions, + UserUpdateOptions as RustUserUpdateOptions, Validatable, }; use iggy_binary_protocol::WireUserHeaders; use iggy_common::{ @@ -40,9 +47,10 @@ ConsumerGroupMember as RustConsumerGroupMember, ConsumerOffsetInfo as RustConsumerOffsetInfo, GlobalPermissions as RustGlobalPermissions, HeaderEntry as RustHeaderEntry, HeaderField as RustHeaderField, HeaderKind as RustHeaderKind, IdentityInfo as RustIdentityInfo, - Permissions as RustPermissions, Stats as RustStats, StreamPermissions as RustStreamPermissions, - TopicPermissions as RustTopicPermissions, TransportEndpoints as RustTransportEndpoints, - UserInfo as RustUserInfo, UserInfoDetails as RustUserInfoDetails, UserStatus as RustUserStatus, + IggyError, Permissions as RustPermissions, Stats as RustStats, + StreamPermissions as RustStreamPermissions, TopicPermissions as RustTopicPermissions, + TransportEndpoints as RustTransportEndpoints, UserInfo as RustUserInfo, + UserInfoDetails as RustUserInfoDetails, UserStatus as RustUserStatus, }; use crate::ffi; @@ -160,6 +168,7 @@ status: ffi::UserStatus::from(user.status), username: user.username, options: resource_options_to_ffi(&user.options, true), + derived_options: resource_options_to_ffi(&user.options, false), } } } @@ -175,6 +184,7 @@ has_permissions, permissions: ffi::Permissions::from(user.permissions.unwrap_or_default()), options: resource_options_to_ffi(&user.options, true), + derived_options: resource_options_to_ffi(&user.options, false), } } } @@ -529,23 +539,60 @@ /// handed in here is rendered rather than passed through: sending `Uint64` /// 134217728 for `segment_size` and sending `"134217728"` land the same stored /// value. A key this build cannot read at all is dropped rather than guessed. +/// String entries skip the header codec so the Rust SDK reports length bounds +/// with the same error categories it uses for its own raw entries. pub(crate) fn ffi_options_to_raw( options: Vec<ffi::HeaderEntry>, ) -> Result<BTreeMap<String, String>, String> { + let string_kind = RustHeaderKind::String.as_code(); let mut raw = BTreeMap::new(); for entry in options { - let RustHeaderEntry { key, value } = RustHeaderEntry::try_from(entry)?; - let key = key - .as_str() - .map_err(|error| format!("Option key is not a string: {error}"))? - .to_owned(); - if raw.insert(key.clone(), value.to_string_value()).is_some() { + let (key, value) = if entry.key.kind == string_kind && entry.value.kind == string_kind { + let key = String::from_utf8(entry.key.value) + .map_err(|error| format!("Option key is not a string: {error}"))?; + let value = String::from_utf8(entry.value.value) + .map_err(|error| format!("Option value of '{key}' is not a string: {error}"))?; + (key, value) + } else { + let RustHeaderEntry { key, value } = RustHeaderEntry::try_from(entry)?; + let key = key + .as_str() + .map_err(|error| format!("Option key is not a string: {error}"))? + .to_owned(); + (key, value.to_string_value()) + }; + if raw.insert(key.clone(), value).is_some() { return Err(format!("Duplicate option key: {key}")); } } Ok(raw) } +/// Resolve reset key names into the typed set the Rust SDK validates against +/// the scope's catalog. The C++ wrapper sends only catalog names. +pub(crate) fn ffi_reset_keys(names: Vec<String>) -> Result<BTreeSet<OptionKey>, String> { + names + .into_iter() + .map(|name| { + OptionKey::from_str(&name).map_err(|_| format!("Unknown option key to reset: {name}")) + }) + .collect() +} + +impl ffi::StreamCreateOptions { + /// The typed key, refused with `InvalidEncryptionKey` before anything is + /// sent. The error never carries key material. + pub(crate) fn encryption_key(&self) -> Result<Option<StreamEncryptionKey>, IggyError> { + if self.has_encryption_key { + return StreamEncryptionKey::try_from(self.encryption_key.as_slice()).map(Some); + } + if self.has_encryption_key_base64 { + return StreamEncryptionKey::from_str(&self.encryption_key_base64).map(Some); + } + Ok(None) + } +} + fn parse_message_expiry(kind: &str, value: u64) -> Result<RustIggyExpiry, String> { match kind { "server_default" | "default" => Ok(RustIggyExpiry::ServerDefault), @@ -565,6 +612,15 @@ .map_err(|error| format!("invalid compression algorithm '{value}': {error}")) } +fn parse_user_status( + has_status: bool, + status: ffi::UserStatus, +) -> Result<Option<RustUserStatus>, String> { + has_status + .then(|| RustUserStatus::try_from(status)) + .transpose() +} + fn parse_max_topic_size(value: &str) -> Result<RustMaxTopicSize, String> { RustMaxTopicSize::from_str(value) .map_err(|error| format!("invalid max topic size '{value}': {error}")) @@ -577,6 +633,11 @@ kind: spec.kind.as_code(), default_value: spec.default_value, description: spec.description, + updatable: spec.updatable, + resettable: spec.resettable, + secret: spec.secret, + read_only: spec.read_only, + critical: spec.critical, } } } @@ -619,6 +680,20 @@ } else { RustDurability::default() }; + let partition_resize_policy = if options.has_partition_resize_policy { + Some( + RustPartitionResizePolicy::from_str(&options.partition_resize_policy).map_err( + |error| { + format!( + "invalid partition resize policy '{}': {error}", + options.partition_resize_policy + ) + }, + )?, + ) + } else { + None + }; let mut raw = ffi_options_to_raw(options.raw_options)?; if options.has_durability { raw.remove("durability"); @@ -642,7 +717,7 @@ }, durability, consumer_offset_durability, - partition_resize_policy: None, + partition_resize_policy, messages_required_to_save: if options.has_messages_required_to_save { Some(options.messages_required_to_save) } else { @@ -669,11 +744,6 @@ type Error = String; fn try_from(options: ffi::TopicUpdateOptions) -> Result<Self, Self::Error> { - let compression_algorithm = if options.has_compression_algorithm { - Some(parse_compression_algorithm(&options.compression_algorithm)?) - } else { - None - }; let message_expiry = if options.has_message_expiry { Some(parse_message_expiry( &options.message_expiry_kind, @@ -687,26 +757,75 @@ } else { None }; - let raw = ffi_options_to_raw(options.raw_options)?; Ok(RustTopicUpdateOptions { - compression_algorithm, + name: options.has_name.then_some(options.name), message_expiry, max_topic_size, - raw, + reset: ffi_reset_keys(options.reset)?, + raw_reset: options.raw_reset.into_iter().collect(), + raw: ffi_options_to_raw(options.raw_options)?, }) } } +impl TryFrom<ffi::StreamUpdateOptions> for RustStreamUpdateOptions { + type Error = String; + + fn try_from(options: ffi::StreamUpdateOptions) -> Result<Self, Self::Error> { + Ok(RustStreamUpdateOptions { + name: options.has_name.then_some(options.name), + reset: ffi_reset_keys(options.reset)?, + raw_reset: options.raw_reset.into_iter().collect(), + raw: ffi_options_to_raw(options.raw_options)?, + }) + } +} + +impl TryFrom<ffi::UserCreateOptions> for RustUserCreateOptions { + type Error = String; + + fn try_from(options: ffi::UserCreateOptions) -> Result<Self, Self::Error> { + Ok(RustUserCreateOptions { + raw: ffi_options_to_raw(options.raw_options)?, + }) + } +} + +impl TryFrom<ffi::UserUpdateOptions> for RustUserUpdateOptions { + type Error = String; + + fn try_from(options: ffi::UserUpdateOptions) -> Result<Self, Self::Error> { + Ok(RustUserUpdateOptions { + username: options.has_username.then_some(options.username), + status: parse_user_status(options.has_status, options.status)?, + reset: ffi_reset_keys(options.reset)?, + raw_reset: options.raw_reset.into_iter().collect(), + raw: ffi_options_to_raw(options.raw_options)?, + }) + } +} + +impl From<RustStreamEncryptionSource> for ffi::StreamEncryptionSource { + fn from(source: RustStreamEncryptionSource) -> Self { + match source { + RustStreamEncryptionSource::None => ffi::StreamEncryptionSource::None, + RustStreamEncryptionSource::Stream => ffi::StreamEncryptionSource::Stream, + RustStreamEncryptionSource::Server => ffi::StreamEncryptionSource::Server, + } + } +} + impl From<RustTopic> for ffi::Topic { fn from(topic: RustTopic) -> Self { + let resolved = topic.resolved_options(); ffi::Topic { id: topic.id, created_at: topic.created_at.as_micros(), name: topic.name, size_bytes: topic.size.as_bytes_u64(), - message_expiry: u64::from(topic.message_expiry), - compression_algorithm: topic.compression_algorithm.to_string(), - max_topic_size: u64::from(topic.max_topic_size), + message_expiry: u64::from(resolved.message_expiry), + compression_algorithm: resolved.compression_algorithm.to_string(), + max_topic_size: u64::from(resolved.max_topic_size), messages_count: topic.messages_count, partitions_count: topic.partitions_count, options: resource_options_to_ffi(&topic.options, true), @@ -717,14 +836,15 @@ impl From<RustTopicDetails> for ffi::TopicDetails { fn from(topic: RustTopicDetails) -> Self { + let resolved = topic.resolved_options(); ffi::TopicDetails { id: topic.id, created_at: topic.created_at.as_micros(), name: topic.name, size_bytes: topic.size.as_bytes_u64(), - message_expiry: u64::from(topic.message_expiry), - compression_algorithm: topic.compression_algorithm.to_string(), - max_topic_size: u64::from(topic.max_topic_size), + message_expiry: u64::from(resolved.message_expiry), + compression_algorithm: resolved.compression_algorithm.to_string(), + max_topic_size: u64::from(resolved.max_topic_size), messages_count: topic.messages_count, partitions_count: topic.partitions_count, partitions: topic @@ -740,6 +860,9 @@ impl From<RustStream> for ffi::Stream { fn from(stream: RustStream) -> Self { + let source = stream.encryption_key_source(); + let fingerprint = stream.encryption_key_fingerprint(); + let kek_fingerprints = stream.encryption_kek_fingerprints(); ffi::Stream { id: stream.id, created_at: stream.created_at.as_micros(), @@ -748,12 +871,21 @@ messages_count: stream.messages_count, topics_count: stream.topics_count, options: resource_options_to_ffi(&stream.options, true), + derived_options: resource_options_to_ffi(&stream.options, false), + has_encryption_key_source: source.is_some(), + encryption_key_source: source.map_or(ffi::StreamEncryptionSource::None, Into::into), + has_encryption_key_fingerprint: fingerprint.is_some(), + encryption_key_fingerprint: fingerprint.unwrap_or_default(), + encryption_kek_fingerprints: kek_fingerprints, } } } impl From<RustStreamDetails> for ffi::StreamDetails { fn from(stream: RustStreamDetails) -> Self { + let source = stream.encryption_key_source(); + let fingerprint = stream.encryption_key_fingerprint(); + let kek_fingerprints = stream.encryption_kek_fingerprints(); ffi::StreamDetails { id: stream.id, created_at: stream.created_at.as_micros(), @@ -763,6 +895,12 @@ topics_count: stream.topics_count, topics: stream.topics.into_iter().map(ffi::Topic::from).collect(), options: resource_options_to_ffi(&stream.options, true), + derived_options: resource_options_to_ffi(&stream.options, false), + has_encryption_key_source: source.is_some(), + encryption_key_source: source.map_or(ffi::StreamEncryptionSource::None, Into::into), + has_encryption_key_fingerprint: fingerprint.is_some(), + encryption_key_fingerprint: fingerprint.unwrap_or_default(), + encryption_kek_fingerprints: kek_fingerprints, } } } @@ -1052,7 +1190,7 @@ #[cfg(test)] mod tests { use super::*; - use iggy::prelude::IggyTimestamp; + use iggy::prelude::{HeaderKey, HeaderValue, IggyTimestamp, OptionValue, ResourceOptions}; // Distinct values so a swapped field fails the comparison. const CONTEXT: RustPartitionContext = RustPartitionContext { @@ -1070,6 +1208,195 @@ } #[test] + fn given_optional_bridge_fields_when_converting_should_preserve_presence() { + let unnamed = RustStreamUpdateOptions::try_from(ffi::StreamUpdateOptions { + has_name: false, + name: "ignored".to_owned(), + reset: vec!["name".to_owned()], + raw_reset: Vec::new(), + raw_options: Vec::new(), + }) + .expect("stream update"); + assert_eq!( + unnamed, + RustStreamUpdateOptions { + reset: BTreeSet::from([OptionKey::Name]), + ..RustStreamUpdateOptions::default() + } + ); + let empty_name = RustStreamUpdateOptions::try_from(ffi::StreamUpdateOptions { + has_name: true, + name: String::new(), + reset: Vec::new(), + raw_reset: Vec::new(), + raw_options: Vec::new(), + }) + .expect("stream update"); + assert_eq!(empty_name.name, Some(String::new())); + + let topic = RustTopicUpdateOptions::try_from(ffi::TopicUpdateOptions { + has_name: false, + name: "ignored".to_owned(), + has_message_expiry: false, + message_expiry_kind: "duration".to_owned(), + message_expiry_value: 5, + has_max_topic_size: false, + max_topic_size: "unlimited".to_owned(), + reset: vec!["max_topic_size".to_owned()], + raw_reset: Vec::new(), + raw_options: Vec::new(), + }) + .expect("topic update"); + assert_eq!( + topic, + RustTopicUpdateOptions { + reset: BTreeSet::from([OptionKey::MaxTopicSize]), + ..RustTopicUpdateOptions::default() + } + ); + + let user = RustUserUpdateOptions::try_from(ffi::UserUpdateOptions { + has_username: false, + username: "ignored".to_owned(), + has_status: true, + status: ffi::UserStatus::Inactive, + reset: Vec::new(), + raw_reset: Vec::new(), + raw_options: Vec::new(), + }) + .expect("user update"); + assert_eq!( + user, + RustUserUpdateOptions { + status: Some(RustUserStatus::Inactive), + ..RustUserUpdateOptions::default() + } + ); + assert!( + RustUserUpdateOptions::try_from(ffi::UserUpdateOptions { + has_username: false, + username: String::new(), + has_status: false, + status: ffi::UserStatus { repr: 0 }, + reset: vec!["not_a_catalog_key".to_owned()], + raw_reset: Vec::new(), + raw_options: Vec::new(), + }) + .is_err() + ); + + let key: Vec<u8> = (0..32).collect(); + let keyless = ffi::StreamCreateOptions { + has_encryption_key: false, + encryption_key: key.clone(), + has_encryption_key_base64: false, + encryption_key_base64: String::new(), + raw_options: Vec::new(), + }; + assert_eq!(keyless.encryption_key(), Ok(None)); + let keyed = ffi::StreamCreateOptions { + has_encryption_key: true, + ..keyless + }; + assert_eq!( + keyed + .encryption_key() + .map(|key| key.map(|key| key.fingerprint())), + Ok(Some("8441e38538cbfa57".to_owned())) + ); + let short = ffi::StreamCreateOptions { + encryption_key: key[..31].to_vec(), + ..keyed + }; + assert_eq!(short.encryption_key(), Err(IggyError::InvalidEncryptionKey)); + } + + #[test] + fn given_raw_reset_names_when_converting_should_leave_them_to_the_server() { + let topic = RustTopicUpdateOptions::try_from(ffi::TopicUpdateOptions { + has_name: false, + name: String::new(), + has_message_expiry: false, + message_expiry_kind: String::new(), + message_expiry_value: 0, + has_max_topic_size: false, + max_topic_size: String::new(), + reset: vec!["max_topic_size".to_owned()], + raw_reset: vec!["future_key".to_owned()], + raw_options: Vec::new(), + }) + .expect("topic update"); + assert_eq!( + topic, + RustTopicUpdateOptions { + reset: BTreeSet::from([OptionKey::MaxTopicSize]), + raw_reset: BTreeSet::from(["future_key".to_owned()]), + ..RustTopicUpdateOptions::default() + } + ); + } + + #[test] + fn given_raw_entry_outside_codec_bounds_when_converting_should_keep_the_sdk_error_category() { + let entry = |key: &str, value: &str| ffi::HeaderEntry { + key: ffi::HeaderField { + kind: RustHeaderKind::String.as_code(), + value: key.as_bytes().to_vec(), + }, + value: ffi::HeaderField { + kind: RustHeaderKind::String.as_code(), + value: value.as_bytes().to_vec(), + }, + }; + let patch = |raw_options: Vec<ffi::HeaderEntry>| { + RustStreamUpdateOptions::try_from(ffi::StreamUpdateOptions { + has_name: false, + name: String::new(), + reset: Vec::new(), + raw_reset: Vec::new(), + raw_options, + }) + .map(|options| options.to_patch().map(|_| ())) + }; + let long_key = "k".repeat(256); + + assert!(matches!( + patch(vec![entry("future_key", "")]), + Ok(Err(IggyError::InvalidOptionValue(_))) + )); + assert!(matches!( + patch(vec![entry(&long_key, "value")]), + Ok(Err(IggyError::UnsupportedOptionKey(_))) + )); + } + + #[test] + fn given_stream_projection_when_converting_should_split_provenance() { + let mut options = ResourceOptions::new(); + options.insert( + HeaderKey::from(OptionKey::EncryptionKeySource), + OptionValue::derived(HeaderValue::from_str("none").expect("source")), + ); + let stream = ffi::StreamDetails::from(RustStreamDetails { + id: 1, + created_at: IggyTimestamp::from(2), + name: "plain".to_owned(), + size: RustIggyByteSize::from(0), + messages_count: 0, + topics_count: 0, + topics: Vec::new(), + options, + }); + + assert!(stream.options.is_empty()); + assert_eq!(stream.derived_options.len(), 1); + assert!(stream.has_encryption_key_source); + assert!(stream.encryption_key_source == ffi::StreamEncryptionSource::None); + assert!(!stream.has_encryption_key_fingerprint); + assert!(stream.encryption_kek_fingerprints.is_empty()); + } + + #[test] fn should_reject_zero_message_expiry_duration() { assert_eq!( parse_message_expiry("duration", 0).err().as_deref(),
diff --git a/foreign/cpp/src/type_conversions.cpp b/foreign/cpp/src/type_conversions.cpp index 742cc95..20e8a21 100644 --- a/foreign/cpp/src/type_conversions.cpp +++ b/foreign/cpp/src/type_conversions.cpp
@@ -21,6 +21,33 @@ namespace iggy { +namespace { + +std::optional<StreamEncryptionSource> EncryptionKeySourceFromFfi(bool has_source, ffi::StreamEncryptionSource source) { + if (!has_source) { + return std::nullopt; + } + return static_cast<StreamEncryptionSource>(source); +} + +std::optional<std::string> TextFromFfi(bool has_text, const rust::String &text) { + if (!has_text) { + return std::nullopt; + } + return std::string(text.data(), text.size()); +} + +std::vector<std::string> TextsFromFfi(const rust::Vec<rust::String> &texts) { + std::vector<std::string> result; + result.reserve(texts.size()); + for (const auto &text : texts) { + result.emplace_back(text.data(), text.size()); + } + return result; +} + +} // namespace + LoginInfo LoginInfo::FromFfi(ffi::LoginInfo login_info) { std::optional<std::string> access_token; std::optional<std::uint64_t> access_token_expiry; @@ -137,7 +164,7 @@ UserInfo UserInfo::FromFfi(ffi::UserInfo user) { return UserInfo(user.id, user.created_at, static_cast<UserStatus>(user.status), std::string(user.username.c_str(), user.username.size()), - ResourceOptions::FromFfi(std::move(user.options), rust::Vec<ffi::HeaderEntry>{})); + ResourceOptions::FromFfi(std::move(user.options), std::move(user.derived_options))); } UserInfoDetails UserInfoDetails::FromFfi(ffi::UserInfoDetails user) { @@ -147,7 +174,7 @@ } return UserInfoDetails(user.id, user.created_at, static_cast<UserStatus>(user.status), std::string(user.username.c_str(), user.username.size()), std::move(permissions), - ResourceOptions::FromFfi(std::move(user.options), rust::Vec<ffi::HeaderEntry>{})); + ResourceOptions::FromFfi(std::move(user.options), std::move(user.derived_options))); } ffi::Identifier Identifier::ToFfi() const { @@ -281,13 +308,26 @@ return StreamDetails(stream.id, stream.created_at, std::string(stream.name.c_str(), stream.name.size()), stream.size_bytes, stream.messages_count, stream.topics_count, std::move(topics), - ResourceOptions::FromFfi(std::move(stream.options), rust::Vec<ffi::HeaderEntry>{})); + ResourceOptions::FromFfi(std::move(stream.options), std::move(stream.derived_options)), + EncryptionKeySourceFromFfi(stream.has_encryption_key_source, stream.encryption_key_source), + TextFromFfi(stream.has_encryption_key_fingerprint, stream.encryption_key_fingerprint), + TextsFromFfi(stream.encryption_kek_fingerprints)); } Stream Stream::FromFfi(ffi::Stream stream) { return Stream(stream.id, stream.created_at, std::string(stream.name.c_str(), stream.name.size()), stream.size_bytes, stream.messages_count, stream.topics_count, - ResourceOptions::FromFfi(std::move(stream.options), rust::Vec<ffi::HeaderEntry>{})); + ResourceOptions::FromFfi(std::move(stream.options), std::move(stream.derived_options)), + EncryptionKeySourceFromFfi(stream.has_encryption_key_source, stream.encryption_key_source), + TextFromFfi(stream.has_encryption_key_fingerprint, stream.encryption_key_fingerprint), + TextsFromFfi(stream.encryption_kek_fingerprints)); +} + +OptionSpec OptionSpec::FromFfi(ffi::OptionSpec spec) { + return OptionSpec(std::string(spec.key.c_str(), spec.key.size()), static_cast<HeaderKind>(spec.kind), + std::vector<std::uint8_t>(spec.default_value.begin(), spec.default_value.end()), + std::string(spec.description.c_str(), spec.description.size()), spec.updatable, spec.resettable, + spec.secret, spec.read_only, spec.critical); } ConsumerGroupMember ConsumerGroupMember::FromFfi(ffi::ConsumerGroupMember member) {
diff --git a/foreign/cpp/tests/e2e/client.cpp b/foreign/cpp/tests/e2e/client.cpp index 1d12bfd..e108794 100644 --- a/foreign/cpp/tests/e2e/client.cpp +++ b/foreign/cpp/tests/e2e/client.cpp
@@ -497,12 +497,12 @@ iggy::ffi::StreamDetails denied_stream{}; ASSERT_NO_THROW({ iggy::ffi::ErrorInfo error_info{}; - allowed_stream = root_client->create_stream(allowed_stream_name, error_info); + allowed_stream = root_client->create_stream(allowed_stream_name, {}, error_info); }); TrackStream(allowed_stream_name); ASSERT_NO_THROW({ iggy::ffi::ErrorInfo error_info{}; - denied_stream = root_client->create_stream(denied_stream_name, error_info); + denied_stream = root_client->create_stream(denied_stream_name, {}, error_info); }); TrackStream(denied_stream_name); @@ -560,7 +560,7 @@ permissions.streams.push_back(std::move(second_stream)); ASSERT_THROW(client->create_user(username, "secret123", iggy::ffi::UserStatus::Active, true, std::move(permissions), - error_info), + {}, error_info), std::exception); ASSERT_THROW(client->get_user(make_string_identifier(username)), std::exception); } @@ -584,7 +584,7 @@ permissions.streams.push_back(std::move(stream)); ASSERT_THROW(client->create_user(username, "secret123", iggy::ffi::UserStatus::Active, true, std::move(permissions), - error_info), + {}, error_info), std::exception); ASSERT_THROW(client->get_user(make_string_identifier(username)), std::exception); } @@ -605,7 +605,7 @@ ASSERT_NO_THROW(user_client->login_user(username, "secret123", error_info)); ASSERT_THROW(user_client->create_user(target, "secret123", iggy::ffi::UserStatus::Active, false, - iggy::ffi::Permissions{}, error_info), + iggy::ffi::Permissions{}, {}, error_info), std::exception); ASSERT_THROW(root_client->get_user(make_string_identifier(target)), std::exception); } @@ -629,7 +629,7 @@ }); ASSERT_NO_THROW(manager_client->connect()); ASSERT_NO_THROW(manager_client->login_user(manager_username, "secret123", error_info)); - ASSERT_THROW(manager_client->create_stream(denied_stream, error_info), std::exception); + ASSERT_THROW(manager_client->create_stream(denied_stream, {}, error_info), std::exception); iggy::ffi::Permissions child_permissions{}; child_permissions.global.manage_streams = true; @@ -647,7 +647,7 @@ ASSERT_NO_THROW(child_client->connect()); ASSERT_NO_THROW(child_client->login_user(child_username, "child-secret", error_info)); - ASSERT_NO_THROW(child_client->create_stream(child_stream, error_info)); + ASSERT_NO_THROW(child_client->create_stream(child_stream, {}, error_info)); TrackStream(child_stream); } @@ -692,23 +692,27 @@ const auto created = CreateUser(root_client, username, "secret123", iggy::UserStatus::Active); auto client = GetLoggedOutHighLevelClient(); - ASSERT_THROW(client.UpdateUser(iggy::Identifier::String(username), replacement, iggy::UserStatus::Inactive, - iggy::UserUpdateOptions{}), - std::exception); + ASSERT_THROW( + client.UpdateUser(iggy::Identifier::String(username), + iggy::UserUpdateOptions().SetUsername(replacement).SetStatus(iggy::UserStatus::Inactive)), + std::exception); ASSERT_NO_THROW(client.Connect()); - ASSERT_THROW(client.UpdateUser(iggy::Identifier::String(username), replacement, iggy::UserStatus::Inactive, - iggy::UserUpdateOptions{}), - std::exception); + ASSERT_THROW( + client.UpdateUser(iggy::Identifier::String(username), + iggy::UserUpdateOptions().SetUsername(replacement).SetStatus(iggy::UserStatus::Inactive)), + std::exception); ASSERT_NO_THROW(client.Login("iggy", "iggy")); ASSERT_NO_THROW(client.Logout()); - ASSERT_THROW(client.UpdateUser(iggy::Identifier::String(username), replacement, iggy::UserStatus::Inactive, - iggy::UserUpdateOptions{}), - std::exception); + ASSERT_THROW( + client.UpdateUser(iggy::Identifier::String(username), + iggy::UserUpdateOptions().SetUsername(replacement).SetStatus(iggy::UserStatus::Inactive)), + std::exception); ASSERT_NO_THROW(client.Login("iggy", "iggy")); ASSERT_NO_THROW(client.Disconnect()); - ASSERT_THROW(client.UpdateUser(iggy::Identifier::String(username), replacement, iggy::UserStatus::Inactive, - iggy::UserUpdateOptions{}), - std::exception); + ASSERT_THROW( + client.UpdateUser(iggy::Identifier::String(username), + iggy::UserUpdateOptions().SetUsername(replacement).SetStatus(iggy::UserStatus::Inactive)), + std::exception); const auto fetched = root_client.GetUser(iggy::Identifier::String(username)); EXPECT_EQ(fetched.Id(), created.Id()); @@ -724,11 +728,13 @@ const std::string proposed_username = GetRandomName(50); const auto unknown_id = std::numeric_limits<std::uint32_t>::max(); - ASSERT_THROW(client.UpdateUser(iggy::Identifier::String(unknown_username), proposed_username, - iggy::UserStatus::Inactive, iggy::UserUpdateOptions{}), + ASSERT_THROW(client.UpdateUser( + iggy::Identifier::String(unknown_username), + iggy::UserUpdateOptions().SetUsername(proposed_username).SetStatus(iggy::UserStatus::Inactive)), std::exception); - ASSERT_THROW(client.UpdateUser(iggy::Identifier::Numeric(unknown_id), GetRandomName(50), iggy::UserStatus::Inactive, - iggy::UserUpdateOptions{}), + ASSERT_THROW(client.UpdateUser( + iggy::Identifier::Numeric(unknown_id), + iggy::UserUpdateOptions().SetUsername(GetRandomName(50)).SetStatus(iggy::UserStatus::Inactive)), std::exception); ASSERT_THROW(client.GetUser(iggy::Identifier::String(proposed_username)), std::exception); } @@ -740,8 +746,9 @@ const std::string replacement = GetRandomName(50); ASSERT_NO_THROW({ const auto created = CreateUser(client, username, "secret123", iggy::UserStatus::Active); - ASSERT_NO_THROW(client.UpdateUser(iggy::Identifier::String(username), replacement, iggy::UserStatus::Inactive, - iggy::UserUpdateOptions{})); + ASSERT_NO_THROW(client.UpdateUser( + iggy::Identifier::String(username), + iggy::UserUpdateOptions().SetUsername(replacement).SetStatus(iggy::UserStatus::Inactive))); RenameTrackedUser(username, replacement); ASSERT_THROW(client.GetUser(iggy::Identifier::String(username)), std::exception); @@ -759,8 +766,9 @@ const std::string replacement = GetRandomName(50); ASSERT_NO_THROW({ const auto created = CreateUser(client, username, "secret123", iggy::UserStatus::Inactive); - ASSERT_NO_THROW(client.UpdateUser(iggy::Identifier::Numeric(created.Id()), replacement, - iggy::UserStatus::Active, iggy::UserUpdateOptions{})); + ASSERT_NO_THROW( + client.UpdateUser(iggy::Identifier::Numeric(created.Id()), + iggy::UserUpdateOptions().SetUsername(replacement).SetStatus(iggy::UserStatus::Active))); RenameTrackedUser(username, replacement); const auto by_id = client.GetUser(iggy::Identifier::Numeric(created.Id())); @@ -782,14 +790,14 @@ ASSERT_NO_THROW({ const auto created = CreateUser(client, username, "secret123", iggy::UserStatus::Active); - ASSERT_NO_THROW(client.UpdateUser(iggy::Identifier::Numeric(created.Id()), std::nullopt, - iggy::UserStatus::Inactive, iggy::UserUpdateOptions{})); + ASSERT_NO_THROW(client.UpdateUser(iggy::Identifier::Numeric(created.Id()), + iggy::UserUpdateOptions().SetStatus(iggy::UserStatus::Inactive))); const auto status_updated = client.GetUser(iggy::Identifier::Numeric(created.Id())); EXPECT_EQ(status_updated.Username(), username); EXPECT_EQ(status_updated.Status(), iggy::UserStatus::Inactive); - ASSERT_NO_THROW(client.UpdateUser(iggy::Identifier::Numeric(created.Id()), replacement, std::nullopt, - iggy::UserUpdateOptions{})); + ASSERT_NO_THROW(client.UpdateUser(iggy::Identifier::Numeric(created.Id()), + iggy::UserUpdateOptions().SetUsername(replacement))); RenameTrackedUser(username, replacement); const auto username_updated = client.GetUser(iggy::Identifier::Numeric(created.Id())); @@ -798,15 +806,19 @@ }); } -TEST_F(E2E_Client, UpdateUserWithoutChangesIsNoOp) { - RecordProperty("description", "Accepts an empty user update without changing the user."); +TEST_F(E2E_Client, UpdateUserWithoutChangesIsRefusedWithoutChangingUser) { + RecordProperty("description", "Refuses an empty user update with invalid_command and leaves the user unchanged."); auto client = GetLoggedInHighLevelClient(); const std::string username = GetRandomName(50); ASSERT_NO_THROW({ const auto created = CreateUser(client, username, "secret123", iggy::UserStatus::Active); - ASSERT_NO_THROW(client.UpdateUser(iggy::Identifier::Numeric(created.Id()), std::nullopt, std::nullopt, - iggy::UserUpdateOptions{})); + try { + client.UpdateUser(iggy::Identifier::Numeric(created.Id()), iggy::UserUpdateOptions()); + ADD_FAILURE() << "An update that changes nothing must be refused"; + } catch (const iggy::IggyException &error) { + EXPECT_EQ(error.Kind(), "invalid_command"); + } const auto fetched = client.GetUser(iggy::Identifier::Numeric(created.Id())); EXPECT_EQ(fetched.Id(), created.Id()); @@ -828,11 +840,13 @@ ASSERT_NO_THROW({ const auto first = CreateUser(client, first_username, "secret123", iggy::UserStatus::Active); const auto second = CreateUser(client, second_username, "secret123", iggy::UserStatus::Active); - ASSERT_NO_THROW(client.UpdateUser(iggy::Identifier::String(first_username), first_replacement, - iggy::UserStatus::Active, iggy::UserUpdateOptions{})); + ASSERT_NO_THROW(client.UpdateUser( + iggy::Identifier::String(first_username), + iggy::UserUpdateOptions().SetUsername(first_replacement).SetStatus(iggy::UserStatus::Active))); RenameTrackedUser(first_username, first_replacement); - ASSERT_NO_THROW(client.UpdateUser(iggy::Identifier::String(second_username), second_replacement, - iggy::UserStatus::Active, iggy::UserUpdateOptions{})); + ASSERT_NO_THROW(client.UpdateUser( + iggy::Identifier::String(second_username), + iggy::UserUpdateOptions().SetUsername(second_replacement).SetStatus(iggy::UserStatus::Active))); RenameTrackedUser(second_username, second_replacement); const auto fetched_first = client.GetUser(iggy::Identifier::String(first_replacement)); @@ -854,8 +868,9 @@ const auto created = CreateUser(client, source, "secret123", iggy::UserStatus::Active); for (const auto &replacement : invalid_usernames) { - ASSERT_THROW(client.UpdateUser(iggy::Identifier::String(source), replacement, iggy::UserStatus::Active, - iggy::UserUpdateOptions{}), + ASSERT_THROW(client.UpdateUser( + iggy::Identifier::String(source), + iggy::UserUpdateOptions().SetUsername(replacement).SetStatus(iggy::UserStatus::Active)), std::exception); } const auto fetched = client.GetUser(iggy::Identifier::String(source)); @@ -877,8 +892,9 @@ for (std::size_t index = 0; index < 3; ++index) { ASSERT_NO_THROW({ const auto created = CreateUser(client, sources[index], "secret123", iggy::UserStatus::Active); - ASSERT_NO_THROW(client.UpdateUser(iggy::Identifier::String(sources[index]), replacements[index], - iggy::UserStatus::Active, iggy::UserUpdateOptions{})); + ASSERT_NO_THROW(client.UpdateUser( + iggy::Identifier::String(sources[index]), + iggy::UserUpdateOptions().SetUsername(replacements[index]).SetStatus(iggy::UserStatus::Active))); RenameTrackedUser(sources[index], replacements[index]); const auto fetched = client.GetUser(iggy::Identifier::String(replacements[index])); EXPECT_EQ(fetched.Id(), created.Id()); @@ -900,9 +916,9 @@ const auto created = CreateUser(client, username, "secret123", iggy::UserStatus::Active); for (const auto status : statuses) { const std::string replacement = GetRandomName(50); - ASSERT_THROW( - client.UpdateUser(iggy::Identifier::String(username), replacement, status, iggy::UserUpdateOptions{}), - std::exception); + ASSERT_THROW(client.UpdateUser(iggy::Identifier::String(username), + iggy::UserUpdateOptions().SetUsername(replacement).SetStatus(status)), + std::exception); ASSERT_THROW(client.GetUser(iggy::Identifier::String(replacement)), std::exception); } const auto fetched = client.GetUser(iggy::Identifier::String(username)); @@ -920,9 +936,11 @@ ASSERT_NO_THROW({ const auto target = CreateUser(client, target_username, "secret123", iggy::UserStatus::Active); const auto conflict = CreateUser(client, conflict_username, "secret123", iggy::UserStatus::Inactive); - ASSERT_THROW(client.UpdateUser(iggy::Identifier::String(target_username), conflict_username, - iggy::UserStatus::Inactive, iggy::UserUpdateOptions{}), - std::exception); + ASSERT_THROW( + client.UpdateUser( + iggy::Identifier::String(target_username), + iggy::UserUpdateOptions().SetUsername(conflict_username).SetStatus(iggy::UserStatus::Inactive)), + std::exception); const auto fetched_target = client.GetUser(iggy::Identifier::String(target_username)); const auto fetched_conflict = client.GetUser(iggy::Identifier::String(conflict_username)); EXPECT_EQ(fetched_target.Id(), target.Id()); @@ -938,8 +956,9 @@ const std::string username = GetRandomName(50); ASSERT_NO_THROW({ const auto created = CreateUser(client, username, "secret123", iggy::UserStatus::Active); - ASSERT_NO_THROW(client.UpdateUser(iggy::Identifier::String(username), username, iggy::UserStatus::Inactive, - iggy::UserUpdateOptions{})); + ASSERT_NO_THROW( + client.UpdateUser(iggy::Identifier::String(username), + iggy::UserUpdateOptions().SetUsername(username).SetStatus(iggy::UserStatus::Inactive))); const auto fetched = client.GetUser(iggy::Identifier::String(username)); EXPECT_EQ(fetched.Id(), created.Id()); EXPECT_EQ(fetched.Status(), iggy::UserStatus::Inactive); @@ -960,8 +979,9 @@ permissions.SetGlobal(global); ASSERT_NO_THROW({ const auto created = CreateUser(root_client, username, "known-secret", iggy::UserStatus::Active, permissions); - ASSERT_NO_THROW(root_client.UpdateUser(iggy::Identifier::String(username), replacement, - iggy::UserStatus::Active, iggy::UserUpdateOptions{})); + ASSERT_NO_THROW(root_client.UpdateUser( + iggy::Identifier::String(username), + iggy::UserUpdateOptions().SetUsername(replacement).SetStatus(iggy::UserStatus::Active))); RenameTrackedUser(username, replacement); const auto fetched = root_client.GetUser(iggy::Identifier::String(replacement)); EXPECT_EQ(fetched.Id(), created.Id()); @@ -983,12 +1003,14 @@ auto user_client = GetLoggedOutHighLevelClient(); const std::string username = GetRandomName(50); ASSERT_NO_THROW({ CreateUser(root_client, username, "known-secret", iggy::UserStatus::Active); }); - ASSERT_NO_THROW(root_client.UpdateUser(iggy::Identifier::String(username), username, iggy::UserStatus::Inactive, - iggy::UserUpdateOptions{})); + ASSERT_NO_THROW( + root_client.UpdateUser(iggy::Identifier::String(username), + iggy::UserUpdateOptions().SetUsername(username).SetStatus(iggy::UserStatus::Inactive))); ASSERT_NO_THROW(user_client.Connect()); ASSERT_THROW(user_client.Login(username, "known-secret"), std::exception); - ASSERT_NO_THROW(root_client.UpdateUser(iggy::Identifier::String(username), username, iggy::UserStatus::Active, - iggy::UserUpdateOptions{})); + ASSERT_NO_THROW( + root_client.UpdateUser(iggy::Identifier::String(username), + iggy::UserUpdateOptions().SetUsername(username).SetStatus(iggy::UserStatus::Active))); ASSERT_NO_THROW(user_client.Login(username, "known-secret")); } @@ -999,8 +1021,9 @@ const std::string new_username = GetRandomName(50); ASSERT_NO_THROW({ const auto first = CreateUser(client, old_username, "secret123", iggy::UserStatus::Active); - ASSERT_NO_THROW(client.UpdateUser(iggy::Identifier::String(old_username), new_username, - iggy::UserStatus::Active, iggy::UserUpdateOptions{})); + ASSERT_NO_THROW( + client.UpdateUser(iggy::Identifier::String(old_username), + iggy::UserUpdateOptions().SetUsername(new_username).SetStatus(iggy::UserStatus::Active))); RenameTrackedUser(old_username, new_username); const auto second = CreateUser(client, old_username, "secret123", iggy::UserStatus::Active); EXPECT_EQ(client.GetUser(iggy::Identifier::String(new_username)).Username(), new_username); @@ -1015,9 +1038,10 @@ const std::string username = GetRandomName(50); ASSERT_NO_THROW({ const auto target = CreateUser(client, username, "secret123", iggy::UserStatus::Active); - ASSERT_THROW(client.UpdateUser(iggy::Identifier::String(username), "iggy", iggy::UserStatus::Inactive, - iggy::UserUpdateOptions{}), - std::exception); + ASSERT_THROW( + client.UpdateUser(iggy::Identifier::String(username), + iggy::UserUpdateOptions().SetUsername("iggy").SetStatus(iggy::UserStatus::Inactive)), + std::exception); const auto root = client.GetUser(iggy::Identifier::String("iggy")); const auto fetched = client.GetUser(iggy::Identifier::String(username)); EXPECT_EQ(root.Id(), 0u); @@ -1037,11 +1061,11 @@ ASSERT_NO_THROW({ const auto created = CreateUser(client, username, "secret123", iggy::UserStatus::Active); iggy::UserUpdateOptions options; - options.SetRawEntries({{"not_a_real_option", "true"}}); + options.SetRawEntries({{"not_a_real_option", "true"}}) + .SetUsername(replacement) + .SetStatus(iggy::UserStatus::Inactive); - ASSERT_THROW( - client.UpdateUser(iggy::Identifier::String(username), replacement, iggy::UserStatus::Inactive, options), - iggy::IggyException); + ASSERT_THROW(client.UpdateUser(iggy::Identifier::String(username), options), iggy::IggyException); const auto fetched = client.GetUser(iggy::Identifier::String(username)); EXPECT_EQ(fetched.Id(), created.Id()); @@ -1068,11 +1092,13 @@ ASSERT_NO_THROW({ target = CreateUser(root_client, target_username, "secret123", iggy::ffi::UserStatus::Active); }); ASSERT_NO_THROW(caller_client->connect()); ASSERT_NO_THROW(caller_client->login_user(caller_username, "secret123", error_info)); - ASSERT_THROW(caller_client->update_user(make_string_identifier(target_username), true, GetRandomName(50), true, - iggy::ffi::UserStatus::Inactive, {}, error_info), + ASSERT_THROW(caller_client->update_user( + make_string_identifier(target_username), + make_user_update_options(GetRandomName(50), iggy::ffi::UserStatus::Inactive), error_info), std::exception); - ASSERT_THROW(caller_client->update_user(make_string_identifier(caller_username), true, GetRandomName(50), true, - iggy::ffi::UserStatus::Inactive, {}, error_info), + ASSERT_THROW(caller_client->update_user( + make_string_identifier(caller_username), + make_user_update_options(GetRandomName(50), iggy::ffi::UserStatus::Inactive), error_info), std::exception); iggy::ffi::UserInfoDetails fetched{}; ASSERT_NO_THROW({ fetched = root_client->get_user(make_string_identifier(target_username)); }); @@ -1098,8 +1124,9 @@ ASSERT_NO_THROW({ target = CreateUser(root_client, target_username, "secret123", iggy::ffi::UserStatus::Active); }); ASSERT_NO_THROW(manager_client->connect()); ASSERT_NO_THROW(manager_client->login_user(manager_username, "secret123", error_info)); - ASSERT_NO_THROW(manager_client->update_user(make_string_identifier(target_username), true, replacement, true, - iggy::ffi::UserStatus::Inactive, {}, error_info)); + ASSERT_NO_THROW(manager_client->update_user(make_string_identifier(target_username), + make_user_update_options(replacement, iggy::ffi::UserStatus::Inactive), + error_info)); RenameTrackedUser(target_username, replacement); iggy::ffi::UserInfoDetails fetched{}; ASSERT_NO_THROW({ fetched = root_client->get_user(make_string_identifier(replacement)); });
diff --git a/foreign/cpp/tests/e2e/message.cpp b/foreign/cpp/tests/e2e/message.cpp index 59a6c5b..f1ac823 100644 --- a/foreign/cpp/tests/e2e/message.cpp +++ b/foreign/cpp/tests/e2e/message.cpp
@@ -34,7 +34,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -78,7 +78,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -108,7 +108,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -185,7 +185,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -207,7 +207,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -234,7 +234,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -270,7 +270,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -290,7 +290,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -313,7 +313,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -343,7 +343,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -378,7 +378,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -413,7 +413,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -481,7 +481,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -571,7 +571,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -598,7 +598,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -646,7 +646,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -675,7 +675,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -741,7 +741,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -876,7 +876,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -894,7 +894,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -912,7 +912,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -941,7 +941,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -970,7 +970,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -1006,7 +1006,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -1043,7 +1043,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -1086,7 +1086,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -1135,7 +1135,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -1171,7 +1171,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -1221,7 +1221,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -1256,7 +1256,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -1285,7 +1285,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -1303,7 +1303,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -1322,7 +1322,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -1359,7 +1359,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -1424,7 +1424,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -1465,7 +1465,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -1529,7 +1529,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); @@ -1552,7 +1552,7 @@ const std::string stream_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - client->create_stream(stream_name, error_info); + client->create_stream(stream_name, {}, error_info); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName();
diff --git a/foreign/cpp/tests/e2e/resource_options.cpp b/foreign/cpp/tests/e2e/resource_options.cpp new file mode 100644 index 0000000..62f00ea --- /dev/null +++ b/foreign/cpp/tests/e2e/resource_options.cpp
@@ -0,0 +1,736 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include <cstddef> +#include <cstdint> +#include <fstream> +#include <limits> +#include <map> +#include <optional> +#include <string> +#include <utility> +#include <vector> + +#include <gtest/gtest.h> +#include <nlohmann/json.hpp> + +#include "lib.rs.h" +#include "tests/e2e/test_helpers.hpp" + +struct ResourceOptionsTransport { + std::string name; + // Empty selects the TCP builder. + std::string scheme; + std::string address; + + [[nodiscard]] std::string ConnectionString(const std::string &username, const std::string &password) const { + return scheme + "://" + username + ":" + password + "@" + address; + } +}; + +namespace { + +constexpr char kFixturePath[] = "tests/fixtures/resource_options.json"; +constexpr std::size_t kFingerprintHex = 16; +// UpdateResource carries the reset count in one byte. +constexpr std::size_t kMaxResetKeys = 255; + +constexpr iggy::StreamOptionKey kStreamOptionKeys[] = { + iggy::StreamOptionKey::Name, + iggy::StreamOptionKey::EncryptionKey, + iggy::StreamOptionKey::EncryptionKeySource, + iggy::StreamOptionKey::EncryptionKeyFingerprint, + iggy::StreamOptionKey::EncryptionKekFingerprint, +}; + +constexpr iggy::TopicOptionKey kTopicOptionKeys[] = { + iggy::TopicOptionKey::Name, + iggy::TopicOptionKey::CompressionAlgorithm, + iggy::TopicOptionKey::MessageExpiry, + iggy::TopicOptionKey::MaxTopicSize, + iggy::TopicOptionKey::SegmentSize, + iggy::TopicOptionKey::Durability, + iggy::TopicOptionKey::ConsumerOffsetDurability, + iggy::TopicOptionKey::PartitionResizePolicy, + iggy::TopicOptionKey::MessagesRequiredToSave, + iggy::TopicOptionKey::SizeOfMessagesRequiredToSave, + iggy::TopicOptionKey::PreallocateSegments, +}; + +constexpr iggy::UserOptionKey kUserOptionKeys[] = { + iggy::UserOptionKey::Username, + iggy::UserOptionKey::Status, +}; + +const nlohmann::json &Fixture() { + static const nlohmann::json fixture = [] { + std::ifstream file(kFixturePath); + if (!file) { + throw std::runtime_error(std::string("Cannot open ") + kFixturePath); + } + return nlohmann::json::parse(file); + }(); + return fixture; +} + +const nlohmann::json &FixtureCase(const std::string §ion, const std::string &name) { + for (const auto &vector : Fixture().at(section)) { + if (vector.at("case") == name) { + return vector; + } + } + throw std::runtime_error("No fixture case " + section + "." + name); +} + +std::vector<std::uint8_t> FromHex(const std::string &hex) { + std::vector<std::uint8_t> bytes; + bytes.reserve(hex.size() / 2); + for (std::size_t index = 0; index + 1 < hex.size(); index += 2) { + bytes.push_back(static_cast<std::uint8_t>(std::stoul(hex.substr(index, 2), nullptr, 16))); + } + return bytes; +} + +std::vector<std::uint8_t> FixtureKey() { + return FromHex(Fixture().at("stream_key").at("key_hex").get<std::string>()); +} + +std::uint64_t ReadLittleEndian(const std::vector<std::uint8_t> &bytes) { + std::uint64_t value = 0; + for (std::size_t index = bytes.size(); index > 0; --index) { + value = (value << 8) | bytes[index - 1]; + } + return value; +} + +std::string HeaderFieldText(const iggy::HeaderField &field) { + const auto &bytes = field.Value(); + return std::string(bytes.begin(), bytes.end()); +} + +std::optional<std::string> DefaultText(const iggy::OptionSpec &spec) { + const auto &bytes = spec.DefaultValue(); + if (bytes.empty()) { + return std::nullopt; + } + switch (spec.Kind()) { + case iggy::HeaderKind::String: + return std::string(bytes.begin(), bytes.end()); + case iggy::HeaderKind::Bool: + return std::string(bytes.front() == 0 ? "false" : "true"); + case iggy::HeaderKind::Uint32: + case iggy::HeaderKind::Uint64: + return std::to_string(ReadLittleEndian(bytes)); + default: + return std::nullopt; + } +} + +std::uint32_t FlagsOf(const iggy::OptionSpec &spec) { + const auto &flags = Fixture().at("constants").at("option_flags"); + std::uint32_t value{}; + value |= spec.Updatable() ? flags.at("updatable").get<std::uint32_t>() : 0; + value |= spec.Resettable() ? flags.at("resettable").get<std::uint32_t>() : 0; + value |= spec.Secret() ? flags.at("secret").get<std::uint32_t>() : 0; + value |= spec.ReadOnly() ? flags.at("read_only").get<std::uint32_t>() : 0; + value |= spec.Critical() ? flags.at("critical").get<std::uint32_t>() : 0; + return value; +} + +// Names the public enums do not know go through RawReset(), as a caller with +// an older build would send them. +template <typename Options, typename Key, std::size_t Count> +void AddReset(Options &options, const Key (&keys)[Count], const std::string &name) { + for (const auto key : keys) { + if (iggy::to_string(key) == name) { + options.Reset(key); + return; + } + } + options.RawReset(name); +} + +template <typename Operation> +::testing::AssertionResult ThrowsIggyError(Operation &&operation, const nlohmann::json &error) { + const auto code = error.at("code").get<std::uint32_t>(); + const auto kind = error.at("name").get<std::string>(); + try { + std::forward<Operation>(operation)(); + } catch (const iggy::IggyException &exception) { + if (exception.Code() == code && exception.Kind() == kind) { + return ::testing::AssertionSuccess(); + } + return ::testing::AssertionFailure() + << "expected " << kind << " (" << code << "), got '" << exception.what() << "' with code " + << exception.Code().value_or(0) << " and kind '" << exception.Kind() << "'"; + } + return ::testing::AssertionFailure() << "expected " << kind << " (" << code << "), nothing was thrown"; +} + +template <typename Details> +void ExpectNoKeyMaterial(const Details &details) { + for (const auto &key : Fixture().at("constants").at("never_returned_keys")) { + const auto name = key.get<std::string>(); + EXPECT_EQ(details.Options().Explicit().count(name), 0u) << name; + EXPECT_EQ(details.Options().Derived().count(name), 0u) << name; + } +} + +template <typename Details> +void ExpectProjection(const Details &details, + iggy::StreamEncryptionSource source, + const std::optional<std::string> &fingerprint) { + ASSERT_TRUE(details.EncryptionKeySource().has_value()); + EXPECT_EQ(*details.EncryptionKeySource(), source); + ASSERT_TRUE(details.EncryptionKeyFingerprint().has_value()); + EXPECT_EQ(details.EncryptionKeyFingerprint()->size(), kFingerprintHex); + if (fingerprint) { + EXPECT_EQ(*details.EncryptionKeyFingerprint(), *fingerprint); + } + ASSERT_FALSE(details.EncryptionKekFingerprints().empty()); + for (const auto &kek_fingerprint : details.EncryptionKekFingerprints()) { + EXPECT_EQ(kek_fingerprint.size(), kFingerprintHex); + } + EXPECT_EQ(HeaderFieldText(details.Options().Derived().at( + std::string(iggy::to_string(iggy::StreamOptionKey::EncryptionKeySource)))), + iggy::to_string(source)); + ExpectNoKeyMaterial(details); +} + +template <typename Left, typename Right> +void ExpectSameProjection(const Left &left, const Right &right) { + EXPECT_EQ(left.EncryptionKeySource(), right.EncryptionKeySource()); + EXPECT_EQ(left.EncryptionKeyFingerprint(), right.EncryptionKeyFingerprint()); + EXPECT_EQ(left.EncryptionKekFingerprints(), right.EncryptionKekFingerprints()); +} + +std::optional<iggy::Stream> FindStream(iggy::IggyBlockingClient &client, std::uint32_t stream_id) { + for (auto &stream : client.GetStreams()) { + if (stream.Id() == stream_id) { + return stream; + } + } + return std::nullopt; +} + +std::string PrintTransport(const ::testing::TestParamInfo<ResourceOptionsTransport> &info) { + return info.param.name; +} + +} // namespace + +class E2E_ResourceOptions : public E2ETestFixture {}; + +class E2E_ResourceOptionsTransport : public E2ETestFixture, + public ::testing::WithParamInterface<ResourceOptionsTransport> { + protected: + iggy::IggyBlockingClient GetTransportClient() { return GetTransportClient("iggy", "iggy"); } + + iggy::IggyBlockingClient GetTransportClient(const std::string &username, const std::string &password) { + const auto &transport = GetParam(); + auto client = + transport.scheme.empty() + ? iggy::IggyBlockingClient::Builder().WithAutoLogin(username, password).Build() + : iggy::IggyBlockingClient::FromConnectionString(transport.ConnectionString(username, password)); + client.Connect(); + if (transport.name == "http") { + client.Login(username, password); + } + return client; + } + + void SendPayload(std::uint32_t stream_id, std::uint32_t topic_id, const std::string &payload) { + iggy::ffi::Client *client = GetLoggedInClient(); + ASSERT_NE(client, nullptr); + rust::Vec<iggy::ffi::IggyMessageToSend> messages; + messages.push_back(iggy::ffi::make_message(to_payload(payload), rust::Vec<iggy::ffi::HeaderEntry>())); + iggy::ffi::ErrorInfo error_info{}; + ASSERT_NO_THROW(client->send_messages(make_numeric_identifier(stream_id), make_numeric_identifier(topic_id), + "partition_id", partition_id_bytes(0), std::move(messages), error_info)); + } + + std::vector<std::string> PollPayloads(std::uint32_t stream_id, std::uint32_t topic_id) { + iggy::ffi::Client *client = GetLoggedInClient(); + std::vector<std::string> payloads; + if (client == nullptr) { + return payloads; + } + iggy::ffi::ErrorInfo error_info{}; + const auto polled = + client->poll_messages(make_numeric_identifier(stream_id), make_numeric_identifier(topic_id), 0, "consumer", + make_numeric_identifier(1), "offset", 0, 10, false, error_info); + for (const auto &message : polled.messages) { + payloads.emplace_back(message.payload.begin(), message.payload.end()); + } + return payloads; + } +}; + +INSTANTIATE_TEST_SUITE_P(Transports, + E2E_ResourceOptionsTransport, + ::testing::Values(ResourceOptionsTransport{"tcp", "", ""}, + ResourceOptionsTransport{"quic", "iggy+quic", "127.0.0.1:8080"}, + ResourceOptionsTransport{"http", "iggy+http", "127.0.0.1:3000"}, + ResourceOptionsTransport{"websocket", "iggy+ws", "127.0.0.1:8092"}), + PrintTransport); + +TEST_P(E2E_ResourceOptionsTransport, GivenStreamKeyWhenCreatingShouldReadMatchingProjection) { + RecordProperty("description", + "Creates a stream with its own key and reads source, key fingerprint and KEK fingerprints through " + "create, get and list, with no key material in any options block."); + const auto &stream_key = Fixture().at("stream_key"); + const auto fingerprint = stream_key.at("fingerprint").get<std::string>(); + auto client = GetTransportClient(); + const auto name = GetRandomName(40); + + const auto created = + client.CreateStream(name, iggy::StreamCreateOptions().SetEncryptionKey(FromHex(stream_key.at("key_hex")))); + TrackStream(name); + ExpectProjection(created, iggy::StreamEncryptionSource::Stream, fingerprint); + + const auto fetched = client.GetStream(iggy::Identifier::Numeric(created.Id())); + ExpectProjection(fetched, iggy::StreamEncryptionSource::Stream, fingerprint); + ExpectSameProjection(created, fetched); + + const auto listed = FindStream(client, created.Id()); + ASSERT_TRUE(listed.has_value()); + ExpectProjection(*listed, iggy::StreamEncryptionSource::Stream, fingerprint); + ExpectSameProjection(created, *listed); + + EXPECT_EQ(iggy::EncryptionKeyFingerprint(FixtureKey()), fingerprint); +} + +TEST_P(E2E_ResourceOptionsTransport, GivenKeylessStreamWhenReadingShouldReportServerGeneratedKey) { + RecordProperty("description", + "Creates a stream without a key on an encryption-enabled server and reads the server-generated " + "key's source and fingerprints through the typed accessors."); + auto client = GetTransportClient(); + const auto name = GetRandomName(40); + + const auto created = client.CreateStream(name); + TrackStream(name); + ExpectProjection(created, iggy::StreamEncryptionSource::Server, std::nullopt); + + const auto fetched = client.GetStream(iggy::Identifier::String(name)); + ExpectProjection(fetched, iggy::StreamEncryptionSource::Server, std::nullopt); + ExpectSameProjection(created, fetched); + + const auto listed = FindStream(client, created.Id()); + ASSERT_TRUE(listed.has_value()); + ExpectSameProjection(created, *listed); +} + +TEST_P(E2E_ResourceOptionsTransport, GivenStreamRenameWhenUpdatingShouldPreserveEncryptionProjection) { + RecordProperty("description", + "Renames a keyed stream through the generic update and keeps its id, source, fingerprints and " + "readable payloads."); + const auto &stream_key = Fixture().at("stream_key"); + auto client = GetTransportClient(); + const auto name = GetRandomName(40); + const auto renamed = GetRandomName(40); + + const auto created = client.CreateStream( + name, iggy::StreamCreateOptions().SetEncryptionKeyBase64(stream_key.at("key_base64").get<std::string>())); + TrackStream(created.Id()); + const auto topic = + client.CreateTopic(iggy::Identifier::Numeric(created.Id()), "events", iggy::TopicCreateOptions()); + SendPayload(created.Id(), topic.Id(), "before-rename"); + + ASSERT_NO_THROW( + client.UpdateStream(iggy::Identifier::Numeric(created.Id()), iggy::StreamUpdateOptions().SetName(renamed))); + + const auto fetched = client.GetStream(iggy::Identifier::Numeric(created.Id())); + EXPECT_EQ(fetched.Id(), created.Id()); + EXPECT_EQ(fetched.Name(), renamed); + ExpectProjection(fetched, iggy::StreamEncryptionSource::Stream, stream_key.at("fingerprint").get<std::string>()); + ExpectSameProjection(created, fetched); + EXPECT_EQ(PollPayloads(created.Id(), topic.Id()), std::vector<std::string>{"before-rename"}); +} + +TEST_P(E2E_ResourceOptionsTransport, GivenFixedResizePolicyWhenCreatingTopicShouldStoreItAndRefuseResize) { + RecordProperty("description", + "Creates a topic with the typed fixed resize policy, reads it back as an explicit option and " + "refuses a later partition creation."); + auto client = GetTransportClient(); + const auto stream_name = GetRandomName(40); + const auto stream = client.CreateStream(stream_name); + TrackStream(stream_name); + const auto created = + client.CreateTopic(iggy::Identifier::Numeric(stream.Id()), "fixed", + iggy::TopicCreateOptions().SetPartitionResizePolicy(iggy::PartitionResizePolicy::Fixed)); + + const auto key = std::string(iggy::to_string(iggy::TopicOptionKey::PartitionResizePolicy)); + ASSERT_EQ(created.Options().Explicit().count(key), 1u); + EXPECT_EQ(HeaderFieldText(created.Options().Explicit().at(key)), "fixed"); + EXPECT_THROW( + client.CreatePartitions(iggy::Identifier::Numeric(stream.Id()), iggy::Identifier::Numeric(created.Id()), 1), + iggy::IggyException); +} + +TEST_P(E2E_ResourceOptionsTransport, GivenTopicResetWhenUpdatingShouldPreserveUnmentionedValues) { + RecordProperty("description", + "Sets a topic's expiry without a name and resets its max size, keeping the name, compression and " + "the provenance of every other key."); + constexpr std::uint64_t initial_expiry = 3600000000; + constexpr std::uint64_t updated_expiry = 7200000000; + constexpr std::uint64_t max_size = 1073741824; + auto client = GetTransportClient(); + const auto stream_name = GetRandomName(40); + const auto stream = client.CreateStream(stream_name); + TrackStream(stream_name); + const auto created = client.CreateTopic(iggy::Identifier::Numeric(stream.Id()), "patched", + iggy::TopicCreateOptions() + .SetCompressionAlgorithm(iggy::CompressionAlgorithm::Gzip()) + .SetMessageExpiry(iggy::Expiry::Duration(initial_expiry)) + .SetMaxTopicSize(iggy::MaxTopicSize::FromBytes(max_size))); + + ASSERT_NO_THROW(client.UpdateTopic(iggy::Identifier::Numeric(stream.Id()), iggy::Identifier::Numeric(created.Id()), + iggy::TopicUpdateOptions() + .SetMessageExpiry(iggy::Expiry::Duration(updated_expiry)) + .Reset(iggy::TopicOptionKey::MaxTopicSize))); + + const auto topic = client.GetTopic(iggy::Identifier::Numeric(stream.Id()), iggy::Identifier::Numeric(created.Id())); + const auto &options = topic.Options(); + EXPECT_EQ(topic.Name(), "patched"); + EXPECT_EQ(topic.MessageExpiry(), updated_expiry); + EXPECT_EQ(topic.MaxTopicSize(), std::numeric_limits<std::uint64_t>::max()); + EXPECT_EQ(topic.CompressionAlgorithm(), "gzip"); + EXPECT_EQ(options.Explicit().count("message_expiry"), 1u); + EXPECT_EQ(options.Explicit().count("max_topic_size"), 0u); + EXPECT_EQ(options.Derived().count("max_topic_size"), 1u); + EXPECT_EQ(options.Explicit().count("compression_algorithm"), 1u); + for (const auto &[key, value] : created.Options().Derived()) { + if (key != "max_topic_size") { + ASSERT_EQ(options.Derived().count(key), 1u) << key; + EXPECT_EQ(options.Derived().at(key).Value(), value.Value()) << key; + } + } +} + +TEST_P(E2E_ResourceOptionsTransport, GivenPatchAdmissionRefusesWhenUpdatingTopicShouldKeepTopicUnchanged) { + RecordProperty("description", + "Refuses a topic update of only zero sentinels as invalid_command and one with more resets than " + "the count byte carries as unsupported_option_key, leaving the topic unchanged."); + auto client = GetTransportClient(); + const auto stream_name = GetRandomName(40); + const auto stream = client.CreateStream(stream_name); + TrackStream(stream_name); + const auto stream_id = iggy::Identifier::Numeric(stream.Id()); + const auto created = client.CreateTopic(stream_id, "bounded", + iggy::TopicCreateOptions() + .SetMessageExpiry(iggy::Expiry::Duration(3600000000)) + .SetMaxTopicSize(iggy::MaxTopicSize::FromBytes(1073741824))); + const auto topic_id = iggy::Identifier::Numeric(created.Id()); + + const auto sentinel_error = FixtureCase("http_update", "topic_zero_sentinel_only").at("error"); + EXPECT_TRUE(ThrowsIggyError( + [&] { + client.UpdateTopic(stream_id, topic_id, + iggy::TopicUpdateOptions().SetMessageExpiry(iggy::Expiry::ServerDefault())); + }, + sentinel_error)); + EXPECT_TRUE(ThrowsIggyError( + [&] { + client.UpdateTopic(stream_id, topic_id, + iggy::TopicUpdateOptions() + .SetMessageExpiry(iggy::Expiry::ServerDefault()) + .SetMaxTopicSize(iggy::MaxTopicSize::ServerDefault())); + }, + sentinel_error)); + + iggy::TopicUpdateOptions too_many_resets; + too_many_resets.Reset(iggy::TopicOptionKey::MaxTopicSize); + for (std::size_t index = 0; index < kMaxResetKeys; ++index) { + too_many_resets.RawReset("future_" + std::to_string(index)); + } + EXPECT_TRUE(ThrowsIggyError([&] { client.UpdateTopic(stream_id, topic_id, too_many_resets); }, + nlohmann::json{{"name", "unsupported_option_key"}, {"code", 4041}})); + + const auto topic = client.GetTopic(stream_id, topic_id); + EXPECT_EQ(topic.Name(), "bounded"); + EXPECT_EQ(topic.MessageExpiry(), created.MessageExpiry()); + EXPECT_EQ(topic.MaxTopicSize(), created.MaxTopicSize()); + EXPECT_EQ(topic.Options().Explicit().count("max_topic_size"), 1u); +} + +TEST_P(E2E_ResourceOptionsTransport, GivenUserOptionsWhenCreatingAndUpdatingShouldPreserveUserBehavior) { + RecordProperty("description", + "Creates a user with typed empty options, patches its username and status, reads its options with " + "provenance and refuses a duplicate username without a partial change."); + auto client = GetTransportClient(); + const auto username = GetRandomName(30); + const auto renamed = GetRandomName(30); + const auto other_username = GetRandomName(30); + const auto created = + CreateUser(client, username, "user-secret", iggy::UserStatus::Active, std::nullopt, iggy::UserCreateOptions()); + CreateUser(client, other_username, "user-secret", iggy::UserStatus::Active); + EXPECT_TRUE(created.Options().Explicit().empty()); + + ASSERT_NO_THROW( + client.UpdateUser(iggy::Identifier::Numeric(created.Id()), + iggy::UserUpdateOptions().SetUsername(renamed).SetStatus(iggy::UserStatus::Inactive))); + RenameTrackedUser(username, renamed); + const auto updated = client.GetUser(iggy::Identifier::Numeric(created.Id())); + EXPECT_EQ(updated.Username(), renamed); + EXPECT_EQ(updated.Status(), iggy::UserStatus::Inactive); + EXPECT_EQ(updated.CreatedAt(), created.CreatedAt()); + EXPECT_TRUE(updated.Options().Explicit().empty()); + + EXPECT_THROW( + client.UpdateUser(iggy::Identifier::String(other_username), + iggy::UserUpdateOptions().SetUsername(renamed).SetStatus(iggy::UserStatus::Inactive)), + iggy::IggyException); + const auto other = client.GetUser(iggy::Identifier::String(other_username)); + EXPECT_EQ(other.Username(), other_username); + EXPECT_EQ(other.Status(), iggy::UserStatus::Active); +} + +TEST_F(E2E_ResourceOptions, GivenInvalidKeyWhenEncodingShouldRejectBeforeNetworkIo) { + RecordProperty("description", + "Refuses every invalid key vector with invalid_encryption_key on a client that never connected, " + "accepts exactly 32 bytes and keeps key material out of the error text."); + const auto &stream_key = Fixture().at("stream_key"); + const auto key_hex = stream_key.at("key_hex").get<std::string>(); + const auto key_base64 = stream_key.at("key_base64").get<std::string>(); + auto client = GetLoggedOutHighLevelClient(); + + for (const auto &vector : stream_key.at("invalid_keys")) { + SCOPED_TRACE(vector.at("case").get<std::string>()); + const auto &input = vector.at("input"); + iggy::StreamCreateOptions options; + if (input.contains("raw_hex")) { + const auto raw = FromHex(input.at("raw_hex").get<std::string>()); + options.SetEncryptionKey(raw); + EXPECT_TRUE(ThrowsIggyError([&raw] { iggy::EncryptionKeyFingerprint(raw); }, vector.at("error"))); + } else { + options.SetEncryptionKeyBase64(input.at("base64").get<std::string>()); + } + EXPECT_TRUE( + ThrowsIggyError([&client, &options] { client.CreateStream("never-sent", options); }, vector.at("error"))); + try { + client.CreateStream("never-sent", options); + } catch (const iggy::IggyException &error) { + const std::string message = error.what(); + EXPECT_EQ(message.find(key_base64.substr(0, 40)), std::string::npos); + EXPECT_EQ(message.find(key_hex.substr(0, 40)), std::string::npos); + } + } + + EXPECT_EQ(iggy::EncryptionKeyFingerprint(FixtureKey()), stream_key.at("fingerprint").get<std::string>()); + try { + client.CreateStream("never-sent", iggy::StreamCreateOptions().SetEncryptionKeyBase64(key_base64)); + FAIL() << "A client that never connected cannot create a stream"; + } catch (const iggy::IggyException &error) { + EXPECT_NE(error.Kind(), "invalid_encryption_key"); + } +} + +TEST_P(E2E_ResourceOptionsTransport, GivenSharedVectorsWhenEncodingShouldMatchServerContract) { + RecordProperty("description", + "Matches the shared fixture on every transport: update refusals in the server order, raw resets " + "decided by the server and stream create admission."); + const auto &fixture = Fixture(); + EXPECT_EQ(fixture.at("schema_version").get<int>(), 1); + + auto client = GetTransportClient(); + auto root = GetLoggedInHighLevelClient(); + const auto raw_reset_user = CreateUser(root, GetRandomName(30), "raw-reset-secret", iggy::UserStatus::Active).Id(); + for (const auto &vector : fixture.at("update_resource_errors")) { + const auto scope = vector.at("scope").get<std::string>(); + SCOPED_TRACE(vector.at("case").get<std::string>()); + const auto &target = vector.at("target"); + const auto &input = vector.at("input"); + if (scope == "topic") { + iggy::TopicUpdateOptions options; + for (const auto &[key, value] : input.items()) { + if (key == "message_expiry") { + options.SetMessageExpiry(iggy::Expiry::Duration(std::stoull(value.get<std::string>()))); + } else if (key == "max_topic_size") { + options.SetMaxTopicSize(iggy::MaxTopicSize::FromBytes(std::stoull(value.get<std::string>()))); + } else if (key == "name") { + options.SetName(value.get<std::string>()); + } else if (key == "reset") { + for (const auto &reset : value) { + AddReset(options, kTopicOptionKeys, reset.get<std::string>()); + } + } else { + FAIL() << "Unmapped topic input " << key; + } + } + EXPECT_TRUE(ThrowsIggyError( + [&] { + client.UpdateTopic(iggy::Identifier::Numeric(target.at("stream_id").get<std::uint32_t>()), + iggy::Identifier::Numeric(target.at("topic_id").get<std::uint32_t>()), options); + }, + vector.at("error"))); + } else if (scope == "stream") { + iggy::StreamUpdateOptions options; + if (input.contains("name")) { + options.SetName(input.at("name").get<std::string>()); + } + for (const auto &reset : input.value("reset", nlohmann::json::array())) { + AddReset(options, kStreamOptionKeys, reset.get<std::string>()); + } + EXPECT_TRUE(ThrowsIggyError( + [&] { + client.UpdateStream(iggy::Identifier::Numeric(target.at("stream_id").get<std::uint32_t>()), + options); + }, + vector.at("error"))); + } else { + iggy::UserUpdateOptions options; + if (input.contains("username")) { + options.SetUsername(input.at("username").get<std::string>()); + } + for (const auto &reset : input.value("reset", nlohmann::json::array())) { + AddReset(options, kUserOptionKeys, reset.get<std::string>()); + } + // A raw reset reaches the server, so it needs a user that exists. + const auto user_id = + options.RawResetKeys().empty() ? target.at("user_id").get<std::uint32_t>() : raw_reset_user; + EXPECT_TRUE(ThrowsIggyError([&] { client.UpdateUser(iggy::Identifier::Numeric(user_id), options); }, + vector.at("error"))); + } + } + + for (const auto &vector : fixture.at("create_stream")) { + SCOPED_TRACE(vector.at("case").get<std::string>()); + const auto &input = vector.at("input"); + const auto name = GetRandomName(40); + iggy::StreamCreateOptions options; + if (input.contains("encryption_key")) { + options.SetEncryptionKey(FromHex(input.at("encryption_key").at("raw_hex").get<std::string>())); + } + if (input.contains("raw")) { + options.SetRawEntries(input.at("raw").get<std::map<std::string, std::string>>()); + } + if (vector.at("admission_error").is_null()) { + const auto created = client.CreateStream(name, options); + TrackStream(name); + const auto source = input.contains("encryption_key") ? iggy::StreamEncryptionSource::Stream + : iggy::StreamEncryptionSource::Server; + ExpectProjection( + created, source, + input.contains("encryption_key") + ? std::optional<std::string>(fixture.at("stream_key").at("fingerprint").get<std::string>()) + : std::nullopt); + } else { + EXPECT_TRUE(ThrowsIggyError([&] { client.CreateStream(name, options); }, vector.at("admission_error"))); + EXPECT_THROW(client.GetStream(iggy::Identifier::String(name)), iggy::IggyException); + } + } +} + +TEST_P(E2E_ResourceOptionsTransport, GivenLoggedInUserRenameWhenReconnectingShouldKeepNewUsername) { + RecordProperty("description", + "Renames the signed-in user, reconnects with the new name, keeps it after a refused rename and is " + "not affected by a stream rename."); + if (GetParam().name == "http") { + GTEST_SKIP() << "HTTP is stateless and does not reconnect"; + } + auto root = GetLoggedInHighLevelClient(); + const auto username = GetRandomName(30); + const auto renamed = GetRandomName(30); + const auto taken = GetRandomName(30); + const std::string secret = "rename-secret"; + iggy::GlobalPermissions global; + global.SetManageUsers(true).SetManageStreams(true); + iggy::Permissions permissions; + permissions.SetGlobal(global); + CreateUser(root, username, secret, iggy::UserStatus::Active, permissions); + CreateUser(root, taken, secret, iggy::UserStatus::Active); + + auto client = GetTransportClient(username, secret); + + ASSERT_NO_THROW( + client.UpdateUser(iggy::Identifier::String(username), iggy::UserUpdateOptions().SetUsername(renamed))); + RenameTrackedUser(username, renamed); + ASSERT_NO_THROW(client.Disconnect()); + ASSERT_NO_THROW(client.Connect()); + ASSERT_NO_THROW(client.GetMe()); + + EXPECT_THROW(client.UpdateUser(iggy::Identifier::String(renamed), iggy::UserUpdateOptions().SetUsername(taken)), + iggy::IggyException); + ASSERT_NO_THROW(client.Disconnect()); + ASSERT_NO_THROW(client.Connect()); + ASSERT_NO_THROW(client.GetMe()); + + const auto stream_name = GetRandomName(40); + const auto stream = client.CreateStream(stream_name); + TrackStream(stream.Id()); + ASSERT_NO_THROW( + client.UpdateStream(iggy::Identifier::Numeric(stream.Id()), iggy::StreamUpdateOptions().SetName(username))); + ASSERT_NO_THROW(client.Disconnect()); + ASSERT_NO_THROW(client.Connect()); + EXPECT_EQ(root.GetUser(iggy::Identifier::String(renamed)).Username(), renamed); + ASSERT_NO_THROW(client.GetMe()); +} + +TEST_P(E2E_ResourceOptionsTransport, GivenPublicDescribeOptionsWhenCalledShouldReturnCatalogFlags) { + RecordProperty("description", + "Serves every scope's catalog through the public DescribeOptions on every transport with the " + "fixture's keys, kinds, flags and defaults."); + const auto &fixture = Fixture(); + std::map<std::string, std::uint8_t> kind_codes; + for (const auto &row : fixture.at("catalog")) { + kind_codes[row.at("kind").get<std::string>()] = row.at("kind_code").get<std::uint8_t>(); + } + const std::map<std::string, iggy::OptionsScope> scopes = { + {"topic", iggy::OptionsScope::Topic}, + {"stream", iggy::OptionsScope::Stream}, + {"user", iggy::OptionsScope::User}, + }; + auto client = GetTransportClient(); + + for (const auto &described : fixture.at("describe_options")) { + const auto scope = described.at("scope").get<std::string>(); + SCOPED_TRACE(scope); + const auto specs = client.DescribeOptions(scopes.at(scope)); + const auto &expected = described.at("entries"); + ASSERT_EQ(specs.size(), expected.size()); + for (std::size_t index = 0; index < specs.size(); ++index) { + const auto &spec = specs[index]; + const auto &entry = expected[index]; + SCOPED_TRACE(spec.Key()); + EXPECT_EQ(spec.Key(), entry.at("key").get<std::string>()); + EXPECT_EQ(static_cast<std::uint8_t>(spec.Kind()), kind_codes.at(entry.at("kind").get<std::string>())); + EXPECT_EQ(FlagsOf(spec), entry.at("flags").get<std::uint32_t>()); + EXPECT_EQ(DefaultText(spec), entry.at("default").is_null() + ? std::nullopt + : std::optional<std::string>(entry.at("default").get<std::string>())); + EXPECT_FALSE(spec.Description().empty()); + } + } + + const auto stream_specs = client.DescribeOptions(iggy::OptionsScope::Stream); + const auto spec_of = [&stream_specs](iggy::StreamOptionKey key) { + for (const auto &spec : stream_specs) { + if (spec.Key() == iggy::to_string(key)) { + return spec; + } + } + throw std::runtime_error("Missing stream option"); + }; + const auto name = spec_of(iggy::StreamOptionKey::Name); + EXPECT_TRUE(name.Updatable() && !name.Secret() && !name.ReadOnly()); + const auto key = spec_of(iggy::StreamOptionKey::EncryptionKey); + EXPECT_TRUE(!key.Updatable() && !key.Resettable() && key.Secret() && key.Critical() && !key.ReadOnly()); + const auto fingerprint = spec_of(iggy::StreamOptionKey::EncryptionKeyFingerprint); + EXPECT_TRUE(fingerprint.ReadOnly() && !fingerprint.Updatable() && !fingerprint.Secret()); +}
diff --git a/foreign/cpp/tests/e2e/stream.cpp b/foreign/cpp/tests/e2e/stream.cpp index 62bd125..19fe959 100644 --- a/foreign/cpp/tests/e2e/stream.cpp +++ b/foreign/cpp/tests/e2e/stream.cpp
@@ -102,7 +102,8 @@ ForgetTrackedStream(stream_name); TrackStream(stream_id); - ASSERT_NO_THROW(client.UpdateStream(iggy::Identifier::String(stream_name), updated_stream_name)); + ASSERT_NO_THROW(client.UpdateStream(iggy::Identifier::String(stream_name), + iggy::StreamUpdateOptions().SetName(updated_stream_name))); ASSERT_THROW(client.GetStream(iggy::Identifier::String(stream_name)), iggy::IggyException); @@ -126,7 +127,8 @@ TrackStream(stream_name); auto first_read = client.GetStream(iggy::Identifier::String(stream_name)); - ASSERT_NO_THROW(client.UpdateStream(iggy::Identifier::Numeric(first_read.Id()), stream_name)); + ASSERT_NO_THROW(client.UpdateStream(iggy::Identifier::Numeric(first_read.Id()), + iggy::StreamUpdateOptions().SetName(stream_name))); auto second_read = client.GetStream(iggy::Identifier::Numeric(first_read.Id())); EXPECT_EQ(second_read.Id(), first_read.Id()); @@ -147,7 +149,8 @@ TrackStream(stream_name); const auto options = iggy::StreamUpdateOptions().SetRawEntries({{"not_a_real_option", "true"}}); - ASSERT_THROW(client.UpdateStream(iggy::Identifier::String(stream_name), updated_stream_name, options), + ASSERT_THROW(client.UpdateStream(iggy::Identifier::String(stream_name), + iggy::StreamUpdateOptions(options).SetName(updated_stream_name)), iggy::IggyException); const auto stream = client.GetStream(iggy::Identifier::String(stream_name)); @@ -165,14 +168,17 @@ auto unauthenticated_client = GetLoggedOutHighLevelClient(); - ASSERT_THROW(unauthenticated_client.UpdateStream(iggy::Identifier::String(stream_name), updated_stream_name), + ASSERT_THROW(unauthenticated_client.UpdateStream(iggy::Identifier::String(stream_name), + iggy::StreamUpdateOptions().SetName(updated_stream_name)), iggy::IggyException); ASSERT_NO_THROW(unauthenticated_client.Connect()); - ASSERT_THROW(unauthenticated_client.UpdateStream(iggy::Identifier::String(stream_name), updated_stream_name), + ASSERT_THROW(unauthenticated_client.UpdateStream(iggy::Identifier::String(stream_name), + iggy::StreamUpdateOptions().SetName(updated_stream_name)), iggy::IggyException); ASSERT_NO_THROW(unauthenticated_client.Login("iggy", "iggy")); ASSERT_NO_THROW(unauthenticated_client.Disconnect()); - ASSERT_THROW(unauthenticated_client.UpdateStream(iggy::Identifier::String(stream_name), updated_stream_name), + ASSERT_THROW(unauthenticated_client.UpdateStream(iggy::Identifier::String(stream_name), + iggy::StreamUpdateOptions().SetName(updated_stream_name)), iggy::IggyException); } @@ -197,7 +203,8 @@ for (const auto &updated_stream_name : updated_stream_names) { SCOPED_TRACE(updated_stream_name); - ASSERT_NO_THROW(client.UpdateStream(iggy::Identifier::Numeric(stream_id), updated_stream_name)); + ASSERT_NO_THROW(client.UpdateStream(iggy::Identifier::Numeric(stream_id), + iggy::StreamUpdateOptions().SetName(updated_stream_name))); ASSERT_NO_THROW({ const auto stream_details = client.GetStream(iggy::Identifier::Numeric(stream_id)); EXPECT_EQ(stream_details.Name(), updated_stream_name); @@ -210,7 +217,9 @@ const std::string stream_name = GetRandomName(); const std::string updated_stream_name = GetRandomName(); auto client = GetLoggedInHighLevelClient(); - ASSERT_THROW(client.UpdateStream(iggy::Identifier::String(stream_name), updated_stream_name), iggy::IggyException); + ASSERT_THROW(client.UpdateStream(iggy::Identifier::String(stream_name), + iggy::StreamUpdateOptions().SetName(updated_stream_name)), + iggy::IggyException); } TEST_F(E2E_Stream, UpdateStreamWithDuplicateNameThrows) { @@ -223,7 +232,8 @@ ASSERT_NO_THROW(client.CreateStream(second_stream_name)); TrackStream(second_stream_name); - ASSERT_THROW(client.UpdateStream(iggy::Identifier::String(first_stream_name), second_stream_name), + ASSERT_THROW(client.UpdateStream(iggy::Identifier::String(first_stream_name), + iggy::StreamUpdateOptions().SetName(second_stream_name)), iggy::IggyException); } @@ -238,7 +248,9 @@ ASSERT_NO_THROW(client.DeleteStream(iggy::Identifier::String(stream_name))); ForgetTrackedStream(stream_name); - ASSERT_THROW(client.UpdateStream(iggy::Identifier::String(stream_name), updated_stream_name), iggy::IggyException); + ASSERT_THROW(client.UpdateStream(iggy::Identifier::String(stream_name), + iggy::StreamUpdateOptions().SetName(updated_stream_name)), + iggy::IggyException); } TEST_F(E2E_Stream, UpdateStreamOnlyChangesName) { @@ -280,7 +292,8 @@ auto stream_before_update = client.GetStream(iggy::Identifier::Numeric(stream_id)); const auto stats_before_update = client.GetStats(); - ASSERT_NO_THROW(client.UpdateStream(iggy::Identifier::Numeric(stream_id), updated_stream_name)); + ASSERT_NO_THROW(client.UpdateStream(iggy::Identifier::Numeric(stream_id), + iggy::StreamUpdateOptions().SetName(updated_stream_name))); ASSERT_THROW(client.GetStream(iggy::Identifier::String(stream_name)), iggy::IggyException); auto stream_after_update = client.GetStream(iggy::Identifier::Numeric(stream_id)); @@ -336,7 +349,8 @@ }; for (const auto &invalid_stream_name : invalid_stream_names) { SCOPED_TRACE("invalid_stream_name_length=" + std::to_string(invalid_stream_name.size())); - ASSERT_THROW(client.UpdateStream(iggy::Identifier::Numeric(stream_id), invalid_stream_name), + ASSERT_THROW(client.UpdateStream(iggy::Identifier::Numeric(stream_id), + iggy::StreamUpdateOptions().SetName(invalid_stream_name)), iggy::IggyException); } }
diff --git a/foreign/cpp/tests/e2e/test_helpers.hpp b/foreign/cpp/tests/e2e/test_helpers.hpp index 008acbc..a28c358 100644 --- a/foreign/cpp/tests/e2e/test_helpers.hpp +++ b/foreign/cpp/tests/e2e/test_helpers.hpp
@@ -123,6 +123,16 @@ return opts; } +inline iggy::ffi::UserUpdateOptions make_user_update_options(const std::string &username, + const iggy::ffi::UserStatus status) { + iggy::ffi::UserUpdateOptions options{}; + options.has_username = true; + options.username = username; + options.has_status = true; + options.status = status; + return options; +} + // Joining commits the membership at once, but a member owns its partitions only after every // affected partition installs it as owner. Until then group polls and group offset calls fail // with ConsumerGroupPartitionNotOwned (5009). @@ -242,7 +252,7 @@ iggy::ffi::Permissions permissions = {}) { iggy::ffi::ErrorInfo error_info{}; auto user = - client->create_user(username, password, status, has_permissions, std::move(permissions), error_info); + client->create_user(username, password, status, has_permissions, std::move(permissions), {}, error_info); tracked_user_names_.push_back(username); return user; } @@ -251,8 +261,9 @@ const std::string &username, const std::string &password, const iggy::UserStatus status, - std::optional<iggy::Permissions> permissions = std::nullopt) { - auto user = client.CreateUser(username, password, status, permissions); + std::optional<iggy::Permissions> permissions = std::nullopt, + const iggy::UserCreateOptions &options = {}) { + auto user = client.CreateUser(username, password, status, permissions, options); tracked_user_names_.push_back(username); return user; }
diff --git a/foreign/cpp/tests/e2e/topic.cpp b/foreign/cpp/tests/e2e/topic.cpp index 6db3e2a..ce516a0 100644 --- a/foreign/cpp/tests/e2e/topic.cpp +++ b/foreign/cpp/tests/e2e/topic.cpp
@@ -336,8 +336,8 @@ TEST_F(E2E_Topic, DescribeOptionsServesTopicCatalogAndRejectsUnknownScope) { RecordProperty("description", - "Serves the topic option catalog with each key's kind, default and description, returns an empty " - "catalog for the stream scope, and rejects an unknown scope name."); + "Serves the topic option catalog with each key's kind, default and description, serves the " + "stream catalog, and rejects an unknown scope name."); iggy::ffi::Client *client = GetLoggedInClient(); @@ -361,10 +361,11 @@ EXPECT_FALSE(segment_size->default_value.empty()); EXPECT_FALSE(segment_size->description.empty()); - // Streams take no option keys yet, which is an empty catalog rather than a failure. ASSERT_NO_THROW({ const auto stream_options = client->describe_options("stream"); - EXPECT_TRUE(stream_options.empty()); + ASSERT_FALSE(stream_options.empty()); + EXPECT_EQ(static_cast<std::string>(stream_options[0].key), "name"); + EXPECT_TRUE(stream_options[0].updatable); }); ASSERT_THROW(client->describe_options("not_a_scope"), std::exception); @@ -884,7 +885,9 @@ } TEST_F(E2E_Topic, GetTopicsAfterTopicUpdateReturnsUpdatedInputFields) { - RecordProperty("description", "Returns updated user-provided topic fields after a topic update."); + RecordProperty("description", + "Returns updated user-provided topic fields after a topic update and keeps the create-only " + "compression algorithm."); const std::string stream_name = GetRandomName(); const std::string original_topic = GetRandomName(); const std::string updated_topic_name = GetRandomName(); @@ -892,14 +895,14 @@ auto client = GetLoggedInHighLevelClient(); ASSERT_NO_THROW(client.CreateStream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client.CreateTopic(iggy::Identifier::String(stream_name), original_topic, - iggy::TopicCreateOptions().SetPartitionsCount(2))); - ASSERT_NO_THROW(client.UpdateTopic( - iggy::Identifier::String(stream_name), iggy::Identifier::String(original_topic), updated_topic_name, - iggy::TopicUpdateOptions() - .SetCompressionAlgorithm(iggy::CompressionAlgorithm::Gzip()) - .SetMessageExpiry(iggy::Expiry::Duration(1000)) - .SetMaxTopicSize(iggy::MaxTopicSize::FromBytes(1024ULL * 1024ULL * 1024ULL)))); + ASSERT_NO_THROW(client.CreateTopic( + iggy::Identifier::String(stream_name), original_topic, + iggy::TopicCreateOptions().SetPartitionsCount(2).SetCompressionAlgorithm(iggy::CompressionAlgorithm::Gzip()))); + ASSERT_NO_THROW(client.UpdateTopic(iggy::Identifier::String(stream_name), iggy::Identifier::String(original_topic), + iggy::TopicUpdateOptions() + .SetMessageExpiry(iggy::Expiry::Duration(1000)) + .SetMaxTopicSize(iggy::MaxTopicSize::FromBytes(1024ULL * 1024ULL * 1024ULL)) + .SetName(updated_topic_name))); ASSERT_NO_THROW({ const auto topics = client.GetTopics(iggy::Identifier::String(stream_name)); @@ -923,12 +926,11 @@ TrackStream(stream_name); ASSERT_NO_THROW(client.CreateTopic(iggy::Identifier::String(stream_name), original_topic, iggy::TopicCreateOptions().SetPartitionsCount(2))); - ASSERT_NO_THROW(client.UpdateTopic( - iggy::Identifier::String(stream_name), iggy::Identifier::String(original_topic), updated_topic_name, - iggy::TopicUpdateOptions() - .SetCompressionAlgorithm(iggy::CompressionAlgorithm::Gzip()) - .SetMessageExpiry(iggy::Expiry::Duration(1000)) - .SetMaxTopicSize(iggy::MaxTopicSize::FromBytes(1024ULL * 1024ULL * 1024ULL)))); + ASSERT_NO_THROW(client.UpdateTopic(iggy::Identifier::String(stream_name), iggy::Identifier::String(original_topic), + iggy::TopicUpdateOptions() + .SetMessageExpiry(iggy::Expiry::Duration(1000)) + .SetMaxTopicSize(iggy::MaxTopicSize::FromBytes(1024ULL * 1024ULL * 1024ULL)) + .SetName(updated_topic_name))); ASSERT_NO_THROW({ const auto topic_details = @@ -962,12 +964,11 @@ ASSERT_NO_THROW(client.CreateTopic(iggy::Identifier::String(stream_name), original_topic, iggy::TopicCreateOptions().SetPartitionsCount(partitions_count))); - ASSERT_NO_THROW(client.UpdateTopic( - iggy::Identifier::String(stream_name), iggy::Identifier::String(original_topic), updated_topic_name, - iggy::TopicUpdateOptions() - .SetCompressionAlgorithm(iggy::CompressionAlgorithm::Gzip()) - .SetMessageExpiry(iggy::Expiry::Duration(1000)) - .SetMaxTopicSize(iggy::MaxTopicSize::FromBytes(1024ULL * 1024ULL * 1024ULL)))); + ASSERT_NO_THROW(client.UpdateTopic(iggy::Identifier::String(stream_name), iggy::Identifier::String(original_topic), + iggy::TopicUpdateOptions() + .SetMessageExpiry(iggy::Expiry::Duration(1000)) + .SetMaxTopicSize(iggy::MaxTopicSize::FromBytes(1024ULL * 1024ULL * 1024ULL)) + .SetName(updated_topic_name))); ASSERT_NO_THROW({ const auto topic_details = @@ -1003,12 +1004,11 @@ make_numeric_identifier(topic_id), "partition_id", partition_id_bytes(0), std::move(messages), error_info)); - ASSERT_NO_THROW(client.UpdateTopic( - iggy::Identifier::String(stream_name), iggy::Identifier::String(original_topic), updated_topic_name, - iggy::TopicUpdateOptions() - .SetCompressionAlgorithm(iggy::CompressionAlgorithm::Gzip()) - .SetMessageExpiry(iggy::Expiry::Duration(1000)) - .SetMaxTopicSize(iggy::MaxTopicSize::FromBytes(1024ULL * 1024ULL * 1024ULL)))); + ASSERT_NO_THROW(client.UpdateTopic(iggy::Identifier::String(stream_name), iggy::Identifier::String(original_topic), + iggy::TopicUpdateOptions() + .SetMessageExpiry(iggy::Expiry::Duration(1000)) + .SetMaxTopicSize(iggy::MaxTopicSize::FromBytes(1024ULL * 1024ULL * 1024ULL)) + .SetName(updated_topic_name))); ASSERT_NO_THROW({ iggy::ffi::ErrorInfo error_info{}; @@ -1028,14 +1028,6 @@ const std::string stream_name = GetRandomName(); std::string topic_name = GetRandomName(); - struct CompressionOption { - std::string name; - iggy::CompressionAlgorithm value; - }; - const std::vector<CompressionOption> compression_algorithms = { - {"none", iggy::CompressionAlgorithm::None()}, - {"gzip", iggy::CompressionAlgorithm::Gzip()}, - }; struct ExpiryOption { std::string name; iggy::Expiry value; @@ -1066,37 +1058,34 @@ std::uint64_t expected_message_expiry = initial_topic.MessageExpiry(); std::uint64_t expected_max_topic_size = initial_topic.MaxTopicSize(); - for (const auto &compression_algorithm : compression_algorithms) { - for (const auto &expiry_option : expiry_options) { - for (const auto &max_topic_size : max_topic_sizes) { - const std::string updated_topic_name = GetRandomName(); - SCOPED_TRACE("compression=" + compression_algorithm.name + ", expiry_kind=" + expiry_option.name + - ", max_topic_size=" + max_topic_size.name); + for (const auto &expiry_option : expiry_options) { + for (const auto &max_topic_size : max_topic_sizes) { + const std::string updated_topic_name = GetRandomName(); + SCOPED_TRACE("expiry_kind=" + expiry_option.name + ", max_topic_size=" + max_topic_size.name); - ASSERT_NO_THROW(client.UpdateTopic(iggy::Identifier::String(stream_name), - iggy::Identifier::String(topic_name), updated_topic_name, - iggy::TopicUpdateOptions() - .SetCompressionAlgorithm(compression_algorithm.value) - .SetMessageExpiry(expiry_option.value) - .SetMaxTopicSize(max_topic_size.value))); - topic_name = updated_topic_name; + ASSERT_NO_THROW(client.UpdateTopic(iggy::Identifier::String(stream_name), + iggy::Identifier::String(topic_name), + iggy::TopicUpdateOptions() + .SetMessageExpiry(expiry_option.value) + .SetMaxTopicSize(max_topic_size.value) + .SetName(updated_topic_name))); + topic_name = updated_topic_name; - if (expiry_option.value.Kind() != "server_default") { - expected_message_expiry = expiry_option.value.Value(); - } - if (max_topic_size.value.Value() == "unlimited") { - expected_max_topic_size = std::numeric_limits<std::uint64_t>::max(); - } else if (max_topic_size.value.Value() != "server_default") { - expected_max_topic_size = 1024ULL * 1024ULL * 1024ULL; - } - - const auto updated_topic = - client.GetTopic(iggy::Identifier::String(stream_name), iggy::Identifier::String(topic_name)); - EXPECT_EQ(updated_topic.Name(), updated_topic_name); - EXPECT_EQ(updated_topic.CompressionAlgorithm(), compression_algorithm.name); - EXPECT_EQ(updated_topic.MessageExpiry(), expected_message_expiry); - EXPECT_EQ(updated_topic.MaxTopicSize(), expected_max_topic_size); + if (expiry_option.value.Kind() != "server_default") { + expected_message_expiry = expiry_option.value.Value(); } + if (max_topic_size.value.Value() == "unlimited") { + expected_max_topic_size = std::numeric_limits<std::uint64_t>::max(); + } else if (max_topic_size.value.Value() != "server_default") { + expected_max_topic_size = 1024ULL * 1024ULL * 1024ULL; + } + + const auto updated_topic = + client.GetTopic(iggy::Identifier::String(stream_name), iggy::Identifier::String(topic_name)); + EXPECT_EQ(updated_topic.Name(), updated_topic_name); + EXPECT_EQ(updated_topic.CompressionAlgorithm(), initial_topic.CompressionAlgorithm()); + EXPECT_EQ(updated_topic.MessageExpiry(), expected_message_expiry); + EXPECT_EQ(updated_topic.MaxTopicSize(), expected_max_topic_size); } } } @@ -1116,21 +1105,21 @@ const auto created_topic = client.GetTopic(iggy::Identifier::String(stream_name), iggy::Identifier::String(original_topic)); - ASSERT_NO_THROW(client.UpdateTopic( - iggy::Identifier::String(stream_name), iggy::Identifier::Numeric(created_topic.Id()), updated_topic_name, - iggy::TopicUpdateOptions() - .SetCompressionAlgorithm(iggy::CompressionAlgorithm::Gzip()) - .SetMessageExpiry(iggy::Expiry::Duration(1000)) - .SetMaxTopicSize(iggy::MaxTopicSize::FromBytes(1024ULL * 1024ULL * 1024ULL)))); + ASSERT_NO_THROW(client.UpdateTopic(iggy::Identifier::String(stream_name), + iggy::Identifier::Numeric(created_topic.Id()), + iggy::TopicUpdateOptions() + .SetMessageExpiry(iggy::Expiry::Duration(1000)) + .SetMaxTopicSize(iggy::MaxTopicSize::FromBytes(1024ULL * 1024ULL * 1024ULL)) + .SetName(updated_topic_name))); const auto first_update = client.GetTopic(iggy::Identifier::String(stream_name), iggy::Identifier::Numeric(created_topic.Id())); - ASSERT_NO_THROW(client.UpdateTopic( - iggy::Identifier::String(stream_name), iggy::Identifier::Numeric(created_topic.Id()), updated_topic_name, - iggy::TopicUpdateOptions() - .SetCompressionAlgorithm(iggy::CompressionAlgorithm::Gzip()) - .SetMessageExpiry(iggy::Expiry::Duration(1000)) - .SetMaxTopicSize(iggy::MaxTopicSize::FromBytes(1024ULL * 1024ULL * 1024ULL)))); + ASSERT_NO_THROW(client.UpdateTopic(iggy::Identifier::String(stream_name), + iggy::Identifier::Numeric(created_topic.Id()), + iggy::TopicUpdateOptions() + .SetMessageExpiry(iggy::Expiry::Duration(1000)) + .SetMaxTopicSize(iggy::MaxTopicSize::FromBytes(1024ULL * 1024ULL * 1024ULL)) + .SetName(updated_topic_name))); const auto second_update = client.GetTopic(iggy::Identifier::String(stream_name), iggy::Identifier::Numeric(created_topic.Id())); @@ -1157,11 +1146,10 @@ iggy::TopicCreateOptions().SetPartitionsCount(1))); ASSERT_THROW(client.UpdateTopic(iggy::Identifier::String(stream_name), iggy::Identifier::String(first_topic_name), - second_topic_name, iggy::TopicUpdateOptions() - .SetCompressionAlgorithm(iggy::CompressionAlgorithm::Gzip()) .SetMessageExpiry(iggy::Expiry::Duration(1000)) - .SetMaxTopicSize(iggy::MaxTopicSize::FromBytes(1024ULL * 1024ULL * 1024ULL))), + .SetMaxTopicSize(iggy::MaxTopicSize::FromBytes(1024ULL * 1024ULL * 1024ULL)) + .SetName(second_topic_name)), iggy::IggyException); } @@ -1184,14 +1172,12 @@ for (const auto &invalid_topic_name : invalid_topic_names) { SCOPED_TRACE("invalid_topic_name_length=" + std::to_string(invalid_topic_name.size())); - ASSERT_THROW( - client.UpdateTopic(iggy::Identifier::String(stream_name), iggy::Identifier::String(topic_name), - invalid_topic_name, - iggy::TopicUpdateOptions() - .SetCompressionAlgorithm(iggy::CompressionAlgorithm::Gzip()) - .SetMessageExpiry(iggy::Expiry::Duration(1000)) - .SetMaxTopicSize(iggy::MaxTopicSize::FromBytes(1024ULL * 1024ULL * 1024ULL))), - iggy::IggyException); + ASSERT_THROW(client.UpdateTopic(iggy::Identifier::String(stream_name), iggy::Identifier::String(topic_name), + iggy::TopicUpdateOptions() + .SetMessageExpiry(iggy::Expiry::Duration(1000)) + .SetMaxTopicSize(iggy::MaxTopicSize::FromBytes(1024ULL * 1024ULL * 1024ULL)) + .SetName(invalid_topic_name)), + iggy::IggyException); } } @@ -1217,12 +1203,11 @@ client.GetTopic(iggy::Identifier::String(stream_name), iggy::Identifier::String(topic_name)); ASSERT_THROW(client.UpdateTopic(iggy::Identifier::String(stream_name), iggy::Identifier::String(topic_name), - updated_topic_name, iggy::TopicUpdateOptions() - .SetCompressionAlgorithm(iggy::CompressionAlgorithm::None()) .SetMessageExpiry(iggy::Expiry::Duration(2000)) .SetMaxTopicSize(iggy::MaxTopicSize::ServerDefault()) - .SetRawEntries({{"max_topic_size", "not-a-size"}})), + .SetRawEntries({{"max_topic_size", "not-a-size"}}) + .SetName(updated_topic_name)), iggy::IggyException); const auto topic_after_failed_update = @@ -1253,18 +1238,21 @@ auto unauthenticated_client = GetLoggedOutHighLevelClient(); - ASSERT_THROW(unauthenticated_client.UpdateTopic(iggy::Identifier::String(stream_name), - iggy::Identifier::String(topic_name), updated_topic_name), - iggy::IggyException); + ASSERT_THROW( + unauthenticated_client.UpdateTopic(iggy::Identifier::String(stream_name), iggy::Identifier::String(topic_name), + iggy::TopicUpdateOptions().SetName(updated_topic_name)), + iggy::IggyException); ASSERT_NO_THROW(unauthenticated_client.Connect()); - ASSERT_THROW(unauthenticated_client.UpdateTopic(iggy::Identifier::String(stream_name), - iggy::Identifier::String(topic_name), updated_topic_name), - iggy::IggyException); + ASSERT_THROW( + unauthenticated_client.UpdateTopic(iggy::Identifier::String(stream_name), iggy::Identifier::String(topic_name), + iggy::TopicUpdateOptions().SetName(updated_topic_name)), + iggy::IggyException); ASSERT_NO_THROW(unauthenticated_client.Login("iggy", "iggy")); ASSERT_NO_THROW(unauthenticated_client.Disconnect()); - ASSERT_THROW(unauthenticated_client.UpdateTopic(iggy::Identifier::String(stream_name), - iggy::Identifier::String(topic_name), updated_topic_name), - iggy::IggyException); + ASSERT_THROW( + unauthenticated_client.UpdateTopic(iggy::Identifier::String(stream_name), iggy::Identifier::String(topic_name), + iggy::TopicUpdateOptions().SetName(updated_topic_name)), + iggy::IggyException); } TEST_F(E2E_Topic, UpdateTopicOnNonExistentStreamThrows) { @@ -1275,11 +1263,10 @@ auto client = GetLoggedInHighLevelClient(); ASSERT_THROW(client.UpdateTopic(iggy::Identifier::String(stream_name), iggy::Identifier::String(topic_name), - updated_topic_name, iggy::TopicUpdateOptions() - .SetCompressionAlgorithm(iggy::CompressionAlgorithm::Gzip()) .SetMessageExpiry(iggy::Expiry::Duration(1000)) - .SetMaxTopicSize(iggy::MaxTopicSize::FromBytes(1024ULL * 1024ULL * 1024ULL))), + .SetMaxTopicSize(iggy::MaxTopicSize::FromBytes(1024ULL * 1024ULL * 1024ULL)) + .SetName(updated_topic_name)), iggy::IggyException); } @@ -1294,11 +1281,10 @@ TrackStream(stream_name); ASSERT_THROW(client.UpdateTopic(iggy::Identifier::String(stream_name), iggy::Identifier::String(topic_name), - updated_topic_name, iggy::TopicUpdateOptions() - .SetCompressionAlgorithm(iggy::CompressionAlgorithm::Gzip()) .SetMessageExpiry(iggy::Expiry::Duration(1000)) - .SetMaxTopicSize(iggy::MaxTopicSize::FromBytes(1024ULL * 1024ULL * 1024ULL))), + .SetMaxTopicSize(iggy::MaxTopicSize::FromBytes(1024ULL * 1024ULL * 1024ULL)) + .SetName(updated_topic_name)), iggy::IggyException); }
diff --git a/foreign/cpp/tests/fixtures/resource_options.json b/foreign/cpp/tests/fixtures/resource_options.json new file mode 100644 index 0000000..c5cc2ac --- /dev/null +++ b/foreign/cpp/tests/fixtures/resource_options.json
@@ -0,0 +1,1619 @@ +{ + "schema_version": 1, + "generated_by": "core/common/examples/resource_options_fixtures.rs", + "conventions": { + "byte_strings": "Every *_hex field is lowercase hex without separators.", + "errors": "name is the snake_case IggyError name and code its numeric code.", + "max_topic_size": "Bytes. 18446744073709551615 means unlimited.", + "message_expiry": "Microseconds. 18446744073709551615 means never expire.", + "never_returned_keys": "No response, decoded projection or accessor may contain these keys or their values.", + "option_maps": "Option maps use the HTTP JSON form: value as reversible text, kind, and explicit provenance. Request blocks mark every entry explicit.", + "options_block_order": "Options block entries are encoded in ascending key byte order.", + "request_bodies": "body_hex and update_resource_hex are command payloads without framing. constants.command_codes maps each command to its code and VSR operation.", + "reset_order": "The typed encoders and the HTTP handler write reset keys in ascending key byte order.", + "stream_state": "state is the stored server state a stream response was projected from. SDKs need only response_hex and expected.", + "u64_values": "u64 option values and typed u64 inputs are decimal strings, so values above 2^53 keep their precision." + }, + "constants": { + "protocol_version": { + "packed": 12288, + "text": "0.12.0", + "major": 0, + "minor": 12, + "patch": 0 + }, + "protocol_version_min": { + "packed": 12288, + "text": "0.12.0", + "major": 0, + "minor": 12, + "patch": 0 + }, + "command_codes": { + "create_stream": { + "code": 202, + "operation": 128 + }, + "create_topic": { + "code": 302, + "operation": 132 + }, + "create_user": { + "code": 33, + "operation": 141 + }, + "describe_options": { + "code": 13, + "operation": null + }, + "get_stream": { + "code": 200, + "operation": null + }, + "get_streams": { + "code": 201, + "operation": null + }, + "get_topic": { + "code": 300, + "operation": null + }, + "get_topics": { + "code": 301, + "operation": null + }, + "get_user": { + "code": 31, + "operation": null + }, + "get_users": { + "code": 32, + "operation": null + }, + "update_resource": { + "code": 16, + "operation": 150 + } + }, + "error_codes": { + "cannot_modify_root_user": 4108 + }, + "retired_command_codes": { + "update_stream": 204, + "update_topic": 304, + "update_user": 35 + }, + "operations": { + "rewrap_stream_keys": { + "value": 75, + "internal": true, + "request_operation": 75 + }, + "update_resource": { + "value": 150, + "internal": false, + "request_operation": 150 + }, + "update_stream_resolved": { + "value": 72, + "internal": true, + "request_operation": 150 + }, + "update_topic_resolved": { + "value": 73, + "internal": true, + "request_operation": 150 + }, + "update_user_resolved": { + "value": 74, + "internal": true, + "request_operation": 150 + } + }, + "retired_operations": { + "update_stream": 129, + "update_topic": 133, + "update_user": 142 + }, + "scope_codes": { + "stream": 2, + "topic": 1, + "user": 3 + }, + "header_kind_codes": { + "raw": 1, + "string": 2, + "bool": 3, + "uint32": 11, + "uint64": 12 + }, + "option_flags": { + "critical": 16, + "read_only": 8, + "resettable": 2, + "secret": 4, + "updatable": 1 + }, + "user_status_codes": { + "active": 1, + "inactive": 2 + }, + "stream_key_length": 32, + "never_returned_keys": [ + "encryption_key", + "encryption_key_wrapped", + "critical_keys" + ] + }, + "catalog": [ + { + "key": "name", + "scopes": [ + "stream", + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "column", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 1, + "described": true, + "default": null + }, + { + "key": "username", + "scopes": [ + "user" + ], + "kind": "string", + "kind_code": 2, + "storage": "column", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 1, + "described": true, + "default": null + }, + { + "key": "status", + "scopes": [ + "user" + ], + "kind": "string", + "kind_code": 2, + "storage": "column", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 1, + "described": true, + "default": null + }, + { + "key": "compression_algorithm", + "scopes": [ + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "none" + }, + { + "key": "message_expiry", + "scopes": [ + "topic" + ], + "kind": "uint64", + "kind_code": 12, + "storage": "map", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 3, + "described": true, + "default": "18446744073709551615" + }, + { + "key": "max_topic_size", + "scopes": [ + "topic" + ], + "kind": "uint64", + "kind_code": 12, + "storage": "map", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 3, + "described": true, + "default": "18446744073709551615" + }, + { + "key": "segment_size", + "scopes": [ + "topic" + ], + "kind": "uint64", + "kind_code": 12, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "1073741824" + }, + { + "key": "durability", + "scopes": [ + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "replicated" + }, + { + "key": "consumer_offset_durability", + "scopes": [ + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "replicated" + }, + { + "key": "partition_resize_policy", + "scopes": [ + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "mutable" + }, + { + "key": "messages_required_to_save", + "scopes": [ + "topic" + ], + "kind": "uint32", + "kind_code": 11, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "1024" + }, + { + "key": "size_of_messages_required_to_save", + "scopes": [ + "topic" + ], + "kind": "uint64", + "kind_code": 12, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "1048576" + }, + { + "key": "preallocate_segments", + "scopes": [ + "topic" + ], + "kind": "bool", + "kind_code": 3, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "false" + }, + { + "key": "encryption_key", + "scopes": [ + "stream" + ], + "kind": "raw", + "kind_code": 1, + "storage": "map", + "mutability": "create_only", + "visibility": "secret", + "critical": true, + "flags": 20, + "described": true, + "default": null + }, + { + "key": "encryption_key_wrapped", + "scopes": [ + "stream" + ], + "kind": "raw", + "kind_code": 1, + "storage": "map", + "mutability": "create_only", + "visibility": "internal", + "critical": true, + "flags": 16, + "described": false, + "default": null + }, + { + "key": "encryption_key_source", + "scopes": [ + "stream" + ], + "kind": "string", + "kind_code": 2, + "storage": "computed", + "mutability": "read_only", + "visibility": "read_only", + "critical": false, + "flags": 8, + "described": true, + "default": null + }, + { + "key": "encryption_key_fingerprint", + "scopes": [ + "stream" + ], + "kind": "string", + "kind_code": 2, + "storage": "computed", + "mutability": "read_only", + "visibility": "read_only", + "critical": false, + "flags": 8, + "described": true, + "default": null + }, + { + "key": "encryption_kek_fingerprint", + "scopes": [ + "stream" + ], + "kind": "string", + "kind_code": 2, + "storage": "computed", + "mutability": "read_only", + "visibility": "read_only", + "critical": false, + "flags": 8, + "described": true, + "default": null + }, + { + "key": "critical_keys", + "scopes": [ + "stream", + "topic", + "user" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "internal", + "critical": false, + "flags": 0, + "described": false, + "default": null + } + ], + "describe_options": [ + { + "scope": "topic", + "request_hex": "01", + "response_hex": "0b000000046e616d650201000000007e0000005265736f75726365206e616d652c203120746f203235352062797465732c20756e697175652077697468696e2069747320706172656e742e20536574206279207468652063726561746520636f6d6d616e642773206e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e15636f6d7072657373696f6e5f616c676f726974686d0200040000006e6f6e6550000000436f6d7072657373696f6e20616c676f726974686d20286e6f6e652c20677a6970292e2053746f72656420616e64207265706f727465642c206e6f74206170706c69656420746f206d657373616765730e6d6573736167655f6578706972790c0308000000ffffffffffffffff430000004d6573736167652065787069727920696e206d6963726f7365636f6e64732c206f7220612068756d616e74696d6520737472696e672028652e672e20372064617973290e6d61785f746f7069635f73697a650c0308000000ffffffffffffffff9d000000546f7069632d77696465207365616c65642d7365676d656e742073697a65206361702c2073706c6974206163726f737320616c6c20706172746974696f6e732c20696e206279746573206f72206120627974652d73697a6520737472696e672028652e672e203120476942293b2066696e6974652076616c756573206d757374206265206174206c6561737420746865207365676d656e742073697a650c7365676d656e745f73697a650c000800000000000040000000006c0000005365676d656e742073697a6520696e2062797465732c206f72206120627974652d73697a6520737472696e672028652e672e20313238204d6942293b2061203531322d62797465206d756c7469706c652077697468696e20313034383537362e2e3d313037333734313832340a6475726162696c69747902000a0000007265706c6963617465643e0100004d65737361676520636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20412073696e676c65746f6e2071756f72756d20686173206f6e6520636f70792e20496e207265706c6963617465642067726f7570732c20706572736973746564206d65737361676573207573652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c2057414c207265636c616d6174696f6e2e205468652066756c6c20626f64792073697a65207374696c6c20636f756e747320616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e1a636f6e73756d65725f6f66667365745f6475726162696c69747902000a0000007265706c6963617465646e0100004578706c69636974206f666673657420636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20506f6c6c206175746f2d636f6d6d69742072656d61696e73206173796e6368726f6e6f75732e20496e207265706c6963617465642067726f7570732c20706572736973746564206f66667365747320616c736f20656e61626c652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c207265636c616d6174696f6e2c206576656e2077697468207265706c696361746564206d657373616765206475726162696c6974792e2046756c6c20626f64792073697a657320636f756e7420616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e17706172746974696f6e5f726573697a655f706f6c6963790200070000006d757461626c65c8000000496d6d757461626c65206166746572206372656174696f6e3a206d757461626c65207065726d69747320706172746974696f6e206164646974696f6e2f64656c6574696f6e20776974686f7574206120626f756e646564206375746f766572206f72207065722d6b6579206f72646572696e67206163726f7373206c61796f7574733b2066697865642072656a6563747320626f74682e20526574656e74696f6e20616e6420746f7069632064656c6574696f6e2072656d61696e2064657374727563746976652e196d657373616765735f72657175697265645f746f5f736176650b000400000000040000870000004f7264696e617279206d6573736167652d636f756e7420666c75736820747269676765723b20312e2e3d31363737373231362e2052657175697265642070657273697374656e63652c2063617061636974792070726573737572652c20616e64206c6966656379636c65206f7065726174696f6e732063616e20666c757368206561726c6965722173697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c0008000000000010000000000083000000466c75736820746865206a6f75726e616c206f6e636520697420686f6c64732074686973206d616e792062797465732c206f72206120627974652d73697a6520737472696e673b20776869636865766572207468726573686f6c6420747269707320666972737420666c75736865732e204174206d6f7374203130373337343138323414707265616c6c6f636174655f7365676d656e7473030001000000001201000052657175657374207365676d656e745f73697a65206279746573206f66206469736b207265736572766174696f6e207768656e2065616368207365676d656e74206f70656e732e20556e737570706f72746564206f72206661696c6564207265736572766174696f6e732066616c6c206261636b20746f206f7264696e61727920616c6c6f636174696f6e20776974682061207761726e696e672e205265736572766174696f6e2072756e7320696e6c696e65206f6e20746865206f776e696e672073686172642e207365676d656e745f73697a65202a20706172746974696f6e735f636f756e74206973206361707065642061742036383731393437363733362062797465732070657220637265617465", + "entries": [ + { + "key": "name", + "kind": "string", + "flags": 1, + "default": null + }, + { + "key": "compression_algorithm", + "kind": "string", + "flags": 0, + "default": "none" + }, + { + "key": "message_expiry", + "kind": "uint64", + "flags": 3, + "default": "18446744073709551615" + }, + { + "key": "max_topic_size", + "kind": "uint64", + "flags": 3, + "default": "18446744073709551615" + }, + { + "key": "segment_size", + "kind": "uint64", + "flags": 0, + "default": "1073741824" + }, + { + "key": "durability", + "kind": "string", + "flags": 0, + "default": "replicated" + }, + { + "key": "consumer_offset_durability", + "kind": "string", + "flags": 0, + "default": "replicated" + }, + { + "key": "partition_resize_policy", + "kind": "string", + "flags": 0, + "default": "mutable" + }, + { + "key": "messages_required_to_save", + "kind": "uint32", + "flags": 0, + "default": "1024" + }, + { + "key": "size_of_messages_required_to_save", + "kind": "uint64", + "flags": 0, + "default": "1048576" + }, + { + "key": "preallocate_segments", + "kind": "bool", + "flags": 0, + "default": "false" + } + ] + }, + { + "scope": "stream", + "request_hex": "02", + "response_hex": "05000000046e616d650201000000007e0000005265736f75726365206e616d652c203120746f203235352062797465732c20756e697175652077697468696e2069747320706172656e742e20536574206279207468652063726561746520636f6d6d616e642773206e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e0e656e6372797074696f6e5f6b6579011400000000a30100005468652073747265616d2773206f776e206461746120656e6372797074696f6e206b65793a2033322072616e646f6d2062797465732c20726177206f72207374616e6461726420706164646564206261736536342e204372656174652d6f6e6c7920616e642077726974652d6f6e6c793a20746865207365727665722073746f726573206974207772617070656420756e6465722061206b657920646572697665642066726f6d207468652073657276657220656e6372797074696f6e206b657920616e64206e657665722072657475726e732069742e205265717569726573207468652073657276657220656e6372797074696f6e206b657920746f20626520636f6e666967757265642e20576974686f75742069742c206120736572766572207769746820656e6372797074696f6e20656e61626c65642067656e6572617465732061206b657920666f72207468652073747265616d2e20546865206b65792074726176656c7320696e20746865206372656174652072657175657374206f76657220746865207472616e73706f72742074686520636c69656e7420757365732e15656e6372797074696f6e5f6b65795f736f757263650208000000009f00000057686f2063686f7365207468652073747265616d27732061742d72657374206b65793a2073747265616d207768656e2074686520636c69656e742073656e742069742c20736572766572207768656e20746865207365727665722067656e6572617465642069742c206e6f6e65207768656e207468652073747265616d2773206d65737361676573206172652073746f72656420756e656e637279707465641a656e6372797074696f6e5f6b65795f66696e6765727072696e740208000000009f0000005468652066697273742038206279746573206f66205348412d323536206f76657220746865206c6162656c20696767792f73747265616d2d6b65792d66696e6765727072696e742f763120616e64207468652073747265616d2773206b65792c206173203136206c6f776572636173652068657820636861726163746572732e20416273656e74207768656e2074686520736f75726365206973206e6f6e651a656e6372797074696f6e5f6b656b5f66696e6765727072696e74020800000000b800000046696e6765727072696e7473206f6620746865206b65792d656e6372797074696f6e206b65797320746861742077726170207468652073747265616d2773206b65792c20636f6d6d612d7365706172617465643a206f6e65206f7574736964652061206b657920726f746174696f6e2c206f6e652070657220636f6e6669677572656420736572766572206b657920647572696e672069742e20416273656e74207768656e2074686520736f75726365206973206e6f6e65", + "entries": [ + { + "key": "name", + "kind": "string", + "flags": 1, + "default": null + }, + { + "key": "encryption_key", + "kind": "raw", + "flags": 20, + "default": null + }, + { + "key": "encryption_key_source", + "kind": "string", + "flags": 8, + "default": null + }, + { + "key": "encryption_key_fingerprint", + "kind": "string", + "flags": 8, + "default": null + }, + { + "key": "encryption_kek_fingerprint", + "kind": "string", + "flags": 8, + "default": null + } + ] + }, + { + "scope": "user", + "request_hex": "03", + "response_hex": "0200000008757365726e616d650201000000006b00000055736572206e616d652c203320746f2035302062797465732c20756e697175652e20536574206279207468652063726561746520636f6d6d616e64277320757365726e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e067374617475730201000000008f00000055736572207374617475732c20616374697665206f7220696e6163746976652e20536574206279207468652063726561746520636f6d6d616e64277320737461747573206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e20446561637469766174696e672061207573657220656e6473206974732073657373696f6e732e", + "entries": [ + { + "key": "username", + "kind": "string", + "flags": 1, + "default": null + }, + { + "key": "status", + "kind": "string", + "flags": 1, + "default": null + } + ] + } + ], + "stream_key": { + "key_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "key_base64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", + "fingerprint_label": "iggy/stream-key-fingerprint/v1", + "fingerprint": "8441e38538cbfa57", + "invalid_keys": [ + { + "case": "raw_31_bytes", + "input": { + "raw_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + }, + { + "case": "raw_33_bytes", + "input": { + "raw_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + }, + { + "case": "base64_with_whitespace", + "input": { + "base64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=\n" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + }, + { + "case": "base64_missing_padding", + "input": { + "base64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + }, + { + "case": "base64_noncanonical_trailing_bits", + "input": { + "base64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh9=" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + } + ] + }, + "update_resource": [ + { + "case": "topic_set_expiry_without_name_reset_max_topic_size", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "message_expiry": "604800000000", + "reset": [ + "max_topic_size" + ] + }, + "wire_set": { + "message_expiry": { + "value": "604800000000", + "kind": "uint64", + "explicit": true + } + }, + "wire_reset": [ + "max_topic_size" + ], + "body_hex": "01010401000000010402000000010e6d61785f746f7069635f73697a65020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000" + }, + { + "case": "topic_rename_only_named_identifiers", + "scope": "topic", + "target": { + "stream_id": "orders", + "topic_id": "events" + }, + "input": { + "name": "events-v2" + }, + "wire_set": { + "name": { + "value": "events-v2", + "kind": "string", + "explicit": true + } + }, + "wire_reset": [], + "body_hex": "0102066f726465727302066576656e74730002040000006e616d6502090000006576656e74732d7632" + }, + { + "case": "topic_rename_set_expiry_reset_max_topic_size", + "scope": "topic", + "target": { + "stream_id": 7, + "topic_id": "events" + }, + "input": { + "name": "events-v3", + "message_expiry": "86400000000", + "reset": [ + "max_topic_size" + ] + }, + "wire_set": { + "message_expiry": { + "value": "86400000000", + "kind": "uint64", + "explicit": true + }, + "name": { + "value": "events-v3", + "kind": "string", + "explicit": true + } + }, + "wire_reset": [ + "max_topic_size" + ], + "body_hex": "0101040700000002066576656e7473010e6d61785f746f7069635f73697a65020e0000006d6573736167655f6578706972790c080000000060d71d1400000002040000006e616d6502090000006576656e74732d7633" + }, + { + "case": "topic_reset_both_defaults", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "reset": [ + "max_topic_size", + "message_expiry" + ] + }, + "wire_set": {}, + "wire_reset": [ + "max_topic_size", + "message_expiry" + ], + "body_hex": "01010401000000010402000000020e6d61785f746f7069635f73697a650e6d6573736167655f657870697279" + }, + { + "case": "topic_max_topic_size_above_2_pow_53", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "max_topic_size": "9007199254740993" + }, + "wire_set": { + "max_topic_size": { + "value": "9007199254740993", + "kind": "uint64", + "explicit": true + } + }, + "wire_reset": [], + "body_hex": "0101040100000001040200000000020e0000006d61785f746f7069635f73697a650c080000000100000000002000" + }, + { + "case": "stream_rename", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "input": { + "name": "payments-v2" + }, + "wire_set": { + "name": { + "value": "payments-v2", + "kind": "string", + "explicit": true + } + }, + "wire_reset": [], + "body_hex": "020104030000000002040000006e616d65020b0000007061796d656e74732d7632" + }, + { + "case": "user_username_and_status", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "username": "alice2", + "status": "inactive" + }, + "wire_set": { + "status": { + "value": "inactive", + "kind": "string", + "explicit": true + }, + "username": { + "value": "alice2", + "kind": "string", + "explicit": true + } + }, + "wire_reset": [], + "body_hex": "030104050000000002060000007374617475730208000000696e6163746976650208000000757365726e616d650206000000616c69636532" + } + ], + "update_resource_errors": [ + { + "case": "topic_empty_update", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": {}, + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_set_and_reset_same_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "message_expiry": "604800000000", + "reset": [ + "message_expiry" + ] + }, + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_reset_create_only_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "reset": [ + "compression_algorithm" + ] + }, + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "stream_reset_name", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "input": { + "reset": [ + "name" + ] + }, + "error": { + "name": "option_not_resettable", + "code": 4104 + } + }, + { + "case": "user_reset_status", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "reset": [ + "status" + ] + }, + "error": { + "name": "option_not_resettable", + "code": 4104 + } + }, + { + "case": "stream_reset_read_only_key", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "input": { + "reset": [ + "encryption_key_source" + ] + }, + "error": { + "name": "read_only_option", + "code": 4105 + } + }, + { + "case": "user_reset_internal_key", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "reset": [ + "critical_keys" + ] + }, + "error": { + "name": "read_only_option", + "code": 4105 + } + }, + { + "case": "topic_empty_name", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "name": "" + }, + "error": { + "name": "invalid_topic_name", + "code": 2014 + } + }, + { + "case": "topic_empty_name_and_create_only_reset", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "name": "", + "reset": [ + "segment_size" + ] + }, + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "stream_empty_name", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "input": { + "name": "" + }, + "error": { + "name": "invalid_stream_name", + "code": 1013 + } + }, + { + "case": "user_short_username", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "username": "a" + }, + "error": { + "name": "invalid_username", + "code": 43 + } + }, + { + "case": "user_short_username_and_status_reset", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "username": "a", + "reset": [ + "status" + ] + }, + "error": { + "name": "option_not_resettable", + "code": 4104 + } + } + ], + "create_stream": [ + { + "case": "stream_without_key", + "input": { + "name": "orders" + }, + "wire_options": {}, + "body_hex": "066f7264657273", + "http_body": "{\"name\":\"orders\"}", + "admission_error": null + }, + { + "case": "stream_with_key", + "input": { + "name": "orders", + "encryption_key": { + "raw_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f" + } + }, + "wire_options": { + "encryption_key": { + "value": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", + "kind": "raw", + "explicit": true + } + }, + "body_hex": "066f7264657273020e000000656e6372797074696f6e5f6b65790120000000000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "http_body": "{\"name\":\"orders\",\"options\":{\"encryption_key\":\"AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=\"}}", + "admission_error": null + }, + { + "case": "stream_with_raw_extra_key", + "input": { + "name": "orders", + "raw": { + "future_option": "enabled" + } + }, + "wire_options": { + "future_option": { + "value": "enabled", + "kind": "string", + "explicit": true + } + }, + "body_hex": "066f7264657273020d0000006675747572655f6f7074696f6e0207000000656e61626c6564", + "http_body": "{\"name\":\"orders\",\"options\":{\"future_option\":\"enabled\"}}", + "admission_error": { + "name": "unsupported_option_key", + "code": 4041 + } + } + ], + "create_topic": [ + { + "case": "topic_with_options", + "input": { + "stream_id": 1, + "name": "events", + "partitions_count": 3, + "compression_algorithm": "gzip", + "message_expiry": "604800000000", + "max_topic_size": "9007199254740993", + "segment_size": "134217728", + "durability": "persisted", + "consumer_offset_durability": "replicated" + }, + "wire_options": { + "compression_algorithm": { + "value": "gzip", + "kind": "string", + "explicit": true + }, + "consumer_offset_durability": { + "value": "replicated", + "kind": "string", + "explicit": true + }, + "durability": { + "value": "persisted", + "kind": "string", + "explicit": true + }, + "max_topic_size": { + "value": "9007199254740993", + "kind": "uint64", + "explicit": true + }, + "message_expiry": { + "value": "604800000000", + "kind": "uint64", + "explicit": true + }, + "segment_size": { + "value": "134217728", + "kind": "uint64", + "explicit": true + } + }, + "body_hex": "01040100000003000000066576656e74730215000000636f6d7072657373696f6e5f616c676f726974686d0204000000677a6970021a000000636f6e73756d65725f6f66667365745f6475726162696c697479020a0000007265706c696361746564020a0000006475726162696c6974790209000000706572736973746564020e0000006d61785f746f7069635f73697a650c080000000100000000002000020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000020c0000007365676d656e745f73697a650c080000000000000800000000", + "http_body": "{\"partitions_count\":3,\"name\":\"events\",\"options\":{\"compression_algorithm\":\"gzip\",\"consumer_offset_durability\":\"replicated\",\"durability\":\"persisted\",\"max_topic_size\":\"9007199254740993\",\"message_expiry\":604800000000,\"segment_size\":134217728}}", + "admission_error": null + } + ], + "create_user": [ + { + "case": "user_with_empty_options", + "input": { + "username": "alice", + "password": "s3cret-pass", + "status": "active" + }, + "wire_options": {}, + "body_hex": "05616c6963650b7333637265742d706173730100", + "http_body": "{\"username\":\"alice\",\"password\":\"s3cret-pass\",\"status\":\"active\",\"permissions\":null}", + "admission_error": null + } + ], + "http_update": [ + { + "case": "topic_flat_keys", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"name\":\"events-v2\",\"message_expiry\":\"7 days\"}", + "update_resource_hex": "0101040100000001040200000000020e0000006d6573736167655f657870697279020600000037206461797302040000006e616d6502090000006576656e74732d7632", + "outcome": "accepted", + "error": null + }, + { + "case": "topic_reset_array", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":604800000000,\"reset\":[\"max_topic_size\"]}", + "update_resource_hex": "01010401000000010402000000010e6d61785f746f7069635f73697a65020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000", + "outcome": "accepted", + "error": null + }, + { + "case": "topic_null_ignored", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"name\":null,\"max_topic_size\":null,\"message_expiry\":\"1 day\"}", + "update_resource_hex": "0101040100000001040200000000020e0000006d6573736167655f65787069727902050000003120646179", + "outcome": "accepted", + "error": null + }, + { + "case": "topic_u64_above_2_pow_53_as_string", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"max_topic_size\":\"9007199254740993\"}", + "update_resource_hex": "0101040100000001040200000000020e0000006d61785f746f7069635f73697a65021000000039303037313939323534373430393933", + "outcome": "accepted", + "error": null + }, + { + "case": "stream_rename", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "body": "{\"name\":\"payments-v2\"}", + "update_resource_hex": "020104030000000002040000006e616d65020b0000007061796d656e74732d7632", + "outcome": "accepted", + "error": null + }, + { + "case": "user_username_and_status", + "scope": "user", + "target": { + "user_id": 5 + }, + "body": "{\"username\":\"alice2\",\"status\":\"inactive\"}", + "update_resource_hex": "030104050000000002060000007374617475730208000000696e6163746976650208000000757365726e616d650206000000616c69636532", + "outcome": "accepted", + "error": null + }, + { + "case": "topic_duplicate_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":\"1 day\",\"message_expiry\":\"2 days\"}", + "update_resource_hex": null, + "outcome": "json_refused", + "error": null + }, + { + "case": "topic_nested_object", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":{\"value\":\"1 day\"}}", + "update_resource_hex": null, + "outcome": "json_refused", + "error": null + }, + { + "case": "topic_null_only", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"max_topic_size\":null}", + "update_resource_hex": "0101040100000001040200000000", + "outcome": "decode_refused", + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_zero_sentinel_only", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":0}", + "update_resource_hex": "0101040100000001040200000000020e0000006d6573736167655f6578706972790c080000000000000000000000", + "outcome": "admission_refused", + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_reset_names_set_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":\"1 day\",\"reset\":[\"message_expiry\"]}", + "update_resource_hex": "01010401000000010402000000010e6d6573736167655f657870697279020e0000006d6573736167655f65787069727902050000003120646179", + "outcome": "decode_refused", + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_unknown_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"retention\":\"1 day\"}", + "update_resource_hex": "01010401000000010402000000000209000000726574656e74696f6e02050000003120646179", + "outcome": "admission_refused", + "error": { + "name": "unsupported_option_key", + "code": 4041 + } + }, + { + "case": "topic_set_create_only_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"compression_algorithm\":\"gzip\"}", + "update_resource_hex": "01010401000000010402000000000215000000636f6d7072657373696f6e5f616c676f726974686d0204000000677a6970", + "outcome": "admission_refused", + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "topic_reset_create_only_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"reset\":[\"compression_algorithm\"]}", + "update_resource_hex": "010104010000000104020000000115636f6d7072657373696f6e5f616c676f726974686d", + "outcome": "admission_refused", + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "stream_reset_name", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "body": "{\"reset\":[\"name\"]}", + "update_resource_hex": "0201040300000001046e616d65", + "outcome": "admission_refused", + "error": { + "name": "option_not_resettable", + "code": 4104 + } + }, + { + "case": "topic_name_as_number", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"name\":5}", + "update_resource_hex": "010104010000000104020000000002040000006e616d650c080000000500000000000000", + "outcome": "admission_refused", + "error": { + "name": "invalid_topic_name", + "code": 2014 + } + }, + { + "case": "stream_empty_name", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "body": "{\"name\":\"\"}", + "update_resource_hex": "020104030000000002040000006e616d65030100000000", + "outcome": "admission_refused", + "error": { + "name": "invalid_stream_name", + "code": 1013 + } + }, + { + "case": "user_status_as_bool", + "scope": "user", + "target": { + "user_id": 5 + }, + "body": "{\"status\":true}", + "update_resource_hex": "03010405000000000206000000737461747573030100000001", + "outcome": "admission_refused", + "error": { + "name": "invalid_user_status", + "code": 45 + } + }, + { + "case": "topic_create_only_key_as_bool", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"segment_size\":true}", + "update_resource_hex": "0101040100000001040200000000020c0000007365676d656e745f73697a65030100000001", + "outcome": "admission_refused", + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "stream_read_only_key_as_number", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "body": "{\"encryption_key_source\":1}", + "update_resource_hex": "02010403000000000215000000656e6372797074696f6e5f6b65795f736f757263650c080000000100000000000000", + "outcome": "admission_refused", + "error": { + "name": "read_only_option", + "code": 4105 + } + }, + { + "case": "topic_invalid_value", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":\"soon\"}", + "update_resource_hex": "0101040100000001040200000000020e0000006d6573736167655f6578706972790204000000736f6f6e", + "outcome": "admission_refused", + "error": { + "name": "invalid_option_value", + "code": 4042 + } + } + ], + "stream_responses": [ + { + "case": "stream_source_none", + "state": { + "encryption_key_source": "none", + "stored_extra": {} + }, + "response_hex": "0100000000e090663c1306000000000000100000000000000c000000000000000c706c61696e2d73747265616d00000000230000000215000000656e6372797074696f6e5f6b65795f736f7572636502040000006e6f6e65", + "expected": { + "id": 1, + "created_at": 1710000000000000, + "name": "plain-stream", + "topics_count": 0, + "size_bytes": 4096, + "messages_count": 12, + "options": { + "encryption_key_source": { + "value": "none", + "kind": "string", + "explicit": false + } + }, + "encryption_key_source": "none", + "encryption_key_fingerprint": null, + "encryption_kek_fingerprints": [] + } + }, + { + "case": "stream_source_stream_with_unknown_explicit_key", + "state": { + "encryption_key_source": "stream", + "data_key": { + "raw_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f" + }, + "kek_config_keys": [ + "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=" + ], + "stored_extra": { + "future_option": { + "value": "enabled", + "kind": "string", + "explicit": true + } + } + }, + "response_hex": "0200000000e090663c1306000000000000100000000000000c0000000000000011636c69656e742d6b65792d73747265616d1e000000020d0000006675747572655f6f7074696f6e0207000000656e61626c65648d000000021a000000656e6372797074696f6e5f6b656b5f66696e6765727072696e74021000000033396534663066393563333639336337021a000000656e6372797074696f6e5f6b65795f66696e6765727072696e740210000000383434316533383533386362666135370215000000656e6372797074696f6e5f6b65795f736f75726365020600000073747265616d", + "expected": { + "id": 2, + "created_at": 1710000000000000, + "name": "client-key-stream", + "topics_count": 0, + "size_bytes": 4096, + "messages_count": 12, + "options": { + "encryption_kek_fingerprint": { + "value": "39e4f0f95c3693c7", + "kind": "string", + "explicit": false + }, + "encryption_key_fingerprint": { + "value": "8441e38538cbfa57", + "kind": "string", + "explicit": false + }, + "encryption_key_source": { + "value": "stream", + "kind": "string", + "explicit": false + }, + "future_option": { + "value": "enabled", + "kind": "string", + "explicit": true + } + }, + "encryption_key_source": "stream", + "encryption_key_fingerprint": "8441e38538cbfa57", + "encryption_kek_fingerprints": [ + "39e4f0f95c3693c7" + ] + } + }, + { + "case": "stream_source_server_two_keks", + "state": { + "encryption_key_source": "server", + "data_key": { + "raw_hex": "202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f" + }, + "kek_config_keys": [ + "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=", + "AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA=" + ], + "stored_extra": {} + }, + "response_hex": "0300000000e090663c1306000000000000100000000000000c00000000000000117365727665722d6b65792d73747265616d000000009e000000021a000000656e6372797074696f6e5f6b656b5f66696e6765727072696e740221000000333965346630663935633336393363372c64353564343739383239626332393336021a000000656e6372797074696f6e5f6b65795f66696e6765727072696e740210000000306563616666316261323931666530640215000000656e6372797074696f6e5f6b65795f736f757263650206000000736572766572", + "expected": { + "id": 3, + "created_at": 1710000000000000, + "name": "server-key-stream", + "topics_count": 0, + "size_bytes": 4096, + "messages_count": 12, + "options": { + "encryption_kek_fingerprint": { + "value": "39e4f0f95c3693c7,d55d479829bc2936", + "kind": "string", + "explicit": false + }, + "encryption_key_fingerprint": { + "value": "0ecaff1ba291fe0d", + "kind": "string", + "explicit": false + }, + "encryption_key_source": { + "value": "server", + "kind": "string", + "explicit": false + } + }, + "encryption_key_source": "server", + "encryption_key_fingerprint": "0ecaff1ba291fe0d", + "encryption_kek_fingerprints": [ + "39e4f0f95c3693c7", + "d55d479829bc2936" + ] + } + } + ], + "topic_responses": [ + { + "case": "topic_explicit_and_derived", + "response_hex": "0400000000e090663c1306000000000000080000000000000700000000000000066576656e7473430000000215000000636f6d7072657373696f6e5f616c676f726974686d0204000000677a6970020e0000006d6573736167655f6578706972790c0800000000a0e3d08c0000002b010000021a000000636f6e73756d65725f6f66667365745f6475726162696c697479020a0000007265706c696361746564020a0000006475726162696c697479020a0000007265706c696361746564020e0000006d61785f746f7069635f73697a650c08000000ffffffffffffffff02190000006d657373616765735f72657175697265645f746f5f736176650b04000000000400000217000000706172746974696f6e5f726573697a655f706f6c69637902070000006d757461626c650214000000707265616c6c6f636174655f7365676d656e7473030100000000020c0000007365676d656e745f73697a650c080000000000004000000000022100000073697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c080000000000100000000000", + "expected": { + "id": 4, + "created_at": 1710000000000000, + "name": "events", + "partitions_count": 0, + "size_bytes": 2048, + "messages_count": 7, + "options": { + "compression_algorithm": { + "value": "gzip", + "kind": "string", + "explicit": true + }, + "consumer_offset_durability": { + "value": "replicated", + "kind": "string", + "explicit": false + }, + "durability": { + "value": "replicated", + "kind": "string", + "explicit": false + }, + "max_topic_size": { + "value": "18446744073709551615", + "kind": "uint64", + "explicit": false + }, + "message_expiry": { + "value": "604800000000", + "kind": "uint64", + "explicit": true + }, + "messages_required_to_save": { + "value": "1024", + "kind": "uint32", + "explicit": false + }, + "partition_resize_policy": { + "value": "mutable", + "kind": "string", + "explicit": false + }, + "preallocate_segments": { + "value": "false", + "kind": "bool", + "explicit": false + }, + "segment_size": { + "value": "1073741824", + "kind": "uint64", + "explicit": false + }, + "size_of_messages_required_to_save": { + "value": "1048576", + "kind": "uint64", + "explicit": false + } + }, + "compression_algorithm": "gzip", + "message_expiry": "604800000000", + "max_topic_size": "18446744073709551615" + } + } + ], + "user_responses": [ + { + "case": "user_empty_blocks", + "response_hex": "0500000000e090663c1306000105616c696365000000000000000000000000", + "expected": { + "id": 5, + "created_at": 1710000000000000, + "status": "active", + "username": "alice", + "options": {} + } + }, + { + "case": "user_populated_blocks", + "response_hex": "0600000000e090663c1306000203626f621e000000020d0000006675747572655f6f7074696f6e0207000000656e61626c65641e000000020c0000006675747572655f71756f74610c08000000010000000000200000000000", + "expected": { + "id": 6, + "created_at": 1710000000000000, + "status": "inactive", + "username": "bob", + "options": { + "future_option": { + "value": "enabled", + "kind": "string", + "explicit": true + }, + "future_quota": { + "value": "9007199254740993", + "kind": "uint64", + "explicit": false + } + } + } + } + ] +}
diff --git a/foreign/cpp/tests/unit/resource_options.cpp b/foreign/cpp/tests/unit/resource_options.cpp new file mode 100644 index 0000000..73f1e0f --- /dev/null +++ b/foreign/cpp/tests/unit/resource_options.cpp
@@ -0,0 +1,355 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <fstream> +#include <functional> +#include <map> +#include <set> +#include <stdexcept> +#include <string> +#include <utility> +#include <vector> + +#include <gtest/gtest.h> +#include <nlohmann/json.hpp> + +#include "iggy.hpp" + +namespace { + +constexpr char kFixturePath[] = "tests/fixtures/resource_options.json"; + +using SurfaceCheck = std::function<bool()>; +using Surface = std::map<std::string, SurfaceCheck>; + +struct ScopeSurface { + Surface create; + Surface update; + Surface reset; + Surface read_only; +}; + +const nlohmann::json &Fixture() { + static const nlohmann::json fixture = [] { + std::ifstream file(kFixturePath); + if (!file) { + throw std::runtime_error(std::string("Cannot open ") + kFixturePath); + } + return nlohmann::json::parse(file); + }(); + return fixture; +} + +std::vector<std::uint8_t> FromHex(const std::string &hex) { + std::vector<std::uint8_t> bytes; + bytes.reserve(hex.size() / 2); + for (std::size_t index = 0; index + 1 < hex.size(); index += 2) { + bytes.push_back(static_cast<std::uint8_t>(std::stoul(hex.substr(index, 2), nullptr, 16))); + } + return bytes; +} + +// Walks the enum from zero until to_string refuses a value, so a member added +// to the public enum is checked without a second list here. +template <typename Key> +std::vector<std::string> PublicKeyNames() { + std::vector<std::string> names; + for (unsigned value = 0; value <= 255; ++value) { + try { + names.emplace_back(iggy::to_string(static_cast<Key>(value))); + } catch (const std::invalid_argument &) { + break; + } + } + return names; +} + +std::vector<std::string> DescribedKeys(const std::string &scope) { + std::vector<std::string> keys; + for (const auto &row : Fixture().at("catalog")) { + const auto &scopes = row.at("scopes"); + if (row.at("described").get<bool>() && + std::find(scopes.begin(), scopes.end(), nlohmann::json(scope)) != scopes.end()) { + keys.push_back(row.at("key").get<std::string>()); + } + } + return keys; +} + +std::set<std::string> CatalogKeys(const std::string &scope, const std::function<bool(const nlohmann::json &)> &filter) { + std::set<std::string> keys; + for (const auto &key : DescribedKeys(scope)) { + for (const auto &row : Fixture().at("catalog")) { + if (row.at("key") == key && filter(row)) { + keys.insert(key); + } + } + } + return keys; +} + +std::set<std::string> SurfaceKeys(const Surface &surface) { + std::set<std::string> keys; + for (const auto &[key, check] : surface) { + EXPECT_TRUE(check()) << key; + keys.insert(key); + } + return keys; +} + +template <typename Options, typename Key> +SurfaceCheck ResetMember(Key key) { + return [key] { + Options options; + options.Reset(key); + return options.ResetKeys().count(key) == 1; + }; +} + +std::map<std::string, ScopeSurface> TypedSurface() { + using iggy::StreamCreateOptions; + using iggy::StreamUpdateOptions; + using iggy::TopicCreateOptions; + using iggy::TopicUpdateOptions; + using iggy::UserUpdateOptions; + return { + {"topic", + {{ + {"compression_algorithm", + [] { + return TopicCreateOptions() + .SetCompressionAlgorithm(iggy::CompressionAlgorithm::Gzip()) + .CompressionAlgorithm() + .has_value(); + }}, + {"message_expiry", + [] { + return TopicCreateOptions() + .SetMessageExpiry(iggy::Expiry::NeverExpire()) + .MessageExpiry() + .has_value(); + }}, + {"max_topic_size", + [] { + return TopicCreateOptions() + .SetMaxTopicSize(iggy::MaxTopicSize::Unlimited()) + .MaxTopicSize() + .has_value(); + }}, + {"segment_size", [] { return TopicCreateOptions().SetSegmentSize(1048576).SegmentSize().has_value(); }}, + {"durability", + [] { return TopicCreateOptions().SetDurability(iggy::Durability::Persisted).Durability().has_value(); }}, + {"consumer_offset_durability", + [] { + return TopicCreateOptions() + .SetConsumerOffsetDurability(iggy::Durability::Persisted) + .ConsumerOffsetDurability() + .has_value(); + }}, + {"partition_resize_policy", + [] { + return TopicCreateOptions() + .SetPartitionResizePolicy(iggy::PartitionResizePolicy::Fixed) + .PartitionResizePolicy() + .has_value(); + }}, + {"messages_required_to_save", + [] { return TopicCreateOptions().SetMessagesRequiredToSave(1).MessagesRequiredToSave().has_value(); }}, + {"size_of_messages_required_to_save", + [] { + return TopicCreateOptions() + .SetSizeOfMessagesRequiredToSave(1) + .SizeOfMessagesRequiredToSave() + .has_value(); + }}, + {"preallocate_segments", + [] { return TopicCreateOptions().SetPreallocateSegments(true).PreallocateSegments().has_value(); }}, + }, + { + {"name", [] { return TopicUpdateOptions().SetName("renamed").Name().has_value(); }}, + {"message_expiry", + [] { + return TopicUpdateOptions() + .SetMessageExpiry(iggy::Expiry::NeverExpire()) + .MessageExpiry() + .has_value(); + }}, + {"max_topic_size", + [] { + return TopicUpdateOptions() + .SetMaxTopicSize(iggy::MaxTopicSize::Unlimited()) + .MaxTopicSize() + .has_value(); + }}, + }, + { + {"message_expiry", ResetMember<TopicUpdateOptions>(iggy::TopicOptionKey::MessageExpiry)}, + {"max_topic_size", ResetMember<TopicUpdateOptions>(iggy::TopicOptionKey::MaxTopicSize)}, + }, + {}}}, + {"stream", + {{ + {"encryption_key", + [] { + return StreamCreateOptions().SetEncryptionKey(std::vector<std::uint8_t>(32, 7)).HasEncryptionKey(); + }}, + }, + { + {"name", [] { return StreamUpdateOptions().SetName("renamed").Name().has_value(); }}, + }, + {}, + { + {"encryption_key_source", + [] { + static_cast<void>(&iggy::StreamDetails::EncryptionKeySource); + return true; + }}, + {"encryption_key_fingerprint", + [] { + static_cast<void>(&iggy::StreamDetails::EncryptionKeyFingerprint); + return true; + }}, + {"encryption_kek_fingerprint", + [] { + static_cast<void>(&iggy::StreamDetails::EncryptionKekFingerprints); + return true; + }}, + }}}, + {"user", + {{}, + { + {"username", [] { return UserUpdateOptions().SetUsername("renamed").Username().has_value(); }}, + {"status", [] { return UserUpdateOptions().SetStatus(iggy::UserStatus::Inactive).Status().has_value(); }}, + }, + {}, + {}}}, + }; +} + +template <typename Operation> +::testing::AssertionResult ThrowsIggyError(Operation &&operation, const nlohmann::json &error) { + const auto code = error.at("code").get<std::uint32_t>(); + const auto kind = error.at("name").get<std::string>(); + try { + std::forward<Operation>(operation)(); + } catch (const iggy::IggyException &exception) { + if (exception.Code() == code && exception.Kind() == kind) { + return ::testing::AssertionSuccess(); + } + return ::testing::AssertionFailure() << "expected " << kind << " (" << code << "), got '" << exception.what() + << "' with kind '" << exception.Kind() << "'"; + } + return ::testing::AssertionFailure() << "expected " << kind << " (" << code << "), nothing was thrown"; +} + +} // namespace + +TEST(ResourceOptionsFixtureTest, GivenFixtureKeyWhenFingerprintingShouldMatchSharedVector) { + const auto &stream_key = Fixture().at("stream_key"); + const auto expected = stream_key.at("fingerprint").get<std::string>(); + + EXPECT_EQ(iggy::EncryptionKeyFingerprint(FromHex(stream_key.at("key_hex").get<std::string>())), expected); + EXPECT_EQ(iggy::EncryptionKeyFingerprint(stream_key.at("key_base64").get<std::string>()), expected); +} + +TEST(ResourceOptionsFixtureTest, GivenInvalidKeyVectorsWhenFingerprintingShouldRefuseWithoutKeyMaterial) { + const auto &stream_key = Fixture().at("stream_key"); + const auto key_base64 = stream_key.at("key_base64").get<std::string>(); + for (const auto &vector : stream_key.at("invalid_keys")) { + SCOPED_TRACE(vector.at("case").get<std::string>()); + const auto &input = vector.at("input"); + if (input.contains("raw_hex")) { + const auto raw = FromHex(input.at("raw_hex").get<std::string>()); + EXPECT_TRUE(ThrowsIggyError([&raw] { iggy::EncryptionKeyFingerprint(raw); }, vector.at("error"))); + } else { + const auto text = input.at("base64").get<std::string>(); + EXPECT_TRUE(ThrowsIggyError([&text] { iggy::EncryptionKeyFingerprint(text); }, vector.at("error"))); + try { + iggy::EncryptionKeyFingerprint(text); + } catch (const iggy::IggyException &error) { + EXPECT_EQ(std::string(error.what()).find(key_base64.substr(0, 40)), std::string::npos); + } + } + } +} + +TEST(ResourceOptionsFixtureTest, GivenFixtureCatalogWhenListingPublicKeyEnumsShouldMatchDescribedKeysInOrder) { + EXPECT_EQ(PublicKeyNames<iggy::StreamOptionKey>(), DescribedKeys("stream")); + EXPECT_EQ(PublicKeyNames<iggy::TopicOptionKey>(), DescribedKeys("topic")); + EXPECT_EQ(PublicKeyNames<iggy::UserOptionKey>(), DescribedKeys("user")); +} + +TEST(ResourceOptionsFixtureTest, GivenFixtureCatalogWhenComparingTypedSurfaceShouldCoverEveryKeyByMutability) { + const auto flags = Fixture().at("constants").at("option_flags"); + const auto resettable = flags.at("resettable").get<std::uint32_t>(); + for (const auto &[scope, surface] : TypedSurface()) { + SCOPED_TRACE(scope); + EXPECT_EQ(SurfaceKeys(surface.create), CatalogKeys(scope, [](const nlohmann::json &row) { + return row.at("storage") == "map" && + (row.at("mutability") == "create_only" || row.at("mutability") == "updatable"); + })); + EXPECT_EQ(SurfaceKeys(surface.update), + CatalogKeys(scope, [](const nlohmann::json &row) { return row.at("mutability") == "updatable"; })); + EXPECT_EQ(SurfaceKeys(surface.reset), CatalogKeys(scope, [resettable](const nlohmann::json &row) { + return (row.at("flags").get<std::uint32_t>() & resettable) != 0; + })); + EXPECT_EQ(SurfaceKeys(surface.read_only), + CatalogKeys(scope, [](const nlohmann::json &row) { return row.at("mutability") == "read_only"; })); + } +} + +TEST(ResourceOptionsFixtureTest, GivenRawResetNamesWhenBuildingUpdatesShouldKeepThemBesideTypedResets) { + iggy::TopicUpdateOptions topic; + topic.Reset(iggy::TopicOptionKey::MaxTopicSize).RawReset("future_key"); + EXPECT_EQ(topic.ResetKeys(), std::set<iggy::TopicOptionKey>{iggy::TopicOptionKey::MaxTopicSize}); + EXPECT_EQ(topic.RawResetKeys(), std::set<std::string>{"future_key"}); + + iggy::StreamUpdateOptions stream; + EXPECT_TRUE(stream.RawReset("future_key").ResetKeys().empty()); + EXPECT_EQ(stream.RawResetKeys(), std::set<std::string>{"future_key"}); + + iggy::UserUpdateOptions user; + EXPECT_EQ(user.RawReset("future_key").RawResetKeys(), std::set<std::string>{"future_key"}); +} + +TEST(ResourceOptionsFixtureTest, GivenReadmeSampleWhenCompilingShouldMatchThePublicApi) { + // Never invoked, it only has to compile, so the README sample cannot drift from the API. + const auto sample = [](iggy::IggyBlockingClient &client, const std::function<std::string()> &LoadStreamKey) { + const std::string key_base64 = LoadStreamKey(); + const auto stream = + client.CreateStream("payments", iggy::StreamCreateOptions().SetEncryptionKeyBase64(key_base64)); + const bool same_key = stream.EncryptionKeyFingerprint() == iggy::EncryptionKeyFingerprint(key_base64); + + const auto stream_id = iggy::Identifier::Numeric(stream.Id()); + const auto topic = client.CreateTopic(stream_id, "orders", + iggy::TopicCreateOptions() + .SetPartitionsCount(3) + .SetDurability(iggy::Durability::Persisted) + .SetPartitionResizePolicy(iggy::PartitionResizePolicy::Fixed)); + client.UpdateTopic(stream_id, iggy::Identifier::Numeric(topic.Id()), + iggy::TopicUpdateOptions() + .SetMessageExpiry(iggy::Expiry::Duration(3600000000)) + .Reset(iggy::TopicOptionKey::MaxTopicSize)); + return same_key; + }; + static_cast<void>(sample); +}
diff --git a/foreign/cpp/tests/unit/unit_tests.cpp b/foreign/cpp/tests/unit/unit_tests.cpp index 60e21f4..c97b193 100644 --- a/foreign/cpp/tests/unit/unit_tests.cpp +++ b/foreign/cpp/tests/unit/unit_tests.cpp
@@ -17,10 +17,12 @@ * under the License. */ +#include <cstddef> #include <cstdint> #include <limits> #include <map> #include <optional> +#include <set> #include <string> #include <type_traits> #include <utility> @@ -274,19 +276,19 @@ TEST(TopicUpdateOptionsTest, DefaultHasNoValues) { const iggy::TopicUpdateOptions options; - EXPECT_FALSE(options.CompressionAlgorithm().has_value()); + EXPECT_FALSE(options.Name().has_value()); EXPECT_FALSE(options.MessageExpiry().has_value()); EXPECT_FALSE(options.MaxTopicSize().has_value()); EXPECT_TRUE(options.RawEntries().empty()); + EXPECT_TRUE(options.ResetKeys().empty()); } TEST(TopicUpdateOptionsTest, StoresUpdatableFields) { iggy::TopicUpdateOptions options; - options.SetCompressionAlgorithm(iggy::CompressionAlgorithm::Gzip()) + options.SetName("renamed") .SetMessageExpiry(iggy::Expiry::NeverExpire()) .SetMaxTopicSize(iggy::MaxTopicSize::Unlimited()); - ASSERT_TRUE(options.CompressionAlgorithm().has_value()); - EXPECT_EQ(options.CompressionAlgorithm()->Value(), "gzip"); + EXPECT_EQ(options.Name(), std::optional<std::string>("renamed")); ASSERT_TRUE(options.MessageExpiry().has_value()); EXPECT_EQ(options.MessageExpiry()->Kind(), "never_expire"); ASSERT_TRUE(options.MaxTopicSize().has_value()); @@ -327,6 +329,77 @@ EXPECT_EQ(options.RawEntries().at("message_expiry"), "1 day"); } +TEST(TopicUpdateOptionsTest, ResetCollectsTypedKeysOnce) { + iggy::TopicUpdateOptions options; + options.Reset(iggy::TopicOptionKey::MaxTopicSize) + .Reset(iggy::TopicOptionKey::MessageExpiry) + .Reset(iggy::TopicOptionKey::MaxTopicSize); + EXPECT_EQ(options.ResetKeys(), (std::set<iggy::TopicOptionKey>{iggy::TopicOptionKey::MessageExpiry, + iggy::TopicOptionKey::MaxTopicSize})); +} + +TEST(OptionKeyTest, ReturnsCatalogNames) { + EXPECT_EQ(iggy::to_string(iggy::StreamOptionKey::Name), "name"); + EXPECT_EQ(iggy::to_string(iggy::StreamOptionKey::EncryptionKey), "encryption_key"); + EXPECT_EQ(iggy::to_string(iggy::StreamOptionKey::EncryptionKekFingerprint), "encryption_kek_fingerprint"); + EXPECT_EQ(iggy::to_string(iggy::TopicOptionKey::SizeOfMessagesRequiredToSave), "size_of_messages_required_to_save"); + EXPECT_EQ(iggy::to_string(iggy::UserOptionKey::Status), "status"); + EXPECT_EQ(iggy::to_string(iggy::OptionsScope::User), "user"); + EXPECT_EQ(iggy::to_string(iggy::StreamEncryptionSource::Server), "server"); +} + +TEST(StreamCreateOptionsTest, LastKeySetterWins) { + iggy::StreamCreateOptions options; + EXPECT_FALSE(options.HasEncryptionKey()); + options.SetEncryptionKey(std::vector<std::uint8_t>(32, 7)); + EXPECT_TRUE(options.HasEncryptionKey()); + options.SetEncryptionKeyBase64("BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc="); + EXPECT_TRUE(options.HasEncryptionKey()); + options.SetRawEntries({{"future_key", "future_value"}}); + EXPECT_EQ(options.RawEntries().at("future_key"), "future_value"); +} + +TEST(StreamUpdateOptionsTest, StoresNameAndReset) { + iggy::StreamUpdateOptions options; + EXPECT_FALSE(options.Name().has_value()); + options.SetName("renamed").Reset(iggy::StreamOptionKey::EncryptionKey); + EXPECT_EQ(options.Name(), std::optional<std::string>("renamed")); + EXPECT_EQ(options.ResetKeys(), (std::set<iggy::StreamOptionKey>{iggy::StreamOptionKey::EncryptionKey})); +} + +TEST(UserUpdateOptionsTest, StoresUsernameStatusAndReset) { + iggy::UserUpdateOptions options; + EXPECT_FALSE(options.Username().has_value()); + EXPECT_FALSE(options.Status().has_value()); + options.SetUsername("renamed").SetStatus(iggy::UserStatus::Inactive).Reset(iggy::UserOptionKey::Status); + EXPECT_EQ(options.Username(), std::optional<std::string>("renamed")); + EXPECT_EQ(options.Status(), std::optional<iggy::UserStatus>(iggy::UserStatus::Inactive)); + EXPECT_EQ(options.ResetKeys(), (std::set<iggy::UserOptionKey>{iggy::UserOptionKey::Status})); +} + +TEST(UserCreateOptionsTest, RawMapStoresKeys) { + iggy::UserCreateOptions options; + EXPECT_TRUE(options.RawEntries().empty()); + options.SetRawEntries({{"future_key", "future_value"}}); + EXPECT_EQ(options.RawEntries().at("future_key"), "future_value"); +} + +TEST(EncryptionKeyFingerprintTest, AcceptsOnlyThirtyTwoBytes) { + const auto fingerprint = iggy::EncryptionKeyFingerprint(std::vector<std::uint8_t>(32, 7)); + EXPECT_EQ(fingerprint.size(), 16u); + EXPECT_EQ(fingerprint, iggy::EncryptionKeyFingerprint(std::vector<std::uint8_t>(32, 7))); + for (const std::size_t length : {std::size_t{0}, std::size_t{31}, std::size_t{33}}) { + SCOPED_TRACE(length); + try { + (void)iggy::EncryptionKeyFingerprint(std::vector<std::uint8_t>(length, 7)); + ADD_FAILURE() << "A key that is not 32 bytes must be refused"; + } catch (const iggy::IggyException &error) { + EXPECT_EQ(error.Kind(), "invalid_encryption_key"); + EXPECT_EQ(error.Code(), std::optional<std::uint32_t>(70)); + } + } +} + TEST(StreamUpdateOptionsTest, RawMapStoresKeys) { iggy::StreamUpdateOptions options; EXPECT_TRUE(options.RawEntries().empty()); @@ -457,16 +530,19 @@ EXPECT_THROW((void)client.GetUsers(), iggy::IggyException); EXPECT_THROW((void)client.CreateUser("user", "secret123", iggy::UserStatus::Active), iggy::IggyException); EXPECT_THROW(client.DeleteUser(stream), iggy::IggyException); - EXPECT_THROW(client.UpdateUser(stream, "updated-user", iggy::UserStatus::Active, iggy::UserUpdateOptions{}), + EXPECT_THROW(client.UpdateUser( + stream, iggy::UserUpdateOptions().SetUsername("updated-user").SetStatus(iggy::UserStatus::Active)), iggy::IggyException); EXPECT_THROW(client.CreateStream("stream"), iggy::IggyException); - EXPECT_THROW(client.UpdateStream(stream, "updated-stream"), iggy::IggyException); + EXPECT_THROW(client.UpdateStream(stream, iggy::StreamUpdateOptions().SetName("updated-stream")), + iggy::IggyException); EXPECT_THROW(client.GetStreams(), iggy::IggyException); EXPECT_THROW(client.GetStream(stream), iggy::IggyException); EXPECT_THROW(client.DeleteStream(stream), iggy::IggyException); EXPECT_THROW(client.CreateTopic(stream, "topic", iggy::TopicCreateOptions().SetPartitionsCount(1)), iggy::IggyException); - EXPECT_THROW(client.UpdateTopic(stream, topic, "updated-topic"), iggy::IggyException); + EXPECT_THROW(client.UpdateTopic(stream, topic, iggy::TopicUpdateOptions().SetName("updated-topic")), + iggy::IggyException); EXPECT_THROW(client.GetTopics(stream), iggy::IggyException); EXPECT_THROW(client.GetTopic(stream, topic), iggy::IggyException); EXPECT_THROW(client.DeleteTopic(stream, topic), iggy::IggyException); @@ -484,6 +560,7 @@ iggy::IggyException); EXPECT_THROW(client.GetConsumerOffset(consumer, stream, topic, 0), iggy::IggyException); EXPECT_THROW(client.DeleteConsumerOffset(consumer, stream, topic, 0), iggy::IggyException); + EXPECT_THROW((void)client.DescribeOptions(iggy::OptionsScope::Stream), iggy::IggyException); EXPECT_THROW((void)client.GetStats(), iggy::IggyException); EXPECT_THROW((void)client.GetMe(), iggy::IggyException); EXPECT_THROW((void)client.GetClient(1), iggy::IggyException);
diff --git a/foreign/csharp/Benchmarks/Program.cs b/foreign/csharp/Benchmarks/Program.cs index b3e7a42..f093647 100644 --- a/foreign/csharp/Benchmarks/Program.cs +++ b/foreign/csharp/Benchmarks/Program.cs
@@ -18,6 +18,7 @@ using Apache.Iggy; using Apache.Iggy.Benchmarks; using Apache.Iggy.Configuration; +using Apache.Iggy.Contracts; using Apache.Iggy.Enums; using Apache.Iggy.Factory; using Apache.Iggy.IggyClient; @@ -58,10 +59,8 @@ await clients[0].CreateTopicAsync(Identifier.Numeric(startingStreamId + i), $"Test bench topic_{i}", - compressionAlgorithm: CompressionAlgorithm.None, - messageExpiry: TimeSpan.Zero, - maxTopicSize: 2_000_000_000, - partitionsCount: 1); + partitionsCount: 1, + options: new TopicOptions { MaxTopicSize = 2_000_000_000 }); } } catch
diff --git a/foreign/csharp/Iggy_SDK.Tests.BDD/Iggy_SDK.Tests.BDD.csproj b/foreign/csharp/Iggy_SDK.Tests.BDD/Iggy_SDK.Tests.BDD.csproj index e52a301..aa5a38b 100644 --- a/foreign/csharp/Iggy_SDK.Tests.BDD/Iggy_SDK.Tests.BDD.csproj +++ b/foreign/csharp/Iggy_SDK.Tests.BDD/Iggy_SDK.Tests.BDD.csproj
@@ -54,6 +54,10 @@ </ItemGroup> <ItemGroup> + <Content Include="xunit.runner.json" CopyToOutputDirectory="PreserveNewest" /> + </ItemGroup> + + <ItemGroup> <ReqnrollFeatureFile Include="..\..\..\bdd\scenarios\basic_messaging.feature"> <Link>Features\basic_messaging.feature</Link> </ReqnrollFeatureFile> @@ -63,6 +67,12 @@ <ReqnrollFeatureFile Include="..\..\..\bdd\scenarios\raw_command.feature"> <Link>Features\raw_command.feature</Link> </ReqnrollFeatureFile> + <ReqnrollFeatureFile Include="..\..\..\bdd\scenarios\stream_options.feature"> + <Link>Features\stream_options.feature</Link> + </ReqnrollFeatureFile> + <ReqnrollFeatureFile Include="..\..\..\bdd\scenarios\resource_options_patch.feature"> + <Link>Features\resource_options_patch.feature</Link> + </ReqnrollFeatureFile> <ReqnrollFeatureFile Include="..\..\..\bdd\scenarios\session_contract.feature"> <Link>Features\session_contract.feature</Link> </ReqnrollFeatureFile>
diff --git a/foreign/csharp/Iggy_SDK.Tests.BDD/StepDefinitions/LeaderRedirectionSteps.cs b/foreign/csharp/Iggy_SDK.Tests.BDD/StepDefinitions/LeaderRedirectionSteps.cs index be3dece..92b9002 100644 --- a/foreign/csharp/Iggy_SDK.Tests.BDD/StepDefinitions/LeaderRedirectionSteps.cs +++ b/foreign/csharp/Iggy_SDK.Tests.BDD/StepDefinitions/LeaderRedirectionSteps.cs
@@ -33,12 +33,23 @@ public class LeaderRedirectionSteps { private readonly TestContext _context; + private IIggyClient? _streamClient; public LeaderRedirectionSteps(TestContext context) { _context = context; } + // The single-server scenario shares its server with the other features, and this runs before the clients close. + [AfterScenario(Order = 0)] + public async Task DeleteCreatedStream() + { + if (_streamClient is not null && _context.LastStreamId is { } streamId) + { + await _streamClient.DeleteStreamAsync(Identifier.Numeric(streamId)); + } + } + // ---------- Background ---------- [Given(@"I have cluster configuration enabled with (\d+) nodes")] @@ -162,6 +173,7 @@ var client = GetClient("main"); var stream = await client.CreateStreamAsync(streamName); stream.ShouldNotBeNull("Should be able to create stream"); + _streamClient = client; _context.LastStreamId = stream.Id; }
diff --git a/foreign/csharp/Iggy_SDK.Tests.BDD/StepDefinitions/ResourceOptionsSteps.cs b/foreign/csharp/Iggy_SDK.Tests.BDD/StepDefinitions/ResourceOptionsSteps.cs new file mode 100644 index 0000000..2eabc4d --- /dev/null +++ b/foreign/csharp/Iggy_SDK.Tests.BDD/StepDefinitions/ResourceOptionsSteps.cs
@@ -0,0 +1,376 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Text; +using Apache.Iggy.Contracts; +using Apache.Iggy.Contracts.Auth; +using Apache.Iggy.Encryption; +using Apache.Iggy.Enums; +using Apache.Iggy.Headers; +using Apache.Iggy.Kinds; +using Apache.Iggy.Messages; +using Reqnroll; +using Shouldly; +using Partitioning = Apache.Iggy.Kinds.Partitioning; +using TestContext = Apache.Iggy.Tests.BDD.Context.TestContext; + +namespace Apache.Iggy.Tests.BDD.StepDefinitions; + +[Binding] +public sealed class ResourceOptionsSteps +{ + private readonly TestContext _context; + private uint? _streamId; + private uint? _topicId; + private uint? _userId; + private byte[]? _streamKey; + private bool _updateRejected; + + public ResourceOptionsSteps(TestContext context) + { + _context = context; + } + + // Features share one server, and this runs before the default-order hook disposes the client. + [AfterScenario(Order = 0)] + public async Task DeleteCreatedResources() + { + if (_streamId is { } streamId) + { + await _context.IggyClient.DeleteStreamAsync(Identifier.Numeric(streamId)); + } + + if (_userId is { } userId) + { + await _context.IggyClient.DeleteUserAsync(Identifier.Numeric(userId)); + } + } + + [When("I create a stream named {string} with an encryption key of 32 bytes valued {int}")] + public async Task CreateKeyedStream(string name, int value) + { + _streamKey = Enumerable.Repeat((byte)value, 32).ToArray(); + _context.CreatedStream = await _context.IggyClient.CreateStreamAsync(name, + new StreamOptions { EncryptionKey = _streamKey }); + _context.CreatedStream.ShouldNotBeNull(); + _streamId = _context.CreatedStream.Id; + } + + [Given("a stream named {string} with an encryption key of 32 bytes valued {int} exists")] + public Task KeyedStreamExists(string name, int value) => CreateKeyedStream(name, value); + + [When("I create a stream named {string} without an encryption key")] + public async Task CreateKeylessStream(string name) + { + _context.CreatedStream = await _context.IggyClient.CreateStreamAsync(name); + _context.CreatedStream.ShouldNotBeNull(); + _streamId = _context.CreatedStream.Id; + } + + [Given("a stream named {string} exists")] + public Task StreamExists(string name) => CreateKeylessStream(name); + + [Then("the stream should report encryption key source {string}")] + public async Task StreamSource(string source) + { + var stream = await Stream(); + stream.EncryptionKeySource.ShouldBe(Enum.Parse<StreamEncryptionSource>(source, true)); + } + + [Then("the stream encryption key fingerprint should match the key")] + public async Task StreamFingerprintMatches() + { + _streamKey.ShouldNotBeNull(); + (await Stream()).EncryptionKeyFingerprint.ShouldBe(EncryptionKeys.Fingerprint(_streamKey)); + } + + [Then("the stream should report an encryption key fingerprint")] + public async Task StreamFingerprintExists() + { + var fingerprint = (await Stream()).EncryptionKeyFingerprint; + fingerprint.ShouldNotBeNull(); + fingerprint!.ShouldMatch("^[0-9a-f]{16}$"); + } + + [Then("the stream should report at least one KEK fingerprint")] + public async Task KekFingerprintExists() + { + var fingerprints = (await Stream()).EncryptionKekFingerprints; + fingerprints.ShouldNotBeEmpty(); + foreach (var fingerprint in fingerprints) + { + fingerprint.ShouldMatch("^[0-9a-f]{16}$"); + } + } + + [Then("the stream should expose no encryption key material")] + public async Task StreamDoesNotExposeKey() + { + var stream = await Stream(); + foreach (var options in new[] { stream.Options, stream.DerivedOptions }) + { + if (options is null) + { + continue; + } + foreach (var name in new[] { "encryption_key", "encryption_key_wrapped", "critical_keys" }) + { + options.ContainsKey(HeaderKey.FromString(name)).ShouldBeFalse(); + } + } + } + + [When("I create a topic named {string} with {int} partition in the stream")] + public async Task CreateTopic(string name, int partitions) + { + _context.CreatedTopic = await _context.IggyClient.CreateTopicAsync(StreamId(), name, (uint)partitions); + _context.CreatedTopic.ShouldNotBeNull(); + _topicId = _context.CreatedTopic.Id; + } + + [When("I send a message with payload {string} to the topic")] + public Task SendPayload(string payload) => SendPayloadWithHeaders(payload, null); + + [When("I send a message with payload {string} and header {string} set to {string} to the topic")] + public Task SendPayloadWithHeader(string payload, string name, string value) => + SendPayloadWithHeaders(payload, new Dictionary<HeaderKey, HeaderValue> + { + [HeaderKey.FromString(name)] = HeaderValue.FromString(value) + }); + + [Then("polling the topic should return the payload {string}")] + public async Task PollPayload(string expected) + { + var messages = await Poll(); + messages.Count.ShouldBe(1); + Encoding.UTF8.GetString(messages[0].Payload).ShouldBe(expected); + } + + [Then("polling the topic should return the payload {string} and header {string} set to {string}")] + public async Task PollPayloadWithHeader(string expected, string name, string value) + { + var messages = await Poll(); + messages.Count.ShouldBe(1); + Encoding.UTF8.GetString(messages[0].Payload).ShouldBe(expected); + var headers = messages[0].UserHeaders; + headers.ShouldNotBeNull(); + headers![HeaderKey.FromString(name)].ToString().ShouldBe(value); + } + + [When("I rename the stream to {string}")] + public Task RenameStream(string name) => + _context.IggyClient.UpdateStreamAsync(StreamId(), new StreamUpdate { Name = name }); + + [Then("the stream name should be {string}")] + public async Task StreamName(string name) + { + (await Stream()).Name.ShouldBe(name); + } + + [When("I request a reset of the stream option {string}")] + public async Task ResetStreamOption(string name) + { + var key = Enum.Parse<StreamOptionKey>(name.Replace("_", ""), true); + try + { + await _context.IggyClient.UpdateStreamAsync(StreamId(), new StreamUpdate + { + Reset = new HashSet<StreamOptionKey> { key } + }); + } + catch (Exception) + { + _updateRejected = true; + } + } + + [Then("the stream update should be rejected")] + public void StreamUpdateRejected() + { + _updateRejected.ShouldBeTrue(); + } + + [When("I describe the stream options")] + public Task DescribeStreamOptions() => _context.IggyClient.DescribeOptionsAsync(OptionsScope.Stream); + + [Then("the stream option {string} should be updatable and not secret")] + public async Task UpdatableStreamOption(string name) + { + var option = await StreamOption(name); + option.Updatable.ShouldBeTrue(); + option.Secret.ShouldBeFalse(); + } + + [Then("the stream option {string} should be create only and secret")] + public async Task SecretStreamOption(string name) + { + var option = await StreamOption(name); + option.Updatable.ShouldBeFalse(); + option.Secret.ShouldBeTrue(); + } + + [Then("the stream option {string} should be read only")] + public async Task ReadOnlyStreamOption(string name) + { + (await StreamOption(name)).ReadOnly.ShouldBeTrue(); + } + + [Given("a topic named {string} exists in the stream with message expiry {long} microseconds and max topic size {long} bytes")] + public async Task TopicWithOptions(string name, long expiry, long bytes) + { + _context.CreatedTopic = await _context.IggyClient.CreateTopicAsync(StreamId(), name, 1, + new TopicOptions { MessageExpiry = TimeSpan.FromTicks(expiry * 10), MaxTopicSize = (ulong)bytes }); + _context.CreatedTopic.ShouldNotBeNull(); + _topicId = _context.CreatedTopic.Id; + } + + [When("I update the topic message expiry to {long} microseconds without a name")] + public Task UpdateTopicExpiry(long micros) => _context.IggyClient.UpdateTopicAsync(StreamId(), TopicId(), + new TopicUpdate { MessageExpiry = TimeSpan.FromTicks(micros * 10) }); + + [When("I reset the topic option {string}")] + public Task ResetTopicOption(string name) + { + name.ShouldBe("max_topic_size"); + return _context.IggyClient.UpdateTopicAsync(StreamId(), TopicId(), new TopicUpdate + { + Reset = new HashSet<TopicOptionKey> { TopicOptionKey.MaxTopicSize } + }); + } + + [When("I rename the topic to {string}")] + public Task RenameTopic(string name) => _context.IggyClient.UpdateTopicAsync(StreamId(), TopicId(), + new TopicUpdate { Name = name }); + + [Then("the topic name should be {string}")] + public async Task TopicName(string name) + { + (await Topic()).Name.ShouldBe(name); + } + + [Then("the topic message expiry should be {long} microseconds set explicitly")] + public async Task TopicExpiry(long micros) + { + var topic = await Topic(); + topic.Options.ShouldNotBeNull(); + topic.Options[HeaderKey.FromString("message_expiry")].ToUInt64().ShouldBe((ulong)micros); + } + + [Then("the topic max topic size should be {long} bytes set explicitly")] + public async Task TopicMaxSize(long bytes) + { + var topic = await Topic(); + topic.Options.ShouldNotBeNull(); + topic.Options[HeaderKey.FromString("max_topic_size")].ToUInt64().ShouldBe((ulong)bytes); + } + + [Then("the topic max topic size should be derived from the server default")] + public async Task DefaultTopicMaxSize() + { + var topic = await Topic(); + topic.DerivedOptions.ShouldNotBeNull(); + topic.DerivedOptions[HeaderKey.FromString("max_topic_size")].ToUInt64().ShouldBe(ulong.MaxValue); + } + + [Given("a user named {string} with password {string} exists")] + public async Task CreateUser(string name, string password) + { + var user = await _context.IggyClient.CreateUserAsync(name, password, UserStatus.Active, + options: new UserOptions()); + user.ShouldNotBeNull(); + _userId = user.Id; + } + + [When("I update the user username to {string} and status to {string}")] + public Task UpdateUser(string name, string status) => _context.IggyClient.UpdateUserAsync(UserId(), + new UserUpdate { Username = name, Status = Enum.Parse<UserStatus>(status, true) }); + + [Then("the user should have username {string}")] + public async Task UserName(string name) + { + var user = await _context.IggyClient.GetUserAsync(UserId()); + user.ShouldNotBeNull(); + user.Username.ShouldBe(name); + } + + [Then("the user should have status {string}")] + public async Task UserStatusIs(string status) + { + var user = await _context.IggyClient.GetUserAsync(UserId()); + user.ShouldNotBeNull(); + user.Status.ShouldBe(Enum.Parse<UserStatus>(status, true)); + } + + private async Task SendPayloadWithHeaders(string payload, Dictionary<HeaderKey, HeaderValue>? headers) + { + var message = new Message(Guid.NewGuid(), Encoding.UTF8.GetBytes(payload), headers); + await _context.IggyClient.SendMessagesAsync(StreamId(), TopicId(), Partitioning.PartitionId(0), [message]); + } + + private async Task<IReadOnlyList<MessageResponse>> Poll() + { + var response = await _context.IggyClient.PollMessagesAsync(new MessageFetchRequest + { + StreamId = StreamId(), + TopicId = TopicId(), + PartitionId = 0, + PollingStrategy = PollingStrategy.First(), + Consumer = Consumer.New(0), + Count = 10, + AutoCommit = false + }); + return response.Messages.ToList(); + } + + private async Task<StreamResponse> Stream() + { + var stream = await _context.IggyClient.GetStreamByIdAsync(StreamId()); + stream.ShouldNotBeNull(); + return stream; + } + + private async Task<TopicResponse> Topic() + { + var topic = await _context.IggyClient.GetTopicByIdAsync(StreamId(), TopicId()); + topic.ShouldNotBeNull(); + return topic; + } + + private async Task<OptionSpec> StreamOption(string name) + { + var options = await _context.IggyClient.DescribeOptionsAsync(OptionsScope.Stream); + return options.Single(option => option.Key == name); + } + + private Identifier StreamId() + { + _streamId.ShouldNotBeNull(); + return Identifier.Numeric(_streamId.Value); + } + + private Identifier TopicId() + { + _topicId.ShouldNotBeNull(); + return Identifier.Numeric(_topicId.Value); + } + + private Identifier UserId() + { + _userId.ShouldNotBeNull(); + return Identifier.Numeric(_userId.Value); + } +}
diff --git a/foreign/csharp/Iggy_SDK.Tests.BDD/xunit.runner.json b/foreign/csharp/Iggy_SDK.Tests.BDD/xunit.runner.json new file mode 100644 index 0000000..08c512b --- /dev/null +++ b/foreign/csharp/Iggy_SDK.Tests.BDD/xunit.runner.json
@@ -0,0 +1,4 @@ +{ + "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json", + "parallelizeTestCollections": false +}
diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyServerFixture.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyServerFixture.cs index 6b1afdb..f5b60e1 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyServerFixture.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyServerFixture.cs
@@ -89,6 +89,12 @@ } /// <summary> + /// Server encryption key every node shares. With encryption enabled a stream created without its own key + /// gets one the server generates, so the stream options tests read a server sourced key. + /// </summary> + public static readonly string ServerEncryptionKey = "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE="; + + /// <summary> /// Extra environment variables for every node, layered over the cluster's base configuration. /// Heartbeat verification is on for every fixture so the eviction tests share the general server /// instead of starting their own. Override in subclasses to customize. @@ -97,7 +103,9 @@ { { "IGGY_HEARTBEAT_ENABLED", "true" }, { "IGGY_HEARTBEAT_INTERVAL", $"{ServerHeartbeatInterval.TotalSeconds}s" }, - { "IGGY_CONSUMER_GROUP_HEARTBEAT_INTERVAL", $"{SessionReportInterval.TotalMilliseconds}ms" } + { "IGGY_CONSUMER_GROUP_HEARTBEAT_INTERVAL", $"{SessionReportInterval.TotalMilliseconds}ms" }, + { "IGGY_ENCRYPTION_ENABLED", "true" }, + { "IGGY_ENCRYPTION_KEY", ServerEncryptionKey } }; /// <summary>
diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/OptionsTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/OptionsTests.cs index 8ed7f14..82f3ef3 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/OptionsTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/OptionsTests.cs
@@ -15,19 +15,33 @@ // specific language governing permissions and limitations // under the License. +using System.Security.Cryptography; using System.Text; +using System.Text.RegularExpressions; using Apache.Iggy.Contracts; +using Apache.Iggy.Encryption; using Apache.Iggy.Enums; +using Apache.Iggy.Exceptions; using Apache.Iggy.Headers; +using Apache.Iggy.IggyClient; +using Apache.Iggy.Kinds; +using Apache.Iggy.Messages; using Apache.Iggy.Tests.Integrations.Fixtures; using Shouldly; +using Partitioning = Apache.Iggy.Kinds.Partitioning; namespace Apache.Iggy.Tests.Integrations; public class OptionsTests { + private const int OptionNotUpdatable = 4103; + private const int ReadOnlyOption = 4105; + private const int UnsupportedOptionKey = 4041; + private const int InvalidTopicName = 2014; + private static readonly string[] TopicCatalogKeys = [ + "name", "compression_algorithm", "message_expiry", "max_topic_size", @@ -40,6 +54,10 @@ "preallocate_segments" ]; + private static readonly string[] NeverReturnedKeys = ["encryption_key", "encryption_key_wrapped", "critical_keys"]; + + private static readonly Regex Fingerprint = new("^[0-9a-f]{16}$", RegexOptions.Compiled); + [ClassDataSource<IggyServerFixture>(Shared = SharedType.PerAssembly)] public required IggyServerFixture Fixture { get; init; } @@ -93,30 +111,51 @@ specs["preallocate_segments"].Kind.ShouldBe(HeaderKind.Bool); specs["preallocate_segments"].DefaultValue.ShouldBe([0]); + + specs["message_expiry"].Updatable.ShouldBeTrue(); + specs["message_expiry"].Resettable.ShouldBeTrue(); + specs["compression_algorithm"].Updatable.ShouldBeFalse(); + specs["name"].Resettable.ShouldBeFalse(); } [Test] [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))] - public async Task DescribeOptions_Stream_Should_Return_EmptyCatalog(Protocol protocol) + public async Task DescribeOptions_Stream_Should_Report_SensitivityAndMutability(Protocol protocol) { var client = await Fixture.CreateAuthenticatedClient(protocol); IReadOnlyList<OptionSpec> specs = await client.DescribeOptionsAsync(OptionsScope.Stream); - specs.ShouldNotBeNull(); - specs.ShouldBeEmpty(); + specs.Select(spec => spec.Key).ShouldBe([ + "name", "encryption_key", "encryption_key_source", "encryption_key_fingerprint", + "encryption_kek_fingerprint" + ]); + var byKey = specs.ToDictionary(spec => spec.Key); + byKey["name"].Updatable.ShouldBeTrue(); + byKey["name"].Secret.ShouldBeFalse(); + byKey["encryption_key"].Kind.ShouldBe(HeaderKind.Raw); + byKey["encryption_key"].Secret.ShouldBeTrue(); + byKey["encryption_key"].Critical.ShouldBeTrue(); + byKey["encryption_key"].Updatable.ShouldBeFalse(); + byKey["encryption_key"].Resettable.ShouldBeFalse(); + foreach (var key in new[] { "encryption_key_source", "encryption_key_fingerprint", "encryption_kek_fingerprint" }) + { + byKey[key].ReadOnly.ShouldBeTrue(); + byKey[key].Updatable.ShouldBeFalse(); + byKey[key].Secret.ShouldBeFalse(); + } } [Test] [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))] - public async Task DescribeOptions_User_Should_Return_EmptyCatalog(Protocol protocol) + public async Task DescribeOptions_User_Should_Report_UpdatableColumns(Protocol protocol) { var client = await Fixture.CreateAuthenticatedClient(protocol); IReadOnlyList<OptionSpec> specs = await client.DescribeOptionsAsync(OptionsScope.User); - specs.ShouldNotBeNull(); - specs.ShouldBeEmpty(); + specs.Select(spec => spec.Key).ShouldBe(["username", "status"]); + specs.ShouldAllBe(spec => spec.Updatable && !spec.Resettable && !spec.Secret && !spec.ReadOnly); } [Test] @@ -130,14 +169,19 @@ .ToHashSet(); var typedKeys = new TopicOptions { + CompressionAlgorithm = CompressionAlgorithm.Gzip, + MessageExpiry = TimeSpan.FromDays(1), + MaxTopicSize = 2_000_000_000, SegmentSize = 1, - Durability = Apache.Iggy.Enums.Durability.Persisted, + Durability = Durability.Persisted, + PartitionResizePolicy = PartitionResizePolicy.Fixed, MessagesRequiredToSave = 1, SizeOfMessagesRequiredToSave = 1, PreallocateSegments = true }.ToDictionary().Keys; typedKeys.ShouldAllBe(key => catalogKeys.Contains(key)); + catalogKeys.Except(typedKeys).ShouldBe(["name"]); } [Test] @@ -150,14 +194,13 @@ await client.CreateStreamAsync(streamName); IReadOnlyList<OptionSpec> catalog = await client.DescribeOptionsAsync(OptionsScope.Topic); - Dictionary<string, HeaderValue> options = new TopicOptions + var options = new TopicOptions { - Durability = Apache.Iggy.Enums.Durability.Persisted, + Durability = Durability.Persisted, MessagesRequiredToSave = 7 - }.ToDictionary(); + }; - var topic = await client.CreateTopicAsync(Identifier.String(streamName), "opts-topic", 1, - options: options); + var topic = await client.CreateTopicAsync(Identifier.String(streamName), "opts-topic", 1, options); topic.ShouldNotBeNull(); topic.Options.ShouldNotBeNull(); @@ -182,12 +225,250 @@ fetched.ShouldNotBeNull(); fetched.Options!.Single(kv => kv.Key.AsString() == "durability").Value.ToString().ShouldBe("persisted"); fetched.DerivedOptions!.Keys.Select(key => key.AsString()).ShouldContain("segment_size"); + fetched.DerivedOptions!.Single(kv => kv.Key.AsString() == "segment_size").Value.Kind + .ShouldBe(HeaderKind.Uint64); } - // REST carries options as readable strings ("true"), the binary transports as the canonical - // kind (Bool byte, rendered "1"), so a bool assertion has to accept both spellings. - private static bool AsBool(HeaderValue value) + [Test] + [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))] + public async Task given_stream_key_when_creating_should_read_matching_projection(Protocol protocol) { - return value.Kind == HeaderKind.Bool ? value.Value[0] != 0 : bool.Parse(value.ToString()); + var client = await Fixture.CreateAuthenticatedClient(protocol); + var key = RandomNumberGenerator.GetBytes(EncryptionKeys.KEY_LENGTH); + + var created = await client.CreateStreamAsync($"keyed-{Guid.NewGuid():N}", new StreamOptions { EncryptionKey = key }); + + created.ShouldNotBeNull(); + var fetched = await client.GetStreamByIdAsync(Identifier.Numeric(created.Id)); + fetched.ShouldNotBeNull(); + var listed = (await client.GetStreamsAsync()).Single(stream => stream.Id == created.Id); + foreach (var stream in new[] { created, fetched, listed }) + { + stream.EncryptionKeySource.ShouldBe(StreamEncryptionSource.Stream); + stream.EncryptionKeyFingerprint.ShouldBe(EncryptionKeys.Fingerprint(key)); + stream.EncryptionKekFingerprints.ShouldNotBeEmpty(); + stream.EncryptionKekFingerprints.ShouldAllBe(fingerprint => Fingerprint.IsMatch(fingerprint)); + stream.EncryptionKekFingerprints.ShouldBe(created.EncryptionKekFingerprints); + ShouldExposeNoKeyMaterial(stream, key); + } + } + + [Test] + [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))] + public async Task given_keyless_stream_when_reading_should_report_server_generated_key(Protocol protocol) + { + var client = await Fixture.CreateAuthenticatedClient(protocol); + + var first = await client.CreateStreamAsync($"generated-{Guid.NewGuid():N}"); + var second = await client.CreateStreamAsync($"generated-{Guid.NewGuid():N}", new StreamOptions()); + + foreach (var created in new[] { first, second }) + { + created.ShouldNotBeNull(); + var stream = await client.GetStreamByIdAsync(Identifier.Numeric(created.Id)); + stream.ShouldNotBeNull(); + stream.EncryptionKeySource.ShouldBe(StreamEncryptionSource.Server); + stream.EncryptionKeyFingerprint.ShouldNotBeNull(); + Fingerprint.IsMatch(stream.EncryptionKeyFingerprint).ShouldBeTrue(); + stream.EncryptionKekFingerprints.ShouldNotBeEmpty(); + ShouldExposeNoKeyMaterial(stream, []); + } + + first!.EncryptionKeyFingerprint.ShouldNotBe(second!.EncryptionKeyFingerprint); + } + + [Test] + [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))] + public async Task given_stream_rename_when_updating_should_preserve_encryption_projection(Protocol protocol) + { + var client = await Fixture.CreateAuthenticatedClient(protocol); + var key = RandomNumberGenerator.GetBytes(EncryptionKeys.KEY_LENGTH); + var created = await client.CreateStreamAsync($"rename-{Guid.NewGuid():N}", new StreamOptions { EncryptionKey = key }); + created.ShouldNotBeNull(); + var streamId = Identifier.Numeric(created.Id); + await client.CreateTopicAsync(streamId, "events", 1); + var payload = Encoding.UTF8.GetBytes("before-rename"); + await client.SendMessagesAsync(streamId, Identifier.String("events"), Partitioning.PartitionId(0), + [new Message(Guid.NewGuid(), payload)]); + + var renamed = $"renamed-{Guid.NewGuid():N}"; + await client.UpdateStreamAsync(streamId, new StreamUpdate { Name = renamed }); + + var stream = await client.GetStreamByIdAsync(streamId); + stream.ShouldNotBeNull(); + stream.Id.ShouldBe(created.Id); + stream.Name.ShouldBe(renamed); + stream.EncryptionKeySource.ShouldBe(StreamEncryptionSource.Stream); + stream.EncryptionKeyFingerprint.ShouldBe(EncryptionKeys.Fingerprint(key)); + stream.EncryptionKekFingerprints.ShouldBe(created.EncryptionKekFingerprints); + (await PollFirstPayload(client, streamId)).ShouldBe(payload); + + var reset = await Should.ThrowAsync<IggyInvalidStatusCodeException>(client.UpdateStreamAsync(streamId, + new StreamUpdate { Reset = new HashSet<StreamOptionKey> { StreamOptionKey.EncryptionKey } })); + reset.StatusCode.ShouldBe(OptionNotUpdatable); + var unchanged = await client.GetStreamByIdAsync(streamId); + unchanged!.Name.ShouldBe(renamed); + unchanged.EncryptionKeyFingerprint.ShouldBe(EncryptionKeys.Fingerprint(key)); + } + + [Test] + [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))] + public async Task given_topic_reset_when_updating_should_preserve_unmentioned_values(Protocol protocol) + { + var client = await Fixture.CreateAuthenticatedClient(protocol); + var streamName = $"reset-{Guid.NewGuid():N}"; + await client.CreateStreamAsync(streamName); + var streamId = Identifier.String(streamName); + await client.CreateTopicAsync(streamId, "reset-topic", 1, new TopicOptions + { + CompressionAlgorithm = CompressionAlgorithm.Gzip, + MessageExpiry = TimeSpan.FromHours(1), + MaxTopicSize = 2_000_000_000 + }); + + await client.UpdateTopicAsync(streamId, Identifier.String("reset-topic"), new TopicUpdate + { + MessageExpiry = TimeSpan.FromHours(2), + Reset = new HashSet<TopicOptionKey> { TopicOptionKey.MaxTopicSize } + }); + + var topic = await client.GetTopicByIdAsync(streamId, Identifier.String("reset-topic")); + topic.ShouldNotBeNull(); + topic.Name.ShouldBe("reset-topic"); + topic.MessageExpiry.ShouldBe(TimeSpan.FromHours(2)); + topic.MaxTopicSize.ShouldBe(ulong.MaxValue); + topic.CompressionAlgorithm.ShouldBe(CompressionAlgorithm.Gzip); + topic.Options!.Keys.Select(option => option.AsString()).ShouldContain("message_expiry"); + topic.Options!.Keys.Select(option => option.AsString()).ShouldContain("compression_algorithm"); + topic.Options!.Keys.Select(option => option.AsString()).ShouldNotContain("max_topic_size"); + topic.DerivedOptions!.Keys.Select(option => option.AsString()).ShouldContain("max_topic_size"); + } + + [Test] + [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))] + public async Task given_user_options_when_creating_and_updating_should_preserve_user_behavior(Protocol protocol) + { + var client = await Fixture.CreateAuthenticatedClient(protocol); + var username = $"opts-{Guid.NewGuid():N}"[..20]; + var renamed = $"ren-{Guid.NewGuid():N}"[..20]; + + var created = await client.CreateUserAsync(username, "options-password", UserStatus.Active, + options: new UserOptions()); + + created.ShouldNotBeNull(); + (created.Options ?? []).ShouldBeEmpty(); + (created.DerivedOptions ?? []).ShouldBeEmpty(); + + await client.UpdateUserAsync(Identifier.Numeric(created.Id), + new UserUpdate { Username = renamed, Status = UserStatus.Inactive }); + + var user = await client.GetUserAsync(Identifier.String(renamed)); + user.ShouldNotBeNull(); + user.Id.ShouldBe(created.Id); + user.Status.ShouldBe(UserStatus.Inactive); + (user.Options ?? []).ShouldBeEmpty(); + + var otherName = $"oth-{Guid.NewGuid():N}"[..20]; + var other = await client.CreateUserAsync(otherName, "options-password", UserStatus.Active); + other.ShouldNotBeNull(); + await Should.ThrowAsync<IggyInvalidStatusCodeException>(client.UpdateUserAsync(Identifier.Numeric(other.Id), + new UserUpdate { Username = renamed, Status = UserStatus.Inactive })); + + var unchanged = await client.GetUserAsync(Identifier.Numeric(other.Id)); + unchanged.ShouldNotBeNull(); + unchanged.Username.ShouldBe(otherName); + unchanged.Status.ShouldBe(UserStatus.Active); + } + + [Test] + public async Task given_http_status_update_when_deactivating_should_prevent_new_login() + { + var admin = await Fixture.CreateAuthenticatedClient(Protocol.Http); + var username = $"deact-{Guid.NewGuid():N}"[..20]; + const string Password = "deactivate-password"; + await admin.CreateUserAsync(username, Password, UserStatus.Active); + var before = await Fixture.CreateClient(Protocol.Http); + (await before.LoginUserAsync(username, Password)).ShouldNotBeNull(); + + await admin.UpdateUserAsync(Identifier.String(username), new UserUpdate { Status = UserStatus.Inactive }); + + (await admin.GetUserAsync(Identifier.String(username)))!.Status.ShouldBe(UserStatus.Inactive); + var after = await Fixture.CreateClient(Protocol.Http); + await Should.ThrowAsync<IggyInvalidStatusCodeException>(after.LoginUserAsync(username, Password)); + } + + [Test] + [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))] + public async Task given_raw_keys_when_updating_should_report_the_server_category(Protocol protocol) + { + var client = await Fixture.CreateAuthenticatedClient(protocol); + var created = await client.CreateUserAsync($"raw-{Guid.NewGuid():N}"[..20], "options-password", + UserStatus.Active); + var stream = await client.CreateStreamAsync($"raw-{Guid.NewGuid():N}"[..20]); + created.ShouldNotBeNull(); + stream.ShouldNotBeNull(); + + var internalReset = await Should.ThrowAsync<IggyInvalidStatusCodeException>(client.UpdateUserAsync( + Identifier.Numeric(created.Id), new UserUpdate { RawReset = new HashSet<string> { "critical_keys" } })); + internalReset.StatusCode.ShouldBe(ReadOnlyOption); + internalReset.FromServer.ShouldBeTrue(); + + var unknownReset = await Should.ThrowAsync<IggyInvalidStatusCodeException>(client.UpdateStreamAsync( + Identifier.Numeric(stream.Id), new StreamUpdate { RawReset = new HashSet<string> { "retention" } })); + unknownReset.StatusCode.ShouldBe(UnsupportedOptionKey); + unknownReset.FromServer.ShouldBeTrue(); + + var emptyName = await Should.ThrowAsync<IggyInvalidStatusCodeException>(client.UpdateStreamAsync( + Identifier.Numeric(stream.Id), new StreamUpdate + { + Name = string.Empty, + Options = new Dictionary<string, HeaderValue> { ["encryption_key_source"] = HeaderValue.FromString("stream") } + })); + emptyName.StatusCode.ShouldBe(ReadOnlyOption); + emptyName.FromServer.ShouldBeTrue(); + + var topic = await client.CreateTopicAsync(Identifier.Numeric(stream.Id), "events", 1); + topic.ShouldNotBeNull(); + var standIn = await Should.ThrowAsync<IggyInvalidStatusCodeException>(client.UpdateTopicAsync( + Identifier.Numeric(stream.Id), Identifier.Numeric(topic.Id), new TopicUpdate + { + Name = string.Empty, + RawReset = new HashSet<string> { "message_expiry" } + })); + standIn.StatusCode.ShouldBe(InvalidTopicName); + standIn.FromServer.ShouldBeTrue(); + } + + private static async Task<byte[]> PollFirstPayload(IIggyClient client, Identifier streamId) + { + var polled = await client.PollMessagesAsync(new MessageFetchRequest + { + Count = 1, + AutoCommit = false, + Consumer = Consumer.New(1), + PartitionId = 0, + PollingStrategy = PollingStrategy.Offset(0), + StreamId = streamId, + TopicId = Identifier.String("events") + }); + + return polled.Messages.Single().Payload; + } + + private static void ShouldExposeNoKeyMaterial(StreamResponse stream, byte[] key) + { + var options = (stream.Options ?? []).Concat(stream.DerivedOptions ?? []).ToList(); + options.Select(option => option.Key.AsString()).ShouldAllBe(name => !NeverReturnedKeys.Contains(name)); + if (key.Length == 0) + { + return; + } + + var base64 = Convert.ToBase64String(key); + foreach (var (_, value) in options) + { + value.Value.SequenceEqual(key).ShouldBeFalse(); + Encoding.UTF8.GetString(value.Value).ShouldNotContain(base64); + } } }
diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/StreamsTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/StreamsTests.cs index 648b25f..b1709fb 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/StreamsTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/StreamsTests.cs
@@ -133,9 +133,9 @@ var topicName1 = "Topic1"; var topicName2 = "Topic2"; await client.CreateTopicAsync(Identifier.String(streamName), - topicName1, 1, messageExpiry: TimeSpan.FromHours(1)); + topicName1, 1, new TopicOptions { MessageExpiry = TimeSpan.FromHours(1) }); await client.CreateTopicAsync(Identifier.String(streamName), - topicName2, 1, messageExpiry: TimeSpan.FromHours(1)); + topicName2, 1, new TopicOptions { MessageExpiry = TimeSpan.FromHours(1) }); await client.SendMessagesAsync(Identifier.String(streamName), Identifier.String(topicName1), Partitioning.None(), @@ -186,7 +186,7 @@ var streamToUpdate = await client.CreateStreamAsync(name); streamToUpdate.ShouldNotBeNull(); - await client.UpdateStreamAsync(Identifier.String(streamToUpdate.Name), updatedName); + await client.UpdateStreamAsync(Identifier.String(streamToUpdate.Name), new StreamUpdate { Name = updatedName }); var result = await client.GetStreamByIdAsync(Identifier.Numeric(streamToUpdate.Id)); result.ShouldNotBeNull();
diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/TopicsTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/TopicsTests.cs index 87ee535..8285368 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/TopicsTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/TopicsTests.cs
@@ -28,6 +28,13 @@ public class TopicsTests { + private static readonly TopicOptions GzipTenMinutes = new() + { + CompressionAlgorithm = CompressionAlgorithm.Gzip, + MessageExpiry = TimeSpan.FromMinutes(10), + MaxTopicSize = 2_000_000_000 + }; + [ClassDataSource<IggyServerFixture>(Shared = SharedType.PerAssembly)] public required IggyServerFixture Fixture { get; init; } @@ -40,9 +47,7 @@ var streamName = $"topic-create-{Guid.NewGuid():N}"; await client.CreateStreamAsync(streamName); - var response = await client.CreateTopicAsync(Identifier.String(streamName), "Test Topic", 2, - CompressionAlgorithm.Gzip, - TimeSpan.FromMinutes(10), 2_000_000_000); + var response = await client.CreateTopicAsync(Identifier.String(streamName), "Test Topic", 2, GzipTenMinutes); response.ShouldNotBeNull(); response.CreatedAt.UtcDateTime.ShouldBe(DateTimeOffset.UtcNow.UtcDateTime, TimeSpan.FromMinutes(1)); @@ -78,8 +83,7 @@ var streamName = $"topic-get-{Guid.NewGuid():N}"; await client.CreateStreamAsync(streamName); - await client.CreateTopicAsync(Identifier.String(streamName), "Get Topic", 2, - CompressionAlgorithm.Gzip, TimeSpan.FromMinutes(10), 2_000_000_000); + await client.CreateTopicAsync(Identifier.String(streamName), "Get Topic", 2, GzipTenMinutes); var response = await client.GetTopicByIdAsync(Identifier.String(streamName), Identifier.Numeric(0)); @@ -103,8 +107,7 @@ var streamName = $"topic-getname-{Guid.NewGuid():N}"; await client.CreateStreamAsync(streamName); - await client.CreateTopicAsync(Identifier.String(streamName), "Name Topic", 2, - CompressionAlgorithm.Gzip, TimeSpan.FromMinutes(10), 2_000_000_000); + await client.CreateTopicAsync(Identifier.String(streamName), "Name Topic", 2, GzipTenMinutes); var response = await client.GetTopicByIdAsync(Identifier.String(streamName), Identifier.String("Name Topic")); @@ -128,10 +131,8 @@ var streamName = $"topic-list-{Guid.NewGuid():N}"; await client.CreateStreamAsync(streamName); - await client.CreateTopicAsync(Identifier.String(streamName), "List Topic 1", 2, - CompressionAlgorithm.Gzip, TimeSpan.FromMinutes(10), 2_000_000_000); - await client.CreateTopicAsync(Identifier.String(streamName), "List Topic 2", 2, - CompressionAlgorithm.Gzip, TimeSpan.FromMinutes(10), 2_000_000_000); + await client.CreateTopicAsync(Identifier.String(streamName), "List Topic 1", 2, GzipTenMinutes); + await client.CreateTopicAsync(Identifier.String(streamName), "List Topic 2", 2, GzipTenMinutes); IReadOnlyList<TopicResponse> response = await client.GetTopicsAsync(Identifier.String(streamName)); @@ -192,15 +193,20 @@ topicToUpdate.ShouldNotBeNull(); await Should.NotThrowAsync(client.UpdateTopicAsync(Identifier.String(streamName), - Identifier.Numeric(topicToUpdate.Id), "Updated Topic", - CompressionAlgorithm.Gzip, 3_000_000_000, TimeSpan.FromMinutes(10))); + Identifier.Numeric(topicToUpdate.Id), new TopicUpdate + { + Name = "Updated Topic", + MaxTopicSize = 3_000_000_000, + MessageExpiry = TimeSpan.FromMinutes(10) + })); var result = await client.GetTopicByIdAsync(Identifier.String(streamName), Identifier.Numeric(topicToUpdate.Id)); result.ShouldNotBeNull(); result!.Name.ShouldBe("Updated Topic"); result.MessageExpiry.ShouldBe(TimeSpan.FromMinutes(10)); - result.CompressionAlgorithm.ShouldBe(CompressionAlgorithm.Gzip); + // Compression is set only at creation, so the update leaves the default in place. + result.CompressionAlgorithm.ShouldBe(CompressionAlgorithm.None); result.MaxTopicSize.ShouldBe(3_000_000_000u); }
diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/UsersTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/UsersTests.cs index 9f708eb..b0f7350 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/UsersTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/UsersTests.cs
@@ -103,7 +103,8 @@ var newUsername = $"new-{Guid.NewGuid():N}"[..20]; await client.CreateUserAsync(username, "test1", UserStatus.Active); - await Should.NotThrowAsync(client.UpdateUserAsync(Identifier.String(username), newUsername, UserStatus.Active)); + await Should.NotThrowAsync(client.UpdateUserAsync(Identifier.String(username), + new UserUpdate { Username = newUsername, Status = UserStatus.Active })); var user = await client.GetUserAsync(Identifier.String(newUsername));
diff --git a/foreign/csharp/Iggy_SDK/Contracts/Auth/UserResponse.cs b/foreign/csharp/Iggy_SDK/Contracts/Auth/UserResponse.cs index aa20b1e..0bc6789 100644 --- a/foreign/csharp/Iggy_SDK/Contracts/Auth/UserResponse.cs +++ b/foreign/csharp/Iggy_SDK/Contracts/Auth/UserResponse.cs
@@ -18,6 +18,7 @@ using System.Text.Json.Serialization; using Apache.Iggy.Enums; using Apache.Iggy.Headers; +using Apache.Iggy.JsonConverters; namespace Apache.Iggy.Contracts.Auth; @@ -52,8 +53,30 @@ public Permissions? Permissions { get; init; } /// <summary> - /// Options attached to the user at creation. TCP transport only. + /// Options the client set on the user. /// </summary> [JsonIgnore] public Dictionary<HeaderKey, HeaderValue>? Options { get; init; } + + /// <summary> + /// Options the server filled for the user. + /// </summary> + [JsonIgnore] + public Dictionary<HeaderKey, HeaderValue>? DerivedOptions { get; init; } + + /// <summary> + /// REST renders both provenances into one object keyed by option name, so one JSON property + /// fills <see cref="Options" /> and <see cref="DerivedOptions" />. + /// </summary> + [JsonInclude] + [JsonPropertyName("options")] + [JsonConverter(typeof(ResourceOptionsConverter))] + internal ResourceOptions RestOptions + { + init + { + Options = value.Explicit; + DerivedOptions = value.Derived; + } + } }
diff --git a/foreign/csharp/Iggy_SDK/Contracts/Http/Auth/CreateUserRequest.cs b/foreign/csharp/Iggy_SDK/Contracts/Http/Auth/CreateUserRequest.cs index a1f6eea..1ecef98 100644 --- a/foreign/csharp/Iggy_SDK/Contracts/Http/Auth/CreateUserRequest.cs +++ b/foreign/csharp/Iggy_SDK/Contracts/Http/Auth/CreateUserRequest.cs
@@ -15,9 +15,16 @@ // specific language governing permissions and limitations // under the License. +using System.Text.Json.Serialization; using Apache.Iggy.Contracts.Auth; using Apache.Iggy.Enums; namespace Apache.Iggy.Contracts.Http.Auth; -internal sealed record CreateUserRequest(string Username, string Password, UserStatus Status, Permissions? Permissions); +internal sealed record CreateUserRequest( + string Username, + string Password, + UserStatus Status, + Permissions? Permissions, + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + Dictionary<string, object>? Options);
diff --git a/foreign/csharp/Iggy_SDK/Contracts/Http/CreateStreamRequest.cs b/foreign/csharp/Iggy_SDK/Contracts/Http/CreateStreamRequest.cs index 1c8ed56..2cf5e85 100644 --- a/foreign/csharp/Iggy_SDK/Contracts/Http/CreateStreamRequest.cs +++ b/foreign/csharp/Iggy_SDK/Contracts/Http/CreateStreamRequest.cs
@@ -15,6 +15,15 @@ // specific language governing permissions and limitations // under the License. +using System.Text.Json.Serialization; + namespace Apache.Iggy.Contracts.Http; -internal sealed record CreateStreamRequest(string Name); +/// <summary> +/// A stream create over REST. <paramref name="Options" /> holds option values as <see cref="OptionInputs" /> +/// renders them, and stays out of the body when there are none. +/// </summary> +internal sealed record CreateStreamRequest( + string Name, + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + Dictionary<string, object>? Options);
diff --git a/foreign/csharp/Iggy_SDK/Contracts/Http/CreateTopicRequest.cs b/foreign/csharp/Iggy_SDK/Contracts/Http/CreateTopicRequest.cs index 3ad0585..289472a 100644 --- a/foreign/csharp/Iggy_SDK/Contracts/Http/CreateTopicRequest.cs +++ b/foreign/csharp/Iggy_SDK/Contracts/Http/CreateTopicRequest.cs
@@ -15,43 +15,16 @@ // specific language governing permissions and limitations // under the License. -using System.Diagnostics.CodeAnalysis; -using Apache.Iggy.Enums; +using System.Text.Json.Serialization; namespace Apache.Iggy.Contracts.Http; -internal sealed class CreateTopicRequest -{ - public required string Name { get; set; } - public CompressionAlgorithm CompressionAlgorithm { get; set; } = CompressionAlgorithm.None; - public ulong MessageExpiry { get; set; } - public uint PartitionsCount { get; set; } = 1; - public ulong MaxTopicSize { get; set; } - - /// <summary> - /// Option keys with no property of their own, as strings the server parses by the same rules a - /// config file value goes through. The REST body carries them this way; the binary transports - /// send a typed TLV block. - /// </summary> - public Dictionary<string, string> Options { get; set; } = new(); - - public CreateTopicRequest() - { - } - - [SetsRequiredMembers] - public CreateTopicRequest(string name, - CompressionAlgorithm compressionAlgorithm, - ulong messageExpiry, - uint partitionsCount, - ulong maxTopicSize, - Dictionary<string, string>? options = null) - { - Name = name; - CompressionAlgorithm = compressionAlgorithm; - MessageExpiry = messageExpiry; - PartitionsCount = partitionsCount; - MaxTopicSize = maxTopicSize; - Options = options ?? new Dictionary<string, string>(); - } -} +/// <summary> +/// A topic create over REST. Every setting, compression, expiry and size cap included, rides +/// <paramref name="Options" /> as <see cref="OptionInputs" /> renders it. +/// </summary> +internal sealed record CreateTopicRequest( + string Name, + uint PartitionsCount, + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + Dictionary<string, object>? Options);
diff --git a/foreign/csharp/Iggy_SDK/Contracts/Http/OptionInputs.cs b/foreign/csharp/Iggy_SDK/Contracts/Http/OptionInputs.cs new file mode 100644 index 0000000..2d52a70 --- /dev/null +++ b/foreign/csharp/Iggy_SDK/Contracts/Http/OptionInputs.cs
@@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Buffers.Binary; +using Apache.Iggy.Headers; +using Apache.Iggy.JsonConverters; + +namespace Apache.Iggy.Contracts.Http; + +/// <summary> +/// Option values as a REST body carries them: strings and booleans as themselves, unsigned integers as JSON +/// numbers, raw bytes as standard base64. A 64-bit value stays exact because the server reads it as a u64. +/// </summary> +internal static class OptionInputs +{ + /// <summary>Option values keyed by name, in ascending name order.</summary> + internal static Dictionary<string, object> From(IReadOnlyDictionary<string, HeaderValue> options) + { + return options + .OrderBy(option => option.Key, StringComparer.Ordinal) + .ToDictionary(option => option.Key, option => Value(option.Value)); + } + + internal static object Value(HeaderValue value) + { + return value.Kind switch + { + HeaderKind.String => value.ToString(), + HeaderKind.Bool => value.ToBool(), + HeaderKind.Uint8 => (ulong)value.Value[0], + HeaderKind.Uint16 => (ulong)BinaryPrimitives.ReadUInt16LittleEndian(value.Value), + HeaderKind.Uint32 => (ulong)value.ToUInt32(), + HeaderKind.Uint64 => value.ToUInt64(), + HeaderKind.Raw => Convert.ToBase64String(value.Value), + // The server parses any other kind from its text, the same form a config file carries. + _ => ResourceOptionsConverter.EncodeValue(value) + }; + } +}
diff --git a/foreign/csharp/Iggy_SDK/Contracts/Http/UpdateStreamRequest.cs b/foreign/csharp/Iggy_SDK/Contracts/Http/UpdateStreamRequest.cs deleted file mode 100644 index 09f602b..0000000 --- a/foreign/csharp/Iggy_SDK/Contracts/Http/UpdateStreamRequest.cs +++ /dev/null
@@ -1,20 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -namespace Apache.Iggy.Contracts.Http; - -internal sealed record UpdateStreamRequest(string Name);
diff --git a/foreign/csharp/Iggy_SDK/Contracts/Http/UpdateTopicRequest.cs b/foreign/csharp/Iggy_SDK/Contracts/Http/UpdateTopicRequest.cs deleted file mode 100644 index bef38de..0000000 --- a/foreign/csharp/Iggy_SDK/Contracts/Http/UpdateTopicRequest.cs +++ /dev/null
@@ -1,31 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using Apache.Iggy.Enums; - -namespace Apache.Iggy.Contracts.Http; - -/// <summary> -/// A topic update over REST. <paramref name="Options" /> carries keys with no property of their -/// own, as strings the server parses by the same rules a config file value goes through. -/// </summary> -internal record UpdateTopicRequest( - string Name, - CompressionAlgorithm CompressionAlgorithm, - ulong MaxTopicSize, - ulong MessageExpiry, - Dictionary<string, string> Options);
diff --git a/foreign/csharp/Iggy_SDK/Contracts/OptionKeys.cs b/foreign/csharp/Iggy_SDK/Contracts/OptionKeys.cs new file mode 100644 index 0000000..1bbe698 --- /dev/null +++ b/foreign/csharp/Iggy_SDK/Contracts/OptionKeys.cs
@@ -0,0 +1,241 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Text; +using Apache.Iggy.Headers; +using Apache.Iggy.Utils; +using Apache.Iggy.Vsr; + +namespace Apache.Iggy.Contracts; + +/// <summary>Bits of the flags byte a DescribeOptions entry carries.</summary> +[Flags] +internal enum OptionFlags : byte +{ + None = 0, + Updatable = 1, + Resettable = 2, + Secret = 4, + ReadOnly = 8, + Critical = 16 +} + +/// <summary>One client-visible catalog key: its name, canonical kind and mutability.</summary> +internal sealed record OptionDescriptor(string Name, HeaderKind Kind, OptionFlags Flags); + +/// <summary> +/// The server's option catalog as a client sees it, keys in catalog order. Typed options take their key names +/// from here, and an update refuses a reset with the error the server would return for it. +/// </summary> +internal static class OptionKeys +{ + internal const int MAX_RESET_KEYS = 255; + private const int ERROR_KEY_PREVIEW_LENGTH = 64; + + private static readonly OptionDescriptor[] StreamKeys = + [ + new("name", HeaderKind.String, OptionFlags.Updatable), + new("encryption_key", HeaderKind.Raw, OptionFlags.Secret | OptionFlags.Critical), + new("encryption_key_source", HeaderKind.String, OptionFlags.ReadOnly), + new("encryption_key_fingerprint", HeaderKind.String, OptionFlags.ReadOnly), + new("encryption_kek_fingerprint", HeaderKind.String, OptionFlags.ReadOnly) + ]; + + private static readonly OptionDescriptor[] TopicKeys = + [ + new("name", HeaderKind.String, OptionFlags.Updatable), + new("compression_algorithm", HeaderKind.String, OptionFlags.None), + new("message_expiry", HeaderKind.Uint64, OptionFlags.Updatable | OptionFlags.Resettable), + new("max_topic_size", HeaderKind.Uint64, OptionFlags.Updatable | OptionFlags.Resettable), + new("segment_size", HeaderKind.Uint64, OptionFlags.None), + new("durability", HeaderKind.String, OptionFlags.None), + new("consumer_offset_durability", HeaderKind.String, OptionFlags.None), + new("partition_resize_policy", HeaderKind.String, OptionFlags.None), + new("messages_required_to_save", HeaderKind.Uint32, OptionFlags.None), + new("size_of_messages_required_to_save", HeaderKind.Uint64, OptionFlags.None), + new("preallocate_segments", HeaderKind.Bool, OptionFlags.None) + ]; + + private static readonly OptionDescriptor[] UserKeys = + [ + new("username", HeaderKind.String, OptionFlags.Updatable), + new("status", HeaderKind.String, OptionFlags.Updatable) + ]; + + internal static IReadOnlyList<OptionDescriptor> For(OptionsScope scope) + { + return scope switch + { + OptionsScope.Stream => StreamKeys, + OptionsScope.Topic => TopicKeys, + OptionsScope.User => UserKeys, + _ => throw new ArgumentOutOfRangeException(nameof(scope), scope, "Unknown options scope") + }; + } + + internal static OptionDescriptor Describe(StreamOptionKey key) + { + return Find(StreamKeys, key switch + { + StreamOptionKey.Name => "name", + StreamOptionKey.EncryptionKey => "encryption_key", + StreamOptionKey.EncryptionKeySource => "encryption_key_source", + StreamOptionKey.EncryptionKeyFingerprint => "encryption_key_fingerprint", + StreamOptionKey.EncryptionKekFingerprint => "encryption_kek_fingerprint", + _ => throw new ArgumentOutOfRangeException(nameof(key), key, "Unknown stream option key") + }); + } + + internal static OptionDescriptor Describe(TopicOptionKey key) + { + return Find(TopicKeys, key switch + { + TopicOptionKey.Name => "name", + TopicOptionKey.CompressionAlgorithm => "compression_algorithm", + TopicOptionKey.MessageExpiry => "message_expiry", + TopicOptionKey.MaxTopicSize => "max_topic_size", + TopicOptionKey.SegmentSize => "segment_size", + TopicOptionKey.Durability => "durability", + TopicOptionKey.ConsumerOffsetDurability => "consumer_offset_durability", + TopicOptionKey.PartitionResizePolicy => "partition_resize_policy", + TopicOptionKey.MessagesRequiredToSave => "messages_required_to_save", + TopicOptionKey.SizeOfMessagesRequiredToSave => "size_of_messages_required_to_save", + TopicOptionKey.PreallocateSegments => "preallocate_segments", + _ => throw new ArgumentOutOfRangeException(nameof(key), key, "Unknown topic option key") + }); + } + + internal static OptionDescriptor Describe(UserOptionKey key) + { + return Find(UserKeys, key switch + { + UserOptionKey.Username => "username", + UserOptionKey.Status => "status", + _ => throw new ArgumentOutOfRangeException(nameof(key), key, "Unknown user option key") + }); + } + + /// <summary> + /// A name or username as a header value. Text no header value holds travels as a value of a kind the key + /// never accepts, so the refusal comes from admission with the key's own error and in its order. + /// </summary> + internal static HeaderValue Text(string text) + { + var length = Encoding.UTF8.GetByteCount(text); + return length is 0 or > WireName.MAX_LENGTH ? HeaderValue.FromBool(false) : HeaderValue.FromString(text); + } + + /// <summary>Raw entries first, so a typed field the caller inserts afterwards wins on collision.</summary> + internal static Dictionary<string, HeaderValue> Raw(IReadOnlyDictionary<string, HeaderValue>? options) + { + return options is null ? new Dictionary<string, HeaderValue>() : new Dictionary<string, HeaderValue>(options); + } + + /// <summary> + /// Validates an update the way server admission does, in the same order, so a refusal reads the same before + /// and after the network. An update naming any raw key or raw reset gets only the structural checks: the + /// server's catalog may know keys or flags this build does not, so it decides the rest. + /// </summary> + /// <param name="scope">The resource the update targets.</param> + /// <param name="set">The keys to set.</param> + /// <param name="reset">The typed keys to reset.</param> + /// <param name="rawReset">Reset names outside this build's typed keys.</param> + /// <param name="hasRawSet">Whether <paramref name="set" /> holds a raw key.</param> + internal static OptionsPatch Patch(OptionsScope scope, Dictionary<string, HeaderValue> set, + IEnumerable<OptionDescriptor> reset, IEnumerable<string>? rawReset, bool hasRawSet) + { + List<OptionDescriptor> resets = reset.ToList(); + List<string> rawResets = rawReset?.ToList() ?? []; + List<string> names = resets.Select(key => key.Name).Concat(rawResets).Distinct(StringComparer.Ordinal) + .Order(StringComparer.Ordinal).ToList(); + if (set.Count == 0 && names.Count == 0) + { + throw VsrError.Exception(VsrError.INVALID_COMMAND, "An update must set or reset at least one option."); + } + + if (names.FirstOrDefault(set.ContainsKey) is { } both) + { + throw VsrError.Exception(VsrError.INVALID_COMMAND, $"Option {Preview(both)} cannot be both set and reset."); + } + + if (names.FirstOrDefault(name => Encoding.UTF8.GetByteCount(name) is 0 or > WireName.MAX_LENGTH) is { } unfit) + { + throw VsrError.Exception(VsrError.UNSUPPORTED_OPTION_KEY, $"Unsupported option key {Preview(unfit)}."); + } + + if (names.Count > MAX_RESET_KEYS) + { + // The catalog is far smaller than the limit, so such a list always names a key outside it. + var unknown = names.First(name => For(scope).All(key => key.Name != name)); + throw VsrError.Exception(VsrError.UNSUPPORTED_OPTION_KEY, $"Unsupported option key {Preview(unknown)}."); + } + + if (hasRawSet || rawResets.Count > 0) + { + return new OptionsPatch(scope, set, names); + } + + List<OptionDescriptor> keys = set.Keys.Select(name => Find(For(scope), name)).Concat(resets).ToList(); + if (keys.FirstOrDefault(key => key.Flags.HasFlag(OptionFlags.ReadOnly)) is { } readOnly) + { + throw VsrError.Exception(VsrError.READ_ONLY_OPTION, $"Option {readOnly.Name} is read-only."); + } + + if (keys.FirstOrDefault(key => !key.Flags.HasFlag(OptionFlags.Updatable)) is { } createOnly) + { + throw VsrError.Exception(VsrError.OPTION_NOT_UPDATABLE, $"Option {createOnly.Name} is set only at creation."); + } + + if (resets.FirstOrDefault(key => !key.Flags.HasFlag(OptionFlags.Resettable)) is { } fixedKey) + { + throw VsrError.Exception(VsrError.OPTION_NOT_RESETTABLE, $"Option {fixedKey.Name} has no default to reset to."); + } + + ValidateColumns(scope, set); + return new OptionsPatch(scope, set, names); + } + + private static void ValidateColumns(OptionsScope scope, Dictionary<string, HeaderValue> set) + { + if (set.TryGetValue("name", out var name) && name.Kind != HeaderKind.String) + { + throw scope == OptionsScope.Topic + ? VsrError.Exception(VsrError.INVALID_TOPIC_NAME, $"Topic name must be 1 to {WireName.MAX_LENGTH} UTF-8 bytes.") + : VsrError.Exception(VsrError.INVALID_STREAM_NAME, $"Stream name must be 1 to {WireName.MAX_LENGTH} UTF-8 bytes."); + } + + if (set.TryGetValue("username", out var username)) + { + if (username.Kind != HeaderKind.String) + { + throw VsrError.Exception(VsrError.INVALID_USERNAME, "Username does not fit a header value."); + } + + CredentialBounds.ValidateUsername(Encoding.UTF8.GetString(username.Value)); + } + } + + private static OptionDescriptor Find(IReadOnlyList<OptionDescriptor> keys, string name) + { + return keys.First(key => key.Name == name); + } + + private static string Preview(string name) + { + return name.Length > ERROR_KEY_PREVIEW_LENGTH ? name[..ERROR_KEY_PREVIEW_LENGTH] : name; + } +}
diff --git a/foreign/csharp/Iggy_SDK/Contracts/OptionSpec.cs b/foreign/csharp/Iggy_SDK/Contracts/OptionSpec.cs index 7eb9f96..1e523e7 100644 --- a/foreign/csharp/Iggy_SDK/Contracts/OptionSpec.cs +++ b/foreign/csharp/Iggy_SDK/Contracts/OptionSpec.cs
@@ -25,17 +25,17 @@ public enum OptionsScope { /// <summary> - /// Keys a CreateTopic accepts. + /// Topic options. /// </summary> Topic = 1, /// <summary> - /// Keys a CreateStream accepts; none yet. + /// Stream options. /// </summary> Stream = 2, /// <summary> - /// Keys a CreateUser accepts; none yet. + /// User options. /// </summary> User = 3 } @@ -46,19 +46,43 @@ public class OptionSpec { /// <summary> - /// Option key the create command accepts. + /// Option key, as a create, an update or a read names it. /// </summary> public required string Key { get; set; } /// <summary> /// Canonical kind for this key: what the server encodes its default under, and - /// what a value set at create is stored as whatever kind the client sent, since - /// create admission re-encodes the block from its own parse. An update stores - /// the client's bytes verbatim and is the exception. + /// what a value set at create or update is stored as whatever kind the client sent, + /// since admission re-encodes every block from its own parse. /// </summary> public required HeaderKind Kind { get; set; } /// <summary> + /// An update may set the key. A key that is neither updatable nor read-only is set only at creation. + /// </summary> + public bool Updatable { get; set; } + + /// <summary> + /// An update may reset the key to its default. + /// </summary> + public bool Resettable { get; set; } + + /// <summary> + /// Write-only: accepted at creation, never returned. + /// </summary> + public bool Secret { get; set; } + + /// <summary> + /// Computed by the server and returned on reads, never accepted from a client. + /// </summary> + public bool ReadOnly { get; set; } + + /// <summary> + /// A server that does not understand the key refuses to serve the resource rather than ignore it. + /// </summary> + public bool Critical { get; set; } + + /// <summary> /// The key's default in <see cref="Kind" />'s encoding; empty when the key /// has no default. /// </summary>
diff --git a/foreign/csharp/Iggy_SDK/Contracts/OptionValues.cs b/foreign/csharp/Iggy_SDK/Contracts/OptionValues.cs new file mode 100644 index 0000000..8d97d33 --- /dev/null +++ b/foreign/csharp/Iggy_SDK/Contracts/OptionValues.cs
@@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Apache.Iggy.Headers; + +namespace Apache.Iggy.Contracts; + +/// <summary> +/// Typed reads of a response's options. A key lives in one of the two provenance blocks, and an absent key +/// resolves to its catalog default. +/// </summary> +internal static class OptionValues +{ + internal static HeaderValue? Find(Dictionary<HeaderKey, HeaderValue>? explicitOptions, + Dictionary<HeaderKey, HeaderValue>? derivedOptions, TopicOptionKey key) + { + return Find(explicitOptions, derivedOptions, OptionKeys.Describe(key).Name); + } + + internal static string? Text(Dictionary<HeaderKey, HeaderValue>? explicitOptions, + Dictionary<HeaderKey, HeaderValue>? derivedOptions, StreamOptionKey key) + { + return Find(explicitOptions, derivedOptions, OptionKeys.Describe(key).Name) is { Kind: HeaderKind.String } value + ? value.ToString() + : null; + } + + /// <summary>A u64 setting, <see cref="ulong.MaxValue" /> when absent: never expire and unlimited.</summary> + internal static ulong UInt64(Dictionary<HeaderKey, HeaderValue>? explicitOptions, + Dictionary<HeaderKey, HeaderValue>? derivedOptions, TopicOptionKey key) + { + return Find(explicitOptions, derivedOptions, key) is { Kind: HeaderKind.Uint64 } value + ? value.ToUInt64() + : ulong.MaxValue; + } + + private static HeaderValue? Find(Dictionary<HeaderKey, HeaderValue>? explicitOptions, + Dictionary<HeaderKey, HeaderValue>? derivedOptions, string name) + { + var key = HeaderKey.FromString(name); + if (explicitOptions is not null && explicitOptions.TryGetValue(key, out var value)) + { + return value; + } + + return derivedOptions is not null && derivedOptions.TryGetValue(key, out value) ? value : null; + } +}
diff --git a/foreign/csharp/Iggy_SDK/Contracts/OptionsPatch.cs b/foreign/csharp/Iggy_SDK/Contracts/OptionsPatch.cs new file mode 100644 index 0000000..9c74661 --- /dev/null +++ b/foreign/csharp/Iggy_SDK/Contracts/OptionsPatch.cs
@@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Apache.Iggy.Contracts.Http; +using Apache.Iggy.Headers; +using Apache.Iggy.Vsr; + +namespace Apache.Iggy.Contracts; + +/// <summary> +/// A validated update: the keys to set and the keys to reset to their defaults, the reset keys in ascending +/// ordinal (key byte) order. Every key not named keeps its stored value. +/// </summary> +internal sealed record OptionsPatch( + OptionsScope Scope, + IReadOnlyDictionary<string, HeaderValue> Set, + IReadOnlyList<string> Reset) +{ + private const string RESET_FIELD = "reset"; + + /// <summary> + /// The REST update body: one flat object of the keys to set, plus a <c>reset</c> array when there is one. + /// </summary> + internal Dictionary<string, object> ToHttpBody() + { + if (Set.ContainsKey(RESET_FIELD)) + { + throw VsrError.Exception(VsrError.UNSUPPORTED_OPTION_KEY, $"Unsupported option key {RESET_FIELD}."); + } + + var body = new Dictionary<string, object>(OptionInputs.From(Set)); + if (Reset.Count > 0) + { + body[RESET_FIELD] = Reset.Order(StringComparer.Ordinal).ToArray(); + } + + return body; + } +}
diff --git a/foreign/csharp/Iggy_SDK/Contracts/StreamOptionKey.cs b/foreign/csharp/Iggy_SDK/Contracts/StreamOptionKey.cs new file mode 100644 index 0000000..82a5aee --- /dev/null +++ b/foreign/csharp/Iggy_SDK/Contracts/StreamOptionKey.cs
@@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.Iggy.Contracts; + +/// <summary> +/// The client-visible option keys of a stream, in catalog order. Names a key to reset in a +/// <see cref="StreamUpdate" />; no stream key is resettable yet, so a reset is refused before it is sent. +/// </summary> +public enum StreamOptionKey +{ + /// <summary><c>name</c>, updatable.</summary> + Name, + + /// <summary><c>encryption_key</c>, create-only and write-only.</summary> + EncryptionKey, + + /// <summary><c>encryption_key_source</c>, read-only.</summary> + EncryptionKeySource, + + /// <summary><c>encryption_key_fingerprint</c>, read-only.</summary> + EncryptionKeyFingerprint, + + /// <summary><c>encryption_kek_fingerprint</c>, read-only.</summary> + EncryptionKekFingerprint +}
diff --git a/foreign/csharp/Iggy_SDK/Contracts/StreamOptions.cs b/foreign/csharp/Iggy_SDK/Contracts/StreamOptions.cs new file mode 100644 index 0000000..1361a22 --- /dev/null +++ b/foreign/csharp/Iggy_SDK/Contracts/StreamOptions.cs
@@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Text.Json.Serialization; +using Apache.Iggy.Encryption; +using Apache.Iggy.Headers; +using Apache.Iggy.IggyClient; +using Apache.Iggy.JsonConverters; +using Apache.Iggy.Vsr; + +namespace Apache.Iggy.Contracts; + +/// <summary> +/// Options a stream takes at creation, passed to <see cref="IIggyStream.CreateStreamAsync" />. +/// </summary> +public sealed class StreamOptions +{ + /// <summary> + /// The stream's own data encryption key: exactly 32 random bytes, see <see cref="EncryptionKeys" />. + /// </summary> + /// <remarks> + /// The key travels inside the create request over the transport the client uses, so TLS is the caller's + /// responsibility. The server stores it wrapped and never returns it, reads report only + /// <see cref="StreamResponse.EncryptionKeySource" /> and its fingerprints. Left null, a server with + /// encryption enabled generates a key for the stream, otherwise the stream is stored in plaintext. + /// The key cannot be changed after creation, and JSON serialization of these options leaves it out and + /// writes <see cref="Options" /> entries by name only, so a structured logger never writes it. + /// </remarks> + [JsonIgnore] + public byte[]? EncryptionKey { get; init; } + + /// <summary> + /// Option keys with no property of their own, keyed by option name, for keys newer than this build. A + /// property above wins on collision, and the server refuses a key outside its catalog by name. + /// </summary> + [JsonConverter(typeof(RedactedOptionsConverter))] + public IReadOnlyDictionary<string, HeaderValue>? Options { get; init; } + + /// <summary> + /// Renders the options that were set, each under the kind the server's catalog gives its key. + /// </summary> + /// <returns>Option values keyed by option name.</returns> + /// <exception cref="Exceptions.IggyInvalidStatusCodeException">The encryption key is not exactly 32 bytes.</exception> + public Dictionary<string, HeaderValue> ToDictionary() + { + var options = OptionKeys.Raw(Options); + if (EncryptionKey is { } key) + { + if (key.Length != EncryptionKeys.KEY_LENGTH) + { + throw VsrError.Exception(VsrError.INVALID_ENCRYPTION_KEY, + $"A stream encryption key must be exactly {EncryptionKeys.KEY_LENGTH} bytes."); + } + + options[OptionKeys.Describe(StreamOptionKey.EncryptionKey).Name] = HeaderValue.FromBytes([.. key]); + } + + return options; + } +}
diff --git a/foreign/csharp/Iggy_SDK/Contracts/StreamResponse.cs b/foreign/csharp/Iggy_SDK/Contracts/StreamResponse.cs index 8555ea1..972bcf8 100644 --- a/foreign/csharp/Iggy_SDK/Contracts/StreamResponse.cs +++ b/foreign/csharp/Iggy_SDK/Contracts/StreamResponse.cs
@@ -16,6 +16,7 @@ // under the License. using System.Text.Json.Serialization; +using Apache.Iggy.Enums; using Apache.Iggy.Headers; using Apache.Iggy.JsonConverters; @@ -64,8 +65,61 @@ public IEnumerable<TopicResponse> Topics { get; init; } = []; /// <summary> - /// Options attached to the stream at creation. TCP transport only. + /// Options the client set on the stream. The server never returns the encryption key. /// </summary> [JsonIgnore] public Dictionary<HeaderKey, HeaderValue>? Options { get; init; } + + /// <summary> + /// Options the server filled or computes on reads, such as the encryption key source and fingerprints. + /// </summary> + [JsonIgnore] + public Dictionary<HeaderKey, HeaderValue>? DerivedOptions { get; init; } + + /// <summary> + /// Who chose the key that encrypts the stream at rest, or null when the server reported no source. + /// </summary> + [JsonIgnore] + public StreamEncryptionSource? EncryptionKeySource => + OptionValues.Text(Options, DerivedOptions, StreamOptionKey.EncryptionKeySource) switch + { + "none" => StreamEncryptionSource.None, + "stream" => StreamEncryptionSource.Stream, + "server" => StreamEncryptionSource.Server, + _ => null + }; + + /// <summary> + /// Fingerprint of the stream's key, the value <see cref="Encryption.EncryptionKeys.Fingerprint(ReadOnlySpan{byte})" /> + /// computes from it. Null for a plaintext stream. + /// </summary> + [JsonIgnore] + public string? EncryptionKeyFingerprint => + OptionValues.Text(Options, DerivedOptions, StreamOptionKey.EncryptionKeyFingerprint); + + /// <summary> + /// Fingerprints of the server keys that wrap the stream's key: one outside a key rotation, empty for a + /// plaintext stream. + /// </summary> + [JsonIgnore] + public IReadOnlyList<string> EncryptionKekFingerprints => + OptionValues.Text(Options, DerivedOptions, StreamOptionKey.EncryptionKekFingerprint) is { Length: > 0 } joined + ? joined.Split(',') + : []; + + /// <summary> + /// REST renders both provenances into one object keyed by option name, so one JSON property + /// fills <see cref="Options" /> and <see cref="DerivedOptions" />. + /// </summary> + [JsonInclude] + [JsonPropertyName("options")] + [JsonConverter(typeof(ResourceOptionsConverter))] + internal ResourceOptions RestOptions + { + init + { + Options = value.Explicit; + DerivedOptions = value.Derived; + } + } }
diff --git a/foreign/csharp/Iggy_SDK/Contracts/StreamUpdate.cs b/foreign/csharp/Iggy_SDK/Contracts/StreamUpdate.cs new file mode 100644 index 0000000..658a8e1 --- /dev/null +++ b/foreign/csharp/Iggy_SDK/Contracts/StreamUpdate.cs
@@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Text.Json.Serialization; +using Apache.Iggy.Headers; +using Apache.Iggy.IggyClient; +using Apache.Iggy.JsonConverters; + +namespace Apache.Iggy.Contracts; + +/// <summary> +/// The changes <see cref="IIggyStream.UpdateStreamAsync" /> applies to a stream. Every property left null keeps +/// the stream's stored value. +/// </summary> +public sealed class StreamUpdate +{ + private static readonly HashSet<StreamOptionKey> NoReset = []; + + /// <summary>The stream's new name, unique within the server.</summary> + public string? Name { get; init; } + + /// <summary> + /// Keys to reset to their defaults. No stream key has one, so a non-empty set is refused with the error + /// the server returns for it, before anything is sent unless the update carries a raw key. + /// </summary> + public IReadOnlySet<StreamOptionKey>? Reset { get; init; } + + /// <summary> + /// Reset names newer than this build's typed keys. The server decides whether each exists and may be reset, + /// so an update naming one skips the client-side checks beyond the structural ones. + /// </summary> + public IReadOnlySet<string>? RawReset { get; init; } + + /// <summary>Updatable keys newer than this build, keyed by option name. A property above wins on collision.</summary> + [JsonConverter(typeof(RedactedOptionsConverter))] + public IReadOnlyDictionary<string, HeaderValue>? Options { get; init; } + + internal OptionsPatch ToPatch() + { + var set = OptionKeys.Raw(Options); + var hasRawSet = set.Count > 0; + if (Name is not null) + { + set[OptionKeys.Describe(StreamOptionKey.Name).Name] = OptionKeys.Text(Name); + } + + return OptionKeys.Patch(OptionsScope.Stream, set, (Reset ?? NoReset).Select(OptionKeys.Describe), + RawReset, hasRawSet); + } +}
diff --git a/foreign/csharp/Iggy_SDK/Contracts/Tcp/TcpContracts.cs b/foreign/csharp/Iggy_SDK/Contracts/Tcp/TcpContracts.cs index 49e7e6b..9cb6302 100644 --- a/foreign/csharp/Iggy_SDK/Contracts/Tcp/TcpContracts.cs +++ b/foreign/csharp/Iggy_SDK/Contracts/Tcp/TcpContracts.cs
@@ -156,60 +156,22 @@ return bytes; } - internal static byte[] UpdateUser(Identifier userId, string? userName, UserStatus? status) + internal static byte[] UpdateUser(Identifier userId, OptionsPatch patch) { - if (userName is not null) - { - CredentialBounds.ValidateUsername(userName); - } - - var userNameLength = userName is null ? 0 : Encoding.UTF8.GetByteCount(userName); - var length = userId.Length + 2 - + (userName is null ? 1 : 2 + userNameLength) - + (status is not null ? 2 : 1); - Span<byte> bytes = stackalloc byte[length]; - - bytes.WriteBytesFromIdentifier(userId); - var position = userId.Length + 2; - if (userName is not null) - { - bytes[position] = 1; - position += 1; - bytes[position] = (byte)userNameLength; - position += 1; - Encoding.UTF8.GetBytes(userName, - bytes[position..(position + userNameLength)]); - position += userNameLength; - } - else - { - bytes[position] = 0; - position += 1; - } - - if (status is not null) - { - bytes[position++] = 1; - bytes[position++] = (byte)status; - } - else - { - bytes[position++] = 0; - } - - return bytes.ToArray(); + return UpdateResource(patch, userId); } internal static byte[] CreateUser(string userName, string password, UserStatus status, - Permissions? permissions = null) + Permissions? permissions = null, IReadOnlyDictionary<string, HeaderValue>? options = null) { CredentialBounds.ValidateUsername(userName); CredentialBounds.ValidatePassword(password); var userNameLength = Encoding.UTF8.GetByteCount(userName); var passwordLength = Encoding.UTF8.GetByteCount(password); var permissionsBytes = permissions is not null ? GetBytesFromPermissions(permissions) : []; + var optionsBlock = OptionsBlock(options); var bytes = new byte[3 + userNameLength + passwordLength + 1 - + (permissions is not null ? 4 + permissionsBytes.Length : 0)]; + + (permissions is not null ? 4 + permissionsBytes.Length : 0) + optionsBlock.Length]; var position = 0; bytes[position++] = (byte)userNameLength; @@ -221,6 +183,7 @@ bytes[position++] = (byte)status; WritePermissionsBlock(bytes.AsSpan(position), permissions, permissionsBytes); + optionsBlock.CopyTo(bytes.AsSpan(bytes.Length - optionsBlock.Length)); return bytes; } @@ -566,24 +529,20 @@ } } - internal static byte[] CreateStream(string name) + internal static byte[] CreateStream(string name, IReadOnlyDictionary<string, HeaderValue>? options = null) { var nameLength = WireName.ByteCount(name, nameof(name)); - Span<byte> bytes = stackalloc byte[nameLength + 1]; + var optionsBlock = OptionsBlock(options); + var bytes = new byte[1 + nameLength + optionsBlock.Length]; bytes[0] = (byte)nameLength; - Encoding.UTF8.GetBytes(name, bytes[1..]); - return bytes.ToArray(); + Encoding.UTF8.GetBytes(name, bytes.AsSpan(1, nameLength)); + optionsBlock.CopyTo(bytes.AsSpan(1 + nameLength)); + return bytes; } - internal static byte[] UpdateStream(Identifier streamId, string name) + internal static byte[] UpdateStream(Identifier streamId, OptionsPatch patch) { - var nameLength = WireName.ByteCount(name, nameof(name)); - Span<byte> bytes = stackalloc byte[streamId.Length + nameLength + 3]; - bytes.WriteBytesFromIdentifier(streamId); - var position = 2 + streamId.Length; - bytes[position] = (byte)nameLength; - Encoding.UTF8.GetBytes(name, bytes[(position + 1)..]); - return bytes.ToArray(); + return UpdateResource(patch, streamId); } internal static byte[] CreateGroup(Identifier streamId, Identifier topicId, string name) @@ -640,116 +599,88 @@ return bytes.ToArray(); } - internal static byte[] UpdateTopic(Identifier streamId, Identifier topicId, string name, - CompressionAlgorithm compressionAlgorithm, ulong maxTopicSize, ulong messageExpiry, - IReadOnlyDictionary<string, HeaderValue>? extraOptions = null) + internal static byte[] UpdateTopic(Identifier streamId, Identifier topicId, OptionsPatch patch) { - // Settings ride the options block. A default value means the caller did - // not set the key, so it is omitted and the server leaves the topic's - // current value alone. - var options = new Dictionary<HeaderKey, HeaderValue>(); - // Caller keys first, so a named argument overwrites one of them. - if (extraOptions is not null) - { - foreach (var (key, value) in extraOptions) - { - options[HeaderKey.FromString(key)] = value; - } - } - - if (compressionAlgorithm != CompressionAlgorithm.None) - { - options[HeaderKey.FromString("compression_algorithm")] - = HeaderValue.FromString(compressionAlgorithm.ToString().ToLowerInvariant()); - } - - if (messageExpiry != 0) - { - options[HeaderKey.FromString("message_expiry")] = HeaderValue.FromUInt64(messageExpiry); - } - - if (maxTopicSize != 0) - { - options[HeaderKey.FromString("max_topic_size")] = HeaderValue.FromUInt64(maxTopicSize); - } - - var optionsLength = HeadersByteLength(options); - var nameLength = WireName.ByteCount(name, nameof(name)); - var length = 4 + streamId.Length + topicId.Length + 1 + nameLength + optionsLength; - var rented = length > MaxStackAllocBytes ? ArrayPool<byte>.Shared.Rent(length) : null; - try - { - Span<byte> bytes = rented is null ? stackalloc byte[length] : rented.AsSpan(0, length); - bytes.WriteBytesFromStreamAndTopicIdentifiers(streamId, topicId); - var position = 4 + streamId.Length + topicId.Length; - bytes[position] = (byte)nameLength; - Encoding.UTF8.GetBytes(name, bytes[(position + 1)..(position + 1 + nameLength)]); - WriteHeadersTo(bytes[(position + 1 + nameLength)..], options); - return bytes.ToArray(); - } - finally - { - if (rented is not null) - { - ArrayPool<byte>.Shared.Return(rented); - } - } + return UpdateResource(patch, streamId, topicId); } internal static byte[] CreateTopic(Identifier streamId, string name, uint partitionCount, - CompressionAlgorithm compressionAlgorithm, ulong messageExpiry, - ulong maxTopicSize, IReadOnlyDictionary<string, HeaderValue>? extraOptions = null) + IReadOnlyDictionary<string, HeaderValue>? options = null) { - var options = new Dictionary<HeaderKey, HeaderValue>(); - // Caller keys go in first so a named argument overwrites one of them: the - // block must not carry a key twice, or the server refuses it whole. - if (extraOptions is not null) - { - foreach (var (key, value) in extraOptions) - { - options[HeaderKey.FromString(key)] = value; - } - } - - if (compressionAlgorithm != CompressionAlgorithm.None) - { - options[HeaderKey.FromString("compression_algorithm")] - = HeaderValue.FromString(compressionAlgorithm.ToString().ToLowerInvariant()); - } - - if (messageExpiry != 0) - { - options[HeaderKey.FromString("message_expiry")] = HeaderValue.FromUInt64(messageExpiry); - } - - if (maxTopicSize != 0) - { - options[HeaderKey.FromString("max_topic_size")] = HeaderValue.FromUInt64(maxTopicSize); - } - - var optionsLength = HeadersByteLength(options); var nameLength = WireName.ByteCount(name, nameof(name)); - var length = 2 + streamId.Length + 4 + 1 + nameLength + optionsLength; - var rented = length > MaxStackAllocBytes ? ArrayPool<byte>.Shared.Rent(length) : null; - try + var optionsBlock = OptionsBlock(options); + var bytes = new byte[2 + streamId.Length + 4 + 1 + nameLength + optionsBlock.Length]; + bytes.AsSpan().WriteBytesFromIdentifier(streamId); + var position = 2 + streamId.Length; + BinaryPrimitives.WriteUInt32LittleEndian(bytes.AsSpan(position, 4), partitionCount); + position += 4; + bytes[position] = (byte)nameLength; + Encoding.UTF8.GetBytes(name, bytes.AsSpan(position + 1, nameLength)); + optionsBlock.CopyTo(bytes.AsSpan(position + 1 + nameLength)); + return bytes; + } + + /// <summary> + /// <c>[scope:u8][identifiers][reset_count:u8][reset_key:u8 len + name]*[options to end of body]</c>, the + /// identifiers being the stream and topic, the stream, or the user the update targets. + /// </summary> + private static byte[] UpdateResource(OptionsPatch patch, params Identifier[] targets) + { + var optionsBlock = OptionsBlock(patch.Set); + var resetLength = patch.Reset.Sum(key => 1 + WireName.ByteCount(key, nameof(patch.Reset))); + var bytes = new byte[1 + targets.Sum(target => 2 + target.Length) + 1 + resetLength + optionsBlock.Length]; + var position = 0; + bytes[position++] = (byte)patch.Scope; + foreach (var target in targets) { - Span<byte> bytes = rented is null ? stackalloc byte[length] : rented.AsSpan(0, length); - bytes.WriteBytesFromIdentifier(streamId); - var position = 2 + streamId.Length; - BinaryPrimitives.WriteUInt32LittleEndian(bytes[position..(position + 4)], partitionCount); - position += 4; - bytes[position] = (byte)nameLength; - Encoding.UTF8.GetBytes(name, bytes[(position + 1)..(position + 1 + nameLength)]); - WriteHeadersTo(bytes[(position + 1 + nameLength)..], options); - return bytes.ToArray(); + bytes.AsSpan().WriteBytesFromIdentifier(target, position); + position += 2 + target.Length; } - finally + + bytes[position++] = (byte)patch.Reset.Count; + foreach (var key in patch.Reset) { - if (rented is not null) - { - ArrayPool<byte>.Shared.Return(rented); - } + bytes[position] = (byte)Encoding.UTF8.GetBytes(key, bytes.AsSpan(position + 1)); + position += 1 + bytes[position]; } + + optionsBlock.CopyTo(bytes.AsSpan(position)); + return bytes; + } + + /// <summary> + /// An options block in ascending key byte order, the order the server writes its own blocks in, so a + /// request is byte-identical to the one any other SDK sends for the same options. + /// </summary> + private static byte[] OptionsBlock(IReadOnlyDictionary<string, HeaderValue>? options) + { + if (options is null || options.Count == 0) + { + return []; + } + + List<(HeaderKey Key, HeaderValue Value)> entries = options + .Select(option => (HeaderKey.FromString(option.Key), option.Value)) + .ToList(); + entries.Sort((left, right) => left.Key.Value.AsSpan().SequenceCompareTo(right.Key.Value)); + + var block = new byte[entries.Sum(entry => 1 + 4 + entry.Key.Value.Length + 1 + 4 + entry.Value.Value.Length)]; + var position = 0; + foreach (var (key, value) in entries) + { + position = WriteOptionField(block, position, key.Kind, key.Value); + position = WriteOptionField(block, position, value.Kind, value.Value); + } + + return block; + } + + private static int WriteOptionField(byte[] block, int position, HeaderKind kind, byte[] value) + { + block[position] = HeaderKindToByte(kind); + BinaryPrimitives.WriteInt32LittleEndian(block.AsSpan(position + 1, 4), value.Length); + value.CopyTo(block, position + 5); + return position + 5 + value.Length; } internal static byte[] GetTopicById(Identifier streamId, Identifier topicId)
diff --git a/foreign/csharp/Iggy_SDK/Contracts/TopicOptionKey.cs b/foreign/csharp/Iggy_SDK/Contracts/TopicOptionKey.cs new file mode 100644 index 0000000..d227af6 --- /dev/null +++ b/foreign/csharp/Iggy_SDK/Contracts/TopicOptionKey.cs
@@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.Iggy.Contracts; + +/// <summary> +/// The client-visible option keys of a topic, in catalog order. Names a key to reset in a +/// <see cref="TopicUpdate" />: only <see cref="MessageExpiry" /> and <see cref="MaxTopicSize" /> reset to a default. +/// </summary> +public enum TopicOptionKey +{ + /// <summary><c>name</c>, updatable.</summary> + Name, + + /// <summary><c>compression_algorithm</c>, create-only.</summary> + CompressionAlgorithm, + + /// <summary><c>message_expiry</c>, updatable and resettable to never expire.</summary> + MessageExpiry, + + /// <summary><c>max_topic_size</c>, updatable and resettable to unlimited.</summary> + MaxTopicSize, + + /// <summary><c>segment_size</c>, create-only.</summary> + SegmentSize, + + /// <summary><c>durability</c>, create-only.</summary> + Durability, + + /// <summary><c>consumer_offset_durability</c>, create-only.</summary> + ConsumerOffsetDurability, + + /// <summary><c>partition_resize_policy</c>, create-only.</summary> + PartitionResizePolicy, + + /// <summary><c>messages_required_to_save</c>, create-only.</summary> + MessagesRequiredToSave, + + /// <summary><c>size_of_messages_required_to_save</c>, create-only.</summary> + SizeOfMessagesRequiredToSave, + + /// <summary><c>preallocate_segments</c>, create-only.</summary> + PreallocateSegments +}
diff --git a/foreign/csharp/Iggy_SDK/Contracts/TopicOptions.cs b/foreign/csharp/Iggy_SDK/Contracts/TopicOptions.cs index 554fcf9..48e8ed5 100644 --- a/foreign/csharp/Iggy_SDK/Contracts/TopicOptions.cs +++ b/foreign/csharp/Iggy_SDK/Contracts/TopicOptions.cs
@@ -15,32 +15,41 @@ // specific language governing permissions and limitations // under the License. -using Apache.Iggy.Headers; +using System.Text.Json.Serialization; using Apache.Iggy.Enums; +using Apache.Iggy.Headers; using Apache.Iggy.IggyClient; +using Apache.Iggy.JsonConverters; +using Apache.Iggy.Utils; namespace Apache.Iggy.Contracts; /// <summary> -/// Topic options that have no named parameter of their own, typed instead of keyed by hand. +/// Options a topic takes at creation, passed to <see cref="IIggyTopic.CreateTopicAsync" />. /// </summary> /// <remarks> -/// Hand the result of <see cref="ToDictionary" /> to the <c>options</c> parameter of -/// <see cref="IIggyTopic.CreateTopicAsync" />. Every key here is settable at creation only: an -/// update takes just the options that have a parameter of their own and refuses the rest by -/// name, the same way it refuses a key outside the server's catalog, so a mistyped key fails -/// the call rather than being ignored. <see cref="IIggySystem.DescribeOptionsAsync" /> -/// enumerates the keys a given server accepts, with the kind and default of each. -/// A property left null emits no key, which leaves the server default in place. +/// <see cref="MessageExpiry" /> and <see cref="MaxTopicSize" /> can change later through a +/// <see cref="TopicUpdate" />, every other setting is fixed at creation. A property left null emits no key, +/// which leaves the server default in place. <see cref="IIggySystem.DescribeOptionsAsync" /> enumerates the +/// keys a given server accepts, with the kind, default and mutability of each. /// </remarks> public sealed class TopicOptions { - private const string SegmentSizeKey = "segment_size"; - private const string DurabilityKey = "durability"; - private const string ConsumerOffsetDurabilityKey = "consumer_offset_durability"; - private const string MessagesRequiredToSaveKey = "messages_required_to_save"; - private const string SizeOfMessagesRequiredToSaveKey = "size_of_messages_required_to_save"; - private const string PreallocateSegmentsKey = "preallocate_segments"; + /// <summary> + /// Compression algorithm label stored with the topic. Fixed at creation. + /// </summary> + public CompressionAlgorithm? CompressionAlgorithm { get; init; } + + /// <summary> + /// How long a message is kept. <see cref="TimeSpan.MaxValue" /> never expires, zero keeps the server default. + /// </summary> + public TimeSpan? MessageExpiry { get; init; } + + /// <summary> + /// Topic-wide retained size cap in bytes, divided among partitions and at least the segment size. + /// <see cref="ulong.MaxValue" /> is unlimited, zero keeps the server default. + /// </summary> + public ulong? MaxTopicSize { get; init; } /// <summary> /// Size of a single segment in bytes. The server checks it against its own bounds, a @@ -57,6 +66,11 @@ public Durability ConsumerOffsetDurability { get; init; } = Durability.Replicated; /// <summary> + /// Whether partitions can be added or deleted after creation. The server default is mutable. + /// </summary> + public PartitionResizePolicy? PartitionResizePolicy { get; init; } + + /// <summary> /// Attempt a flush after this many messages accumulate in a partition buffer. Must be non-zero. /// </summary> public uint? MessagesRequiredToSave { get; init; } @@ -74,59 +88,108 @@ public bool? PreallocateSegments { get; init; } /// <summary> + /// Option keys with no property of their own, keyed by option name, for keys newer than this build. A + /// property above wins on collision, and the server refuses a key outside its catalog by name. A raw + /// durability is kept unless the matching property asks for persisted durability. + /// </summary> + [JsonConverter(typeof(RedactedOptionsConverter))] + public IReadOnlyDictionary<string, HeaderValue>? Options { get; init; } + + /// <summary> /// Renders the options that were set, each under the kind the server's catalog gives its key. /// </summary> - /// <returns>Option values keyed by option name, including both durability defaults.</returns> + /// <returns>Option values keyed by option name, including both durability policies.</returns> + /// <exception cref="ArgumentException">A raw durability is invalid or contradicts a persisted property.</exception> public Dictionary<string, HeaderValue> ToDictionary() { - var options = new Dictionary<string, HeaderValue>(); + var options = OptionKeys.Raw(Options); + + if (CompressionAlgorithm is { } compressionAlgorithm) + { + options[Name(TopicOptionKey.CompressionAlgorithm)] = HeaderValue.FromString(compressionAlgorithm switch + { + Enums.CompressionAlgorithm.None => "none", + Enums.CompressionAlgorithm.Gzip => "gzip", + _ => throw new ArgumentOutOfRangeException(nameof(CompressionAlgorithm), compressionAlgorithm, + "Unknown compression algorithm") + }); + } + + if (MessageExpiry is { } messageExpiry && DurationHelpers.ToDuration(messageExpiry) is not 0 and var micros) + { + options[Name(TopicOptionKey.MessageExpiry)] = HeaderValue.FromUInt64(micros); + } + + if (MaxTopicSize is { } maxTopicSize and not 0) + { + options[Name(TopicOptionKey.MaxTopicSize)] = HeaderValue.FromUInt64(maxTopicSize); + } if (SegmentSize is { } segmentSize) { - options[SegmentSizeKey] = HeaderValue.FromUInt64(segmentSize); + options[Name(TopicOptionKey.SegmentSize)] = HeaderValue.FromUInt64(segmentSize); } - options[DurabilityKey] = HeaderValue.FromString(EncodeDurability(Durability)); - options[ConsumerOffsetDurabilityKey] = HeaderValue.FromString(EncodeDurability(ConsumerOffsetDurability)); + options[Name(TopicOptionKey.Durability)] = DurabilityValue(TopicOptionKey.Durability, Durability); + options[Name(TopicOptionKey.ConsumerOffsetDurability)] = + DurabilityValue(TopicOptionKey.ConsumerOffsetDurability, ConsumerOffsetDurability); + + if (PartitionResizePolicy is { } partitionResizePolicy) + { + options[Name(TopicOptionKey.PartitionResizePolicy)] = HeaderValue.FromString(partitionResizePolicy switch + { + Enums.PartitionResizePolicy.Mutable => "mutable", + Enums.PartitionResizePolicy.Fixed => "fixed", + _ => throw new ArgumentOutOfRangeException(nameof(PartitionResizePolicy), partitionResizePolicy, + "Unknown partition resize policy") + }); + } if (MessagesRequiredToSave is { } messagesRequiredToSave) { - options[MessagesRequiredToSaveKey] = HeaderValue.FromUInt32(messagesRequiredToSave); + options[Name(TopicOptionKey.MessagesRequiredToSave)] = HeaderValue.FromUInt32(messagesRequiredToSave); } if (SizeOfMessagesRequiredToSave is { } sizeOfMessagesRequiredToSave) { - options[SizeOfMessagesRequiredToSaveKey] = HeaderValue.FromUInt64(sizeOfMessagesRequiredToSave); + options[Name(TopicOptionKey.SizeOfMessagesRequiredToSave)] = + HeaderValue.FromUInt64(sizeOfMessagesRequiredToSave); } if (PreallocateSegments is { } preallocateSegments) { - options[PreallocateSegmentsKey] = HeaderValue.FromBool(preallocateSegments); + options[Name(TopicOptionKey.PreallocateSegments)] = HeaderValue.FromBool(preallocateSegments); } return options; } - internal static Dictionary<string, HeaderValue> WithDurabilityDefaults(IReadOnlyDictionary<string, HeaderValue>? source) + + private HeaderValue DurabilityValue(TopicOptionKey key, Durability policy) { - var options = source is null ? new Dictionary<string, HeaderValue>() : new Dictionary<string, HeaderValue>(source); - foreach (var key in new[] { DurabilityKey, ConsumerOffsetDurabilityKey }) + var name = Name(key); + if (Options is not null && Options.TryGetValue(name, out var raw)) { - if (!options.TryGetValue(key, out var value)) + var rawPolicy = raw.Kind == HeaderKind.String + ? raw.ToString() switch + { + "replicated" => Durability.Replicated, + "persisted" => Durability.Persisted, + _ => (Durability?)null + } + : null; + if (rawPolicy is null || (policy == Durability.Persisted && rawPolicy != policy)) { - options[key] = HeaderValue.FromString("replicated"); + throw new ArgumentException($"Invalid {name}", nameof(Options)); } - else if (value.Kind != HeaderKind.String || value.ToString() is not ("replicated" or "persisted")) - { - throw new ArgumentException($"Invalid {key}", nameof(source)); - } + + policy = rawPolicy.Value; } - return options; + + return HeaderValue.FromString(policy == Durability.Persisted ? "persisted" : "replicated"); } - private static string EncodeDurability(Durability value) => value switch + private static string Name(TopicOptionKey key) { - Durability.Replicated => "replicated", - Durability.Persisted => "persisted", - _ => throw new ArgumentOutOfRangeException(nameof(value), value, "Unknown durability") - }; + return OptionKeys.Describe(key).Name; + } }
diff --git a/foreign/csharp/Iggy_SDK/Contracts/TopicResponse.cs b/foreign/csharp/Iggy_SDK/Contracts/TopicResponse.cs index fd508e6..b542154 100644 --- a/foreign/csharp/Iggy_SDK/Contracts/TopicResponse.cs +++ b/foreign/csharp/Iggy_SDK/Contracts/TopicResponse.cs
@@ -19,6 +19,7 @@ using Apache.Iggy.Enums; using Apache.Iggy.Headers; using Apache.Iggy.JsonConverters; +using Apache.Iggy.Utils; namespace Apache.Iggy.Contracts; @@ -44,9 +45,16 @@ public required string Name { get; init; } /// <summary> - /// Compression algorithm used for the topic. + /// Compression algorithm label stored with the topic, read from its options. /// </summary> - public CompressionAlgorithm CompressionAlgorithm { get; set; } + [JsonIgnore] + public CompressionAlgorithm CompressionAlgorithm => + OptionValues.Find(Options, DerivedOptions, TopicOptionKey.CompressionAlgorithm) switch + { + { Kind: HeaderKind.String } value when value.ToString() == "gzip" => CompressionAlgorithm.Gzip, + { Kind: HeaderKind.Uint8 } value => (CompressionAlgorithm)value.Value[0], + _ => CompressionAlgorithm.None + }; /// <summary> /// Topic size in bytes. @@ -55,15 +63,17 @@ public required ulong Size { get; init; } /// <summary> - /// Message expiry in milliseconds. + /// How long a message is kept, read from the topic's options. <see cref="TimeSpan.MaxValue" /> never expires. /// </summary> - [JsonConverter(typeof(TimeSpanConverter))] - public TimeSpan MessageExpiry { get; init; } + [JsonIgnore] + public TimeSpan MessageExpiry => + DurationHelpers.FromDuration(OptionValues.UInt64(Options, DerivedOptions, TopicOptionKey.MessageExpiry)); /// <summary> - /// Maximum topic size in bytes. + /// Topic-wide size cap in bytes, read from the topic's options. <see cref="ulong.MaxValue" /> is unlimited. /// </summary> - public required ulong MaxTopicSize { get; init; } + [JsonIgnore] + public ulong MaxTopicSize => OptionValues.UInt64(Options, DerivedOptions, TopicOptionKey.MaxTopicSize); /// <summary> /// Number of messages in the topic. @@ -82,17 +92,15 @@ public IEnumerable<PartitionResponse>? Partitions { get; init; } /// <summary> - /// Options explicitly set by the client at topic creation. Over REST the values arrive in - /// their readable string form, so every one of them is <see cref="HeaderKind.String" />; - /// the binary transports carry the server's canonical kind per key. + /// Options the client set, at creation or through an update, each in the server's canonical kind for + /// its key on every transport. /// </summary> [JsonIgnore] public Dictionary<HeaderKey, HeaderValue>? Options { get; init; } /// <summary> - /// Options resolved from server defaults at topic creation. Over REST the values arrive in - /// their readable string form, so every one of them is <see cref="HeaderKind.String" />; - /// the binary transports carry the server's canonical kind per key. + /// Options resolved from server defaults, at creation or by a reset, each in the server's canonical + /// kind for its key on every transport. /// </summary> [JsonIgnore] public Dictionary<HeaderKey, HeaderValue>? DerivedOptions { get; init; }
diff --git a/foreign/csharp/Iggy_SDK/Contracts/TopicUpdate.cs b/foreign/csharp/Iggy_SDK/Contracts/TopicUpdate.cs new file mode 100644 index 0000000..89c06a5 --- /dev/null +++ b/foreign/csharp/Iggy_SDK/Contracts/TopicUpdate.cs
@@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Text.Json.Serialization; +using Apache.Iggy.Headers; +using Apache.Iggy.IggyClient; +using Apache.Iggy.JsonConverters; +using Apache.Iggy.Utils; + +namespace Apache.Iggy.Contracts; + +/// <summary> +/// The changes <see cref="IIggyTopic.UpdateTopicAsync" /> applies to a topic. Every property left null keeps the +/// topic's stored value. Settings fixed at creation, compression among them, have no property here. +/// </summary> +public sealed class TopicUpdate +{ + private static readonly HashSet<TopicOptionKey> NoReset = []; + + /// <summary>The topic's new name, unique within its stream.</summary> + public string? Name { get; init; } + + /// <summary> + /// The new message expiry. <see cref="TimeSpan.MaxValue" /> never expires, zero changes nothing. Reset + /// <see cref="TopicOptionKey.MessageExpiry" /> to return to the default. + /// </summary> + public TimeSpan? MessageExpiry { get; init; } + + /// <summary> + /// The new topic-wide size cap in bytes, at least the segment size. <see cref="ulong.MaxValue" /> is + /// unlimited, zero changes nothing. Reset <see cref="TopicOptionKey.MaxTopicSize" /> to return to the default. + /// </summary> + public ulong? MaxTopicSize { get; init; } + + /// <summary> + /// Keys to reset to their defaults: <see cref="TopicOptionKey.MessageExpiry" /> or + /// <see cref="TopicOptionKey.MaxTopicSize" />. Any other key is refused, before anything is sent unless + /// the update carries a raw key. + /// </summary> + public IReadOnlySet<TopicOptionKey>? Reset { get; init; } + + /// <summary> + /// Reset names newer than this build's typed keys. The server decides whether each exists and may be reset, + /// so an update naming one skips the client-side checks beyond the structural ones. + /// </summary> + public IReadOnlySet<string>? RawReset { get; init; } + + /// <summary>Updatable keys newer than this build, keyed by option name. A property above wins on collision.</summary> + [JsonConverter(typeof(RedactedOptionsConverter))] + public IReadOnlyDictionary<string, HeaderValue>? Options { get; init; } + + internal OptionsPatch ToPatch() + { + var set = OptionKeys.Raw(Options); + var hasRawSet = set.Count > 0; + if (Name is not null) + { + set[OptionKeys.Describe(TopicOptionKey.Name).Name] = OptionKeys.Text(Name); + } + + if (MessageExpiry is { } messageExpiry && DurationHelpers.ToDuration(messageExpiry) is not 0 and var micros) + { + set[OptionKeys.Describe(TopicOptionKey.MessageExpiry).Name] = HeaderValue.FromUInt64(micros); + } + + if (MaxTopicSize is { } maxTopicSize and not 0) + { + set[OptionKeys.Describe(TopicOptionKey.MaxTopicSize).Name] = HeaderValue.FromUInt64(maxTopicSize); + } + + return OptionKeys.Patch(OptionsScope.Topic, set, (Reset ?? NoReset).Select(OptionKeys.Describe), + RawReset, hasRawSet); + } +}
diff --git a/core/binary_protocol/src/responses/users/update_user.rs b/foreign/csharp/Iggy_SDK/Contracts/UserOptionKey.cs similarity index 64% copy from core/binary_protocol/src/responses/users/update_user.rs copy to foreign/csharp/Iggy_SDK/Contracts/UserOptionKey.cs index 2b4a4e4..3c02112 100644 --- a/core/binary_protocol/src/responses/users/update_user.rs +++ b/foreign/csharp/Iggy_SDK/Contracts/UserOptionKey.cs
@@ -15,4 +15,17 @@ // specific language governing permissions and limitations // under the License. -pub type UpdateUserResponse = super::EmptyResponse; +namespace Apache.Iggy.Contracts; + +/// <summary> +/// The client-visible option keys of a user, in catalog order. Names a key to reset in a +/// <see cref="UserUpdate" />; no user key is resettable yet, so a reset is refused before it is sent. +/// </summary> +public enum UserOptionKey +{ + /// <summary><c>username</c>, updatable.</summary> + Username, + + /// <summary><c>status</c>, updatable.</summary> + Status +}
diff --git a/foreign/csharp/Iggy_SDK/Contracts/UserOptions.cs b/foreign/csharp/Iggy_SDK/Contracts/UserOptions.cs new file mode 100644 index 0000000..0ccb980 --- /dev/null +++ b/foreign/csharp/Iggy_SDK/Contracts/UserOptions.cs
@@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Text.Json.Serialization; +using Apache.Iggy.Headers; +using Apache.Iggy.IggyClient; +using Apache.Iggy.JsonConverters; + +namespace Apache.Iggy.Contracts; + +/// <summary> +/// Options a user takes at creation, passed to <see cref="IIggyUsers.CreateUserAsync" />. The catalog defines +/// no user key yet, so this carries only keys newer than this build. +/// </summary> +public sealed class UserOptions +{ + /// <summary> + /// Option keys keyed by option name. The server refuses a key outside its catalog by name. + /// </summary> + [JsonConverter(typeof(RedactedOptionsConverter))] + public IReadOnlyDictionary<string, HeaderValue>? Options { get; init; } + + /// <summary>Renders the options that were set.</summary> + /// <returns>Option values keyed by option name.</returns> + public Dictionary<string, HeaderValue> ToDictionary() + { + return OptionKeys.Raw(Options); + } +}
diff --git a/foreign/csharp/Iggy_SDK/Contracts/UserUpdate.cs b/foreign/csharp/Iggy_SDK/Contracts/UserUpdate.cs new file mode 100644 index 0000000..fd3bc06 --- /dev/null +++ b/foreign/csharp/Iggy_SDK/Contracts/UserUpdate.cs
@@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Text.Json.Serialization; +using Apache.Iggy.Enums; +using Apache.Iggy.Headers; +using Apache.Iggy.IggyClient; +using Apache.Iggy.JsonConverters; + +namespace Apache.Iggy.Contracts; + +/// <summary> +/// The changes <see cref="IIggyUsers.UpdateUserAsync" /> applies to a user. Every property left null keeps the +/// user's stored value. +/// </summary> +public sealed class UserUpdate +{ + private static readonly HashSet<UserOptionKey> NoReset = []; + + /// <summary>The user's new username, unique within the server.</summary> + public string? Username { get; init; } + + /// <summary>The user's new status. Deactivating a user ends its sessions and refuses its next login.</summary> + public UserStatus? Status { get; init; } + + /// <summary> + /// Keys to reset to their defaults. No user key has one, so a non-empty set is refused with the error the + /// server returns for it, before anything is sent unless the update carries a raw key. + /// </summary> + public IReadOnlySet<UserOptionKey>? Reset { get; init; } + + /// <summary> + /// Reset names newer than this build's typed keys. The server decides whether each exists and may be reset, + /// so an update naming one skips the client-side checks beyond the structural ones. + /// </summary> + public IReadOnlySet<string>? RawReset { get; init; } + + /// <summary>Updatable keys newer than this build, keyed by option name. A property above wins on collision.</summary> + [JsonConverter(typeof(RedactedOptionsConverter))] + public IReadOnlyDictionary<string, HeaderValue>? Options { get; init; } + + internal OptionsPatch ToPatch() + { + var set = OptionKeys.Raw(Options); + var hasRawSet = set.Count > 0; + if (Username is not null) + { + set[OptionKeys.Describe(UserOptionKey.Username).Name] = OptionKeys.Text(Username); + } + + if (Status is { } status) + { + set[OptionKeys.Describe(UserOptionKey.Status).Name] = HeaderValue.FromString(status switch + { + UserStatus.Active => "active", + UserStatus.Inactive => "inactive", + _ => throw new ArgumentOutOfRangeException(nameof(Status), status, "Unknown user status") + }); + } + + return OptionKeys.Patch(OptionsScope.User, set, (Reset ?? NoReset).Select(OptionKeys.Describe), + RawReset, hasRawSet); + } +}
diff --git a/foreign/csharp/Iggy_SDK/Encryption/EncryptionKeys.cs b/foreign/csharp/Iggy_SDK/Encryption/EncryptionKeys.cs new file mode 100644 index 0000000..7fee84f --- /dev/null +++ b/foreign/csharp/Iggy_SDK/Encryption/EncryptionKeys.cs
@@ -0,0 +1,108 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Security.Cryptography; + +namespace Apache.Iggy.Encryption; + +/// <summary> +/// Helpers for the per-stream encryption key passed in <see cref="Contracts.StreamOptions.EncryptionKey" />. +/// </summary> +/// <remarks> +/// A stream key is 32 random bytes. It travels inside the create request over whatever transport the client +/// uses, so protecting it on the wire with TLS is the caller's choice. The server stores it wrapped and never +/// returns it: reads report only its source and fingerprints. Never derive a key from a password, the +/// fingerprint would then let anyone who can read the stream test guesses offline. +/// </remarks> +public static class EncryptionKeys +{ + /// <summary>Length of a stream encryption key in bytes.</summary> + public const int KEY_LENGTH = 32; + + private static readonly byte[] FingerprintLabel = "iggy/stream-key-fingerprint/v1"u8.ToArray(); + + /// <summary> + /// The fingerprint a stream reports in <see cref="Contracts.StreamResponse.EncryptionKeyFingerprint" /> for + /// this key: the first 8 bytes of SHA-256 over a fixed label and the key, as 16 lowercase hex characters. + /// </summary> + /// <param name="key">The 32 key bytes.</param> + /// <exception cref="ArgumentException">The key is not exactly 32 bytes.</exception> + public static string Fingerprint(ReadOnlySpan<byte> key) + { + if (key.Length != KEY_LENGTH) + { + throw new ArgumentException($"A stream encryption key must be exactly {KEY_LENGTH} bytes.", nameof(key)); + } + + var input = new byte[FingerprintLabel.Length + KEY_LENGTH]; + try + { + FingerprintLabel.CopyTo(input, 0); + key.CopyTo(input.AsSpan(FingerprintLabel.Length)); + return Convert.ToHexString(SHA256.HashData(input), 0, 8).ToLowerInvariant(); + } + finally + { + CryptographicOperations.ZeroMemory(input); + } + } + + /// <summary>The fingerprint of a key given in canonical standard base64.</summary> + /// <param name="base64Key">The key as padded standard base64 of exactly 32 bytes.</param> + /// <exception cref="ArgumentException">The text is not the canonical base64 of 32 bytes.</exception> + public static string Fingerprint(string base64Key) + { + var key = FromBase64(base64Key); + try + { + return Fingerprint(key); + } + finally + { + CryptographicOperations.ZeroMemory(key); + } + } + + /// <summary> + /// Decodes a key from strict canonical standard base64: padded, no whitespace, no URL-safe alphabet and + /// no stray bits in the last character. + /// </summary> + /// <param name="base64Key">The key as padded standard base64 of exactly 32 bytes.</param> + /// <returns>The 32 key bytes.</returns> + /// <exception cref="ArgumentException">The text is not the canonical base64 of 32 bytes.</exception> + public static byte[] FromBase64(string base64Key) + { + // Room for one extra decoded byte, so a longer key fails the length check instead of the decode. + var buffer = new byte[KEY_LENGTH + 1]; + try + { + if (!Convert.TryFromBase64String(base64Key, buffer, out var written) || written != KEY_LENGTH + || Convert.ToBase64String(buffer, 0, KEY_LENGTH) != base64Key) + { + throw new ArgumentException( + $"A stream encryption key must be the canonical padded base64 of exactly {KEY_LENGTH} bytes.", + nameof(base64Key)); + } + + return buffer[..KEY_LENGTH]; + } + finally + { + CryptographicOperations.ZeroMemory(buffer); + } + } +}
diff --git a/core/binary_protocol/src/responses/users/update_user.rs b/foreign/csharp/Iggy_SDK/Enums/PartitionResizePolicy.cs similarity index 70% copy from core/binary_protocol/src/responses/users/update_user.rs copy to foreign/csharp/Iggy_SDK/Enums/PartitionResizePolicy.cs index 2b4a4e4..85ec011 100644 --- a/core/binary_protocol/src/responses/users/update_user.rs +++ b/foreign/csharp/Iggy_SDK/Enums/PartitionResizePolicy.cs
@@ -15,4 +15,14 @@ // specific language governing permissions and limitations // under the License. -pub type UpdateUserResponse = super::EmptyResponse; +namespace Apache.Iggy.Enums; + +/// <summary>Whether a topic accepts partition additions and deletions after it is created.</summary> +public enum PartitionResizePolicy +{ + /// <summary>Partitions may be added or deleted.</summary> + Mutable, + + /// <summary>The partition set is fixed at creation.</summary> + Fixed +}
diff --git a/foreign/csharp/Iggy_SDK/Contracts/Http/Auth/UpdateUserRequest.cs b/foreign/csharp/Iggy_SDK/Enums/StreamEncryptionSource.cs similarity index 65% copy from foreign/csharp/Iggy_SDK/Contracts/Http/Auth/UpdateUserRequest.cs copy to foreign/csharp/Iggy_SDK/Enums/StreamEncryptionSource.cs index 7ef9c59..946d7e6 100644 --- a/foreign/csharp/Iggy_SDK/Contracts/Http/Auth/UpdateUserRequest.cs +++ b/foreign/csharp/Iggy_SDK/Enums/StreamEncryptionSource.cs
@@ -15,8 +15,17 @@ // specific language governing permissions and limitations // under the License. -using Apache.Iggy.Enums; +namespace Apache.Iggy.Enums; -namespace Apache.Iggy.Contracts.Http.Auth; +/// <summary>Who chose the key that encrypts a stream's data at rest.</summary> +public enum StreamEncryptionSource +{ + /// <summary>The stream is stored in plaintext.</summary> + None, -internal sealed record UpdateUserRequest(string? Username, UserStatus? UserStatus); + /// <summary>The client supplied the key when it created the stream.</summary> + Stream, + + /// <summary>The server generated the key because its encryption was enabled.</summary> + Server +}
diff --git a/foreign/csharp/Iggy_SDK/IggyClient/IIggyStream.cs b/foreign/csharp/Iggy_SDK/IggyClient/IIggyStream.cs index de8a5ba..5bbb01f 100644 --- a/foreign/csharp/Iggy_SDK/IggyClient/IIggyStream.cs +++ b/foreign/csharp/Iggy_SDK/IggyClient/IIggyStream.cs
@@ -26,15 +26,19 @@ public interface IIggyStream { /// <summary> - /// Creates a new stream with the specified name. + /// Creates a new stream with the specified name and options. /// </summary> /// <remarks> /// The stream name must be unique within the Iggy instance and has a maximum length of 255 UTF-8 bytes. + /// An encryption key in <paramref name="options" /> is checked before anything is sent. Without one, a + /// server with encryption enabled generates a key for the stream. /// </remarks> /// <param name="name">The unique name of the stream to create.</param> + /// <param name="options">The stream's options, such as its own encryption key. Null uses the server defaults.</param> /// <param name="token">The cancellation token to cancel the operation.</param> /// <returns>A task that represents the asynchronous operation and returns the created stream information.</returns> - Task<StreamResponse?> CreateStreamAsync(string name, CancellationToken token = default); + Task<StreamResponse?> CreateStreamAsync(string name, StreamOptions? options = null, + CancellationToken token = default); /// <summary> /// Retrieves detailed information about a specific stream by its identifier. @@ -48,16 +52,17 @@ Task<StreamResponse?> GetStreamByIdAsync(Identifier streamId, CancellationToken token = default); /// <summary> - /// Updates the name of an existing stream. + /// Applies the changes in <paramref name="update" /> to an existing stream, leaving every other setting as is. /// </summary> /// <remarks> - /// The new stream name must be unique within the Iggy instance. + /// The new stream name must be unique within the Iggy instance. The encryption key is fixed at creation: + /// a reset is refused before anything is sent. /// </remarks> /// <param name="streamId">The identifier of the stream to update (numeric ID or name).</param> - /// <param name="name">The new name for the stream.</param> + /// <param name="update">The keys to set and reset. At least one is required.</param> /// <param name="token">The cancellation token to cancel the operation.</param> /// <returns>A task that represents the asynchronous operation.</returns> - Task UpdateStreamAsync(Identifier streamId, string name, CancellationToken token = default); + Task UpdateStreamAsync(Identifier streamId, StreamUpdate update, CancellationToken token = default); /// <summary> /// Retrieves information about all streams.
diff --git a/foreign/csharp/Iggy_SDK/IggyClient/IIggySystem.cs b/foreign/csharp/Iggy_SDK/IggyClient/IIggySystem.cs index 4b505eb..4640646 100644 --- a/foreign/csharp/Iggy_SDK/IggyClient/IIggySystem.cs +++ b/foreign/csharp/Iggy_SDK/IggyClient/IIggySystem.cs
@@ -73,20 +73,20 @@ Task<StatsResponse?> GetStatsAsync(CancellationToken token = default); /// <summary> - /// Retrieves the option catalog for a resource scope: every key its create - /// command accepts, with the kind, default and bounds of each. + /// Retrieves the option catalog for a resource scope: every key a client can set, + /// update, reset or read, with the kind, default, bounds and mutability of each. /// </summary> /// <remarks> /// This is the discovery surface for options. A key outside the catalog is - /// refused at create, and binary transports carry only the error code, so a - /// client has no other way to learn which keys this server knows. Available - /// only for TCP. + /// refused, and binary transports carry only the error code, so a client has + /// no other way to learn which keys this server knows. Available over TCP and + /// HTTP. /// </remarks> /// <param name="scope">The resource whose catalog to describe.</param> /// <param name="token">The cancellation token to cancel the operation.</param> /// <returns> /// A task that represents the asynchronous operation and returns the catalog - /// entries, empty for a scope with no keys yet. + /// entries in catalog order. /// </returns> Task<IReadOnlyList<OptionSpec>> DescribeOptionsAsync(OptionsScope scope, CancellationToken token = default);
diff --git a/foreign/csharp/Iggy_SDK/IggyClient/IIggyTopic.cs b/foreign/csharp/Iggy_SDK/IggyClient/IIggyTopic.cs index 01c535b..84ffa24 100644 --- a/foreign/csharp/Iggy_SDK/IggyClient/IIggyTopic.cs +++ b/foreign/csharp/Iggy_SDK/IggyClient/IIggyTopic.cs
@@ -16,8 +16,6 @@ // under the License. using Apache.Iggy.Contracts; -using Apache.Iggy.Enums; -using Apache.Iggy.Headers; namespace Apache.Iggy.IggyClient; @@ -52,53 +50,36 @@ /// </summary> /// <remarks> /// The topic name must be unique within the stream. Topics contain one or more partitions for distributing messages. - /// Additional parameters control message expiry, compression, replication, and maximum size. + /// <paramref name="options" /> controls compression, message expiry, maximum size, durability and the other + /// settings of the server's catalog. Call <see cref="IIggySystem.DescribeOptionsAsync" /> to see which keys a + /// server accepts. /// </remarks> /// <param name="streamId">The identifier of the stream where the topic will be created (numeric ID or name).</param> /// <param name="name">The unique name of the topic (max 255 UTF-8 bytes).</param> /// <param name="partitionsCount">The number of partitions for the topic (max 1000).</param> - /// <param name="compressionAlgorithm">The compression algorithm to use for messages (default: None).</param> - /// <param name="messageExpiry">The message expiry period (0 for server default, MaxValue for never expire).</param> - /// <param name="maxTopicSize">The topic-wide retained size limit in bytes, divided among partitions. Zero uses the server default at creation.</param> - /// <param name="options"> - /// Option keys with no parameter of their own, keyed by option name. Reaches a key the server - /// catalog gained after this build shipped; a named parameter above wins on collision, and a key - /// outside the catalog is refused by name. <see cref="TopicOptions" /> builds this map for the - /// keys the catalog ships with. Call - /// <see cref="IIggySystem.DescribeOptionsAsync" /> to see which keys a server accepts. - /// </param> + /// <param name="options">The topic's settings. Null sends both durability policies as replicated and leaves the rest to the server defaults.</param> /// <param name="token">The cancellation token to cancel the operation.</param> /// <returns> /// A task that represents the asynchronous operation and returns the created topic information, or null if /// creation failed. /// </returns> Task<TopicResponse?> CreateTopicAsync(Identifier streamId, string name, uint partitionsCount, - CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, - TimeSpan? messageExpiry = null, ulong maxTopicSize = 0, - IReadOnlyDictionary<string, HeaderValue>? options = null, CancellationToken token = default); + TopicOptions? options = null, CancellationToken token = default); /// <summary> - /// Updates the configuration of an existing topic. + /// Applies the changes in <paramref name="update" /> to an existing topic, leaving every other setting as is. /// </summary> /// <remarks> - /// This method allows updating topic properties such as name, compression algorithm, size limits and message - /// expiry. + /// An update renames the topic, sets its message expiry and size cap, and resets those two to the server + /// defaults. Settings fixed at creation have no property on <see cref="TopicUpdate" />, and a reset of one + /// is refused before anything is sent. /// </remarks> /// <param name="streamId">The identifier of the stream containing the topic (numeric ID or name).</param> /// <param name="topicId">The identifier of the topic to update (numeric ID or name).</param> - /// <param name="name">The new name for the topic (max 255 UTF-8 bytes).</param> - /// <param name="compressionAlgorithm">The new compression algorithm to use (default: None).</param> - /// <param name="maxTopicSize">The retention size update in bytes. TCP omits zero; HTTP sends the default sentinel.</param> - /// <param name="messageExpiry">The new message expiry period (0 for server default, MaxValue for never expire).</param> - /// <param name="options"> - /// Option keys with no parameter of their own. The server refuses any key an update may not - /// change, by name; a key left out keeps its current value. - /// </param> + /// <param name="update">The keys to set and reset. At least one is required.</param> /// <param name="token">The cancellation token to cancel the operation.</param> /// <returns>A task that represents the asynchronous operation.</returns> - Task UpdateTopicAsync(Identifier streamId, Identifier topicId, string name, - CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, ulong maxTopicSize = 0, - TimeSpan? messageExpiry = null, IReadOnlyDictionary<string, HeaderValue>? options = null, + Task UpdateTopicAsync(Identifier streamId, Identifier topicId, TopicUpdate update, CancellationToken token = default); /// <summary>
diff --git a/foreign/csharp/Iggy_SDK/IggyClient/IIggyUsers.cs b/foreign/csharp/Iggy_SDK/IggyClient/IIggyUsers.cs index 6369ef3..11e997d 100644 --- a/foreign/csharp/Iggy_SDK/IggyClient/IIggyUsers.cs +++ b/foreign/csharp/Iggy_SDK/IggyClient/IIggyUsers.cs
@@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +using Apache.Iggy.Contracts; using Apache.Iggy.Contracts.Auth; using Apache.Iggy.Enums; @@ -48,13 +49,14 @@ /// <param name="password">The password for the user.</param> /// <param name="status">The initial status of the user (active or inactive).</param> /// <param name="permissions">The permissions granted to the user (optional).</param> + /// <param name="options">The user's options (optional). The catalog defines no user key yet.</param> /// <param name="token">The cancellation token to cancel the operation.</param> /// <returns> /// A task that represents the asynchronous operation and returns the created user information, or null if /// creation failed. /// </returns> Task<UserResponse?> CreateUserAsync(string userName, string password, UserStatus status, Permissions? permissions = null, - CancellationToken token = default); + UserOptions? options = null, CancellationToken token = default); /// <summary> /// Deletes an existing user from the system. @@ -65,15 +67,17 @@ Task DeleteUserAsync(Identifier userId, CancellationToken token = default); /// <summary> - /// Updates user properties such as username and status. + /// Applies the changes in <paramref name="update" /> to a user, such as its username and status. /// </summary> + /// <remarks> + /// Renaming the signed-in user also renames the credentials a reconnecting TCP client signs in with. + /// Deactivating a user ends its sessions and refuses its next login. + /// </remarks> /// <param name="userId">The identifier of the user to update (numeric ID or username).</param> - /// <param name="userName">The new username (optional).</param> - /// <param name="status">The new status (optional).</param> + /// <param name="update">The keys to set and reset. At least one is required.</param> /// <param name="token">The cancellation token to cancel the operation.</param> /// <returns>A task representing the asynchronous operation.</returns> - Task UpdateUserAsync(Identifier userId, string? userName = null, UserStatus? status = null, - CancellationToken token = default); + Task UpdateUserAsync(Identifier userId, UserUpdate update, CancellationToken token = default); /// <summary> /// Updates the permissions for a specific user.
diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs index 619161e..61e77bb 100644 --- a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs +++ b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs
@@ -75,9 +75,11 @@ } /// <inheritdoc /> - public async Task<StreamResponse?> CreateStreamAsync(string name, CancellationToken token = default) + public async Task<StreamResponse?> CreateStreamAsync(string name, StreamOptions? options = null, + CancellationToken token = default) { - var json = JsonSerializer.Serialize(new CreateStreamRequest(name), _jsonSerializerOptions); + var json = JsonSerializer.Serialize(new CreateStreamRequest(name, CreateOptions(options?.ToDictionary())), + _jsonSerializerOptions); var data = new StringContent(json, Encoding.UTF8, "application/json"); @@ -121,16 +123,10 @@ } /// <inheritdoc /> - public async Task UpdateStreamAsync(Identifier streamId, string name, CancellationToken token = default) + public async Task UpdateStreamAsync(Identifier streamId, StreamUpdate update, CancellationToken token = default) { - var json = JsonSerializer.Serialize(new UpdateStreamRequest(name), _jsonSerializerOptions); - - var data = new StringContent(json, Encoding.UTF8, "application/json"); - var response = await _httpClient.PutAsync($"/streams/{streamId}", data, token); - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } + ArgumentNullException.ThrowIfNull(update); + await PutUpdateAsync($"/streams/{streamId}", update.ToPatch(), token); } /// <inheritdoc /> @@ -150,21 +146,10 @@ /// <inheritdoc /> public async Task<TopicResponse?> CreateTopicAsync(Identifier streamId, string name, uint partitionsCount, - CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, - TimeSpan? messageExpiry = null, ulong maxTopicSize = 0, - IReadOnlyDictionary<string, HeaderValue>? options = null, CancellationToken token = default) + TopicOptions? options = null, CancellationToken token = default) { - - options = TopicOptions.WithDurabilityDefaults(options); - var json = JsonSerializer.Serialize(new CreateTopicRequest - { - Name = name, - CompressionAlgorithm = compressionAlgorithm, - MaxTopicSize = maxTopicSize, - MessageExpiry = DurationHelpers.ToDuration(messageExpiry), - PartitionsCount = partitionsCount, - Options = ToStringOptions(options) - }, _jsonSerializerOptions); + var json = JsonSerializer.Serialize(new CreateTopicRequest(name, partitionsCount, + CreateOptions((options ?? new TopicOptions()).ToDictionary())), _jsonSerializerOptions); var data = new StringContent(json, Encoding.UTF8, "application/json"); var response = await _httpClient.PostAsync($"/streams/{streamId}/topics", data, token); @@ -180,22 +165,11 @@ } /// <inheritdoc /> - public async Task UpdateTopicAsync(Identifier streamId, Identifier topicId, string name, - CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, - ulong maxTopicSize = 0, TimeSpan? messageExpiry = null, - IReadOnlyDictionary<string, HeaderValue>? options = null, + public async Task UpdateTopicAsync(Identifier streamId, Identifier topicId, TopicUpdate update, CancellationToken token = default) { - var json = JsonSerializer.Serialize( - new UpdateTopicRequest(name, compressionAlgorithm, maxTopicSize, - DurationHelpers.ToDuration(messageExpiry), ToStringOptions(options)), - _jsonSerializerOptions); - var data = new StringContent(json, Encoding.UTF8, "application/json"); - var response = await _httpClient.PutAsync($"/streams/{streamId}/topics/{topicId}", data, token); - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } + ArgumentNullException.ThrowIfNull(update); + await PutUpdateAsync($"/streams/{streamId}/topics/{topicId}", update.ToPatch(), token); } /// <inheritdoc /> @@ -513,7 +487,7 @@ /// catalog straight from its own byte vector. /// </summary> private sealed record HttpOptionSpec(string Key, string Kind, IReadOnlyList<byte>? DefaultValue, - string? Description) + string? Description, bool Updatable, bool Resettable, bool Secret, bool ReadOnly, bool Critical) { internal OptionSpec ToOptionSpec() { @@ -521,39 +495,36 @@ { Key = Key, Kind = UserHeadersConverter.ParseHeaderKind(Kind), + Updatable = Updatable, + Resettable = Resettable, + Secret = Secret, + ReadOnly = ReadOnly, + Critical = Critical, DefaultValue = DefaultValue is null ? [] : [.. DefaultValue], Description = Description ?? string.Empty }; } } - /// <summary> - /// Renders option values as the strings the REST body carries them in. - /// - /// The binary transports send a typed TLV block, but the JSON body takes a plain string map the - /// server parses by the same rules a config file value goes through, so a typed value handed in - /// here is rendered rather than passed through. - /// </summary> - private static Dictionary<string, string> ToStringOptions(IReadOnlyDictionary<string, HeaderValue>? options) + /// <summary>The <c>options</c> member of a create body, left out when there is nothing to send.</summary> + private static Dictionary<string, object>? CreateOptions(IReadOnlyDictionary<string, HeaderValue>? options) { - if (options is null) - { - return new Dictionary<string, string>(); - } - - return options.ToDictionary(entry => entry.Key, entry => ToStringValue(entry.Value)); + return options is { Count: > 0 } ? OptionInputs.From(options) : null; } - private static string ToStringValue(HeaderValue value) + /// <summary> + /// PUT of the flat update body. A key the body leaves out keeps its value, a key in <c>reset</c> returns to + /// its default. + /// </summary> + private async Task PutUpdateAsync(string path, OptionsPatch patch, CancellationToken token) { - // A Bool header renders as "1" or "0", which the server's option parser refuses. It takes - // the words a config file would carry. - if (value.Kind is HeaderKind.Bool) + var json = JsonSerializer.Serialize(patch.ToHttpBody(), _jsonSerializerOptions); + var data = new StringContent(json, Encoding.UTF8, "application/json"); + var response = await _httpClient.PutAsync(path, data, token); + if (!response.IsSuccessStatusCode) { - return value.ToBool() ? "true" : "false"; + await HandleResponseAsync(response); } - - return value.ToString(); } /// <inheritdoc /> @@ -776,10 +747,10 @@ /// <inheritdoc /> public async Task<UserResponse?> CreateUserAsync(string userName, string password, UserStatus status, - Permissions? permissions = null, CancellationToken token = default) + Permissions? permissions = null, UserOptions? options = null, CancellationToken token = default) { - var json = JsonSerializer.Serialize(new CreateUserRequest(userName, password, status, permissions), - _jsonSerializerOptions); + var json = JsonSerializer.Serialize(new CreateUserRequest(userName, password, status, permissions, + CreateOptions(options?.ToDictionary())), _jsonSerializerOptions); var content = new StringContent(json, Encoding.UTF8, "application/json"); var response = await _httpClient.PostAsync("/users", content, token); @@ -803,16 +774,10 @@ } /// <inheritdoc /> - public async Task UpdateUserAsync(Identifier userId, string? userName = null, UserStatus? status = null, - CancellationToken token = default) + public async Task UpdateUserAsync(Identifier userId, UserUpdate update, CancellationToken token = default) { - var json = JsonSerializer.Serialize(new UpdateUserRequest(userName, status), _jsonSerializerOptions); - var content = new StringContent(json, Encoding.UTF8, "application/json"); - var response = await _httpClient.PutAsync($"/users/{userId}", content, token); - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response); - } + ArgumentNullException.ThrowIfNull(update); + await PutUpdateAsync($"/users/{userId}", update.ToPatch(), token); } /// <inheritdoc />
diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.PollRouting.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.PollRouting.cs index 66cc7d8..daba7e9 100644 --- a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.PollRouting.cs +++ b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.PollRouting.cs
@@ -41,6 +41,7 @@ private const int ConsumerSessionWatermarkOffset = 24; private const int IdentifierPrefixSize = 2; private const byte ConsumerGroupWireKind = 2; + private const byte StringKind = 2; private readonly object _pollRoutingGate = new(); private readonly Dictionary<PartitionRouteKey, CachedRoute> _partitionRoutes = []; @@ -914,27 +915,56 @@ // Typed, raw and replayed calls share this success path, and only the request body carries the new credentials. private void RefreshCommittedCredentials(int code, ReadOnlySpan<byte> body, ulong generation) { - if (code is not (CommandCodes.CHANGE_PASSWORD_CODE or CommandCodes.UPDATE_USER_CODE)) - { - return; - } - - var position = 0; - var user = ReadIdentifier(body, ref position); if (code == CommandCodes.CHANGE_PASSWORD_CODE) { + var position = 0; + var user = ReadIdentifier(body, ref position); var currentPasswordLength = body[position]; RefreshRememberedCredentials(user, null, ReadName(body, position + 1 + currentPasswordLength), generation); } - else if (body[position] == 1) + else if (code == CommandCodes.UPDATE_RESOURCE_CODE && body.Length > 0 && body[0] == (byte)OptionsScope.User) { - RefreshRememberedCredentials(user, ReadName(body, position + 1), null, generation); + var position = 1; + var user = ReadIdentifier(body, ref position); + var resetCount = body[position++]; + for (var reset = 0; reset < resetCount; reset++) + { + position += 1 + body[position]; + } + + if (PatchedUsername(body[position..]) is { } username) + { + RefreshRememberedCredentials(user, username, null, generation); + } } static string ReadName(ReadOnlySpan<byte> body, int position) => Encoding.UTF8.GetString(body.Slice(position + 1, body[position])); } + /// <summary>The username a user update sets, read from the options block that ends its body.</summary> + private static string? PatchedUsername(ReadOnlySpan<byte> options) + { + var position = 0; + while (position < options.Length) + { + var keyKind = options[position]; + var keyLength = BinaryPrimitives.ReadInt32LittleEndian(options[(position + 1)..]); + var key = options.Slice(position + 5, keyLength); + position += 5 + keyLength; + var valueKind = options[position]; + var valueLength = BinaryPrimitives.ReadInt32LittleEndian(options[(position + 1)..]); + var value = options.Slice(position + 5, valueLength); + position += 5 + valueLength; + if (keyKind == StringKind && valueKind == StringKind && key.SequenceEqual("username"u8)) + { + return Encoding.UTF8.GetString(value); + } + } + + return null; + } + private void RefreshRememberedCredentials(Identifier user, string? username, string? password, ulong generation) { lock (_pollRoutingGate)
diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs index 652b86c..566e3be 100644 --- a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs +++ b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs
@@ -164,9 +164,10 @@ } /// <inheritdoc /> - public async Task<StreamResponse?> CreateStreamAsync(string name, CancellationToken token = default) + public async Task<StreamResponse?> CreateStreamAsync(string name, StreamOptions? options = null, + CancellationToken token = default) { - var message = TcpContracts.CreateStream(name); + var message = TcpContracts.CreateStream(name, options?.ToDictionary()); using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(CommandCodes.CREATE_STREAM_CODE, message, token: token); @@ -209,10 +210,11 @@ } /// <inheritdoc /> - public async Task UpdateStreamAsync(Identifier streamId, string name, CancellationToken token = default) + public async Task UpdateStreamAsync(Identifier streamId, StreamUpdate update, CancellationToken token = default) { - var message = TcpContracts.UpdateStream(streamId, name); - await SendAckAsync(CommandCodes.UPDATE_STREAM_CODE, message, token); + ArgumentNullException.ThrowIfNull(update); + var message = TcpContracts.UpdateStream(streamId, update.ToPatch()); + await SendAckAsync(CommandCodes.UPDATE_RESOURCE_CODE, message, token); } /// <inheritdoc /> @@ -257,15 +259,10 @@ /// <inheritdoc /> public async Task<TopicResponse?> CreateTopicAsync(Identifier streamId, string name, uint partitionsCount, - CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, - TimeSpan? messageExpiry = null, ulong maxTopicSize = 0, - IReadOnlyDictionary<string, HeaderValue>? options = null, CancellationToken token = default) + TopicOptions? options = null, CancellationToken token = default) { - - options = TopicOptions.WithDurabilityDefaults(options); - var messageExpiryValue = DurationHelpers.ToDuration(messageExpiry); - var message = TcpContracts.CreateTopic(streamId, name, partitionsCount, compressionAlgorithm, - messageExpiryValue, maxTopicSize, options); + var message = TcpContracts.CreateTopic(streamId, name, partitionsCount, + (options ?? new TopicOptions()).ToDictionary()); using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(CommandCodes.CREATE_TOPIC_CODE, message, token: token); @@ -278,16 +275,12 @@ } /// <inheritdoc /> - public async Task UpdateTopicAsync(Identifier streamId, Identifier topicId, string name, - CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, - ulong maxTopicSize = 0, TimeSpan? messageExpiry = null, - IReadOnlyDictionary<string, HeaderValue>? options = null, + public async Task UpdateTopicAsync(Identifier streamId, Identifier topicId, TopicUpdate update, CancellationToken token = default) { - var messageExpiryValue = DurationHelpers.ToDuration(messageExpiry); - var message = TcpContracts.UpdateTopic(streamId, topicId, name, compressionAlgorithm, maxTopicSize, - messageExpiryValue, options); - await SendAckAsync(CommandCodes.UPDATE_TOPIC_CODE, message, token); + ArgumentNullException.ThrowIfNull(update); + var message = TcpContracts.UpdateTopic(streamId, topicId, update.ToPatch()); + await SendAckAsync(CommandCodes.UPDATE_RESOURCE_CODE, message, token); } /// <inheritdoc /> @@ -688,9 +681,9 @@ /// <inheritdoc /> public async Task<UserResponse?> CreateUserAsync(string userName, string password, UserStatus status, - Permissions? permissions = null, CancellationToken token = default) + Permissions? permissions = null, UserOptions? options = null, CancellationToken token = default) { - var message = TcpContracts.CreateUser(userName, password, status, permissions); + var message = TcpContracts.CreateUser(userName, password, status, permissions, options?.ToDictionary()); using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(CommandCodes.CREATE_USER_CODE, message, token: token); @@ -710,11 +703,11 @@ } /// <inheritdoc /> - public async Task UpdateUserAsync(Identifier userId, string? userName = null, UserStatus? status = null, - CancellationToken token = default) + public async Task UpdateUserAsync(Identifier userId, UserUpdate update, CancellationToken token = default) { - var message = TcpContracts.UpdateUser(userId, userName, status); - await SendAckAsync(CommandCodes.UPDATE_USER_CODE, message, token); + ArgumentNullException.ThrowIfNull(update); + var message = TcpContracts.UpdateUser(userId, update.ToPatch()); + await SendAckAsync(CommandCodes.UPDATE_RESOURCE_CODE, message, token); } /// <inheritdoc />
diff --git a/foreign/csharp/Iggy_SDK/Iggy_SDK.csproj b/foreign/csharp/Iggy_SDK/Iggy_SDK.csproj index 04c0af6..cc60480 100644 --- a/foreign/csharp/Iggy_SDK/Iggy_SDK.csproj +++ b/foreign/csharp/Iggy_SDK/Iggy_SDK.csproj
@@ -26,7 +26,7 @@ <TargetFrameworks>net8.0;net10.0</TargetFrameworks> <AssemblyName>Apache.Iggy</AssemblyName> <RootNamespace>Apache.Iggy</RootNamespace> - <Version>0.9.1-edge.1</Version> + <Version>0.9.1-edge.2</Version> <GenerateDocumentationFile>true</GenerateDocumentationFile> </PropertyGroup>
diff --git a/foreign/csharp/Iggy_SDK/JsonConverters/RedactedOptionsConverter.cs b/foreign/csharp/Iggy_SDK/JsonConverters/RedactedOptionsConverter.cs new file mode 100644 index 0000000..697336a --- /dev/null +++ b/foreign/csharp/Iggy_SDK/JsonConverters/RedactedOptionsConverter.cs
@@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Text.Json; +using System.Text.Json.Serialization; +using Apache.Iggy.Headers; + +namespace Apache.Iggy.JsonConverters; + +/// <summary> +/// Writes raw option entries by name only, so a structured logger never records a value. This build cannot tell +/// which keys newer than it are secret, so every value is hidden. Requests never serialize through here. +/// </summary> +internal sealed class RedactedOptionsConverter : JsonConverter<IReadOnlyDictionary<string, HeaderValue>> +{ + private const string REDACTED = "<redacted>"; + + public override IReadOnlyDictionary<string, HeaderValue>? Read(ref Utf8JsonReader reader, Type typeToConvert, + JsonSerializerOptions options) + { + return JsonSerializer.Deserialize<Dictionary<string, HeaderValue>>(ref reader, options); + } + + public override void Write(Utf8JsonWriter writer, IReadOnlyDictionary<string, HeaderValue> value, + JsonSerializerOptions options) + { + writer.WriteStartObject(); + foreach (var name in value.Keys) + { + writer.WriteString(name, REDACTED); + } + + writer.WriteEndObject(); + } +}
diff --git a/foreign/csharp/Iggy_SDK/JsonConverters/ResourceOptionsConverter.cs b/foreign/csharp/Iggy_SDK/JsonConverters/ResourceOptionsConverter.cs index adc1aad..4d815d5 100644 --- a/foreign/csharp/Iggy_SDK/JsonConverters/ResourceOptionsConverter.cs +++ b/foreign/csharp/Iggy_SDK/JsonConverters/ResourceOptionsConverter.cs
@@ -15,6 +15,10 @@ // specific language governing permissions and limitations // under the License. +using System.Buffers.Binary; +using System.Globalization; +using System.Numerics; +using System.Text; using System.Text.Json; using System.Text.Json.Serialization; using Apache.Iggy.Headers; @@ -23,7 +27,7 @@ /// <summary> /// A resource's options split by provenance, the way the binary transports send them: what a -/// client set at creation, and what the server resolved from its own defaults. +/// client set, and what the server resolved from its defaults or computes on reads. /// </summary> internal readonly record struct ResourceOptions( Dictionary<HeaderKey, HeaderValue> Explicit, @@ -31,11 +35,9 @@ /// <summary> /// Reads the one object a REST response renders both provenances into, keyed by option name, -/// each entry a readable value plus the flag saying who set it: -/// <c>{"segment_size":{"value":"1073741824","explicit":false}}</c>. The value arrives as the -/// string a config file or a create body would carry it in, so every value maps to -/// <see cref="HeaderKind.String" /> and the server's canonical kind for the key is not -/// recoverable from it. +/// each entry a reversible text, its kind and the flag saying who set it: +/// <c>{"segment_size":{"value":"1073741824","kind":"uint64","explicit":false}}</c>. The text decodes back to the +/// exact value a binary transport carries, so a key reads with the same kind over either. /// </summary> internal sealed class ResourceOptionsConverter : JsonConverter<ResourceOptions> { @@ -83,7 +85,7 @@ } /// <summary> - /// Options travel out as the plain string map a create or update body takes, never in this + /// Options travel out as the plain value map a create or update body takes, never in this /// shape, so nothing serializes through here. /// </summary> public override void Write(Utf8JsonWriter writer, ResourceOptions value, JsonSerializerOptions options) @@ -91,6 +93,113 @@ throw new NotSupportedException("Resource options are read from a response, not written to a request."); } + /// <summary> + /// The reversible text of a value for its kind: UTF-8 for strings, <c>true</c> or <c>false</c>, base-10 + /// integers, standard padded base64 for raw bytes, and <c>0x</c> followed by the IEEE-754 bits in + /// fixed-width lowercase hex for floats. + /// </summary> + internal static string EncodeValue(HeaderValue value) + { + var bytes = value.Value; + return value.Kind switch + { + HeaderKind.Raw => Convert.ToBase64String(bytes), + HeaderKind.String => Encoding.UTF8.GetString(bytes), + HeaderKind.Bool => bytes[0] switch + { + 0 => "false", + 1 => "true", + _ => throw new FormatException($"Bool option value {bytes[0]} is neither 0 nor 1.") + }, + HeaderKind.Int8 => ((sbyte)bytes[0]).ToString(CultureInfo.InvariantCulture), + HeaderKind.Int16 => BinaryPrimitives.ReadInt16LittleEndian(bytes).ToString(CultureInfo.InvariantCulture), + HeaderKind.Int32 => BinaryPrimitives.ReadInt32LittleEndian(bytes).ToString(CultureInfo.InvariantCulture), + HeaderKind.Int64 => BinaryPrimitives.ReadInt64LittleEndian(bytes).ToString(CultureInfo.InvariantCulture), + HeaderKind.Int128 => BinaryPrimitives.ReadInt128LittleEndian(bytes).ToString(CultureInfo.InvariantCulture), + HeaderKind.Uint8 => bytes[0].ToString(CultureInfo.InvariantCulture), + HeaderKind.Uint16 => BinaryPrimitives.ReadUInt16LittleEndian(bytes).ToString(CultureInfo.InvariantCulture), + HeaderKind.Uint32 => BinaryPrimitives.ReadUInt32LittleEndian(bytes).ToString(CultureInfo.InvariantCulture), + HeaderKind.Uint64 => BinaryPrimitives.ReadUInt64LittleEndian(bytes).ToString(CultureInfo.InvariantCulture), + HeaderKind.Uint128 => BinaryPrimitives.ReadUInt128LittleEndian(bytes).ToString(CultureInfo.InvariantCulture), + HeaderKind.Float => "0x" + BinaryPrimitives.ReadUInt32LittleEndian(bytes).ToString("x8", CultureInfo.InvariantCulture), + HeaderKind.Double => "0x" + BinaryPrimitives.ReadUInt64LittleEndian(bytes).ToString("x16", CultureInfo.InvariantCulture), + _ => throw new FormatException($"Option value kind {value.Kind} has no text form.") + }; + } + + /// <summary> + /// The value <paramref name="text" /> encodes for <paramref name="kind" />, the inverse of + /// <see cref="EncodeValue" />. Text that is not the canonical form of a value is refused, so one value has + /// one JSON form. + /// </summary> + internal static HeaderValue DecodeValue(HeaderKind kind, string text) + { + HeaderValue value; + try + { + value = kind switch + { + HeaderKind.Raw => HeaderValue.FromBytes(Convert.FromBase64String(text)), + HeaderKind.String => HeaderValue.FromString(text), + HeaderKind.Bool => HeaderValue.FromBool(text switch + { + "true" => true, + "false" => false, + _ => throw new FormatException("Bool option value must be true or false.") + }), + HeaderKind.Int8 => Integer<sbyte>(kind, text, (bytes, parsed) => bytes[0] = (byte)parsed), + HeaderKind.Int16 => Integer<short>(kind, text, BinaryPrimitives.WriteInt16LittleEndian), + HeaderKind.Int32 => Integer<int>(kind, text, BinaryPrimitives.WriteInt32LittleEndian), + HeaderKind.Int64 => Integer<long>(kind, text, BinaryPrimitives.WriteInt64LittleEndian), + HeaderKind.Int128 => Integer<Int128>(kind, text, BinaryPrimitives.WriteInt128LittleEndian), + HeaderKind.Uint8 => Integer<byte>(kind, text, (bytes, parsed) => bytes[0] = parsed), + HeaderKind.Uint16 => Integer<ushort>(kind, text, BinaryPrimitives.WriteUInt16LittleEndian), + HeaderKind.Uint32 => Integer<uint>(kind, text, BinaryPrimitives.WriteUInt32LittleEndian), + HeaderKind.Uint64 => Integer<ulong>(kind, text, BinaryPrimitives.WriteUInt64LittleEndian), + HeaderKind.Uint128 => Integer<UInt128>(kind, text, BinaryPrimitives.WriteUInt128LittleEndian), + HeaderKind.Float => FloatBits<uint>(kind, text, 8, BinaryPrimitives.WriteUInt32LittleEndian), + HeaderKind.Double => FloatBits<ulong>(kind, text, 16, BinaryPrimitives.WriteUInt64LittleEndian), + _ => throw new FormatException($"Option value kind {kind} has no text form.") + }; + } + catch (Exception error) when (error is FormatException or OverflowException or ArgumentException) + { + throw new JsonException($"Option value is not a valid {kind}.", error); + } + + if (EncodeValue(value) != text) + { + throw new JsonException($"Option value is not the canonical text of a {kind}."); + } + + return value; + } + + private delegate void Writer<in T>(Span<byte> destination, T value); + + private static HeaderValue Integer<T>(HeaderKind kind, string text, Writer<T> write) + where T : IBinaryInteger<T> + { + var parsed = T.Parse(text, NumberStyles.AllowLeadingSign, CultureInfo.InvariantCulture); + var bytes = new byte[parsed.GetByteCount()]; + write(bytes, parsed); + return new HeaderValue { Kind = kind, Value = bytes }; + } + + private static HeaderValue FloatBits<T>(HeaderKind kind, string text, int digits, Writer<T> write) + where T : IBinaryInteger<T> + { + if (!text.StartsWith("0x", StringComparison.Ordinal) || text.Length != digits + 2) + { + throw new FormatException($"Float option value must be 0x followed by {digits} hex digits."); + } + + var parsed = T.Parse(text.AsSpan(2), NumberStyles.AllowHexSpecifier, CultureInfo.InvariantCulture); + var bytes = new byte[parsed.GetByteCount()]; + write(bytes, parsed); + return new HeaderValue { Kind = kind, Value = bytes }; + } + private static (HeaderValue value, bool isExplicit) ReadOptionValue(ref Utf8JsonReader reader, string key) { if (reader.TokenType != JsonTokenType.StartObject) @@ -99,6 +208,7 @@ } string? value = null; + string? kind = null; bool? isExplicit = null; while (reader.Read()) @@ -121,6 +231,9 @@ case "value": value = reader.GetString(); break; + case "kind": + kind = reader.GetString(); + break; case "explicit": isExplicit = reader.GetBoolean(); break; @@ -130,11 +243,11 @@ } } - if (value is null || isExplicit is null) + if (value is null || kind is null || isExplicit is null) { - throw new JsonException($"Option '{key}' must have both 'value' and 'explicit' properties."); + throw new JsonException($"Option '{key}' must have 'value', 'kind' and 'explicit' properties."); } - return (HeaderValue.FromString(value), isExplicit.Value); + return (DecodeValue(UserHeadersConverter.ParseHeaderKind(kind), value), isExplicit.Value); } }
diff --git a/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs b/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs index c202b8c..0d4b814 100644 --- a/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs +++ b/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs
@@ -38,8 +38,8 @@ private const int CONSUMER_GROUP_INFO_SIZE = 12; private const int CACHE_METRICS_ENTRY_SIZE = 32; private const int OPEN_FILES_FIELDS_SIZE = 8 + 8; - private const int MIN_TOPIC_SIZE = 50 + 4 + 4; - private const int MIN_OPTION_SPEC_SIZE = 1 + 1 + 4 + 4; + private const int MIN_TOPIC_SIZE = 33 + 4 + 4; + private const int MIN_OPTION_SPEC_SIZE = 1 + 1 + 1 + 4 + 4; private const int CLUSTER_NODE_TAIL_SIZE = 4 * 2 + 1 + 1; private const int MIN_CLUSTER_NODE_SIZE = 1 + 4 + 4 + CLUSTER_NODE_TAIL_SIZE; @@ -121,7 +121,8 @@ CreatedAt = response.CreatedAt, Username = response.Username, Status = response.Status, - Options = response.Options + Options = response.Options, + DerivedOptions = response.DerivedOptions }; } @@ -132,7 +133,8 @@ Username = response.Username, Status = response.Status, Permissions = null, - Options = response.Options + Options = response.Options, + DerivedOptions = response.DerivedOptions }; } @@ -240,6 +242,8 @@ var readBytes = 4 + 8 + 1 + 1 + usernameLength; var options = MapOptions(payload, position + readBytes, out var optionsReadBytes); readBytes += optionsReadBytes; + var derivedOptions = MapOptions(payload, position + readBytes, out var derivedOptionsReadBytes); + readBytes += derivedOptionsReadBytes; return (new UserResponse { @@ -247,7 +251,8 @@ CreatedAt = createdAt, Status = userStatus, Username = username, - Options = options + Options = options, + DerivedOptions = derivedOptions }, readBytes); } @@ -1002,7 +1007,8 @@ CreatedAt = stream.CreatedAt, MessagesCount = stream.MessagesCount, Size = stream.Size, - Options = stream.Options + Options = stream.Options, + DerivedOptions = stream.DerivedOptions }; } @@ -1019,6 +1025,8 @@ var readBytes = 4 + 4 + 8 + 8 + 8 + 1 + nameLength; var options = MapOptions(payload, position + readBytes, out var optionsReadBytes); readBytes += optionsReadBytes; + var derivedOptions = MapOptions(payload, position + readBytes, out var derivedOptionsReadBytes); + readBytes += derivedOptionsReadBytes; return ( new StreamResponse @@ -1029,7 +1037,8 @@ Size = sizeBytes, MessagesCount = messagesCount, CreatedAt = DateTimeOffsetUtils.FromUnixTimeMicroSeconds(createdAt).LocalDateTime, - Options = options + Options = options, + DerivedOptions = derivedOptions }, readBytes); } @@ -1068,12 +1077,9 @@ Id = topic.Id, Name = topic.Name, PartitionsCount = topic.PartitionsCount, - CompressionAlgorithm = topic.CompressionAlgorithm, CreatedAt = topic.CreatedAt, - MessageExpiry = topic.MessageExpiry, MessagesCount = topic.MessagesCount, Size = topic.Size, - MaxTopicSize = topic.MaxTopicSize, Partitions = partitions, Options = topic.Options, DerivedOptions = topic.DerivedOptions @@ -1085,14 +1091,11 @@ var id = BinaryPrimitives.ReadUInt32LittleEndian(payload[position..(position + 4)]); var createdAt = BinaryPrimitives.ReadUInt64LittleEndian(payload[(position + 4)..(position + 12)]); var partitionsCount = BinaryPrimitives.ReadUInt32LittleEndian(payload[(position + 12)..(position + 16)]); - var messageExpiry = BinaryPrimitives.ReadUInt64LittleEndian(payload[(position + 16)..(position + 24)]); - var compressionAlgorithm = payload[position + 24]; - var maxTopicSize = BinaryPrimitives.ReadUInt64LittleEndian(payload[(position + 25)..(position + 33)]); - var sizeBytes = BinaryPrimitives.ReadUInt64LittleEndian(payload[(position + 33)..(position + 41)]); - var messagesCount = BinaryPrimitives.ReadUInt64LittleEndian(payload[(position + 41)..(position + 49)]); - var nameLength = (int)payload[position + 49]; - var name = Encoding.UTF8.GetString(payload[(position + 50)..(position + 50 + nameLength)]); - var readBytes = 4 + 8 + 4 + 8 + 1 + 8 + 8 + 8 + 1 + nameLength; + var sizeBytes = BinaryPrimitives.ReadUInt64LittleEndian(payload[(position + 16)..(position + 24)]); + var messagesCount = BinaryPrimitives.ReadUInt64LittleEndian(payload[(position + 24)..(position + 32)]); + var nameLength = (int)payload[position + 32]; + var name = Encoding.UTF8.GetString(payload[(position + 33)..(position + 33 + nameLength)]); + var readBytes = 4 + 8 + 4 + 8 + 8 + 1 + nameLength; var options = MapOptions(payload, position + readBytes, out var optionsReadBytes); readBytes += optionsReadBytes; var derivedOptions = MapOptions(payload, position + readBytes, out var derivedOptionsReadBytes); @@ -1104,12 +1107,9 @@ Id = id, PartitionsCount = partitionsCount, Name = name, - CompressionAlgorithm = (CompressionAlgorithm)compressionAlgorithm, MessagesCount = messagesCount, Size = sizeBytes, CreatedAt = DateTimeOffsetUtils.FromUnixTimeMicroSeconds(createdAt).LocalDateTime, - MessageExpiry = DurationHelpers.FromDuration(messageExpiry), - MaxTopicSize = maxTopicSize, Options = options, DerivedOptions = derivedOptions }, readBytes); @@ -1377,8 +1377,16 @@ var key = Encoding.UTF8.GetString(payload[position..(position + keyLength)]); position += keyLength; + if (position + 2 > payload.Length) + { + throw new MalformedResponseException( + $"Malformed DescribeOptions response: option kind and flags at offset {position} overrun the " + + $"{payload.Length}-byte payload"); + } + var kind = payload[position]; - position += 1; + var flags = (OptionFlags)payload[position + 1]; + position += 2; var defaultLength = ReadLength(payload, position, "Option default value"); position += 4; @@ -1391,6 +1399,11 @@ { Key = key, Kind = MapHeaderKind(kind), + Updatable = flags.HasFlag(OptionFlags.Updatable), + Resettable = flags.HasFlag(OptionFlags.Resettable), + Secret = flags.HasFlag(OptionFlags.Secret), + ReadOnly = flags.HasFlag(OptionFlags.ReadOnly), + Critical = flags.HasFlag(OptionFlags.Critical), DefaultValue = defaultValue, Description = description });
diff --git a/foreign/csharp/Iggy_SDK/Publishers/IggyPublisher.cs b/foreign/csharp/Iggy_SDK/Publishers/IggyPublisher.cs index 146cdd1..c7cb559 100644 --- a/foreign/csharp/Iggy_SDK/Publishers/IggyPublisher.cs +++ b/foreign/csharp/Iggy_SDK/Publishers/IggyPublisher.cs
@@ -229,11 +229,11 @@ if (Config.StreamId.Kind is IdKind.String) { - await Client.CreateStreamAsync(Config.StreamId.GetString(), ct); + await Client.CreateStreamAsync(Config.StreamId.GetString(), token: ct); } else { - await Client.CreateStreamAsync(Config.StreamName, ct); + await Client.CreateStreamAsync(Config.StreamName, token: ct); } LogStreamCreated(Config.StreamId); @@ -260,18 +260,16 @@ LogCreatingTopic(Config.TopicId, Config.TopicName, Config.StreamId); - if (Config.TopicId.Kind is IdKind.String) + var options = new TopicOptions { - await Client.CreateTopicAsync(Config.StreamId, Config.TopicId.GetString(), - Config.TopicPartitionsCount, Config.TopicCompressionAlgorithm, - Config.TopicMessageExpiry, Config.TopicMaxTopicSize, token: ct); - } - else - { - await Client.CreateTopicAsync(Config.StreamId, Config.TopicName, Config.TopicPartitionsCount, - Config.TopicCompressionAlgorithm, - Config.TopicMessageExpiry, Config.TopicMaxTopicSize, token: ct); - } + CompressionAlgorithm = Config.TopicCompressionAlgorithm == CompressionAlgorithm.None + ? null + : Config.TopicCompressionAlgorithm, + MessageExpiry = Config.TopicMessageExpiry, + MaxTopicSize = Config.TopicMaxTopicSize + }; + var topicName = Config.TopicId.Kind is IdKind.String ? Config.TopicId.GetString() : Config.TopicName; + await Client.CreateTopicAsync(Config.StreamId, topicName, Config.TopicPartitionsCount, options, ct); LogTopicCreated(Config.TopicId, Config.StreamId); }
diff --git a/foreign/csharp/Iggy_SDK/Utils/CommandCodes.cs b/foreign/csharp/Iggy_SDK/Utils/CommandCodes.cs index 21a9b6d..f1f2349 100644 --- a/foreign/csharp/Iggy_SDK/Utils/CommandCodes.cs +++ b/foreign/csharp/Iggy_SDK/Utils/CommandCodes.cs
@@ -25,6 +25,7 @@ internal const int GET_CLUSTER_METADATA_CODE = 12; internal const int DESCRIBE_OPTIONS_CODE = 13; internal const int BIND_SESSION_CODE = 15; + internal const int UPDATE_RESOURCE_CODE = 16; internal const int GET_ME_CODE = 20; internal const int GET_CLIENT_CODE = 21; internal const int GET_CLIENTS_CODE = 22; @@ -32,7 +33,7 @@ internal const int GET_USERS_CODE = 32; internal const int CREATE_USER_CODE = 33; internal const int DELETE_USER_CODE = 34; - internal const int UPDATE_USER_CODE = 35; + // 35 was UpdateUser and must never be reused. internal const int UPDATE_PERMISSIONS_CODE = 36; internal const int CHANGE_PASSWORD_CODE = 37; internal const int LOGIN_USER_CODE = 38; @@ -55,12 +56,12 @@ internal const int GET_STREAMS_CODE = 201; internal const int CREATE_STREAM_CODE = 202; internal const int DELETE_STREAM_CODE = 203; - internal const int UPDATE_STREAM_CODE = 204; + // 204 was UpdateStream and must never be reused. internal const int GET_TOPIC_CODE = 300; internal const int GET_TOPICS_CODE = 301; internal const int CREATE_TOPIC_CODE = 302; internal const int DELETE_TOPIC_CODE = 303; - internal const int UPDATE_TOPIC_CODE = 304; + // 304 was UpdateTopic and must never be reused. internal const int CREATE_PARTITIONS_CODE = 402; internal const int DELETE_PARTITIONS_CODE = 403; internal const int DELETE_SEGMENTS_CODE = 503;
diff --git a/foreign/csharp/Iggy_SDK/Vsr/ConsensusSession.cs b/foreign/csharp/Iggy_SDK/Vsr/ConsensusSession.cs index 8984e75..34001c7 100644 --- a/foreign/csharp/Iggy_SDK/Vsr/ConsensusSession.cs +++ b/foreign/csharp/Iggy_SDK/Vsr/ConsensusSession.cs
@@ -338,7 +338,7 @@ _registerPending = true; - return new SessionFrame(_clientId, 0, 0); + return new SessionFrame(_clientId, 0, 0, RandomRequestIdentifier()); } private SessionFrame RequestFrameLocked(VsrOperation operation) @@ -354,7 +354,23 @@ var requestId = _requestCounter; _requestCounter = checked(_requestCounter + 1); - return new SessionFrame(_clientId, requestId, sessionId); + var requestChecksum = operation != VsrOperation.NonReplicated + && (byte)operation < (byte)VsrOperation.SendMessages + ? RandomRequestIdentifier() : UInt128.Zero; + return new SessionFrame(_clientId, requestId, sessionId, requestChecksum); + } + + private static UInt128 RandomRequestIdentifier() + { + Span<byte> bytes = stackalloc byte[16]; + UInt128 identifier; + do + { + RandomNumberGenerator.Fill(bytes); + identifier = BinaryPrimitives.ReadUInt128LittleEndian(bytes); + } while (identifier == UInt128.Zero); + + return identifier; } private void ReArmLocked() @@ -388,4 +404,5 @@ } /// <summary>The session identity one request header is encoded from, resolved as a single atomic snapshot.</summary> -internal readonly record struct SessionFrame(UInt128 ClientId, ulong RequestId, ulong SessionId); +internal readonly record struct SessionFrame(UInt128 ClientId, ulong RequestId, ulong SessionId, + UInt128 RequestChecksum = default);
diff --git a/foreign/csharp/Iggy_SDK/Vsr/LoginRegister.cs b/foreign/csharp/Iggy_SDK/Vsr/LoginRegister.cs index 47afaed..ba2091b 100644 --- a/foreign/csharp/Iggy_SDK/Vsr/LoginRegister.cs +++ b/foreign/csharp/Iggy_SDK/Vsr/LoginRegister.cs
@@ -34,8 +34,8 @@ /// <summary>Explicit wire version from <c>core/binary_protocol/src/version.rs</c>.</summary> internal const int PROTOCOL_VERSION_MAJOR = 0; - internal const int PROTOCOL_VERSION_MINOR = 11; - internal const int PROTOCOL_VERSION_PATCH = 1; + internal const int PROTOCOL_VERSION_MINOR = 12; + internal const int PROTOCOL_VERSION_PATCH = 0; /// <summary>Packed protocol version: <c>major << 20 | minor << 10 | patch</c>, 10 bits each.</summary> internal const uint PROTOCOL_VERSION =
diff --git a/foreign/csharp/Iggy_SDK/Vsr/VsrConnection.cs b/foreign/csharp/Iggy_SDK/Vsr/VsrConnection.cs index f19b461..e559bfa 100644 --- a/foreign/csharp/Iggy_SDK/Vsr/VsrConnection.cs +++ b/foreign/csharp/Iggy_SDK/Vsr/VsrConnection.cs
@@ -346,12 +346,7 @@ continue; } var operation = VsrHeader.ReadReplyOperation(header); - var expected = VsrOperations.ForCode(request.Code); - var alias = (expected, operation) is - (VsrOperation.CreateTopic, VsrOperation.CreateTopicWithAssignments) - or (VsrOperation.CreatePartitions, VsrOperation.CreatePartitionsWithAssignments) - or (VsrOperation.DeleteSegments, VsrOperation.TruncatePartition); - if (operation != expected && !alias) + if (operation.RequestOperation() != VsrOperations.ForCode(request.Code)) { throw VsrError.Exception(VsrError.INVALID_COMMAND, "Reply operation does not match its request."); }
diff --git a/foreign/csharp/Iggy_SDK/Vsr/VsrError.cs b/foreign/csharp/Iggy_SDK/Vsr/VsrError.cs index ea9fa6e..b2c88eb 100644 --- a/foreign/csharp/Iggy_SDK/Vsr/VsrError.cs +++ b/foreign/csharp/Iggy_SDK/Vsr/VsrError.cs
@@ -42,6 +42,7 @@ internal const int PERSONAL_ACCESS_TOKEN_EXPIRED = 54; internal const int TRANSIENT_NOT_COMMITTED = 57; internal const int TRANSIENT_NOT_ACCEPTED = 58; + internal const int INVALID_ENCRYPTION_KEY = 70; internal const int INVALID_SIZE_BYTES = 80; internal const int REQUEST_TOO_OLD = 85; internal const int HISTORY_UNAVAILABLE = 87; @@ -50,12 +51,19 @@ internal const int EMPTY_RESPONSE = 304; internal const int STREAM_ID_NOT_FOUND = 1009; internal const int STREAM_NAME_NOT_FOUND = 1010; + internal const int INVALID_STREAM_NAME = 1013; internal const int TOPIC_ID_NOT_FOUND = 2010; internal const int TOPIC_NAME_NOT_FOUND = 2011; + internal const int INVALID_TOPIC_NAME = 2014; internal const int CONSUMER_GROUP_MEMBER_NOT_FOUND = 5006; internal const int PARTITION_NOT_FOUND = 3007; internal const int CONSUMER_GROUP_PARTITION_NOT_OWNED = 5009; internal const int TOO_MANY_CONSUMER_GROUPS = 5010; + internal const int UNSUPPORTED_OPTION_KEY = 4041; + internal const int OPTION_NOT_UPDATABLE = 4103; + internal const int OPTION_NOT_RESETTABLE = 4104; + internal const int READ_ONLY_OPTION = 4105; + internal const int CANNOT_MODIFY_ROOT_USER = 4108; internal const int INCOMPATIBLE_PROTOCOL_VERSION = 14003; internal const int SESSION_MISMATCH = 14004; internal const int REQUEST_ID_EXHAUSTED = 14005;
diff --git a/foreign/csharp/Iggy_SDK/Vsr/VsrHeader.cs b/foreign/csharp/Iggy_SDK/Vsr/VsrHeader.cs index 772d705..1d91274 100644 --- a/foreign/csharp/Iggy_SDK/Vsr/VsrHeader.cs +++ b/foreign/csharp/Iggy_SDK/Vsr/VsrHeader.cs
@@ -22,10 +22,8 @@ /// <summary> /// The 256-byte consensus header, read and written by wire offset. Offsets mirror -/// <c>core/binary_protocol/src/consensus/header.rs</c>. Checksums stay zero: the frame and body checksums -/// are not read on the client request path, and a zero <c>request_checksum</c> is unstamped, which skips -/// the server's payload verification. Stamping it is optional. The Rust SDK stamps metadata, session and -/// DeleteSegments operations. This SDK does not stamp it yet. +/// <c>core/binary_protocol/src/consensus/header.rs</c>. Metadata and session requests carry a random +/// per-request identifier in <c>request_checksum</c>, retained across retries. Partition operations leave it zero. /// </summary> internal static class VsrHeader { @@ -98,6 +96,7 @@ BinaryPrimitives.WriteUInt32LittleEndian(header[SIZE_OFFSET..], (uint)totalSize); header[COMMAND_OFFSET] = (byte)Command.Request; WriteUInt128(header[REQUEST_CLIENT_OFFSET..], frame.ClientId); + WriteUInt128(header[REQUEST_CHECKSUM_OFFSET..], frame.RequestChecksum); BinaryPrimitives.WriteUInt64LittleEndian(header[REQUEST_TIMESTAMP_OFFSET..], 0); BinaryPrimitives.WriteUInt64LittleEndian(header[REQUEST_ID_OFFSET..], frame.RequestId); header[REQUEST_OPERATION_OFFSET] = (byte)operation;
diff --git a/foreign/csharp/Iggy_SDK/Vsr/VsrOperation.cs b/foreign/csharp/Iggy_SDK/Vsr/VsrOperation.cs index 69dfb9d..05f2c09 100644 --- a/foreign/csharp/Iggy_SDK/Vsr/VsrOperation.cs +++ b/foreign/csharp/Iggy_SDK/Vsr/VsrOperation.cs
@@ -37,12 +37,16 @@ TruncatePartition = 68, FinalizeSession = 70, CompleteLifecycle = 71, + UpdateStreamResolved = 72, + UpdateTopicResolved = 73, + UpdateUserResolved = 74, + RewrapStreamKeys = 75, CreateStream = 128, - UpdateStream = 129, + // 129 was UpdateStream. Never reuse it. DeleteStream = 130, CreateTopic = 132, - UpdateTopic = 133, + // 133 was UpdateTopic. Never reuse it. DeleteTopic = 134, CreatePartitions = 136, DeletePartitions = 137, @@ -50,7 +54,7 @@ CreateConsumerGroup = 139, DeleteConsumerGroup = 140, CreateUser = 141, - UpdateUser = 142, + // 142 was UpdateUser. Never reuse it. DeleteUser = 143, ChangePassword = 144, UpdatePermissions = 145, @@ -58,6 +62,7 @@ DeletePersonalAccessToken = 147, JoinConsumerGroup = 148, LeaveConsumerGroup = 149, + UpdateResource = 150, SendMessages = 160, StoreConsumerOffset = 161, @@ -120,7 +125,6 @@ CommandCodes.LOGOUT_USER_CODE => VsrOperation.Logout, CommandCodes.CREATE_USER_CODE => VsrOperation.CreateUser, CommandCodes.DELETE_USER_CODE => VsrOperation.DeleteUser, - CommandCodes.UPDATE_USER_CODE => VsrOperation.UpdateUser, CommandCodes.UPDATE_PERMISSIONS_CODE => VsrOperation.UpdatePermissions, CommandCodes.CHANGE_PASSWORD_CODE => VsrOperation.ChangePassword, CommandCodes.CREATE_PERSONAL_ACCESS_TOKEN_CODE => VsrOperation.CreatePersonalAccessToken, @@ -130,10 +134,9 @@ CommandCodes.DELETE_CONSUMER_OFFSET_CODE => VsrOperation.DeleteConsumerOffset, CommandCodes.CREATE_STREAM_CODE => VsrOperation.CreateStream, CommandCodes.DELETE_STREAM_CODE => VsrOperation.DeleteStream, - CommandCodes.UPDATE_STREAM_CODE => VsrOperation.UpdateStream, CommandCodes.CREATE_TOPIC_CODE => VsrOperation.CreateTopic, CommandCodes.DELETE_TOPIC_CODE => VsrOperation.DeleteTopic, - CommandCodes.UPDATE_TOPIC_CODE => VsrOperation.UpdateTopic, + CommandCodes.UPDATE_RESOURCE_CODE => VsrOperation.UpdateResource, CommandCodes.CREATE_PARTITIONS_CODE => VsrOperation.CreatePartitions, CommandCodes.DELETE_PARTITIONS_CODE => VsrOperation.DeletePartitions, CommandCodes.DELETE_SEGMENTS_CODE => VsrOperation.DeleteSegments, @@ -173,23 +176,27 @@ /// <summary> /// Whether the byte is a declared discriminant. Replies carry a server-controlled operation byte, so - /// an undeclared value is rejected rather than classified by range. + /// an undeclared value, a retired one included, is rejected rather than classified by range. /// </summary> internal static bool IsKnown(byte value) { - return (VsrOperation)value switch + return Enum.IsDefined((VsrOperation)value); + } + + /// <summary> + /// The operation a request carried before prepare rewrote it, mirroring <c>Operation::request_operation</c>. + /// A committed reply names the rewritten operation, so a reply correlates with its request through this. + /// </summary> + internal static VsrOperation RequestOperation(this VsrOperation operation) + { + return operation switch { - VsrOperation.Reserved or VsrOperation.Register or VsrOperation.NonReplicated or VsrOperation.Logout => - true, - >= VsrOperation.CreateTopicWithAssignments and <= VsrOperation.TruncatePartition => true, - VsrOperation.FinalizeSession or VsrOperation.CompleteLifecycle or VsrOperation.RetireSession - or VsrOperation.InstallConsumerGroupOwner or VsrOperation.TransitionPartitionHistory or VsrOperation.RetireConsumerGroupOwners => true, - >= VsrOperation.CreateStream and <= VsrOperation.DeleteStream => true, - >= VsrOperation.CreateTopic and <= VsrOperation.DeleteTopic => true, - >= VsrOperation.CreatePartitions and <= VsrOperation.LeaveConsumerGroup => true, - VsrOperation.SendMessages or VsrOperation.StoreConsumerOffset or VsrOperation.DeleteConsumerOffset => - true, - _ => false + VsrOperation.CreateTopicWithAssignments => VsrOperation.CreateTopic, + VsrOperation.CreatePartitionsWithAssignments => VsrOperation.CreatePartitions, + VsrOperation.UpdateStreamResolved or VsrOperation.UpdateTopicResolved or VsrOperation.UpdateUserResolved => + VsrOperation.UpdateResource, + VsrOperation.TruncatePartition => VsrOperation.DeleteSegments, + _ => operation }; } @@ -211,24 +218,22 @@ { return (operation.IsInternal() && (byte)operation < (byte)VsrOperation.SendMessages) || operation is VsrOperation.CreateStream - or VsrOperation.UpdateStream or VsrOperation.DeleteStream or VsrOperation.CreateTopic - or VsrOperation.UpdateTopic or VsrOperation.DeleteTopic or VsrOperation.CreatePartitions or VsrOperation.DeletePartitions or VsrOperation.CreateConsumerGroup or VsrOperation.DeleteConsumerGroup or VsrOperation.CreateUser - or VsrOperation.UpdateUser or VsrOperation.DeleteUser or VsrOperation.ChangePassword or VsrOperation.UpdatePermissions or VsrOperation.CreatePersonalAccessToken or VsrOperation.DeletePersonalAccessToken or VsrOperation.JoinConsumerGroup - or VsrOperation.LeaveConsumerGroup; + or VsrOperation.LeaveConsumerGroup + or VsrOperation.UpdateResource; } /// <summary>
diff --git a/foreign/csharp/Iggy_SDK_Tests/ClientTests/ExampleHelpersTests.cs b/foreign/csharp/Iggy_SDK_Tests/ClientTests/ExampleHelpersTests.cs index f0069bf..9130137 100644 --- a/foreign/csharp/Iggy_SDK_Tests/ClientTests/ExampleHelpersTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/ClientTests/ExampleHelpersTests.cs
@@ -64,14 +64,13 @@ var token = TestContext.Current.CancellationToken; if (topic) { - client.Setup(value => value.CreateTopicAsync(streamId, topicId.GetString(), 1, - CompressionAlgorithm.None, null, 0, null, token)) + client.Setup(value => value.CreateTopicAsync(streamId, topicId.GetString(), 1, null, token)) .ThrowsAsync(error); await ExampleHelpers.EnsureTopicExists(client.Object, streamId, topicId, topicId.GetString(), 1, token); } else { - client.Setup(value => value.CreateStreamAsync(streamId.GetString(), token)) + client.Setup(value => value.CreateStreamAsync(streamId.GetString(), null, token)) .ThrowsAsync(error); await ExampleHelpers.EnsureStreamExists(client.Object, streamId, streamId.GetString(), token); }
diff --git a/foreign/csharp/Iggy_SDK_Tests/ClientTests/HttpPartitionContextTests.cs b/foreign/csharp/Iggy_SDK_Tests/ClientTests/HttpPartitionContextTests.cs index fc01a0c..bd6c960 100644 --- a/foreign/csharp/Iggy_SDK_Tests/ClientTests/HttpPartitionContextTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/ClientTests/HttpPartitionContextTests.cs
@@ -85,7 +85,7 @@ var client = Client(handler); await SendAsync(client); - await client.CreateStreamAsync("created", TestContext.Current.CancellationToken); + await client.CreateStreamAsync("created", token: TestContext.Current.CancellationToken); await client.CreateTopicAsync(Identifier.Numeric(1), "created", 1, token: TestContext.Current.CancellationToken); await SendAsync(client);
diff --git a/foreign/csharp/Iggy_SDK_Tests/ClientTests/HttpTopicOptionsTests.cs b/foreign/csharp/Iggy_SDK_Tests/ClientTests/HttpTopicOptionsTests.cs index 0db5259..394aacb 100644 --- a/foreign/csharp/Iggy_SDK_Tests/ClientTests/HttpTopicOptionsTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/ClientTests/HttpTopicOptionsTests.cs
@@ -17,6 +17,7 @@ using System.Net; using System.Text; +using System.Text.Json; using Apache.Iggy.Contracts; using Apache.Iggy.Exceptions; using Apache.Iggy.Headers; @@ -34,15 +35,12 @@ "created_at": 1750000000000000, "name": "topic", "size": "0 B", - "message_expiry": 0, - "compression_algorithm": "none", - "max_topic_size": 0, "messages_count": 0, "partitions_count": 1, "partitions": [], "options": { - "preallocate_segments": { "value": "true", "explicit": true }, - "segment_size": { "value": "134217728", "explicit": false } + "preallocate_segments": { "value": "true", "kind": "bool", "explicit": true }, + "segment_size": { "value": "134217728", "kind": "uint64", "explicit": false } } } """; @@ -88,11 +86,13 @@ Assert.NotNull(topic); var explicitOption = Assert.Single(topic.Options!); Assert.Equal("preallocate_segments", explicitOption.Key.AsString()); - Assert.Equal("true", explicitOption.Value.ToString()); + Assert.Equal(HeaderKind.Bool, explicitOption.Value.Kind); + Assert.True(explicitOption.Value.ToBool()); var derivedOption = Assert.Single(topic.DerivedOptions!); Assert.Equal("segment_size", derivedOption.Key.AsString()); - Assert.Equal("134217728", derivedOption.Value.ToString()); + Assert.Equal(HeaderKind.Uint64, derivedOption.Value.Kind); + Assert.Equal(134217728UL, derivedOption.Value.ToUInt64()); } [Fact] @@ -102,12 +102,14 @@ var client = new HttpMessageStream(new HttpClient(handler) { BaseAddress = new Uri("http://localhost") }); await client.CreateTopicAsync(StreamId, "topic", 1, - options: new TopicOptions { Durability = Apache.Iggy.Enums.Durability.Persisted, SegmentSize = 134217728 }.ToDictionary(), + options: new TopicOptions { Durability = Apache.Iggy.Enums.Durability.Persisted, SegmentSize = 134217728 }, token: TestContext.Current.CancellationToken); - Assert.Contains("\"durability\":\"persisted\"", handler.RequestBody); - Assert.Contains("\"consumer_offset_durability\":\"replicated\"", handler.RequestBody); - Assert.Contains("\"segment_size\":\"134217728\"", handler.RequestBody); + using var body = JsonDocument.Parse(handler.RequestBody); + var options = body.RootElement.GetProperty("options"); + Assert.Equal("persisted", options.GetProperty("durability").GetString()); + Assert.Equal("replicated", options.GetProperty("consumer_offset_durability").GetString()); + Assert.Equal(134217728UL, options.GetProperty("segment_size").GetUInt64()); } [Fact] @@ -119,8 +121,8 @@ var topic = await client.GetTopicByIdAsync(StreamId, Identifier.Numeric(1), TestContext.Current.CancellationToken); - Assert.Equal(HeaderKind.String, Assert.Single(topic!.Options!).Value.Kind); - Assert.Equal(HeaderKind.String, Assert.Single(topic.DerivedOptions!).Value.Kind); + Assert.Equal(HeaderKind.Bool, Assert.Single(topic!.Options!).Value.Kind); + Assert.Equal(HeaderKind.Uint64, Assert.Single(topic.DerivedOptions!).Value.Kind); } [Fact]
diff --git a/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/PartitionIncarnationTests.cs b/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/PartitionIncarnationTests.cs index 56f0f92..ed9972e 100644 --- a/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/PartitionIncarnationTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/PartitionIncarnationTests.cs
@@ -63,7 +63,7 @@ Assert.DoesNotContain((OriginalOffset, Recreated), stores); Assert.Equal(0ul, Assert.IsType<ReceivedMessage>(delivered).CurrentOffset); - Assert.Equal([(OriginalOffset, Original)], stores); + Assert.Equal([(OriginalOffset, Original)], stores.ToArray()); } [Fact] @@ -90,7 +90,7 @@ Assert.DoesNotContain((OriginalOffset, Recreated), stores); Assert.Equal(0ul, Assert.IsType<ReceivedRentedMessage>(delivered).CurrentOffset); - Assert.Equal([(OriginalOffset, Original)], stores); + Assert.Equal([(OriginalOffset, Original)], stores.ToArray()); } [Fact] @@ -117,7 +117,7 @@ await consumer.StoreOffsetAsync(early.CurrentOffset, early.PartitionId, early.Context, TestContext.Current.CancellationToken); - Assert.Equal([(OriginalOffset, Original)], stores); + Assert.Equal([(OriginalOffset, Original)], stores.ToArray()); } public enum OffsetMutation
diff --git a/foreign/csharp/Iggy_SDK_Tests/ContractsTests/TopicContractsTests.cs b/foreign/csharp/Iggy_SDK_Tests/ContractsTests/TopicContractsTests.cs index 20c49ba..8a26e41 100644 --- a/foreign/csharp/Iggy_SDK_Tests/ContractsTests/TopicContractsTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/ContractsTests/TopicContractsTests.cs
@@ -17,8 +17,8 @@ using System.Buffers.Binary; using System.Text; +using Apache.Iggy.Contracts; using Apache.Iggy.Contracts.Tcp; -using Apache.Iggy.Enums; using Apache.Iggy.Headers; namespace Apache.Iggy.Tests.ContractsTests; @@ -36,8 +36,7 @@ ["segment_size"] = HeaderValue.FromUInt64(1024 * 1024) }; - var bytes = TcpContracts.CreateTopic(Identifier.Numeric(StreamId), NonAsciiName, 3, - CompressionAlgorithm.None, 0, 0, options); + var bytes = TcpContracts.CreateTopic(Identifier.Numeric(StreamId), NonAsciiName, 3, options); // [stream id: kind + length + 4][partition count: 4][name length: 1][name][options] Assert.Equal(3u, BinaryPrimitives.ReadUInt32LittleEndian(bytes.AsSpan(6, 4))); @@ -45,18 +44,18 @@ } [Fact] - public void UpdateTopic_WithANonAsciiName_PrefixesTheUtf8ByteCount() + public void UpdateTopic_WithANonAsciiName_SetsTheNameInTheOptionsBlock() { - var options = new Dictionary<string, HeaderValue> - { - ["max_topic_size"] = HeaderValue.FromUInt64(2_000_000) - }; + var update = new TopicUpdate { Name = NonAsciiName, MaxTopicSize = 2_000_000 }; - var bytes = TcpContracts.UpdateTopic(Identifier.Numeric(StreamId), Identifier.Numeric(7), NonAsciiName, - CompressionAlgorithm.None, 0, 0, options); + var bytes = TcpContracts.UpdateTopic(Identifier.Numeric(StreamId), Identifier.Numeric(7), update.ToPatch()); - // [stream id: kind + length + 4][topic id: kind + length + 4][name length: 1][name][options] - AssertNameAndOptions(bytes, 12, NonAsciiName, options); + // [scope: 1][stream id: kind + length + 4][topic id: kind + length + 4][reset count: 1][options] + Assert.Equal((byte)OptionsScope.Topic, bytes[0]); + Assert.Equal(0, bytes[13]); + var options = Mappers.BinaryMapper.MapHeaders(bytes.AsSpan(14)); + Assert.Equal(NonAsciiName, options[HeaderKey.FromString("name")].ToString()); + Assert.Equal(2_000_000UL, options[HeaderKey.FromString("max_topic_size")].ToUInt64()); } [Fact] @@ -65,8 +64,7 @@ // 129 characters, but 258 UTF-8 bytes: a character count would let this through. var name = new string('ż', 129); - Assert.Throws<ArgumentException>(() => TcpContracts.CreateTopic(Identifier.Numeric(StreamId), name, 1, - CompressionAlgorithm.None, 0, 0)); + Assert.Throws<ArgumentException>(() => TcpContracts.CreateTopic(Identifier.Numeric(StreamId), name, 1)); } [Fact] @@ -78,8 +76,7 @@ options[$"option_{i}"] = HeaderValue.FromUInt64((ulong)i); } - var bytes = TcpContracts.CreateTopic(Identifier.Numeric(StreamId), "topic", 1, CompressionAlgorithm.None, 0, - 0, options); + var bytes = TcpContracts.CreateTopic(Identifier.Numeric(StreamId), "topic", 1, options); AssertNameAndOptions(bytes, 10, "topic", options); }
diff --git a/foreign/csharp/Iggy_SDK_Tests/ContractsTests/WireNameLengthContractsTests.cs b/foreign/csharp/Iggy_SDK_Tests/ContractsTests/WireNameLengthContractsTests.cs index 0b62ec8..2ff1262 100644 --- a/foreign/csharp/Iggy_SDK_Tests/ContractsTests/WireNameLengthContractsTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/ContractsTests/WireNameLengthContractsTests.cs
@@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +using System.Text; +using Apache.Iggy.Contracts; using Apache.Iggy.Contracts.Tcp; using Apache.Iggy.Enums; using Apache.Iggy.Exceptions; @@ -36,7 +38,6 @@ public static TheoryData<string, string, Func<byte[]>> EmptyCases => new() { { nameof(TcpContracts.CreateStream), "name", () => TcpContracts.CreateStream("") }, - { nameof(TcpContracts.UpdateStream), "name", () => TcpContracts.UpdateStream(Id, "") }, { nameof(TcpContracts.CreateGroup), "name", () => TcpContracts.CreateGroup(Id, Id, "") }, { nameof(TcpContracts.CreatePersonalAccessToken), "name", () => TcpContracts.CreatePersonalAccessToken("", null) }, { nameof(TcpContracts.DeletePersonalRequestToken), "name", () => TcpContracts.DeletePersonalRequestToken("") }, @@ -48,7 +49,6 @@ public static TheoryData<string, string, Func<byte[]>> OverLimitCases => new() { { nameof(TcpContracts.CreateStream), "name", () => TcpContracts.CreateStream(OverLimit) }, - { nameof(TcpContracts.UpdateStream), "name", () => TcpContracts.UpdateStream(Id, OverLimit) }, { nameof(TcpContracts.CreateGroup), "name", () => TcpContracts.CreateGroup(Id, Id, OverLimit) }, { nameof(TcpContracts.CreatePersonalAccessToken), "name", () => TcpContracts.CreatePersonalAccessToken(OverLimit, null) }, { nameof(TcpContracts.DeletePersonalRequestToken), "name", () => TcpContracts.DeletePersonalRequestToken(OverLimit) }, @@ -61,7 +61,6 @@ public static TheoryData<string, int, Func<byte[]>> AtLimitCases => new() { { nameof(TcpContracts.CreateStream), 0, () => TcpContracts.CreateStream(AtLimit) }, - { nameof(TcpContracts.UpdateStream), 2 + Id.Length, () => TcpContracts.UpdateStream(Id, AtLimit) }, { nameof(TcpContracts.CreateGroup), 2 * (2 + Id.Length), () => TcpContracts.CreateGroup(Id, Id, AtLimit) }, { nameof(TcpContracts.CreatePersonalAccessToken), 0, () => TcpContracts.CreatePersonalAccessToken(AtLimit, null) }, { nameof(TcpContracts.DeletePersonalRequestToken), 0, () => TcpContracts.DeletePersonalRequestToken(AtLimit) }, @@ -78,8 +77,8 @@ { nameof(TcpContracts.CreateUser) + "/long userName", VsrError.INVALID_USERNAME, () => TcpContracts.CreateUser(new string('a', 51), "pass", UserStatus.Active) }, { nameof(TcpContracts.CreateUser) + "/empty password", VsrError.INVALID_PASSWORD, () => TcpContracts.CreateUser("user", "", UserStatus.Active) }, { nameof(TcpContracts.CreateUser) + "/long password", VsrError.INVALID_PASSWORD, () => TcpContracts.CreateUser("user", new string('a', 101), UserStatus.Active) }, - { nameof(TcpContracts.UpdateUser) + "/empty userName", VsrError.INVALID_USERNAME, () => TcpContracts.UpdateUser(Id, "", null) }, - { nameof(TcpContracts.UpdateUser) + "/long userName", VsrError.INVALID_USERNAME, () => TcpContracts.UpdateUser(Id, new string('a', 51), null) }, + { nameof(TcpContracts.UpdateUser) + "/empty userName", VsrError.INVALID_USERNAME, () => TcpContracts.UpdateUser(Id, new UserUpdate { Username = "" }.ToPatch()) }, + { nameof(TcpContracts.UpdateUser) + "/long userName", VsrError.INVALID_USERNAME, () => TcpContracts.UpdateUser(Id, new UserUpdate { Username = new string('a', 51) }.ToPatch()) }, { nameof(TcpContracts.ChangePassword) + "/empty current", VsrError.INVALID_PASSWORD, () => TcpContracts.ChangePassword(Id, "", "new") }, { nameof(TcpContracts.ChangePassword) + "/long current", VsrError.INVALID_PASSWORD, () => TcpContracts.ChangePassword(Id, new string('a', 101), "new") }, { nameof(TcpContracts.ChangePassword) + "/empty new", VsrError.INVALID_PASSWORD, () => TcpContracts.ChangePassword(Id, "old", "") }, @@ -105,6 +104,16 @@ Assert.Equal(parameterName, exception.ParamName); } + [Fact] + public void UpdateStream_WithANameNoHeaderValueHolds_RefusesWithTheServerNameError() + { + foreach (var name in new[] { "", OverLimit }) + { + var exception = Assert.Throws<IggyInvalidStatusCodeException>(() => new StreamUpdate { Name = name }.ToPatch()); + Assert.Equal(VsrError.INVALID_STREAM_NAME, exception.StatusCode); + } + } + [Theory] [MemberData(nameof(AtLimitCases))] public void Contract_WithAStringOfExactly255Bytes_PrefixesTheFullLength(string contract, int prefixOffset, @@ -146,19 +155,27 @@ } [Fact] - public void UpdateUser_WithStatusOnly_SerializesExactlyOneNameFlagAndStatusPair() + public void UpdateUser_WithStatusOnly_SerializesOnlyTheStatusOption() { - var bytes = TcpContracts.UpdateUser(Id, null, UserStatus.Inactive); + var bytes = TcpContracts.UpdateUser(Id, new UserUpdate { Status = UserStatus.Inactive }.ToPatch()); - Assert.Equal(new byte[] { 1, 4, 1, 0, 0, 0, 0, 1, (byte)UserStatus.Inactive }, bytes); + Assert.Equal([3, 1, 4, 1, 0, 0, 0, 0, .. StringOption("status", "inactive")], bytes); } [Fact] - public void UpdateUser_WithNameOnly_SerializesExactlyOneNameAndStatusFlag() + public void UpdateUser_WithNameOnly_SerializesOnlyTheUsernameOption() { - var bytes = TcpContracts.UpdateUser(Id, "abc", null); + var bytes = TcpContracts.UpdateUser(Id, new UserUpdate { Username = "abc" }.ToPatch()); - Assert.Equal(new byte[] { 1, 4, 1, 0, 0, 0, 1, 3, (byte)'a', (byte)'b', (byte)'c', 0 }, bytes); + Assert.Equal([3, 1, 4, 1, 0, 0, 0, 0, .. StringOption("username", "abc")], bytes); + } + + [Fact] + public void UpdateStream_WithANameOfExactly255Bytes_CarriesTheFullLength() + { + var bytes = TcpContracts.UpdateStream(Id, new StreamUpdate { Name = AtLimit }.ToPatch()); + + Assert.Equal([2, 1, 4, 1, 0, 0, 0, 0, .. StringOption("name", AtLimit)], bytes); } [Fact] @@ -181,14 +198,13 @@ [Fact] public void UpdateStream_WithANonAsciiStringIdentifier_PlacesTheNameAfterTheUtf8Bytes() { - var bytes = TcpContracts.UpdateStream(Identifier.String("café"), "topic"); + var bytes = TcpContracts.UpdateStream(Identifier.String("café"), new StreamUpdate { Name = "topic" }.ToPatch()); Assert.Equal( - new byte[] - { - 2, 5, (byte)'c', (byte)'a', (byte)'f', 0xC3, 0xA9, - 5, (byte)'t', (byte)'o', (byte)'p', (byte)'i', (byte)'c' - }, bytes); + [ + 2, 2, 5, (byte)'c', (byte)'a', (byte)'f', 0xC3, 0xA9, 0, + .. StringOption("name", "topic") + ], bytes); } [Fact] @@ -214,4 +230,16 @@ Assert.Equal(255, identifier.Length); } + + /// <summary>One options block entry with a string key and a string value.</summary> + private static byte[] StringOption(string key, string value) + { + var keyBytes = Encoding.UTF8.GetBytes(key); + var valueBytes = Encoding.UTF8.GetBytes(value); + return + [ + 2, .. BitConverter.GetBytes(keyBytes.Length), .. keyBytes, + 2, .. BitConverter.GetBytes(valueBytes.Length), .. valueBytes + ]; + } }
diff --git a/foreign/csharp/Iggy_SDK_Tests/Fixtures/resource_options.json b/foreign/csharp/Iggy_SDK_Tests/Fixtures/resource_options.json new file mode 100644 index 0000000..c5cc2ac --- /dev/null +++ b/foreign/csharp/Iggy_SDK_Tests/Fixtures/resource_options.json
@@ -0,0 +1,1619 @@ +{ + "schema_version": 1, + "generated_by": "core/common/examples/resource_options_fixtures.rs", + "conventions": { + "byte_strings": "Every *_hex field is lowercase hex without separators.", + "errors": "name is the snake_case IggyError name and code its numeric code.", + "max_topic_size": "Bytes. 18446744073709551615 means unlimited.", + "message_expiry": "Microseconds. 18446744073709551615 means never expire.", + "never_returned_keys": "No response, decoded projection or accessor may contain these keys or their values.", + "option_maps": "Option maps use the HTTP JSON form: value as reversible text, kind, and explicit provenance. Request blocks mark every entry explicit.", + "options_block_order": "Options block entries are encoded in ascending key byte order.", + "request_bodies": "body_hex and update_resource_hex are command payloads without framing. constants.command_codes maps each command to its code and VSR operation.", + "reset_order": "The typed encoders and the HTTP handler write reset keys in ascending key byte order.", + "stream_state": "state is the stored server state a stream response was projected from. SDKs need only response_hex and expected.", + "u64_values": "u64 option values and typed u64 inputs are decimal strings, so values above 2^53 keep their precision." + }, + "constants": { + "protocol_version": { + "packed": 12288, + "text": "0.12.0", + "major": 0, + "minor": 12, + "patch": 0 + }, + "protocol_version_min": { + "packed": 12288, + "text": "0.12.0", + "major": 0, + "minor": 12, + "patch": 0 + }, + "command_codes": { + "create_stream": { + "code": 202, + "operation": 128 + }, + "create_topic": { + "code": 302, + "operation": 132 + }, + "create_user": { + "code": 33, + "operation": 141 + }, + "describe_options": { + "code": 13, + "operation": null + }, + "get_stream": { + "code": 200, + "operation": null + }, + "get_streams": { + "code": 201, + "operation": null + }, + "get_topic": { + "code": 300, + "operation": null + }, + "get_topics": { + "code": 301, + "operation": null + }, + "get_user": { + "code": 31, + "operation": null + }, + "get_users": { + "code": 32, + "operation": null + }, + "update_resource": { + "code": 16, + "operation": 150 + } + }, + "error_codes": { + "cannot_modify_root_user": 4108 + }, + "retired_command_codes": { + "update_stream": 204, + "update_topic": 304, + "update_user": 35 + }, + "operations": { + "rewrap_stream_keys": { + "value": 75, + "internal": true, + "request_operation": 75 + }, + "update_resource": { + "value": 150, + "internal": false, + "request_operation": 150 + }, + "update_stream_resolved": { + "value": 72, + "internal": true, + "request_operation": 150 + }, + "update_topic_resolved": { + "value": 73, + "internal": true, + "request_operation": 150 + }, + "update_user_resolved": { + "value": 74, + "internal": true, + "request_operation": 150 + } + }, + "retired_operations": { + "update_stream": 129, + "update_topic": 133, + "update_user": 142 + }, + "scope_codes": { + "stream": 2, + "topic": 1, + "user": 3 + }, + "header_kind_codes": { + "raw": 1, + "string": 2, + "bool": 3, + "uint32": 11, + "uint64": 12 + }, + "option_flags": { + "critical": 16, + "read_only": 8, + "resettable": 2, + "secret": 4, + "updatable": 1 + }, + "user_status_codes": { + "active": 1, + "inactive": 2 + }, + "stream_key_length": 32, + "never_returned_keys": [ + "encryption_key", + "encryption_key_wrapped", + "critical_keys" + ] + }, + "catalog": [ + { + "key": "name", + "scopes": [ + "stream", + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "column", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 1, + "described": true, + "default": null + }, + { + "key": "username", + "scopes": [ + "user" + ], + "kind": "string", + "kind_code": 2, + "storage": "column", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 1, + "described": true, + "default": null + }, + { + "key": "status", + "scopes": [ + "user" + ], + "kind": "string", + "kind_code": 2, + "storage": "column", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 1, + "described": true, + "default": null + }, + { + "key": "compression_algorithm", + "scopes": [ + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "none" + }, + { + "key": "message_expiry", + "scopes": [ + "topic" + ], + "kind": "uint64", + "kind_code": 12, + "storage": "map", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 3, + "described": true, + "default": "18446744073709551615" + }, + { + "key": "max_topic_size", + "scopes": [ + "topic" + ], + "kind": "uint64", + "kind_code": 12, + "storage": "map", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 3, + "described": true, + "default": "18446744073709551615" + }, + { + "key": "segment_size", + "scopes": [ + "topic" + ], + "kind": "uint64", + "kind_code": 12, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "1073741824" + }, + { + "key": "durability", + "scopes": [ + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "replicated" + }, + { + "key": "consumer_offset_durability", + "scopes": [ + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "replicated" + }, + { + "key": "partition_resize_policy", + "scopes": [ + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "mutable" + }, + { + "key": "messages_required_to_save", + "scopes": [ + "topic" + ], + "kind": "uint32", + "kind_code": 11, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "1024" + }, + { + "key": "size_of_messages_required_to_save", + "scopes": [ + "topic" + ], + "kind": "uint64", + "kind_code": 12, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "1048576" + }, + { + "key": "preallocate_segments", + "scopes": [ + "topic" + ], + "kind": "bool", + "kind_code": 3, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "false" + }, + { + "key": "encryption_key", + "scopes": [ + "stream" + ], + "kind": "raw", + "kind_code": 1, + "storage": "map", + "mutability": "create_only", + "visibility": "secret", + "critical": true, + "flags": 20, + "described": true, + "default": null + }, + { + "key": "encryption_key_wrapped", + "scopes": [ + "stream" + ], + "kind": "raw", + "kind_code": 1, + "storage": "map", + "mutability": "create_only", + "visibility": "internal", + "critical": true, + "flags": 16, + "described": false, + "default": null + }, + { + "key": "encryption_key_source", + "scopes": [ + "stream" + ], + "kind": "string", + "kind_code": 2, + "storage": "computed", + "mutability": "read_only", + "visibility": "read_only", + "critical": false, + "flags": 8, + "described": true, + "default": null + }, + { + "key": "encryption_key_fingerprint", + "scopes": [ + "stream" + ], + "kind": "string", + "kind_code": 2, + "storage": "computed", + "mutability": "read_only", + "visibility": "read_only", + "critical": false, + "flags": 8, + "described": true, + "default": null + }, + { + "key": "encryption_kek_fingerprint", + "scopes": [ + "stream" + ], + "kind": "string", + "kind_code": 2, + "storage": "computed", + "mutability": "read_only", + "visibility": "read_only", + "critical": false, + "flags": 8, + "described": true, + "default": null + }, + { + "key": "critical_keys", + "scopes": [ + "stream", + "topic", + "user" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "internal", + "critical": false, + "flags": 0, + "described": false, + "default": null + } + ], + "describe_options": [ + { + "scope": "topic", + "request_hex": "01", + "response_hex": "0b000000046e616d650201000000007e0000005265736f75726365206e616d652c203120746f203235352062797465732c20756e697175652077697468696e2069747320706172656e742e20536574206279207468652063726561746520636f6d6d616e642773206e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e15636f6d7072657373696f6e5f616c676f726974686d0200040000006e6f6e6550000000436f6d7072657373696f6e20616c676f726974686d20286e6f6e652c20677a6970292e2053746f72656420616e64207265706f727465642c206e6f74206170706c69656420746f206d657373616765730e6d6573736167655f6578706972790c0308000000ffffffffffffffff430000004d6573736167652065787069727920696e206d6963726f7365636f6e64732c206f7220612068756d616e74696d6520737472696e672028652e672e20372064617973290e6d61785f746f7069635f73697a650c0308000000ffffffffffffffff9d000000546f7069632d77696465207365616c65642d7365676d656e742073697a65206361702c2073706c6974206163726f737320616c6c20706172746974696f6e732c20696e206279746573206f72206120627974652d73697a6520737472696e672028652e672e203120476942293b2066696e6974652076616c756573206d757374206265206174206c6561737420746865207365676d656e742073697a650c7365676d656e745f73697a650c000800000000000040000000006c0000005365676d656e742073697a6520696e2062797465732c206f72206120627974652d73697a6520737472696e672028652e672e20313238204d6942293b2061203531322d62797465206d756c7469706c652077697468696e20313034383537362e2e3d313037333734313832340a6475726162696c69747902000a0000007265706c6963617465643e0100004d65737361676520636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20412073696e676c65746f6e2071756f72756d20686173206f6e6520636f70792e20496e207265706c6963617465642067726f7570732c20706572736973746564206d65737361676573207573652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c2057414c207265636c616d6174696f6e2e205468652066756c6c20626f64792073697a65207374696c6c20636f756e747320616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e1a636f6e73756d65725f6f66667365745f6475726162696c69747902000a0000007265706c6963617465646e0100004578706c69636974206f666673657420636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20506f6c6c206175746f2d636f6d6d69742072656d61696e73206173796e6368726f6e6f75732e20496e207265706c6963617465642067726f7570732c20706572736973746564206f66667365747320616c736f20656e61626c652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c207265636c616d6174696f6e2c206576656e2077697468207265706c696361746564206d657373616765206475726162696c6974792e2046756c6c20626f64792073697a657320636f756e7420616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e17706172746974696f6e5f726573697a655f706f6c6963790200070000006d757461626c65c8000000496d6d757461626c65206166746572206372656174696f6e3a206d757461626c65207065726d69747320706172746974696f6e206164646974696f6e2f64656c6574696f6e20776974686f7574206120626f756e646564206375746f766572206f72207065722d6b6579206f72646572696e67206163726f7373206c61796f7574733b2066697865642072656a6563747320626f74682e20526574656e74696f6e20616e6420746f7069632064656c6574696f6e2072656d61696e2064657374727563746976652e196d657373616765735f72657175697265645f746f5f736176650b000400000000040000870000004f7264696e617279206d6573736167652d636f756e7420666c75736820747269676765723b20312e2e3d31363737373231362e2052657175697265642070657273697374656e63652c2063617061636974792070726573737572652c20616e64206c6966656379636c65206f7065726174696f6e732063616e20666c757368206561726c6965722173697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c0008000000000010000000000083000000466c75736820746865206a6f75726e616c206f6e636520697420686f6c64732074686973206d616e792062797465732c206f72206120627974652d73697a6520737472696e673b20776869636865766572207468726573686f6c6420747269707320666972737420666c75736865732e204174206d6f7374203130373337343138323414707265616c6c6f636174655f7365676d656e7473030001000000001201000052657175657374207365676d656e745f73697a65206279746573206f66206469736b207265736572766174696f6e207768656e2065616368207365676d656e74206f70656e732e20556e737570706f72746564206f72206661696c6564207265736572766174696f6e732066616c6c206261636b20746f206f7264696e61727920616c6c6f636174696f6e20776974682061207761726e696e672e205265736572766174696f6e2072756e7320696e6c696e65206f6e20746865206f776e696e672073686172642e207365676d656e745f73697a65202a20706172746974696f6e735f636f756e74206973206361707065642061742036383731393437363733362062797465732070657220637265617465", + "entries": [ + { + "key": "name", + "kind": "string", + "flags": 1, + "default": null + }, + { + "key": "compression_algorithm", + "kind": "string", + "flags": 0, + "default": "none" + }, + { + "key": "message_expiry", + "kind": "uint64", + "flags": 3, + "default": "18446744073709551615" + }, + { + "key": "max_topic_size", + "kind": "uint64", + "flags": 3, + "default": "18446744073709551615" + }, + { + "key": "segment_size", + "kind": "uint64", + "flags": 0, + "default": "1073741824" + }, + { + "key": "durability", + "kind": "string", + "flags": 0, + "default": "replicated" + }, + { + "key": "consumer_offset_durability", + "kind": "string", + "flags": 0, + "default": "replicated" + }, + { + "key": "partition_resize_policy", + "kind": "string", + "flags": 0, + "default": "mutable" + }, + { + "key": "messages_required_to_save", + "kind": "uint32", + "flags": 0, + "default": "1024" + }, + { + "key": "size_of_messages_required_to_save", + "kind": "uint64", + "flags": 0, + "default": "1048576" + }, + { + "key": "preallocate_segments", + "kind": "bool", + "flags": 0, + "default": "false" + } + ] + }, + { + "scope": "stream", + "request_hex": "02", + "response_hex": "05000000046e616d650201000000007e0000005265736f75726365206e616d652c203120746f203235352062797465732c20756e697175652077697468696e2069747320706172656e742e20536574206279207468652063726561746520636f6d6d616e642773206e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e0e656e6372797074696f6e5f6b6579011400000000a30100005468652073747265616d2773206f776e206461746120656e6372797074696f6e206b65793a2033322072616e646f6d2062797465732c20726177206f72207374616e6461726420706164646564206261736536342e204372656174652d6f6e6c7920616e642077726974652d6f6e6c793a20746865207365727665722073746f726573206974207772617070656420756e6465722061206b657920646572697665642066726f6d207468652073657276657220656e6372797074696f6e206b657920616e64206e657665722072657475726e732069742e205265717569726573207468652073657276657220656e6372797074696f6e206b657920746f20626520636f6e666967757265642e20576974686f75742069742c206120736572766572207769746820656e6372797074696f6e20656e61626c65642067656e6572617465732061206b657920666f72207468652073747265616d2e20546865206b65792074726176656c7320696e20746865206372656174652072657175657374206f76657220746865207472616e73706f72742074686520636c69656e7420757365732e15656e6372797074696f6e5f6b65795f736f757263650208000000009f00000057686f2063686f7365207468652073747265616d27732061742d72657374206b65793a2073747265616d207768656e2074686520636c69656e742073656e742069742c20736572766572207768656e20746865207365727665722067656e6572617465642069742c206e6f6e65207768656e207468652073747265616d2773206d65737361676573206172652073746f72656420756e656e637279707465641a656e6372797074696f6e5f6b65795f66696e6765727072696e740208000000009f0000005468652066697273742038206279746573206f66205348412d323536206f76657220746865206c6162656c20696767792f73747265616d2d6b65792d66696e6765727072696e742f763120616e64207468652073747265616d2773206b65792c206173203136206c6f776572636173652068657820636861726163746572732e20416273656e74207768656e2074686520736f75726365206973206e6f6e651a656e6372797074696f6e5f6b656b5f66696e6765727072696e74020800000000b800000046696e6765727072696e7473206f6620746865206b65792d656e6372797074696f6e206b65797320746861742077726170207468652073747265616d2773206b65792c20636f6d6d612d7365706172617465643a206f6e65206f7574736964652061206b657920726f746174696f6e2c206f6e652070657220636f6e6669677572656420736572766572206b657920647572696e672069742e20416273656e74207768656e2074686520736f75726365206973206e6f6e65", + "entries": [ + { + "key": "name", + "kind": "string", + "flags": 1, + "default": null + }, + { + "key": "encryption_key", + "kind": "raw", + "flags": 20, + "default": null + }, + { + "key": "encryption_key_source", + "kind": "string", + "flags": 8, + "default": null + }, + { + "key": "encryption_key_fingerprint", + "kind": "string", + "flags": 8, + "default": null + }, + { + "key": "encryption_kek_fingerprint", + "kind": "string", + "flags": 8, + "default": null + } + ] + }, + { + "scope": "user", + "request_hex": "03", + "response_hex": "0200000008757365726e616d650201000000006b00000055736572206e616d652c203320746f2035302062797465732c20756e697175652e20536574206279207468652063726561746520636f6d6d616e64277320757365726e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e067374617475730201000000008f00000055736572207374617475732c20616374697665206f7220696e6163746976652e20536574206279207468652063726561746520636f6d6d616e64277320737461747573206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e20446561637469766174696e672061207573657220656e6473206974732073657373696f6e732e", + "entries": [ + { + "key": "username", + "kind": "string", + "flags": 1, + "default": null + }, + { + "key": "status", + "kind": "string", + "flags": 1, + "default": null + } + ] + } + ], + "stream_key": { + "key_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "key_base64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", + "fingerprint_label": "iggy/stream-key-fingerprint/v1", + "fingerprint": "8441e38538cbfa57", + "invalid_keys": [ + { + "case": "raw_31_bytes", + "input": { + "raw_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + }, + { + "case": "raw_33_bytes", + "input": { + "raw_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + }, + { + "case": "base64_with_whitespace", + "input": { + "base64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=\n" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + }, + { + "case": "base64_missing_padding", + "input": { + "base64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + }, + { + "case": "base64_noncanonical_trailing_bits", + "input": { + "base64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh9=" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + } + ] + }, + "update_resource": [ + { + "case": "topic_set_expiry_without_name_reset_max_topic_size", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "message_expiry": "604800000000", + "reset": [ + "max_topic_size" + ] + }, + "wire_set": { + "message_expiry": { + "value": "604800000000", + "kind": "uint64", + "explicit": true + } + }, + "wire_reset": [ + "max_topic_size" + ], + "body_hex": "01010401000000010402000000010e6d61785f746f7069635f73697a65020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000" + }, + { + "case": "topic_rename_only_named_identifiers", + "scope": "topic", + "target": { + "stream_id": "orders", + "topic_id": "events" + }, + "input": { + "name": "events-v2" + }, + "wire_set": { + "name": { + "value": "events-v2", + "kind": "string", + "explicit": true + } + }, + "wire_reset": [], + "body_hex": "0102066f726465727302066576656e74730002040000006e616d6502090000006576656e74732d7632" + }, + { + "case": "topic_rename_set_expiry_reset_max_topic_size", + "scope": "topic", + "target": { + "stream_id": 7, + "topic_id": "events" + }, + "input": { + "name": "events-v3", + "message_expiry": "86400000000", + "reset": [ + "max_topic_size" + ] + }, + "wire_set": { + "message_expiry": { + "value": "86400000000", + "kind": "uint64", + "explicit": true + }, + "name": { + "value": "events-v3", + "kind": "string", + "explicit": true + } + }, + "wire_reset": [ + "max_topic_size" + ], + "body_hex": "0101040700000002066576656e7473010e6d61785f746f7069635f73697a65020e0000006d6573736167655f6578706972790c080000000060d71d1400000002040000006e616d6502090000006576656e74732d7633" + }, + { + "case": "topic_reset_both_defaults", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "reset": [ + "max_topic_size", + "message_expiry" + ] + }, + "wire_set": {}, + "wire_reset": [ + "max_topic_size", + "message_expiry" + ], + "body_hex": "01010401000000010402000000020e6d61785f746f7069635f73697a650e6d6573736167655f657870697279" + }, + { + "case": "topic_max_topic_size_above_2_pow_53", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "max_topic_size": "9007199254740993" + }, + "wire_set": { + "max_topic_size": { + "value": "9007199254740993", + "kind": "uint64", + "explicit": true + } + }, + "wire_reset": [], + "body_hex": "0101040100000001040200000000020e0000006d61785f746f7069635f73697a650c080000000100000000002000" + }, + { + "case": "stream_rename", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "input": { + "name": "payments-v2" + }, + "wire_set": { + "name": { + "value": "payments-v2", + "kind": "string", + "explicit": true + } + }, + "wire_reset": [], + "body_hex": "020104030000000002040000006e616d65020b0000007061796d656e74732d7632" + }, + { + "case": "user_username_and_status", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "username": "alice2", + "status": "inactive" + }, + "wire_set": { + "status": { + "value": "inactive", + "kind": "string", + "explicit": true + }, + "username": { + "value": "alice2", + "kind": "string", + "explicit": true + } + }, + "wire_reset": [], + "body_hex": "030104050000000002060000007374617475730208000000696e6163746976650208000000757365726e616d650206000000616c69636532" + } + ], + "update_resource_errors": [ + { + "case": "topic_empty_update", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": {}, + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_set_and_reset_same_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "message_expiry": "604800000000", + "reset": [ + "message_expiry" + ] + }, + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_reset_create_only_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "reset": [ + "compression_algorithm" + ] + }, + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "stream_reset_name", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "input": { + "reset": [ + "name" + ] + }, + "error": { + "name": "option_not_resettable", + "code": 4104 + } + }, + { + "case": "user_reset_status", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "reset": [ + "status" + ] + }, + "error": { + "name": "option_not_resettable", + "code": 4104 + } + }, + { + "case": "stream_reset_read_only_key", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "input": { + "reset": [ + "encryption_key_source" + ] + }, + "error": { + "name": "read_only_option", + "code": 4105 + } + }, + { + "case": "user_reset_internal_key", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "reset": [ + "critical_keys" + ] + }, + "error": { + "name": "read_only_option", + "code": 4105 + } + }, + { + "case": "topic_empty_name", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "name": "" + }, + "error": { + "name": "invalid_topic_name", + "code": 2014 + } + }, + { + "case": "topic_empty_name_and_create_only_reset", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "name": "", + "reset": [ + "segment_size" + ] + }, + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "stream_empty_name", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "input": { + "name": "" + }, + "error": { + "name": "invalid_stream_name", + "code": 1013 + } + }, + { + "case": "user_short_username", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "username": "a" + }, + "error": { + "name": "invalid_username", + "code": 43 + } + }, + { + "case": "user_short_username_and_status_reset", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "username": "a", + "reset": [ + "status" + ] + }, + "error": { + "name": "option_not_resettable", + "code": 4104 + } + } + ], + "create_stream": [ + { + "case": "stream_without_key", + "input": { + "name": "orders" + }, + "wire_options": {}, + "body_hex": "066f7264657273", + "http_body": "{\"name\":\"orders\"}", + "admission_error": null + }, + { + "case": "stream_with_key", + "input": { + "name": "orders", + "encryption_key": { + "raw_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f" + } + }, + "wire_options": { + "encryption_key": { + "value": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", + "kind": "raw", + "explicit": true + } + }, + "body_hex": "066f7264657273020e000000656e6372797074696f6e5f6b65790120000000000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "http_body": "{\"name\":\"orders\",\"options\":{\"encryption_key\":\"AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=\"}}", + "admission_error": null + }, + { + "case": "stream_with_raw_extra_key", + "input": { + "name": "orders", + "raw": { + "future_option": "enabled" + } + }, + "wire_options": { + "future_option": { + "value": "enabled", + "kind": "string", + "explicit": true + } + }, + "body_hex": "066f7264657273020d0000006675747572655f6f7074696f6e0207000000656e61626c6564", + "http_body": "{\"name\":\"orders\",\"options\":{\"future_option\":\"enabled\"}}", + "admission_error": { + "name": "unsupported_option_key", + "code": 4041 + } + } + ], + "create_topic": [ + { + "case": "topic_with_options", + "input": { + "stream_id": 1, + "name": "events", + "partitions_count": 3, + "compression_algorithm": "gzip", + "message_expiry": "604800000000", + "max_topic_size": "9007199254740993", + "segment_size": "134217728", + "durability": "persisted", + "consumer_offset_durability": "replicated" + }, + "wire_options": { + "compression_algorithm": { + "value": "gzip", + "kind": "string", + "explicit": true + }, + "consumer_offset_durability": { + "value": "replicated", + "kind": "string", + "explicit": true + }, + "durability": { + "value": "persisted", + "kind": "string", + "explicit": true + }, + "max_topic_size": { + "value": "9007199254740993", + "kind": "uint64", + "explicit": true + }, + "message_expiry": { + "value": "604800000000", + "kind": "uint64", + "explicit": true + }, + "segment_size": { + "value": "134217728", + "kind": "uint64", + "explicit": true + } + }, + "body_hex": "01040100000003000000066576656e74730215000000636f6d7072657373696f6e5f616c676f726974686d0204000000677a6970021a000000636f6e73756d65725f6f66667365745f6475726162696c697479020a0000007265706c696361746564020a0000006475726162696c6974790209000000706572736973746564020e0000006d61785f746f7069635f73697a650c080000000100000000002000020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000020c0000007365676d656e745f73697a650c080000000000000800000000", + "http_body": "{\"partitions_count\":3,\"name\":\"events\",\"options\":{\"compression_algorithm\":\"gzip\",\"consumer_offset_durability\":\"replicated\",\"durability\":\"persisted\",\"max_topic_size\":\"9007199254740993\",\"message_expiry\":604800000000,\"segment_size\":134217728}}", + "admission_error": null + } + ], + "create_user": [ + { + "case": "user_with_empty_options", + "input": { + "username": "alice", + "password": "s3cret-pass", + "status": "active" + }, + "wire_options": {}, + "body_hex": "05616c6963650b7333637265742d706173730100", + "http_body": "{\"username\":\"alice\",\"password\":\"s3cret-pass\",\"status\":\"active\",\"permissions\":null}", + "admission_error": null + } + ], + "http_update": [ + { + "case": "topic_flat_keys", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"name\":\"events-v2\",\"message_expiry\":\"7 days\"}", + "update_resource_hex": "0101040100000001040200000000020e0000006d6573736167655f657870697279020600000037206461797302040000006e616d6502090000006576656e74732d7632", + "outcome": "accepted", + "error": null + }, + { + "case": "topic_reset_array", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":604800000000,\"reset\":[\"max_topic_size\"]}", + "update_resource_hex": "01010401000000010402000000010e6d61785f746f7069635f73697a65020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000", + "outcome": "accepted", + "error": null + }, + { + "case": "topic_null_ignored", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"name\":null,\"max_topic_size\":null,\"message_expiry\":\"1 day\"}", + "update_resource_hex": "0101040100000001040200000000020e0000006d6573736167655f65787069727902050000003120646179", + "outcome": "accepted", + "error": null + }, + { + "case": "topic_u64_above_2_pow_53_as_string", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"max_topic_size\":\"9007199254740993\"}", + "update_resource_hex": "0101040100000001040200000000020e0000006d61785f746f7069635f73697a65021000000039303037313939323534373430393933", + "outcome": "accepted", + "error": null + }, + { + "case": "stream_rename", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "body": "{\"name\":\"payments-v2\"}", + "update_resource_hex": "020104030000000002040000006e616d65020b0000007061796d656e74732d7632", + "outcome": "accepted", + "error": null + }, + { + "case": "user_username_and_status", + "scope": "user", + "target": { + "user_id": 5 + }, + "body": "{\"username\":\"alice2\",\"status\":\"inactive\"}", + "update_resource_hex": "030104050000000002060000007374617475730208000000696e6163746976650208000000757365726e616d650206000000616c69636532", + "outcome": "accepted", + "error": null + }, + { + "case": "topic_duplicate_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":\"1 day\",\"message_expiry\":\"2 days\"}", + "update_resource_hex": null, + "outcome": "json_refused", + "error": null + }, + { + "case": "topic_nested_object", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":{\"value\":\"1 day\"}}", + "update_resource_hex": null, + "outcome": "json_refused", + "error": null + }, + { + "case": "topic_null_only", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"max_topic_size\":null}", + "update_resource_hex": "0101040100000001040200000000", + "outcome": "decode_refused", + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_zero_sentinel_only", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":0}", + "update_resource_hex": "0101040100000001040200000000020e0000006d6573736167655f6578706972790c080000000000000000000000", + "outcome": "admission_refused", + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_reset_names_set_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":\"1 day\",\"reset\":[\"message_expiry\"]}", + "update_resource_hex": "01010401000000010402000000010e6d6573736167655f657870697279020e0000006d6573736167655f65787069727902050000003120646179", + "outcome": "decode_refused", + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_unknown_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"retention\":\"1 day\"}", + "update_resource_hex": "01010401000000010402000000000209000000726574656e74696f6e02050000003120646179", + "outcome": "admission_refused", + "error": { + "name": "unsupported_option_key", + "code": 4041 + } + }, + { + "case": "topic_set_create_only_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"compression_algorithm\":\"gzip\"}", + "update_resource_hex": "01010401000000010402000000000215000000636f6d7072657373696f6e5f616c676f726974686d0204000000677a6970", + "outcome": "admission_refused", + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "topic_reset_create_only_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"reset\":[\"compression_algorithm\"]}", + "update_resource_hex": "010104010000000104020000000115636f6d7072657373696f6e5f616c676f726974686d", + "outcome": "admission_refused", + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "stream_reset_name", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "body": "{\"reset\":[\"name\"]}", + "update_resource_hex": "0201040300000001046e616d65", + "outcome": "admission_refused", + "error": { + "name": "option_not_resettable", + "code": 4104 + } + }, + { + "case": "topic_name_as_number", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"name\":5}", + "update_resource_hex": "010104010000000104020000000002040000006e616d650c080000000500000000000000", + "outcome": "admission_refused", + "error": { + "name": "invalid_topic_name", + "code": 2014 + } + }, + { + "case": "stream_empty_name", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "body": "{\"name\":\"\"}", + "update_resource_hex": "020104030000000002040000006e616d65030100000000", + "outcome": "admission_refused", + "error": { + "name": "invalid_stream_name", + "code": 1013 + } + }, + { + "case": "user_status_as_bool", + "scope": "user", + "target": { + "user_id": 5 + }, + "body": "{\"status\":true}", + "update_resource_hex": "03010405000000000206000000737461747573030100000001", + "outcome": "admission_refused", + "error": { + "name": "invalid_user_status", + "code": 45 + } + }, + { + "case": "topic_create_only_key_as_bool", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"segment_size\":true}", + "update_resource_hex": "0101040100000001040200000000020c0000007365676d656e745f73697a65030100000001", + "outcome": "admission_refused", + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "stream_read_only_key_as_number", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "body": "{\"encryption_key_source\":1}", + "update_resource_hex": "02010403000000000215000000656e6372797074696f6e5f6b65795f736f757263650c080000000100000000000000", + "outcome": "admission_refused", + "error": { + "name": "read_only_option", + "code": 4105 + } + }, + { + "case": "topic_invalid_value", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":\"soon\"}", + "update_resource_hex": "0101040100000001040200000000020e0000006d6573736167655f6578706972790204000000736f6f6e", + "outcome": "admission_refused", + "error": { + "name": "invalid_option_value", + "code": 4042 + } + } + ], + "stream_responses": [ + { + "case": "stream_source_none", + "state": { + "encryption_key_source": "none", + "stored_extra": {} + }, + "response_hex": "0100000000e090663c1306000000000000100000000000000c000000000000000c706c61696e2d73747265616d00000000230000000215000000656e6372797074696f6e5f6b65795f736f7572636502040000006e6f6e65", + "expected": { + "id": 1, + "created_at": 1710000000000000, + "name": "plain-stream", + "topics_count": 0, + "size_bytes": 4096, + "messages_count": 12, + "options": { + "encryption_key_source": { + "value": "none", + "kind": "string", + "explicit": false + } + }, + "encryption_key_source": "none", + "encryption_key_fingerprint": null, + "encryption_kek_fingerprints": [] + } + }, + { + "case": "stream_source_stream_with_unknown_explicit_key", + "state": { + "encryption_key_source": "stream", + "data_key": { + "raw_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f" + }, + "kek_config_keys": [ + "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=" + ], + "stored_extra": { + "future_option": { + "value": "enabled", + "kind": "string", + "explicit": true + } + } + }, + "response_hex": "0200000000e090663c1306000000000000100000000000000c0000000000000011636c69656e742d6b65792d73747265616d1e000000020d0000006675747572655f6f7074696f6e0207000000656e61626c65648d000000021a000000656e6372797074696f6e5f6b656b5f66696e6765727072696e74021000000033396534663066393563333639336337021a000000656e6372797074696f6e5f6b65795f66696e6765727072696e740210000000383434316533383533386362666135370215000000656e6372797074696f6e5f6b65795f736f75726365020600000073747265616d", + "expected": { + "id": 2, + "created_at": 1710000000000000, + "name": "client-key-stream", + "topics_count": 0, + "size_bytes": 4096, + "messages_count": 12, + "options": { + "encryption_kek_fingerprint": { + "value": "39e4f0f95c3693c7", + "kind": "string", + "explicit": false + }, + "encryption_key_fingerprint": { + "value": "8441e38538cbfa57", + "kind": "string", + "explicit": false + }, + "encryption_key_source": { + "value": "stream", + "kind": "string", + "explicit": false + }, + "future_option": { + "value": "enabled", + "kind": "string", + "explicit": true + } + }, + "encryption_key_source": "stream", + "encryption_key_fingerprint": "8441e38538cbfa57", + "encryption_kek_fingerprints": [ + "39e4f0f95c3693c7" + ] + } + }, + { + "case": "stream_source_server_two_keks", + "state": { + "encryption_key_source": "server", + "data_key": { + "raw_hex": "202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f" + }, + "kek_config_keys": [ + "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=", + "AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA=" + ], + "stored_extra": {} + }, + "response_hex": "0300000000e090663c1306000000000000100000000000000c00000000000000117365727665722d6b65792d73747265616d000000009e000000021a000000656e6372797074696f6e5f6b656b5f66696e6765727072696e740221000000333965346630663935633336393363372c64353564343739383239626332393336021a000000656e6372797074696f6e5f6b65795f66696e6765727072696e740210000000306563616666316261323931666530640215000000656e6372797074696f6e5f6b65795f736f757263650206000000736572766572", + "expected": { + "id": 3, + "created_at": 1710000000000000, + "name": "server-key-stream", + "topics_count": 0, + "size_bytes": 4096, + "messages_count": 12, + "options": { + "encryption_kek_fingerprint": { + "value": "39e4f0f95c3693c7,d55d479829bc2936", + "kind": "string", + "explicit": false + }, + "encryption_key_fingerprint": { + "value": "0ecaff1ba291fe0d", + "kind": "string", + "explicit": false + }, + "encryption_key_source": { + "value": "server", + "kind": "string", + "explicit": false + } + }, + "encryption_key_source": "server", + "encryption_key_fingerprint": "0ecaff1ba291fe0d", + "encryption_kek_fingerprints": [ + "39e4f0f95c3693c7", + "d55d479829bc2936" + ] + } + } + ], + "topic_responses": [ + { + "case": "topic_explicit_and_derived", + "response_hex": "0400000000e090663c1306000000000000080000000000000700000000000000066576656e7473430000000215000000636f6d7072657373696f6e5f616c676f726974686d0204000000677a6970020e0000006d6573736167655f6578706972790c0800000000a0e3d08c0000002b010000021a000000636f6e73756d65725f6f66667365745f6475726162696c697479020a0000007265706c696361746564020a0000006475726162696c697479020a0000007265706c696361746564020e0000006d61785f746f7069635f73697a650c08000000ffffffffffffffff02190000006d657373616765735f72657175697265645f746f5f736176650b04000000000400000217000000706172746974696f6e5f726573697a655f706f6c69637902070000006d757461626c650214000000707265616c6c6f636174655f7365676d656e7473030100000000020c0000007365676d656e745f73697a650c080000000000004000000000022100000073697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c080000000000100000000000", + "expected": { + "id": 4, + "created_at": 1710000000000000, + "name": "events", + "partitions_count": 0, + "size_bytes": 2048, + "messages_count": 7, + "options": { + "compression_algorithm": { + "value": "gzip", + "kind": "string", + "explicit": true + }, + "consumer_offset_durability": { + "value": "replicated", + "kind": "string", + "explicit": false + }, + "durability": { + "value": "replicated", + "kind": "string", + "explicit": false + }, + "max_topic_size": { + "value": "18446744073709551615", + "kind": "uint64", + "explicit": false + }, + "message_expiry": { + "value": "604800000000", + "kind": "uint64", + "explicit": true + }, + "messages_required_to_save": { + "value": "1024", + "kind": "uint32", + "explicit": false + }, + "partition_resize_policy": { + "value": "mutable", + "kind": "string", + "explicit": false + }, + "preallocate_segments": { + "value": "false", + "kind": "bool", + "explicit": false + }, + "segment_size": { + "value": "1073741824", + "kind": "uint64", + "explicit": false + }, + "size_of_messages_required_to_save": { + "value": "1048576", + "kind": "uint64", + "explicit": false + } + }, + "compression_algorithm": "gzip", + "message_expiry": "604800000000", + "max_topic_size": "18446744073709551615" + } + } + ], + "user_responses": [ + { + "case": "user_empty_blocks", + "response_hex": "0500000000e090663c1306000105616c696365000000000000000000000000", + "expected": { + "id": 5, + "created_at": 1710000000000000, + "status": "active", + "username": "alice", + "options": {} + } + }, + { + "case": "user_populated_blocks", + "response_hex": "0600000000e090663c1306000203626f621e000000020d0000006675747572655f6f7074696f6e0207000000656e61626c65641e000000020c0000006675747572655f71756f74610c08000000010000000000200000000000", + "expected": { + "id": 6, + "created_at": 1710000000000000, + "status": "inactive", + "username": "bob", + "options": { + "future_option": { + "value": "enabled", + "kind": "string", + "explicit": true + }, + "future_quota": { + "value": "9007199254740993", + "kind": "uint64", + "explicit": false + } + } + } + } + ] +}
diff --git a/foreign/csharp/Iggy_SDK_Tests/Iggy_SDK_Tests.csproj b/foreign/csharp/Iggy_SDK_Tests/Iggy_SDK_Tests.csproj index f088fc0..f2ca489 100644 --- a/foreign/csharp/Iggy_SDK_Tests/Iggy_SDK_Tests.csproj +++ b/foreign/csharp/Iggy_SDK_Tests/Iggy_SDK_Tests.csproj
@@ -48,4 +48,10 @@ <Compile Include="../../../examples/csharp/src/Iggy_SDK.Examples.Shared/ExampleHelpers.cs" Link="Examples/ExampleHelpers.cs" /> </ItemGroup> + <ItemGroup> + <None Include="Fixtures/resource_options.json"> + <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> + </None> + </ItemGroup> + </Project>
diff --git a/foreign/csharp/Iggy_SDK_Tests/MapperTests/BinaryMapper.cs b/foreign/csharp/Iggy_SDK_Tests/MapperTests/BinaryMapper.cs index 9def08c..68a0977 100644 --- a/foreign/csharp/Iggy_SDK_Tests/MapperTests/BinaryMapper.cs +++ b/foreign/csharp/Iggy_SDK_Tests/MapperTests/BinaryMapper.cs
@@ -188,18 +188,14 @@ var topicsCount = 1u; var streamPayload = BinaryFactory.CreateStreamPayload(id, topicsCount, name, sizeBytes, messagesCount, createdAt); - var (topicId1, partitionsCount1, topicName1, messageExpiry1, topicSizeBytes1, messagesCountTopic1, - createdAtTopic, maxTopicSize) = + var (topicId1, partitionsCount1, topicName1, _, topicSizeBytes1, messagesCountTopic1, _, _) = TopicFactory.CreateTopicResponseFields(); var topicPayload1 = BinaryFactory.CreateTopicPayload(topicId1, partitionsCount1, - messageExpiry1, topicName1, topicSizeBytes1, messagesCountTopic1, - createdAt, - maxTopicSize, - 1); + createdAt); var topicCombinedPayload = new byte[topicPayload1.Length]; topicPayload1.CopyTo(topicCombinedPayload.AsSpan()); @@ -233,16 +229,15 @@ public void MapTopics_ReturnsValidTopicsResponses() { // Arrange - var (id1, partitionsCount1, name1, messageExpiry1, sizeBytesTopic1, messagesCountTopic1, createdAt, - maxTopicSize1) = + var (id1, partitionsCount1, name1, _, sizeBytesTopic1, messagesCountTopic1, createdAt, _) = TopicFactory.CreateTopicResponseFields(); - var payload1 = BinaryFactory.CreateTopicPayload(id1, partitionsCount1, messageExpiry1, name1, - sizeBytesTopic1, messagesCountTopic1, createdAt, maxTopicSize1, 1); - var (id2, partitionsCount2, name2, messageExpiry2, sizeBytesTopic2, messagesCountTopic2, createdAt2, - maxTopicSize2) = + var payload1 = BinaryFactory.CreateTopicPayload(id1, partitionsCount1, name1, + sizeBytesTopic1, messagesCountTopic1, createdAt); + var (id2, partitionsCount2, name2, _, sizeBytesTopic2, messagesCountTopic2, createdAt2, _) = TopicFactory.CreateTopicResponseFields(); - var payload2 = BinaryFactory.CreateTopicPayload(id2, partitionsCount2, messageExpiry2, name2, - sizeBytesTopic2, messagesCountTopic2, createdAt2, maxTopicSize2, 2); + var payload2 = BinaryFactory.CreateTopicPayload(id2, partitionsCount2, name2, + sizeBytesTopic2, messagesCountTopic2, createdAt2, + BinaryFactory.CreateOptionEntry(2, "compression_algorithm", 2, "gzip"u8.ToArray())); // GetTopics replies start with the topics count. var combinedPayload = new byte[4 + payload1.Length + payload2.Length]; @@ -280,8 +275,10 @@ // Arrange var (topicId, partitionsCount, topicName, messageExpiry, sizeBytes, messagesCount, createdAt2, maxTopicSize) = TopicFactory.CreateTopicResponseFields(); - var topicPayload = BinaryFactory.CreateTopicPayload(topicId, partitionsCount, messageExpiry, topicName, - sizeBytes, messagesCount, createdAt2, maxTopicSize, 1); + var topicPayload = BinaryFactory.CreateTopicPayload(topicId, partitionsCount, topicName, + sizeBytes, messagesCount, createdAt2, + BinaryFactory.CreateOptionEntry(2, "message_expiry", 12, BitConverter.GetBytes((ulong)messageExpiry)), + BinaryFactory.CreateOptionEntry(2, "max_topic_size", 12, BitConverter.GetBytes(maxTopicSize))); var combinedPayload = new byte[topicPayload.Length]; topicPayload.CopyTo(combinedPayload.AsSpan()); @@ -297,18 +294,20 @@ Assert.Equal(topicId, response.Id); Assert.Equal(topicName, response.Name); Assert.Equal(CompressionAlgorithm.None, response.CompressionAlgorithm); + Assert.Equal(TimeSpan.FromMicroseconds(messageExpiry), response.MessageExpiry); + Assert.Equal(maxTopicSize, response.MaxTopicSize); } [Fact] public void MapTopic_ReadsAPartitionRecordWithItsContext() { - var (topicId, partitionsCount, topicName, messageExpiry, sizeBytes, messagesCount, createdAt, - maxTopicSize) = TopicFactory.CreateTopicResponseFields(); + var (topicId, partitionsCount, topicName, _, sizeBytes, messagesCount, createdAt, _) = + TopicFactory.CreateTopicResponseFields(); var context = new PartitionContext(17, 9, 52); byte[] payload = [ - ..BinaryFactory.CreateTopicPayload(topicId, partitionsCount, messageExpiry, topicName, sizeBytes, - messagesCount, createdAt, maxTopicSize, 1), + ..BinaryFactory.CreateTopicPayload(topicId, partitionsCount, topicName, sizeBytes, messagesCount, + createdAt), ..BinaryFactory.CreatePartitionPayload(3, createdAt, 7, 101, 2048, 102, context) ]; @@ -330,14 +329,14 @@ // Arrange: an option kind a newer server may introduce, between two this build knows. const byte stringKind = 2; const byte unknownKind = 200; - var (topicId, partitionsCount, topicName, messageExpiry, sizeBytes, messagesCount, createdAt, - maxTopicSize) = TopicFactory.CreateTopicResponseFields(); + var (topicId, partitionsCount, topicName, _, sizeBytes, messagesCount, createdAt, _) = + TopicFactory.CreateTopicResponseFields(); var options = new List<byte>(); options.AddRange(BinaryFactory.CreateOptionEntry(stringKind, "segment_size", stringKind, "1GB"u8.ToArray())); options.AddRange(BinaryFactory.CreateOptionEntry(stringKind, "future_option", unknownKind, [0xAA, 0xBB])); options.AddRange(BinaryFactory.CreateOptionEntry(stringKind, "preallocate_segments", 3, [1])); - var topicPayload = BinaryFactory.CreateTopicPayload(topicId, partitionsCount, messageExpiry, topicName, - sizeBytes, messagesCount, createdAt, maxTopicSize, 1, options.ToArray()); + var topicPayload = BinaryFactory.CreateTopicPayload(topicId, partitionsCount, topicName, + sizeBytes, messagesCount, createdAt, options.ToArray()); // Act var response = Mappers.BinaryMapper.MapTopic(topicPayload); @@ -359,11 +358,11 @@ // Arrange: a Uint64 value carrying three bytes, which the Rust decoder rejects too. const byte stringKind = 2; const byte uint64Kind = 12; - var (topicId, partitionsCount, topicName, messageExpiry, sizeBytes, messagesCount, createdAt, - maxTopicSize) = TopicFactory.CreateTopicResponseFields(); + var (topicId, partitionsCount, topicName, _, sizeBytes, messagesCount, createdAt, _) = + TopicFactory.CreateTopicResponseFields(); var options = BinaryFactory.CreateOptionEntry(stringKind, "segment_size", uint64Kind, [1, 2, 3]); - var topicPayload = BinaryFactory.CreateTopicPayload(topicId, partitionsCount, messageExpiry, topicName, - sizeBytes, messagesCount, createdAt, maxTopicSize, 1, options); + var topicPayload = BinaryFactory.CreateTopicPayload(topicId, partitionsCount, topicName, + sizeBytes, messagesCount, createdAt, options); // Act + Assert Assert.Throws<MalformedResponseException>(() => Mappers.BinaryMapper.MapTopic(topicPayload)); @@ -372,7 +371,7 @@ [Fact] public void MapOptionSpecs_ReturnsTheCatalogWithKindsAndDefaults() { - // Arrange: [count][key_len][key][kind][default_len][default][description_len][description] + // Arrange: [count][key_len][key][kind][flags][default_len][default][description_len][description] const string key = "segment_size"; const string description = "Segment size in bytes"; var defaultValue = new byte[8]; @@ -383,6 +382,7 @@ payload.Add((byte)key.Length); payload.AddRange(Encoding.UTF8.GetBytes(key)); payload.Add(12); // Uint64 wire code + payload.Add(0b11); // updatable and resettable payload.AddRange(BitConverter.GetBytes((uint)defaultValue.Length)); payload.AddRange(defaultValue); payload.AddRange(BitConverter.GetBytes((uint)description.Length)); @@ -395,6 +395,11 @@ var spec = Assert.Single(specs); Assert.Equal(key, spec.Key); Assert.Equal(HeaderKind.Uint64, spec.Kind); + Assert.True(spec.Updatable); + Assert.True(spec.Resettable); + Assert.False(spec.Secret); + Assert.False(spec.ReadOnly); + Assert.False(spec.Critical); Assert.Equal(defaultValue, spec.DefaultValue); Assert.Equal(description, spec.Description); } @@ -408,6 +413,7 @@ payload.Add(4); payload.AddRange(Encoding.UTF8.GetBytes("size")); payload.Add(12); + payload.Add(0); payload.AddRange(BitConverter.GetBytes(64u)); // claims 64 bytes that are not there Assert.Throws<MalformedResponseException>(() =>
diff --git a/foreign/csharp/Iggy_SDK_Tests/MapperTests/OptionsBlockGoldenVectorTests.cs b/foreign/csharp/Iggy_SDK_Tests/MapperTests/OptionsBlockGoldenVectorTests.cs index 6e41cb2..5d9bec2 100644 --- a/foreign/csharp/Iggy_SDK_Tests/MapperTests/OptionsBlockGoldenVectorTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/MapperTests/OptionsBlockGoldenVectorTests.cs
@@ -84,7 +84,7 @@ public void MapTopic_WithOptionsLengthPrefixCutShort_ThrowsMalformedResponse() { var payload = TopicPayloadWithOptions(GoldenOptionsBlock); - var truncated = payload[..(50 + "topic".Length + 2)]; + var truncated = payload[..(33 + "topic".Length + 2)]; Assert.Throws<MalformedResponseException>(() => Mappers.BinaryMapper.MapTopic(truncated)); } @@ -96,20 +96,17 @@ private static byte[] TopicPayloadWithOptions(byte[] options) { var name = Encoding.UTF8.GetBytes("topic"); - var payload = new byte[50 + name.Length + 4 + options.Length + 4]; + var payload = new byte[33 + name.Length + 4 + options.Length + 4]; BinaryPrimitives.WriteUInt32LittleEndian(payload, 1); BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(4), 1750000000000000); BinaryPrimitives.WriteUInt32LittleEndian(payload.AsSpan(12), 1); BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(16), 0); - payload[24] = 1; - BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(25), 0); - BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(33), 0); - BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(41), 0); - payload[49] = (byte)name.Length; - name.CopyTo(payload.AsSpan(50)); + BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(24), 0); + payload[32] = (byte)name.Length; + name.CopyTo(payload.AsSpan(33)); - var position = 50 + name.Length; + var position = 33 + name.Length; BinaryPrimitives.WriteUInt32LittleEndian(payload.AsSpan(position), (uint)options.Length); options.CopyTo(payload.AsSpan(position + 4)); BinaryPrimitives.WriteUInt32LittleEndian(payload.AsSpan(position + 4 + options.Length), 0);
diff --git a/foreign/csharp/Iggy_SDK_Tests/MapperTests/ResourceOptionsConverterTests.cs b/foreign/csharp/Iggy_SDK_Tests/MapperTests/ResourceOptionsConverterTests.cs index a30af7b..68833af 100644 --- a/foreign/csharp/Iggy_SDK_Tests/MapperTests/ResourceOptionsConverterTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/MapperTests/ResourceOptionsConverterTests.cs
@@ -46,17 +46,14 @@ "created_at": 1750000000000000, "name": "topic", "size": "0 B", - "message_expiry": 0, - "compression_algorithm": "none", - "max_topic_size": 0, "messages_count": 0, "partitions_count": 1, "partitions": [], "options": { - "preallocate_segments": { "value": "true", "explicit": true }, - "segment_size": { "value": "134217728", "explicit": true }, - "messages_required_to_save": { "value": "1024", "explicit": false }, - "preallocate_segments": { "value": "false", "explicit": false } + "preallocate_segments": { "value": "true", "kind": "bool", "explicit": true }, + "segment_size": { "value": "134217728", "kind": "uint64", "explicit": true }, + "messages_required_to_save": { "value": "1024", "kind": "uint32", "explicit": false }, + "durability": { "value": "replicated", "kind": "string", "explicit": false } } } """; @@ -70,36 +67,34 @@ Assert.NotNull(topic.DerivedOptions); Assert.Equal(2, topic.Options.Count); - Assert.Equal("true", topic.Options[HeaderKey.FromString("preallocate_segments")].ToString()); - Assert.Equal("134217728", topic.Options[HeaderKey.FromString("segment_size")].ToString()); + Assert.True(topic.Options[HeaderKey.FromString("preallocate_segments")].ToBool()); + Assert.Equal(134217728UL, topic.Options[HeaderKey.FromString("segment_size")].ToUInt64()); Assert.Equal(2, topic.DerivedOptions.Count); - Assert.Equal("1024", topic.DerivedOptions[HeaderKey.FromString("messages_required_to_save")].ToString()); - Assert.Equal("false", topic.DerivedOptions[HeaderKey.FromString("preallocate_segments")].ToString()); + Assert.Equal(1024U, topic.DerivedOptions[HeaderKey.FromString("messages_required_to_save")].ToUInt32()); + Assert.Equal("replicated", topic.DerivedOptions[HeaderKey.FromString("durability")].ToString()); } [Fact] - public void DeserializeTopic_ReadsEveryValueAsAString() + public void given_uint64_http_option_when_reading_should_keep_kind() { - // A byte count arrives rendered, so its canonical Uint64 kind is not recoverable over REST. const string json = """ { "id": 1, "created_at": 1750000000000000, "name": "topic", "size": "0 B", - "max_topic_size": 0, "messages_count": 0, "partitions_count": 1, - "options": { "segment_size": { "value": "134217728", "explicit": true } } + "options": { "segment_size": { "value": "134217728", "kind": "uint64", "explicit": true } } } """; var topic = JsonSerializer.Deserialize<TopicResponse>(json, HttpOptions); var value = Assert.Single(topic!.Options!).Value; - Assert.Equal(HeaderKind.String, value.Kind); - Assert.Throws<InvalidOperationException>(() => value.ToUInt64()); + Assert.Equal(HeaderKind.Uint64, value.Kind); + Assert.Equal(134217728UL, value.ToUInt64()); } [Fact] @@ -135,7 +130,7 @@ [Fact] public void Read_RejectsAnEntryWithoutProvenance() { - const string json = """{ "segment_size": { "value": "134217728" } }"""; + const string json = """{ "segment_size": { "value": "134217728", "kind": "uint64" } }"""; Assert.Throws<JsonException>(() => JsonSerializer.Deserialize<ResourceOptions>(json, ConverterOptions)); } @@ -158,20 +153,18 @@ "created_at": 1750000000000000, "name": "first", "size": "0 B", - "max_topic_size": 0, "messages_count": 0, "partitions_count": 1, - "options": { "preallocate_segments": { "value": "true", "explicit": true } } + "options": { "preallocate_segments": { "value": "true", "kind": "bool", "explicit": true } } }, { "id": 2, "created_at": 1750000000000000, "name": "second", "size": "0 B", - "max_topic_size": 0, "messages_count": 0, "partitions_count": 1, - "options": { "preallocate_segments": { "value": "false", "explicit": false } } + "options": { "preallocate_segments": { "value": "false", "kind": "bool", "explicit": false } } } ] """;
diff --git a/foreign/csharp/Iggy_SDK_Tests/MapperTests/ResourceOptionsTests.cs b/foreign/csharp/Iggy_SDK_Tests/MapperTests/ResourceOptionsTests.cs new file mode 100644 index 0000000..fec5f12 --- /dev/null +++ b/foreign/csharp/Iggy_SDK_Tests/MapperTests/ResourceOptionsTests.cs
@@ -0,0 +1,1187 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Globalization; +using System.Reflection; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using Apache.Iggy.Configuration; +using Apache.Iggy.Contracts; +using Apache.Iggy.Contracts.Auth; +using Apache.Iggy.Contracts.Tcp; +using Apache.Iggy.Encryption; +using Apache.Iggy.Enums; +using Apache.Iggy.Exceptions; +using Apache.Iggy.Headers; +using Apache.Iggy.IggyClient.Implementations; +using Apache.Iggy.JsonConverters; +using Apache.Iggy.Tests.ClientTests; +using Apache.Iggy.Utils; +using Apache.Iggy.Vsr; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Apache.Iggy.Tests.MapperTests; + +/// <summary> +/// The cross-SDK resource options contract. Every expectation comes from the shared fixture the Rust +/// encoders generate, so a byte this SDK writes or reads differently from the server fails here. +/// </summary> +public sealed class ResourceOptionsTests +{ + private const string StreamJson = """ + {"id":1,"created_at":1750000000000000,"name":"orders","size":"0 B", + "messages_count":0,"topics_count":0} + """; + + private const string TopicJson = """ + {"id":1,"created_at":1750000000000000,"name":"events","size":"0 B", + "messages_count":0,"partitions_count":3,"partitions":[]} + """; + + private const string UserJson = """ + {"id":5,"created_at":1750000000000000,"status":"active","username":"alice"} + """; + + private static readonly JsonElement Fixture = LoadFixture(); + + private static readonly JsonSerializerOptions HttpJsonOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, + Converters = { new JsonStringEnumConverter(JsonNamingPolicy.SnakeCaseLower) } + }; + + private static readonly Dictionary<string, Func<StreamResponse, object?>> StreamReadOnlyAccessors = new() + { + ["encryption_key_source"] = stream => stream.EncryptionKeySource, + ["encryption_key_fingerprint"] = stream => stream.EncryptionKeyFingerprint, + ["encryption_kek_fingerprint"] = stream => stream.EncryptionKekFingerprints.SingleOrDefault() + }; + + private static readonly byte[] RawKey = Enumerable.Range(1, EncryptionKeys.KEY_LENGTH).Select(value => (byte)value).ToArray(); + + private static readonly string[] HttpUpdateCases = ["topic_reset_array", "stream_rename", "user_username_and_status"]; + + public static TheoryData<string, string> SharedVectors + { + get + { + var vectors = new TheoryData<string, string> { { "constants", "constants" } }; + foreach (var scope in new[] { "stream", "topic", "user" }) + { + vectors.Add("catalog_scope", scope); + vectors.Add("typed_fields", scope); + } + + AddCases(vectors, "catalog", "key"); + AddCases(vectors, "describe_options", "scope"); + vectors.Add("stream_key", "fingerprint"); + foreach (var section in new[] + { + "update_resource", "update_resource_errors", "create_stream", "create_topic", "create_user", + "stream_responses", "topic_responses", "user_responses" + }) + { + AddCases(vectors, section, "case"); + } + + foreach (var vectorCase in HttpUpdateCases) + { + vectors.Add("http_update", vectorCase); + } + + return vectors; + } + } + + public static TheoryData<string> RawOptionTypes => + [ + nameof(StreamOptions), nameof(StreamUpdate), nameof(TopicOptions), nameof(TopicUpdate), nameof(UserOptions), + nameof(UserUpdate) + ]; + + public static TheoryData<string> InvalidKeys + { + get + { + var cases = new TheoryData<string>(); + foreach (var entry in Fixture.GetProperty("stream_key").GetProperty("invalid_keys").EnumerateArray()) + { + cases.Add(entry.GetProperty("case").GetString()!); + } + + return cases; + } + } + + [Theory] + [MemberData(nameof(SharedVectors))] + public async Task given_shared_vectors_when_encoding_should_match_server_contract(string section, + string vectorCase) + { + switch (section) + { + case "constants": + AssertConstants(Fixture.GetProperty("constants")); + break; + case "catalog_scope": + AssertCatalogScope(vectorCase); + break; + case "typed_fields": + AssertTypedFields(vectorCase); + break; + case "catalog": + AssertCatalogRow(Find(section, "key", vectorCase)); + break; + case "describe_options": + AssertDescribeOptions(Find(section, "scope", vectorCase)); + break; + case "stream_key": + AssertStreamKey(Fixture.GetProperty("stream_key")); + break; + case "update_resource": + await AssertUpdateResource(Find(section, "case", vectorCase)); + break; + case "update_resource_errors": + await AssertUpdateResourceError(Find(section, "case", vectorCase)); + break; + case "create_stream": + await AssertCreateStream(Find(section, "case", vectorCase)); + break; + case "create_topic": + await AssertCreateTopic(Find(section, "case", vectorCase)); + break; + case "create_user": + await AssertCreateUser(Find(section, "case", vectorCase)); + break; + case "http_update": + await AssertHttpUpdate(Find(section, "case", vectorCase)); + break; + case "stream_responses": + AssertStreamResponse(Find(section, "case", vectorCase)); + break; + case "topic_responses": + AssertTopicResponse(Find(section, "case", vectorCase)); + break; + case "user_responses": + AssertUserResponse(Find(section, "case", vectorCase)); + break; + default: + Assert.Fail($"Unknown fixture section {section}."); + break; + } + } + + [Theory] + [MemberData(nameof(InvalidKeys))] + public async Task given_invalid_key_when_encoding_should_reject_before_network_io(string vectorCase) + { + var vector = Fixture.GetProperty("stream_key").GetProperty("invalid_keys").EnumerateArray() + .Single(entry => entry.GetProperty("case").GetString() == vectorCase); + var input = vector.GetProperty("input"); + var expectedCode = vector.GetProperty("error").GetProperty("code").GetInt32(); + + if (input.TryGetProperty("base64", out var base64)) + { + var text = base64.GetString()!; + var parse = Assert.Throws<ArgumentException>(() => EncryptionKeys.FromBase64(text)); + Assert.DoesNotContain(text.Trim(), parse.Message); + Assert.Throws<ArgumentException>(() => EncryptionKeys.Fingerprint(text)); + return; + } + + var key = Convert.FromHexString(input.GetProperty("raw_hex").GetString()!); + Assert.Throws<ArgumentException>(() => EncryptionKeys.Fingerprint(key)); + var options = new StreamOptions { EncryptionKey = key }; + AssertRedacted($"{options}", key); + + var handler = new HttpTopicOptionsTests.StubHandler(StreamJson); + using var http = new HttpMessageStream(new HttpClient(handler) { BaseAddress = new Uri("http://localhost") }); + var httpError = await Assert.ThrowsAsync<IggyInvalidStatusCodeException>(() => + http.CreateStreamAsync("orders", options, TestContext.Current.CancellationToken)); + Assert.Equal(expectedCode, httpError.StatusCode); + Assert.False(httpError.FromServer); + Assert.Equal(string.Empty, handler.RequestPath); + AssertRedacted(httpError.Message, key); + + // Never connected: an encoder that ran after the network call would surface a connection error instead. + using var tcp = new TcpMessageStream(new IggyClientConfigurator + { + BaseAddress = "127.0.0.1:1", + Protocol = Protocol.Tcp + }, NullLoggerFactory.Instance); + var tcpError = await Assert.ThrowsAsync<IggyInvalidStatusCodeException>(() => + tcp.CreateStreamAsync("orders", options, TestContext.Current.CancellationToken)); + Assert.Equal(expectedCode, tcpError.StatusCode); + AssertRedacted(tcpError.Message, key); + + var valid = new byte[EncryptionKeys.KEY_LENGTH]; + Assert.Equal(16, EncryptionKeys.Fingerprint(valid).Length); + Assert.Equal(valid, new StreamOptions { EncryptionKey = valid }.ToDictionary()["encryption_key"].Value); + } + + [Fact] + public void given_raw_set_key_and_typed_reset_when_patching_should_leave_the_verdict_to_the_server() + { + var patch = new StreamUpdate + { + Options = new Dictionary<string, HeaderValue> + { + ["encryption_key_source"] = HeaderValue.FromString("stream") + }, + Reset = new HashSet<StreamOptionKey> { StreamOptionKey.Name } + }.ToPatch(); + + Assert.Equal(["encryption_key_source"], patch.Set.Keys); + Assert.Equal(["name"], patch.Reset); + } + + [Fact] + public void given_raw_reset_when_patching_should_send_it_for_the_server_to_decide() + { + var patch = new TopicUpdate + { + Reset = new HashSet<TopicOptionKey> { TopicOptionKey.MessageExpiry }, + RawReset = new HashSet<string> { "retention", "message_expiry" } + }.ToPatch(); + + Assert.Empty(patch.Set); + Assert.Equal(["message_expiry", "retention"], patch.Reset); + } + + [Fact] + public void given_raw_reset_no_wire_name_holds_when_patching_should_refuse_unsupported_key() + { + var empty = Assert.Throws<IggyInvalidStatusCodeException>(() => + new TopicUpdate { RawReset = new HashSet<string> { string.Empty } }.ToPatch()); + Assert.Equal(VsrError.UNSUPPORTED_OPTION_KEY, empty.StatusCode); + + var tooMany = Assert.Throws<IggyInvalidStatusCodeException>(() => new TopicUpdate + { + RawReset = Enumerable.Range(0, OptionKeys.MAX_RESET_KEYS + 1) + .Select(index => $"key_{index:D3}").ToHashSet() + }.ToPatch()); + Assert.Equal(VsrError.UNSUPPORTED_OPTION_KEY, tooMany.StatusCode); + Assert.Contains("key_000", tooMany.Message); + } + + [Fact] + public void given_raw_set_key_also_reset_when_patching_should_refuse_invalid_command() + { + var both = Assert.Throws<IggyInvalidStatusCodeException>(() => new TopicUpdate + { + Options = new Dictionary<string, HeaderValue> { ["retention"] = HeaderValue.FromString("1 day") }, + RawReset = new HashSet<string> { "retention" } + }.ToPatch()); + Assert.Equal(VsrError.INVALID_COMMAND, both.StatusCode); + } + + [Fact] + public void given_unencodable_name_beside_raw_key_when_patching_should_send_a_stand_in() + { + var patch = new TopicUpdate + { + Name = string.Empty, + Options = new Dictionary<string, HeaderValue> { ["retention"] = HeaderValue.FromString("1 day") } + }.ToPatch(); + + Assert.Equal(HeaderKind.Bool, patch.Set["name"].Kind); + } + + [Fact] + public void given_raw_set_key_named_reset_when_building_http_body_should_refuse_unsupported_key() + { + var patch = new TopicUpdate + { + Options = new Dictionary<string, HeaderValue> { ["reset"] = HeaderValue.FromString("max_topic_size") } + }.ToPatch(); + + var refused = Assert.Throws<IggyInvalidStatusCodeException>(() => patch.ToHttpBody()); + Assert.Equal(VsrError.UNSUPPORTED_OPTION_KEY, refused.StatusCode); + } + + [Fact] + public void given_stream_options_with_key_when_serializing_to_json_should_not_write_the_key() + { + var key = Enumerable.Range(1, EncryptionKeys.KEY_LENGTH).Select(value => (byte)value).ToArray(); + var options = new StreamOptions { EncryptionKey = key }; + + AssertRedacted(JsonSerializer.Serialize(options), key); + AssertRedacted(JsonSerializer.Serialize(options, HttpJsonOptions), key); + } + + [Theory] + [MemberData(nameof(RawOptionTypes))] + public void given_raw_secret_entries_when_serializing_to_json_should_write_names_without_values( + string type) + { + var raw = new Dictionary<string, HeaderValue> + { + ["encryption_key"] = HeaderValue.FromBytes(RawKey), + ["future_secret"] = HeaderValue.FromString(Convert.ToBase64String(RawKey)) + }; + object options = type switch + { + nameof(StreamOptions) => new StreamOptions { Options = raw }, + nameof(StreamUpdate) => new StreamUpdate { Options = raw }, + nameof(TopicOptions) => new TopicOptions { Options = raw }, + nameof(TopicUpdate) => new TopicUpdate { Options = raw }, + nameof(UserOptions) => new UserOptions { Options = raw }, + nameof(UserUpdate) => new UserUpdate { Options = raw }, + _ => throw new ArgumentOutOfRangeException(nameof(type), type, "Unknown options type") + }; + + foreach (var json in new[] + { + JsonSerializer.Serialize(options, options.GetType()), + JsonSerializer.Serialize(options, options.GetType(), HttpJsonOptions) + }) + { + AssertRedacted(json, RawKey); + AssertRedacted(json, Encoding.UTF8.GetBytes(Convert.ToBase64String(RawKey))); + Assert.Contains("\"encryption_key\"", json); + Assert.Contains("\"future_secret\"", json); + } + } + + private static void AssertConstants(JsonElement constants) + { + Assert.Equal(VsrError.CANNOT_MODIFY_ROOT_USER, + constants.GetProperty("error_codes").GetProperty("cannot_modify_root_user").GetInt32()); + var version = constants.GetProperty("protocol_version"); + Assert.Equal(LoginRegister.PROTOCOL_VERSION, version.GetProperty("packed").GetUInt32()); + Assert.Equal(LoginRegister.PROTOCOL_VERSION_MAJOR, version.GetProperty("major").GetInt32()); + Assert.Equal(LoginRegister.PROTOCOL_VERSION_MINOR, version.GetProperty("minor").GetInt32()); + Assert.Equal(LoginRegister.PROTOCOL_VERSION_PATCH, version.GetProperty("patch").GetInt32()); + + Dictionary<string, int> codes = typeof(CommandCodes) + .GetFields(BindingFlags.NonPublic | BindingFlags.Static) + .Where(field => field.IsLiteral) + .ToDictionary(field => field.Name, field => (int)field.GetRawConstantValue()!); + foreach (var command in constants.GetProperty("command_codes").EnumerateObject()) + { + var code = command.Value.GetProperty("code").GetInt32(); + Assert.Equal(code, codes[command.Name.ToUpperInvariant() + "_CODE"]); + var operation = command.Value.GetProperty("operation"); + var expected = operation.ValueKind == JsonValueKind.Null + ? VsrOperation.NonReplicated + : (VsrOperation)operation.GetByte(); + Assert.Equal(expected, VsrOperations.ForCode(code)); + } + + foreach (var retired in constants.GetProperty("retired_command_codes").EnumerateObject()) + { + var code = retired.Value.GetInt32(); + Assert.DoesNotContain(code, codes.Values); + Assert.Equal(VsrOperation.NonReplicated, VsrOperations.ForCode(code)); + } + + foreach (var operation in constants.GetProperty("operations").EnumerateObject()) + { + var value = operation.Value.GetProperty("value").GetByte(); + Assert.True(VsrOperations.IsKnown(value), $"{operation.Name} must be a known operation"); + Assert.Equal(operation.Value.GetProperty("internal").GetBoolean(), ((VsrOperation)value).IsInternal()); + Assert.Equal((VsrOperation)operation.Value.GetProperty("request_operation").GetByte(), + ((VsrOperation)value).RequestOperation()); + } + + foreach (var retired in constants.GetProperty("retired_operations").EnumerateObject()) + { + Assert.False(VsrOperations.IsKnown(retired.Value.GetByte()), $"{retired.Name} must stay retired"); + } + + foreach (var scope in constants.GetProperty("scope_codes").EnumerateObject()) + { + Assert.Equal(scope.Value.GetByte(), (byte)Enum.Parse<OptionsScope>(scope.Name, true)); + } + + foreach (var kind in constants.GetProperty("header_kind_codes").EnumerateObject()) + { + Assert.Equal(UserHeadersConverter.ParseHeaderKind(kind.Name), Mappers.BinaryMapper.MapHeaderKind(kind.Value.GetByte())); + } + + var flags = constants.GetProperty("option_flags"); + Assert.Equal((byte)OptionFlags.Updatable, flags.GetProperty("updatable").GetByte()); + Assert.Equal((byte)OptionFlags.Resettable, flags.GetProperty("resettable").GetByte()); + Assert.Equal((byte)OptionFlags.Secret, flags.GetProperty("secret").GetByte()); + Assert.Equal((byte)OptionFlags.ReadOnly, flags.GetProperty("read_only").GetByte()); + Assert.Equal((byte)OptionFlags.Critical, flags.GetProperty("critical").GetByte()); + + foreach (var status in constants.GetProperty("user_status_codes").EnumerateObject()) + { + Assert.Equal(status.Value.GetInt32(), (int)Enum.Parse<UserStatus>(status.Name, true)); + } + + Assert.Equal(EncryptionKeys.KEY_LENGTH, constants.GetProperty("stream_key_length").GetInt32()); + } + + private static void AssertCatalogScope(string scopeName) + { + var scope = Enum.Parse<OptionsScope>(scopeName, true); + var expected = Fixture.GetProperty("catalog").EnumerateArray() + .Where(row => row.GetProperty("described").GetBoolean() && InScope(row, scopeName)) + .Select(row => row.GetProperty("key").GetString()!) + .ToList(); + + Assert.Equal(expected, OptionKeys.For(scope).Select(descriptor => descriptor.Name).ToList()); + } + + private static void AssertTypedFields(string scopeName) + { + List<JsonElement> rows = Fixture.GetProperty("catalog").EnumerateArray() + .Where(row => InScope(row, scopeName) && row.GetProperty("described").GetBoolean()) + .ToList(); + var createKeys = rows + .Where(row => row.GetProperty("storage").GetString() == "map" + && row.GetProperty("mutability").GetString() is "create_only" or "updatable") + .Select(row => row.GetProperty("key").GetString()!) + .ToHashSet(); + var updateKeys = rows + .Where(row => row.GetProperty("mutability").GetString() == "updatable") + .Select(row => row.GetProperty("key").GetString()!) + .ToHashSet(); + + var resettableKeys = rows + .Where(row => (row.GetProperty("flags").GetByte() & (byte)OptionFlags.Resettable) != 0) + .Select(row => row.GetProperty("key").GetString()!) + .ToHashSet(); + var readOnlyKeys = rows + .Where(row => row.GetProperty("mutability").GetString() == "read_only") + .Select(row => row.GetProperty("key").GetString()!) + .ToHashSet(); + var describedKeys = rows.Select(row => row.GetProperty("key").GetString()!).ToHashSet(); + + switch (scopeName) + { + case "stream": + AssertKeyMembers<StreamOptionKey>(describedKeys, resettableKeys, OptionKeys.Describe, + reset => new StreamUpdate { Reset = reset }.ToPatch()); + Assert.Equal(readOnlyKeys, StreamReadOnlyAccessors.Keys.ToHashSet()); + foreach (var (key, accessor) in StreamReadOnlyAccessors) + { + var response = new StreamResponse + { + Id = 1, + Name = "orders", + Size = 0, + CreatedAt = DateTimeOffset.UnixEpoch, + MessagesCount = 0, + TopicsCount = 0, + DerivedOptions = new Dictionary<HeaderKey, HeaderValue> + { + [HeaderKey.FromString(key)] = HeaderValue.FromString("stream") + } + }; + Assert.NotNull(accessor(response)); + } + + Assert.Equal(createKeys, + new StreamOptions { EncryptionKey = new byte[EncryptionKeys.KEY_LENGTH] }.ToDictionary().Keys + .ToHashSet()); + Assert.Equal(updateKeys, new StreamUpdate { Name = "renamed" }.ToPatch().Set.Keys.ToHashSet()); + break; + case "topic": + AssertKeyMembers<TopicOptionKey>(describedKeys, resettableKeys, OptionKeys.Describe, + reset => new TopicUpdate { Reset = reset }.ToPatch()); + Assert.Empty(readOnlyKeys); + Assert.Equal(createKeys, new TopicOptions + { + CompressionAlgorithm = CompressionAlgorithm.Gzip, + MessageExpiry = TimeSpan.FromDays(7), + MaxTopicSize = 1_073_741_824, + SegmentSize = 134_217_728, + Durability = Durability.Persisted, + ConsumerOffsetDurability = Durability.Persisted, + PartitionResizePolicy = PartitionResizePolicy.Fixed, + MessagesRequiredToSave = 64, + SizeOfMessagesRequiredToSave = 4096, + PreallocateSegments = true + }.ToDictionary().Keys.ToHashSet()); + Assert.Equal(updateKeys, new TopicUpdate + { + Name = "renamed", + MessageExpiry = TimeSpan.FromDays(1), + MaxTopicSize = 1_073_741_824 + }.ToPatch().Set.Keys.ToHashSet()); + break; + case "user": + AssertKeyMembers<UserOptionKey>(describedKeys, resettableKeys, OptionKeys.Describe, + reset => new UserUpdate { Reset = reset }.ToPatch()); + Assert.Empty(readOnlyKeys); + Assert.Equal(createKeys, new UserOptions().ToDictionary().Keys.ToHashSet()); + Assert.Equal(updateKeys, + new UserUpdate { Username = "renamed", Status = UserStatus.Inactive }.ToPatch().Set.Keys + .ToHashSet()); + break; + } + } + + /// <summary> + /// Pins every key member to its catalog name by the member's own name, so reordering either list cannot move + /// a member onto another key, and checks there is a member per described key and a reset per resettable one. + /// </summary> + private static void AssertKeyMembers<TKey>(HashSet<string> describedKeys, HashSet<string> resettableKeys, + Func<TKey, OptionDescriptor> describe, Func<IReadOnlySet<TKey>, OptionsPatch> reset) where TKey : struct, Enum + { + foreach (var member in Enum.GetValues<TKey>()) + { + Assert.Equal(JsonNamingPolicy.SnakeCaseLower.ConvertName(member.ToString()), describe(member).Name); + } + + Assert.Equal(describedKeys, Enum.GetValues<TKey>().Select(member => describe(member).Name).ToHashSet()); + var resettable = Enum.GetValues<TKey>().Where(member => resettableKeys.Contains(describe(member).Name)) + .ToHashSet(); + if (resettable.Count > 0) + { + Assert.Equal(resettableKeys, reset(resettable).Reset.ToHashSet()); + } + } + + private static void AssertCatalogRow(JsonElement row) + { + var key = row.GetProperty("key").GetString(); + var described = row.GetProperty("described").GetBoolean(); + foreach (var scope in Enum.GetValues<OptionsScope>()) + { + var descriptor = OptionKeys.For(scope).SingleOrDefault(entry => entry.Name == key); + if (!described || !InScope(row, scope.ToString().ToLowerInvariant())) + { + Assert.Null(descriptor); + continue; + } + + Assert.NotNull(descriptor); + Assert.Equal(Mappers.BinaryMapper.MapHeaderKind(row.GetProperty("kind_code").GetByte()), descriptor.Kind); + Assert.Equal(row.GetProperty("flags").GetByte(), (byte)descriptor.Flags); + } + } + + private static void AssertDescribeOptions(JsonElement vector) + { + var scope = Enum.Parse<OptionsScope>(vector.GetProperty("scope").GetString()!, true); + Assert.Equal(Hex(vector, "request_hex"), new[] { (byte)scope }); + + IReadOnlyList<OptionSpec> specs = Mappers.BinaryMapper.MapOptionSpecs(Hex(vector, "response_hex")); + List<JsonElement> entries = vector.GetProperty("entries").EnumerateArray().ToList(); + Assert.Equal(entries.Count, specs.Count); + for (var index = 0; index < entries.Count; index++) + { + var entry = entries[index]; + var spec = specs[index]; + Assert.Equal(entry.GetProperty("key").GetString(), spec.Key); + Assert.Equal(UserHeadersConverter.ParseHeaderKind(entry.GetProperty("kind").GetString()), spec.Kind); + Assert.Equal(entry.GetProperty("flags").GetByte(), Flags(spec)); + var defaultValue = entry.GetProperty("default"); + if (defaultValue.ValueKind == JsonValueKind.Null) + { + Assert.Empty(spec.DefaultValue); + } + else + { + Assert.Equal(defaultValue.GetString(), + ResourceOptionsConverter.EncodeValue(new HeaderValue { Kind = spec.Kind, Value = spec.DefaultValue })); + } + + Assert.False(string.IsNullOrWhiteSpace(spec.Description)); + } + } + + private static void AssertStreamKey(JsonElement streamKey) + { + var key = Convert.FromHexString(streamKey.GetProperty("key_hex").GetString()!); + var base64 = streamKey.GetProperty("key_base64").GetString()!; + var fingerprint = streamKey.GetProperty("fingerprint").GetString(); + + Assert.Equal(fingerprint, EncryptionKeys.Fingerprint(key)); + Assert.Equal(fingerprint, EncryptionKeys.Fingerprint(base64)); + Assert.Equal(key, EncryptionKeys.FromBase64(base64)); + + var value = new StreamOptions { EncryptionKey = key }.ToDictionary()["encryption_key"]; + Assert.Equal(HeaderKind.Raw, value.Kind); + Assert.Equal(key, value.Value); + } + + private static async Task AssertUpdateResource(JsonElement vector) + { + var scope = vector.GetProperty("scope").GetString()!; + var target = vector.GetProperty("target"); + var input = vector.GetProperty("input"); + + var (wire, path, send) = UpdateRequest(scope, target, input); + Assert.Equal(Hex(vector, "body_hex"), wire); + + var handler = new HttpTopicOptionsTests.StubHandler(string.Empty); + using var http = new HttpMessageStream(new HttpClient(handler) { BaseAddress = new Uri("http://localhost") }); + await send(http); + Assert.Equal(path, handler.RequestPath); + AssertUpdateBody(JsonDocument.Parse(handler.RequestBody).RootElement, vector.GetProperty("wire_set"), + vector.GetProperty("wire_reset")); + } + + private static async Task AssertUpdateResourceError(JsonElement vector) + { + var scope = vector.GetProperty("scope").GetString()!; + var expectedCode = vector.GetProperty("error").GetProperty("code").GetInt32(); + if (HasRawReset(scope, vector.GetProperty("input"))) + { + // A reset this build has no typed key for is the server's to judge, so the request goes out. + var (_, path, sendRaw) = UpdateRequest(scope, vector.GetProperty("target"), vector.GetProperty("input")); + var stub = new HttpTopicOptionsTests.StubHandler(string.Empty); + using var rawHttp = new HttpMessageStream(new HttpClient(stub) { BaseAddress = new Uri("http://localhost") }); + await sendRaw(rawHttp); + Assert.Equal(path, stub.RequestPath); + return; + } + + var encode = Assert.Throws<IggyInvalidStatusCodeException>(() => + UpdateRequest(scope, vector.GetProperty("target"), vector.GetProperty("input"))); + Assert.Equal(expectedCode, encode.StatusCode); + Assert.False(encode.FromServer); + + var handler = new HttpTopicOptionsTests.StubHandler(string.Empty); + using var http = new HttpMessageStream(new HttpClient(handler) { BaseAddress = new Uri("http://localhost") }); + var send = UpdateCall(scope, vector.GetProperty("target"), vector.GetProperty("input")); + var refused = await Assert.ThrowsAsync<IggyInvalidStatusCodeException>(() => send(http)); + Assert.Equal(expectedCode, refused.StatusCode); + Assert.Equal(string.Empty, handler.RequestPath); + + using var tcp = new TcpMessageStream(new IggyClientConfigurator + { + BaseAddress = "127.0.0.1:1", + Protocol = Protocol.Tcp + }, NullLoggerFactory.Instance); + var tcpRefused = await Assert.ThrowsAsync<IggyInvalidStatusCodeException>(() => send(tcp)); + Assert.Equal(expectedCode, tcpRefused.StatusCode); + } + + private static async Task AssertCreateStream(JsonElement vector) + { + var input = vector.GetProperty("input"); + var name = input.GetProperty("name").GetString()!; + var options = new StreamOptions + { + EncryptionKey = input.TryGetProperty("encryption_key", out var key) + ? Convert.FromHexString(key.GetProperty("raw_hex").GetString()!) + : null, + Options = RawOptions(input) + }; + + Assert.Equal(Hex(vector, "body_hex"), TcpContracts.CreateStream(name, options.ToDictionary())); + + var handler = new HttpTopicOptionsTests.StubHandler(StreamJson); + using var http = new HttpMessageStream(new HttpClient(handler) { BaseAddress = new Uri("http://localhost") }); + await http.CreateStreamAsync(name, options, TestContext.Current.CancellationToken); + Assert.Equal("/streams", handler.RequestPath); + var body = JsonDocument.Parse(handler.RequestBody).RootElement; + Assert.Equal(name, body.GetProperty("name").GetString()); + AssertCreateOptionsBody(body, vector.GetProperty("wire_options")); + AssertRedacted($"{options}", options.EncryptionKey ?? []); + } + + private static async Task AssertCreateTopic(JsonElement vector) + { + var input = vector.GetProperty("input"); + var streamId = TargetIdentifier(input.GetProperty("stream_id")); + var name = input.GetProperty("name").GetString()!; + var partitionsCount = input.GetProperty("partitions_count").GetUInt32(); + var options = new TopicOptions + { + CompressionAlgorithm = Enum.Parse<CompressionAlgorithm>(input.GetProperty("compression_algorithm").GetString()!, true), + MessageExpiry = DurationHelpers.FromDuration(ParseU64(input, "message_expiry")), + MaxTopicSize = ParseU64(input, "max_topic_size"), + SegmentSize = ParseU64(input, "segment_size"), + Durability = Enum.Parse<Durability>(input.GetProperty("durability").GetString()!, true), + ConsumerOffsetDurability = + Enum.Parse<Durability>(input.GetProperty("consumer_offset_durability").GetString()!, true) + }; + + Assert.Equal(Hex(vector, "body_hex"), + TcpContracts.CreateTopic(streamId, name, partitionsCount, options.ToDictionary())); + + var handler = new HttpTopicOptionsTests.StubHandler(TopicJson); + using var http = new HttpMessageStream(new HttpClient(handler) { BaseAddress = new Uri("http://localhost") }); + await http.CreateTopicAsync(streamId, name, partitionsCount, options, TestContext.Current.CancellationToken); + Assert.Equal($"/streams/{streamId}/topics", handler.RequestPath); + var body = JsonDocument.Parse(handler.RequestBody).RootElement; + Assert.Equal(name, body.GetProperty("name").GetString()); + Assert.Equal(partitionsCount, body.GetProperty("partitions_count").GetUInt32()); + AssertCreateOptionsBody(body, vector.GetProperty("wire_options")); + } + + private static async Task AssertCreateUser(JsonElement vector) + { + var input = vector.GetProperty("input"); + var username = input.GetProperty("username").GetString()!; + var password = input.GetProperty("password").GetString()!; + var status = Enum.Parse<UserStatus>(input.GetProperty("status").GetString()!, true); + var options = new UserOptions { Options = RawOptions(input) }; + + Assert.Equal(Hex(vector, "body_hex"), + TcpContracts.CreateUser(username, password, status, null, options.ToDictionary())); + + var handler = new HttpTopicOptionsTests.StubHandler(UserJson); + using var http = new HttpMessageStream(new HttpClient(handler) { BaseAddress = new Uri("http://localhost") }); + await http.CreateUserAsync(username, password, status, options: options, + token: TestContext.Current.CancellationToken); + Assert.Equal("/users", handler.RequestPath); + var body = JsonDocument.Parse(handler.RequestBody).RootElement; + Assert.Equal(username, body.GetProperty("username").GetString()); + Assert.Equal(input.GetProperty("status").GetString(), body.GetProperty("status").GetString()); + AssertCreateOptionsBody(body, vector.GetProperty("wire_options")); + } + + private static async Task AssertHttpUpdate(JsonElement vector) + { + var scope = vector.GetProperty("scope").GetString()!; + var expectedBody = JsonDocument.Parse(vector.GetProperty("body").GetString()!).RootElement; + var input = TypedInput(expectedBody); + + var (wire, path, send) = UpdateRequest(scope, vector.GetProperty("target"), input); + Assert.Equal(Hex(vector, "update_resource_hex"), wire); + + var handler = new HttpTopicOptionsTests.StubHandler(string.Empty); + using var http = new HttpMessageStream(new HttpClient(handler) { BaseAddress = new Uri("http://localhost") }); + await send(http); + Assert.Equal(path, handler.RequestPath); + var actual = JsonDocument.Parse(handler.RequestBody).RootElement; + Assert.Equal(Members(expectedBody), Members(actual)); + } + + private static void AssertStreamResponse(JsonElement vector) + { + var expected = vector.GetProperty("expected"); + var binary = Mappers.BinaryMapper.MapStream(Hex(vector, "response_hex")); + var rest = JsonSerializer.Deserialize<StreamResponse>(RestJson(expected, "topics_count"), HttpJsonOptions)!; + + foreach (var stream in new[] { binary, rest }) + { + Assert.Equal(expected.GetProperty("id").GetUInt32(), stream.Id); + Assert.Equal(expected.GetProperty("name").GetString(), stream.Name); + Assert.Equal(expected.GetProperty("topics_count").GetUInt32(), stream.TopicsCount); + Assert.Equal(expected.GetProperty("size_bytes").GetUInt64(), stream.Size); + Assert.Equal(expected.GetProperty("messages_count").GetUInt64(), stream.MessagesCount); + Assert.Equal(Projection(expected.GetProperty("options")), Projection(stream.Options, stream.DerivedOptions)); + Assert.Equal(Enum.Parse<StreamEncryptionSource>(expected.GetProperty("encryption_key_source").GetString()!, true), + stream.EncryptionKeySource); + var fingerprint = expected.GetProperty("encryption_key_fingerprint"); + Assert.Equal(fingerprint.ValueKind == JsonValueKind.Null ? null : fingerprint.GetString(), + stream.EncryptionKeyFingerprint); + Assert.Equal(expected.GetProperty("encryption_kek_fingerprints").EnumerateArray() + .Select(entry => entry.GetString()!).ToList(), stream.EncryptionKekFingerprints); + AssertNeverReturned(stream.Options, stream.DerivedOptions); + } + + Assert.Equal(MicrosSinceEpoch(binary.CreatedAt), expected.GetProperty("created_at").GetUInt64()); + } + + private static void AssertTopicResponse(JsonElement vector) + { + var expected = vector.GetProperty("expected"); + var binary = Mappers.BinaryMapper.MapTopic(Hex(vector, "response_hex")); + var rest = JsonSerializer.Deserialize<TopicResponse>(RestJson(expected, "partitions_count"), HttpJsonOptions)!; + + foreach (var topic in new[] { binary, rest }) + { + Assert.Equal(expected.GetProperty("id").GetUInt32(), topic.Id); + Assert.Equal(expected.GetProperty("name").GetString(), topic.Name); + Assert.Equal(expected.GetProperty("partitions_count").GetUInt32(), topic.PartitionsCount); + Assert.Equal(expected.GetProperty("size_bytes").GetUInt64(), topic.Size); + Assert.Equal(expected.GetProperty("messages_count").GetUInt64(), topic.MessagesCount); + Assert.Equal(Projection(expected.GetProperty("options")), Projection(topic.Options, topic.DerivedOptions)); + Assert.Equal(Enum.Parse<CompressionAlgorithm>(expected.GetProperty("compression_algorithm").GetString()!, true), + topic.CompressionAlgorithm); + Assert.Equal(ParseU64(expected, "message_expiry"), DurationHelpers.ToDuration(topic.MessageExpiry)); + Assert.Equal(ParseU64(expected, "max_topic_size"), topic.MaxTopicSize); + AssertNeverReturned(topic.Options, topic.DerivedOptions); + } + + Assert.Equal(MicrosSinceEpoch(binary.CreatedAt), expected.GetProperty("created_at").GetUInt64()); + } + + private static void AssertUserResponse(JsonElement vector) + { + var expected = vector.GetProperty("expected"); + var binary = Mappers.BinaryMapper.MapUser(Hex(vector, "response_hex")); + var rest = JsonSerializer.Deserialize<UserResponse>( + $$""" + {"id":{{expected.GetProperty("id").GetUInt32()}}, + "created_at":{{expected.GetProperty("created_at").GetUInt64()}}, + "status":"{{expected.GetProperty("status").GetString()}}", + "username":"{{expected.GetProperty("username").GetString()}}", + "options":{{expected.GetProperty("options").GetRawText()}}} + """, HttpJsonOptions)!; + + foreach (var user in new[] { binary, rest }) + { + Assert.Equal(expected.GetProperty("id").GetUInt32(), user.Id); + Assert.Equal(expected.GetProperty("created_at").GetUInt64(), user.CreatedAt); + Assert.Equal(Enum.Parse<UserStatus>(expected.GetProperty("status").GetString()!, true), user.Status); + Assert.Equal(expected.GetProperty("username").GetString(), user.Username); + Assert.Equal(Projection(expected.GetProperty("options")), Projection(user.Options, user.DerivedOptions)); + AssertNeverReturned(user.Options, user.DerivedOptions); + } + } + + /// <summary> + /// Builds the typed update a vector describes and returns its TCP payload, its REST path and the call + /// that sends it through a client. + /// </summary> + private static (byte[] Wire, string Path, Func<Apache.Iggy.IggyClient.IIggyClient, Task> Send) UpdateRequest( + string scope, JsonElement target, JsonElement input) + { + var send = UpdateCall(scope, target, input); + switch (scope) + { + case "topic": + { + var streamId = TargetIdentifier(target.GetProperty("stream_id")); + var topicId = TargetIdentifier(target.GetProperty("topic_id")); + return (TcpContracts.UpdateTopic(streamId, topicId, TopicUpdateFrom(input).ToPatch()), + $"/streams/{streamId}/topics/{topicId}", send); + } + case "stream": + { + var streamId = TargetIdentifier(target.GetProperty("stream_id")); + return (TcpContracts.UpdateStream(streamId, StreamUpdateFrom(input).ToPatch()), + $"/streams/{streamId}", send); + } + default: + { + var userId = TargetIdentifier(target.GetProperty("user_id")); + return (TcpContracts.UpdateUser(userId, UserUpdateFrom(input).ToPatch()), $"/users/{userId}", send); + } + } + } + + private static Func<Apache.Iggy.IggyClient.IIggyClient, Task> UpdateCall(string scope, JsonElement target, + JsonElement input) + { + var token = TestContext.Current.CancellationToken; + return scope switch + { + "topic" => client => client.UpdateTopicAsync(TargetIdentifier(target.GetProperty("stream_id")), + TargetIdentifier(target.GetProperty("topic_id")), TopicUpdateFrom(input), token), + "stream" => client => client.UpdateStreamAsync(TargetIdentifier(target.GetProperty("stream_id")), + StreamUpdateFrom(input), token), + _ => client => client.UpdateUserAsync(TargetIdentifier(target.GetProperty("user_id")), + UserUpdateFrom(input), token) + }; + } + + private static TopicUpdate TopicUpdateFrom(JsonElement input) + { + return new TopicUpdate + { + Name = input.TryGetProperty("name", out var name) ? name.GetString() : null, + MessageExpiry = input.TryGetProperty("message_expiry", out _) + ? DurationHelpers.FromDuration(ParseU64(input, "message_expiry")) + : null, + MaxTopicSize = input.TryGetProperty("max_topic_size", out _) ? ParseU64(input, "max_topic_size") : null, + Reset = ResetKeys<TopicOptionKey>(input), + RawReset = RawResetNames<TopicOptionKey>(input) + }; + } + + private static StreamUpdate StreamUpdateFrom(JsonElement input) + { + return new StreamUpdate + { + Name = input.TryGetProperty("name", out var name) ? name.GetString() : null, + Reset = ResetKeys<StreamOptionKey>(input), + RawReset = RawResetNames<StreamOptionKey>(input) + }; + } + + private static UserUpdate UserUpdateFrom(JsonElement input) + { + return new UserUpdate + { + Username = input.TryGetProperty("username", out var username) ? username.GetString() : null, + Status = input.TryGetProperty("status", out var status) + ? Enum.Parse<UserStatus>(status.GetString()!, true) + : null, + Reset = ResetKeys<UserOptionKey>(input), + RawReset = RawResetNames<UserOptionKey>(input) + }; + } + + private static bool HasRawReset(string scope, JsonElement input) + { + return scope switch + { + "topic" => RawResetNames<TopicOptionKey>(input) is not null, + "stream" => RawResetNames<StreamOptionKey>(input) is not null, + _ => RawResetNames<UserOptionKey>(input) is not null + }; + } + + private static HashSet<TKey>? ResetKeys<TKey>(JsonElement input) where TKey : struct, Enum + { + if (!input.TryGetProperty("reset", out var reset)) + { + return null; + } + + return reset.EnumerateArray() + .Select(name => TypedKey<TKey>(name.GetString()!)) + .OfType<TKey>() + .ToHashSet(); + } + + private static HashSet<string>? RawResetNames<TKey>(JsonElement input) where TKey : struct, Enum + { + if (!input.TryGetProperty("reset", out var reset)) + { + return null; + } + + var raw = reset.EnumerateArray().Select(name => name.GetString()!) + .Where(name => TypedKey<TKey>(name) is null) + .ToHashSet(); + return raw.Count == 0 ? null : raw; + } + + private static TKey? TypedKey<TKey>(string name) where TKey : struct, Enum + { + return Enum.GetValues<TKey>().Select(member => (TKey?)member) + .FirstOrDefault(member => JsonNamingPolicy.SnakeCaseLower.ConvertName(member.ToString()!) == name); + } + + /// <summary> + /// A REST update body read back as the typed input that produces it: decimal numbers are typed values, + /// every other member keeps its JSON text. + /// </summary> + private static JsonElement TypedInput(JsonElement body) + { + var members = body.EnumerateObject().ToDictionary(member => member.Name, member => + member.Value.ValueKind == JsonValueKind.Number + ? JsonSerializer.SerializeToElement(member.Value.GetUInt64().ToString(CultureInfo.InvariantCulture)) + : member.Value); + return JsonSerializer.SerializeToElement(members); + } + + private static void AssertUpdateBody(JsonElement body, JsonElement wireSet, JsonElement wireReset) + { + var expectedReset = wireReset.EnumerateArray().Select(name => name.GetString()!).ToHashSet(); + var setMembers = body.EnumerateObject().Where(member => member.Name != "reset").ToList(); + + Assert.Equal(wireSet.EnumerateObject().Select(member => member.Name).ToHashSet(), + setMembers.Select(member => member.Name).ToHashSet()); + foreach (var member in setMembers) + { + AssertInputValue(member.Value, wireSet.GetProperty(member.Name)); + } + + if (expectedReset.Count == 0) + { + Assert.False(body.TryGetProperty("reset", out _), "an empty reset list stays out of the body"); + } + else + { + Assert.Equal(expectedReset, + body.GetProperty("reset").EnumerateArray().Select(name => name.GetString()!).ToHashSet()); + } + } + + private static void AssertCreateOptionsBody(JsonElement body, JsonElement wireOptions) + { + var expected = wireOptions.EnumerateObject().ToList(); + if (expected.Count == 0) + { + Assert.False(body.TryGetProperty("options", out _), "empty options stay out of the body"); + return; + } + + var options = body.GetProperty("options"); + Assert.Equal(expected.Select(member => member.Name).ToHashSet(), + options.EnumerateObject().Select(member => member.Name).ToHashSet()); + foreach (var member in expected) + { + AssertInputValue(options.GetProperty(member.Name), member.Value); + } + } + + private static void AssertInputValue(JsonElement actual, JsonElement expectedEntry) + { + var text = expectedEntry.GetProperty("value").GetString()!; + switch (expectedEntry.GetProperty("kind").GetString()) + { + case "uint32" or "uint64": + Assert.Equal(JsonValueKind.Number, actual.ValueKind); + Assert.Equal(ulong.Parse(text, CultureInfo.InvariantCulture), actual.GetUInt64()); + break; + case "bool": + Assert.Equal(bool.Parse(text), actual.GetBoolean()); + break; + default: + Assert.Equal(JsonValueKind.String, actual.ValueKind); + Assert.Equal(text, actual.GetString()); + break; + } + } + + private static List<string> Members(JsonElement body) + { + return body.EnumerateObject() + .Select(member => member.Name == "reset" + ? $"reset={string.Join(',', member.Value.EnumerateArray().Select(name => name.GetString()).Order(StringComparer.Ordinal))}" + : $"{member.Name}={member.Value.GetRawText()}") + .Order(StringComparer.Ordinal) + .ToList(); + } + + private static List<string> Projection(JsonElement options) + { + return options.EnumerateObject() + .Select(member => + $"{member.Name}|{member.Value.GetProperty("value").GetString()}|{member.Value.GetProperty("kind").GetString()}|{member.Value.GetProperty("explicit").GetBoolean()}") + .Order(StringComparer.Ordinal) + .ToList(); + } + + private static List<string> Projection(Dictionary<HeaderKey, HeaderValue>? explicitOptions, + Dictionary<HeaderKey, HeaderValue>? derivedOptions) + { + IEnumerable<string> Render(Dictionary<HeaderKey, HeaderValue>? options, bool isExplicit) + { + return (options ?? []).Select(option => + $"{option.Key.AsString()}|{ResourceOptionsConverter.EncodeValue(option.Value)}|{KindName(option.Value.Kind)}|{isExplicit}"); + } + + return Render(explicitOptions, true).Concat(Render(derivedOptions, false)) + .Order(StringComparer.Ordinal) + .ToList(); + } + + private static void AssertNeverReturned(Dictionary<HeaderKey, HeaderValue>? explicitOptions, + Dictionary<HeaderKey, HeaderValue>? derivedOptions) + { + var returned = (explicitOptions ?? []).Keys.Concat((derivedOptions ?? []).Keys) + .Select(key => key.AsString()) + .ToHashSet(); + foreach (var key in Fixture.GetProperty("constants").GetProperty("never_returned_keys").EnumerateArray()) + { + Assert.DoesNotContain(key.GetString()!, returned); + } + } + + /// <summary>The REST JSON of a response projection: its fixed fields plus the shared options object.</summary> + private static string RestJson(JsonElement expected, string countField) + { + return $$""" + {"id":{{expected.GetProperty("id").GetUInt32()}}, + "created_at":{{expected.GetProperty("created_at").GetUInt64()}}, + "name":"{{expected.GetProperty("name").GetString()}}", + "size":"{{expected.GetProperty("size_bytes").GetUInt64()}} B", + "messages_count":{{expected.GetProperty("messages_count").GetUInt64()}}, + "{{countField}}":{{expected.GetProperty(countField).GetUInt32()}}, + "options":{{expected.GetProperty("options").GetRawText()}}} + """; + } + + private static IReadOnlyDictionary<string, HeaderValue>? RawOptions(JsonElement input) + { + if (!input.TryGetProperty("raw", out var raw)) + { + return null; + } + + return raw.EnumerateObject() + .ToDictionary(member => member.Name, member => HeaderValue.FromString(member.Value.GetString()!)); + } + + private static void AssertRedacted(string text, byte[] key) + { + if (key.Length == 0) + { + return; + } + + Assert.DoesNotContain(Convert.ToHexString(key), text, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain(Convert.ToBase64String(key), text); + } + + private static byte Flags(OptionSpec spec) + { + var flags = OptionFlags.None; + flags |= spec.Updatable ? OptionFlags.Updatable : OptionFlags.None; + flags |= spec.Resettable ? OptionFlags.Resettable : OptionFlags.None; + flags |= spec.Secret ? OptionFlags.Secret : OptionFlags.None; + flags |= spec.ReadOnly ? OptionFlags.ReadOnly : OptionFlags.None; + flags |= spec.Critical ? OptionFlags.Critical : OptionFlags.None; + return (byte)flags; + } + + private static string KindName(HeaderKind kind) + { + return kind switch + { + HeaderKind.Float => "float32", + HeaderKind.Double => "float64", + _ => kind.ToString().ToLowerInvariant() + }; + } + + private static ulong MicrosSinceEpoch(DateTimeOffset timestamp) + { + return (ulong)((timestamp.UtcTicks - DateTimeOffset.UnixEpoch.UtcTicks) / TimeSpan.TicksPerMicrosecond); + } + + private static Identifier TargetIdentifier(JsonElement value) + { + return value.ValueKind == JsonValueKind.Number + ? Identifier.Numeric(value.GetUInt32()) + : Identifier.String(value.GetString()!); + } + + private static ulong ParseU64(JsonElement element, string property) + { + return ulong.Parse(element.GetProperty(property).GetString()!, CultureInfo.InvariantCulture); + } + + private static bool InScope(JsonElement row, string scope) + { + return row.GetProperty("scopes").EnumerateArray().Any(entry => entry.GetString() == scope); + } + + private static byte[] Hex(JsonElement vector, string property) + { + return Convert.FromHexString(vector.GetProperty(property).GetString()!); + } + + private static JsonElement Find(string section, string property, string value) + { + return Fixture.GetProperty(section).EnumerateArray() + .Single(entry => entry.GetProperty(property).GetString() == value); + } + + private static void AddCases(TheoryData<string, string> vectors, string section, string property) + { + foreach (var entry in Fixture.GetProperty(section).EnumerateArray()) + { + vectors.Add(section, entry.GetProperty(property).GetString()!); + } + } + + private static JsonElement LoadFixture() + { + var path = Path.Combine(AppContext.BaseDirectory, "Fixtures", "resource_options.json"); + using var document = JsonDocument.Parse(File.ReadAllText(path)); + Assert.Equal(1, document.RootElement.GetProperty("schema_version").GetInt32()); + return document.RootElement.Clone(); + } +}
diff --git a/foreign/csharp/Iggy_SDK_Tests/PublisherTests/IggyTypedPublisherTests.cs b/foreign/csharp/Iggy_SDK_Tests/PublisherTests/IggyTypedPublisherTests.cs index d29c98d..6704ffb 100644 --- a/foreign/csharp/Iggy_SDK_Tests/PublisherTests/IggyTypedPublisherTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/PublisherTests/IggyTypedPublisherTests.cs
@@ -428,7 +428,6 @@ Name = "topic", Size = 0, CreatedAt = default, - MaxTopicSize = 0, MessagesCount = 0, PartitionsCount = 1 };
diff --git a/foreign/csharp/Iggy_SDK_Tests/Utils/BinaryFactory.cs b/foreign/csharp/Iggy_SDK_Tests/Utils/BinaryFactory.cs index e791131..794008e 100644 --- a/foreign/csharp/Iggy_SDK_Tests/Utils/BinaryFactory.cs +++ b/foreign/csharp/Iggy_SDK_Tests/Utils/BinaryFactory.cs
@@ -92,7 +92,7 @@ ulong messagesCount, ulong createdAt) { var nameBytes = Encoding.UTF8.GetBytes(name); - var totalSize = 4 + 4 + 8 + 8 + 1 + 8 + nameBytes.Length + 4; + var totalSize = 4 + 4 + 8 + 8 + 1 + 8 + nameBytes.Length + 4 + 4; var payload = new byte[totalSize]; BinaryPrimitives.WriteUInt32LittleEndian(payload, id); BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(4), createdAt); @@ -101,35 +101,35 @@ BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(24), messagesCount); payload[32] = (byte)nameBytes.Length; nameBytes.CopyTo(payload.AsSpan(33)); - // Empty length-prefixed options block. + // Empty length-prefixed explicit and derived options blocks. BinaryPrimitives.WriteUInt32LittleEndian(payload.AsSpan(33 + nameBytes.Length), 0); + BinaryPrimitives.WriteUInt32LittleEndian(payload.AsSpan(37 + nameBytes.Length), 0); return payload; } - internal static byte[] CreateTopicPayload(uint id, uint partitionsCount, uint messageExpiry, string name, - ulong sizeBytes, ulong messagesCount, ulong createdAt, ulong maxTopicSize, int compressionType, - byte[]? options = null) + internal static byte[] CreateTopicPayload(uint id, uint partitionsCount, string name, ulong sizeBytes, + ulong messagesCount, ulong createdAt, byte[]? options = null, byte[]? derivedOptions = null) { var nameBytes = Encoding.UTF8.GetBytes(name); var optionsBytes = options ?? []; - var totalSize = 4 + 8 + 4 + 8 + 1 + 8 + 8 + 8 + 1 + nameBytes.Length + 4 + optionsBytes.Length + 4; + var derivedBytes = derivedOptions ?? []; + var totalSize = 4 + 8 + 4 + 8 + 8 + 1 + nameBytes.Length + 4 + optionsBytes.Length + 4 + derivedBytes.Length; var payload = new byte[totalSize]; BinaryPrimitives.WriteUInt32LittleEndian(payload, id); BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(4), createdAt); BinaryPrimitives.WriteUInt32LittleEndian(payload.AsSpan(12), partitionsCount); - BinaryPrimitives.WriteInt64LittleEndian(payload.AsSpan(16), messageExpiry); - payload[24] = (byte)compressionType; - BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(25), maxTopicSize); - BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(33), sizeBytes); - BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(41), messagesCount); - payload[49] = (byte)nameBytes.Length; - nameBytes.CopyTo(payload.AsSpan(50)); - // Length-prefixed explicit options block, then an empty derived one. - var position = 50 + nameBytes.Length; + BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(16), sizeBytes); + BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(24), messagesCount); + payload[32] = (byte)nameBytes.Length; + nameBytes.CopyTo(payload.AsSpan(33)); + // Length-prefixed explicit options block, then the derived one. + var position = 33 + nameBytes.Length; BinaryPrimitives.WriteUInt32LittleEndian(payload.AsSpan(position), (uint)optionsBytes.Length); optionsBytes.CopyTo(payload.AsSpan(position + 4)); - BinaryPrimitives.WriteUInt32LittleEndian(payload.AsSpan(position + 4 + optionsBytes.Length), 0); + position += 4 + optionsBytes.Length; + BinaryPrimitives.WriteUInt32LittleEndian(payload.AsSpan(position), (uint)derivedBytes.Length); + derivedBytes.CopyTo(payload.AsSpan(position + 4)); return payload; }
diff --git a/foreign/csharp/Iggy_SDK_Tests/Utils/Users/UsersFactory.cs b/foreign/csharp/Iggy_SDK_Tests/Utils/Users/UsersFactory.cs index d307e38..ced278e 100644 --- a/foreign/csharp/Iggy_SDK_Tests/Utils/Users/UsersFactory.cs +++ b/foreign/csharp/Iggy_SDK_Tests/Utils/Users/UsersFactory.cs
@@ -30,7 +30,8 @@ return new CreateUserRequest(username ?? Utility.RandomString(Random.Shared.Next(5, 25)), password ?? Utility.RandomString(Random.Shared.Next(5, 25)), UserStatus.Active, - permissions ?? CreatePermissions()); + permissions ?? CreatePermissions(), + null); } internal static Dictionary<uint, StreamPermissions> CreateStreamPermissions(uint streamId = 1, uint topicId = 1)
diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/ConsensusSessionTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/ConsensusSessionTests.cs index 44b842f..96b5eae 100644 --- a/foreign/csharp/Iggy_SDK_Tests/VsrTests/ConsensusSessionTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/VsrTests/ConsensusSessionTests.cs
@@ -16,6 +16,7 @@ // under the License. using System.Reflection; +using System.Buffers.Binary; using Apache.Iggy.Exceptions; using Apache.Iggy.Vsr; @@ -122,6 +123,26 @@ } [Fact] + public void MetadataRequestIdentifier_IsUniqueAndStableAcrossReencoding() + { + var session = new ConsensusSession(1); + session.Resolve(VsrOperation.Register); + session.Bind(10, new byte[LoginRegister.BIND_SECRET_BYTES]); + var first = session.Resolve(VsrOperation.CreateStream); + var second = session.Resolve(VsrOperation.CreateStream); + Assert.NotEqual(UInt128.Zero, first.RequestChecksum); + Assert.NotEqual(first.RequestChecksum, second.RequestChecksum); + + var original = new byte[VsrHeader.HEADER_SIZE]; + var retry = new byte[VsrHeader.HEADER_SIZE]; + VsrHeader.EncodeRequestHeader(original, session, 202, Array.Empty<byte>(), retainedFrame: first); + VsrHeader.EncodeRequestHeader(retry, session, 202, Array.Empty<byte>(), retainedFrame: first); + Assert.Equal(first.RequestChecksum, + BinaryPrimitives.ReadUInt128LittleEndian(original.AsSpan(VsrHeader.REQUEST_CHECKSUM_OFFSET))); + Assert.Equal(original, retry); + } + + [Fact] public void NextRequestId_BeforeBindThrows() { var session = new ConsensusSession(1); @@ -147,7 +168,11 @@ counter.SetValue(session, ulong.MaxValue - 1); var lastFrame = session.Resolve((VsrOperation)operation); - Assert.Equal(new SessionFrame(clientId, ulong.MaxValue - 1, 10), lastFrame); + Assert.Equal(clientId, lastFrame.ClientId); + Assert.Equal(ulong.MaxValue - 1, lastFrame.RequestId); + Assert.Equal(10UL, lastFrame.SessionId); + Assert.Equal(operation == (byte)VsrOperation.CreateStream, + lastFrame.RequestChecksum != UInt128.Zero); Assert.Equal(ulong.MaxValue, session.RequestCounter); for (var attempt = 0; attempt < 2; attempt++)
diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/EndpointFailoverTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/EndpointFailoverTests.cs index 95a5e80..61a3c0b 100644 --- a/foreign/csharp/Iggy_SDK_Tests/VsrTests/EndpointFailoverTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/VsrTests/EndpointFailoverTests.cs
@@ -370,7 +370,8 @@ await client.LoginUserAsync("iggy", "iggy", TestContext.Current.CancellationToken); if (changeCredentials) { - await client.UpdateUserAsync(Identifier.Numeric(7), "renamed", token: TestContext.Current.CancellationToken); + await client.UpdateUserAsync(Identifier.Numeric(7), new UserUpdate { Username = "renamed" }, + TestContext.Current.CancellationToken); await client.ChangePasswordAsync(Identifier.String("renamed"), "iggy", "new-secret", TestContext.Current.CancellationToken); }
diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/NativeRecoveryTests.SignIn.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/NativeRecoveryTests.SignIn.cs index 7f04015..0563212 100644 --- a/foreign/csharp/Iggy_SDK_Tests/VsrTests/NativeRecoveryTests.SignIn.cs +++ b/foreign/csharp/Iggy_SDK_Tests/VsrTests/NativeRecoveryTests.SignIn.cs
@@ -625,7 +625,7 @@ ? Reply(request.Operation, [.. new byte[4], .. BinaryFactory.CreateStreamPayload(1, 0, "created", 0, 0, 0)]) : Reply(request.Operation, [], VsrError.TRANSIENT_NOT_ACCEPTED); }; - var create = client.CreateStreamAsync("created", cluster.Token); + var create = client.CreateStreamAsync("created", token: cluster.Token); await metadata.Task.WaitAsync(cluster.Token); Assert.Equal(0, Volatile.Read(ref coordinatorMetadata)); Assert.True(Volatile.Read(ref primaryMetadata) > 0);
diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/NativeRecoveryTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/NativeRecoveryTests.cs index c28338a..5acafd1 100644 --- a/foreign/csharp/Iggy_SDK_Tests/VsrTests/NativeRecoveryTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/VsrTests/NativeRecoveryTests.cs
@@ -1222,7 +1222,7 @@ } var change = changePassword ? client.ChangePasswordAsync(Identifier.String("user"), "secret", "rotated", cancellation.Token) - : client.UpdateUserAsync(Identifier.String("user"), "renamed", token: cancellation.Token); + : client.UpdateUserAsync(Identifier.String("user"), new UserUpdate { Username = "renamed" }, cancellation.Token); if (resolve) { var uncertain = await Assert.ThrowsAsync<VsrRequestOutcomeUnknownException>(() => change); @@ -1255,6 +1255,72 @@ registration.Body); } + /// <summary> + /// Every update commits under the resolved operation of its scope. Only a committed username change of the + /// signed-in user moves the credentials recovery signs in with. A rejected rename, a rename of another + /// user and a stream or topic rename all keep them. + /// </summary> + [Theory] + [InlineData("self", 0, "renamed")] + [InlineData("self", VsrError.UNAUTHORIZED, "user")] + [InlineData("other", 0, "user")] + [InlineData("stream", 0, "user")] + [InlineData("topic", 0, "user")] + public async Task given_logged_in_user_rename_when_reconnecting_should_keep_new_username(string target, + int status, string username) + { + using var cluster = new RecoveryCluster(); + using var client = await cluster.ConnectAsync(heartbeat: TimeSpan.FromMilliseconds(25)); + var resolved = target switch + { + "stream" => VsrOperation.UpdateStreamResolved, + "topic" => VsrOperation.UpdateTopicResolved, + _ => VsrOperation.UpdateUserResolved + }; + var updates = new ConcurrentQueue<MockRequest>(); + cluster.OnMetadata = request => + { + updates.Enqueue(request); + var reply = Result(request, status); + reply[REPLY_OPERATION_OFFSET] = (byte)resolved; + return reply; + }; + var update = target switch + { + "self" => client.UpdateUserAsync(Identifier.String("user"), new UserUpdate { Username = "renamed" }, + cluster.Token), + "other" => client.UpdateUserAsync(Identifier.String("someone"), new UserUpdate { Username = "renamed" }, + cluster.Token), + "stream" => client.UpdateStreamAsync(Stream, new StreamUpdate { Name = "renamed" }, cluster.Token), + _ => client.UpdateTopicAsync(Stream, Topic, new TopicUpdate { Name = "renamed" }, cluster.Token) + }; + if (status == 0) + { + await update; + } + else + { + Assert.True((await Assert.ThrowsAsync<IggyInvalidStatusCodeException>(() => update)).Committed); + } + + Assert.Equal((byte)VsrOperation.UpdateResource, Assert.Single(updates).Operation); + var registrations = new ConcurrentQueue<MockRequest>(); + cluster.BeforeReply = (request, _) => + { + if (request.Operation == (byte)VsrOperation.Register) + { + registrations.Enqueue(request); + } + }; + + cluster.PingEviction = EvictionReason.SessionError; + await cluster.UntilAsync(() => !registrations.IsEmpty); + + var registration = Assert.Single(registrations); + Assert.Equal(LoginRegister.Serialize(username, "secret", LoginRegister.ReadBindSecret(registration.Body)), + registration.Body); + } + [Fact] public async Task given_mutated_raw_credentials_when_the_session_recovers_should_use_the_submitted_password() { @@ -1628,7 +1694,7 @@ private static Task WriteAsync(TcpMessageStream client, bool send, CancellationToken token) => send ? client.SendMessagesAsync(Stream, Topic, Partitioning.PartitionId(0), new Message(Guid.NewGuid(), "payload"u8.ToArray()), token) - : client.CreateStreamAsync("created", token); + : client.CreateStreamAsync("created", token: token); private static Task PartitionWriteAsync(TcpMessageStream client, bool send, CancellationToken token) => send ? WriteAsync(client, send: true, token)
diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/PartitionContextTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/PartitionContextTests.cs index 1316e8b..4fd2a6a 100644 --- a/foreign/csharp/Iggy_SDK_Tests/VsrTests/PartitionContextTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/VsrTests/PartitionContextTests.cs
@@ -356,7 +356,7 @@ using var node = new MockNode(); node.Serve(request => { - if (request.Operation == (byte)VsrOperation.UpdateStream) + if (request.Operation == (byte)VsrOperation.UpdateResource) { var reply = Reply(request.Operation, new byte[4]); BinaryPrimitives.WriteUInt64LittleEndian(reply.AsSpan(VsrHeader.REPLY_COMMIT_OFFSET), UpdateCommit); @@ -371,7 +371,8 @@ await SignInAsync(client); await SendAsync(client); - await client.UpdateStreamAsync(Identifier.Numeric(1), "renamed", TestContext.Current.CancellationToken); + await client.UpdateStreamAsync(Identifier.Numeric(1), new StreamUpdate { Name = "renamed" }, + TestContext.Current.CancellationToken); await SendAsync(client); Assert.Equal(2, node.Requests(CommandCodes.GET_PARTITION_ROUTE_CODE)); @@ -393,7 +394,7 @@ await SignInAsync(client); await SendAsync(client); - await client.CreateStreamAsync("created", TestContext.Current.CancellationToken); + await client.CreateStreamAsync("created", token: TestContext.Current.CancellationToken); await client.CreateTopicAsync(Identifier.Numeric(1), "created", 1, token: TestContext.Current.CancellationToken); await SendAsync(client);
diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/PrimaryPollingTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/PrimaryPollingTests.cs index 3ae425b..00d7db4 100644 --- a/foreign/csharp/Iggy_SDK_Tests/VsrTests/PrimaryPollingTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/VsrTests/PrimaryPollingTests.cs
@@ -786,7 +786,8 @@ using var cluster = new PollCluster(); using var client = await cluster.ConnectAsync(); await PollAsync(client); - await client.UpdateUserAsync(Identifier.Numeric(7), "renamed", token: TestContext.Current.CancellationToken); + await client.UpdateUserAsync(Identifier.Numeric(7), new UserUpdate { Username = "renamed" }, + TestContext.Current.CancellationToken); await client.ChangePasswordAsync(Identifier.String("renamed"), "secret", "new-secret", TestContext.Current.CancellationToken); await client.PollMessagesAsync(Stream, Topic, 1, Consumer.Group(3), PollingStrategy.Next(), 10, true, @@ -1146,11 +1147,11 @@ { using var cluster = new PollCluster(); using var client = await cluster.ConnectAsync(); - await client.UpdateStreamAsync(Stream, "renamed", TestContext.Current.CancellationToken); + await client.UpdateStreamAsync(Stream, new StreamUpdate { Name = "renamed" }, TestContext.Current.CancellationToken); await client.StoreOffsetAsync(Consumer.New(3), Stream, Topic, 10, 1, TestContext.Current.CancellationToken); var update = Assert.Single(cluster.CoordinatorRequests, - request => request.Operation == (byte)VsrOperation.UpdateStream); + request => request.Operation == (byte)VsrOperation.UpdateResource); Assert.True(Assert.Single(cluster.OffsetWrites).RequestId > update.RequestId); }
diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrMultiplexingTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrMultiplexingTests.cs index 3edf06a..7946ff2 100644 --- a/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrMultiplexingTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrMultiplexingTests.cs
@@ -261,6 +261,73 @@ } } + /// <summary> + /// The server commits an update under the resolved operation of its scope, so the reply names 72, 73 or 74 + /// while the request carried 150. The rewrite must not relax the request identity the reply has to prove. + /// </summary> + [Theory] + [InlineData((byte)VsrOperation.UpdateStreamResolved, false)] + [InlineData((byte)VsrOperation.UpdateTopicResolved, false)] + [InlineData((byte)VsrOperation.UpdateUserResolved, false)] + [InlineData((byte)VsrOperation.UpdateUserResolved, true)] + public async Task given_resolved_update_reply_when_correlating_should_accept_only_matching_request_identity( + byte resolved, bool wrongChecksum) + { + const ulong RequestStamp = 11; + using var pair = await LinkPair.ConnectAsync(); + var session = new ConsensusSession(7, 42, new byte[LoginRegister.BIND_SECRET_BYTES]); + var lanes = new MutationLanes(session, new IggyClientConfigurator { BaseAddress = "127.0.0.1:1", Protocol = Protocol.Tcp }); + var operation = await lanes.EnterAsync(MutationLane.Metadata, CommandCodes.UPDATE_RESOURCE_CODE, + new byte[] { 2, 1, 4, 3, 0, 0, 0, 0 }, default, true, session.Generation, TestContext.Current.CancellationToken); + try + { + BinaryPrimitives.WriteUInt128LittleEndian(operation.Frame.Bytes.AsSpan(VsrHeader.REQUEST_CHECKSUM_OFFSET), RequestStamp); + var pending = pair.SendFrozenAsync(operation); + var request = await pair.ReadAsync(); + Assert.Equal((byte)VsrOperation.UpdateResource, request.Operation); + await pair.ReplyAsync(request, new byte[4], VsrHeader.COMMITTED_RESULT_MARKER, + checksum: wrongChecksum ? RequestStamp + 1 : RequestStamp, operation: resolved); + var result = await pending; + using var reply = result.Response; + Assert.Equal(!wrongChecksum, result.Committed); + if (wrongChecksum) + { + var error = Assert.IsType<IggyInvalidStatusCodeException>(result.Error); + Assert.False(error.Committed); + } + else + { + Assert.Null(result.Error); + } + } + finally + { + lanes.Complete(operation); + } + + Assert.Equal(0, pair.Dropped); + + // A resolved update answers only the generic update: another request, or the update answered by an + // operation that is not its rewrite, fails the link instead of completing the call. + foreach (var (code, replyOperation) in new[] + { + (CommandCodes.CREATE_STREAM_CODE, resolved), + (CommandCodes.UPDATE_RESOURCE_CODE, (byte)VsrOperation.RewrapStreamKeys), + (CommandCodes.UPDATE_RESOURCE_CODE, (byte)VsrOperation.CreateTopicWithAssignments) + }) + { + using var mismatched = await LinkPair.ConnectAsync(); + var call = mismatched.SendAsync(code, TestContext.Current.CancellationToken); + await mismatched.ReplyAsync(await mismatched.ReadAsync(), new byte[4], VsrHeader.COMMITTED_RESULT_MARKER, + operation: replyOperation); + var refused = await call; + Assert.NotNull(refused.Error); + Assert.Null(refused.Response); + await mismatched.DropNotified.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(1, mismatched.Dropped); + } + } + [Fact] public async Task given_a_sensitive_frozen_request_when_its_reply_is_disposed_should_clear_the_pooled_token() { @@ -347,9 +414,10 @@ }; } - internal async Task ReplyAsync(MockRequest request, byte[] body, byte marker = 0, UInt128? checksum = null) + internal async Task ReplyAsync(MockRequest request, byte[] body, byte marker = 0, UInt128? checksum = null, + byte? operation = null) { - var reply = Reply(request.Operation, body); + var reply = Reply(operation ?? request.Operation, body); reply[VsrHeader.REPLY_COMMITTED_OFFSET] = marker; CorrelateReply(request, reply); if (checksum is { } stamp)
diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrOperationTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrOperationTests.cs index 077ff36..b555fa6 100644 --- a/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrOperationTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrOperationTests.cs
@@ -27,7 +27,6 @@ [InlineData(CommandCodes.LOGOUT_USER_CODE, (byte)VsrOperation.Logout)] [InlineData(CommandCodes.CREATE_USER_CODE, (byte)VsrOperation.CreateUser)] [InlineData(CommandCodes.DELETE_USER_CODE, (byte)VsrOperation.DeleteUser)] - [InlineData(CommandCodes.UPDATE_USER_CODE, (byte)VsrOperation.UpdateUser)] [InlineData(CommandCodes.UPDATE_PERMISSIONS_CODE, (byte)VsrOperation.UpdatePermissions)] [InlineData(CommandCodes.CHANGE_PASSWORD_CODE, (byte)VsrOperation.ChangePassword)] [InlineData(CommandCodes.CREATE_PERSONAL_ACCESS_TOKEN_CODE, (byte)VsrOperation.CreatePersonalAccessToken)] @@ -37,10 +36,9 @@ [InlineData(CommandCodes.DELETE_CONSUMER_OFFSET_CODE, (byte)VsrOperation.DeleteConsumerOffset)] [InlineData(CommandCodes.CREATE_STREAM_CODE, (byte)VsrOperation.CreateStream)] [InlineData(CommandCodes.DELETE_STREAM_CODE, (byte)VsrOperation.DeleteStream)] - [InlineData(CommandCodes.UPDATE_STREAM_CODE, (byte)VsrOperation.UpdateStream)] [InlineData(CommandCodes.CREATE_TOPIC_CODE, (byte)VsrOperation.CreateTopic)] [InlineData(CommandCodes.DELETE_TOPIC_CODE, (byte)VsrOperation.DeleteTopic)] - [InlineData(CommandCodes.UPDATE_TOPIC_CODE, (byte)VsrOperation.UpdateTopic)] + [InlineData(CommandCodes.UPDATE_RESOURCE_CODE, (byte)VsrOperation.UpdateResource)] [InlineData(CommandCodes.CREATE_PARTITIONS_CODE, (byte)VsrOperation.CreatePartitions)] [InlineData(CommandCodes.DELETE_PARTITIONS_CODE, (byte)VsrOperation.DeletePartitions)] [InlineData(CommandCodes.DELETE_SEGMENTS_CODE, (byte)VsrOperation.DeleteSegments)] @@ -58,6 +56,9 @@ [InlineData(CommandCodes.POLL_MESSAGES_CODE)] [InlineData(CommandCodes.GET_STREAM_CODE)] [InlineData(CommandCodes.GET_CONSUMER_OFFSET_CODE)] + [InlineData(35)] + [InlineData(204)] + [InlineData(304)] [InlineData(9999)] public void ForCode_ReadsAndUnknownCodesRideNonReplicated(int code) { @@ -114,8 +115,11 @@ public void IsKnown_RejectsUndefinedDiscriminants() { Assert.True(VsrOperations.IsKnown((byte)VsrOperation.SendMessages)); + Assert.False(VsrOperations.IsKnown(129)); Assert.False(VsrOperations.IsKnown(131)); + Assert.False(VsrOperations.IsKnown(133)); Assert.False(VsrOperations.IsKnown(135)); + Assert.False(VsrOperations.IsKnown(142)); Assert.False(VsrOperations.IsKnown(163)); Assert.False(VsrOperations.IsKnown(164)); Assert.False(VsrOperations.IsKnown(165)); @@ -125,17 +129,14 @@ /// <summary>Every arm of the control-plane table: shard 0 owns these, so a miss would route to a data shard.</summary> [Theory] [InlineData((byte)VsrOperation.CreateStream)] - [InlineData((byte)VsrOperation.UpdateStream)] [InlineData((byte)VsrOperation.DeleteStream)] [InlineData((byte)VsrOperation.CreateTopic)] - [InlineData((byte)VsrOperation.UpdateTopic)] [InlineData((byte)VsrOperation.DeleteTopic)] [InlineData((byte)VsrOperation.CreatePartitions)] [InlineData((byte)VsrOperation.DeletePartitions)] [InlineData((byte)VsrOperation.CreateConsumerGroup)] [InlineData((byte)VsrOperation.DeleteConsumerGroup)] [InlineData((byte)VsrOperation.CreateUser)] - [InlineData((byte)VsrOperation.UpdateUser)] [InlineData((byte)VsrOperation.DeleteUser)] [InlineData((byte)VsrOperation.ChangePassword)] [InlineData((byte)VsrOperation.UpdatePermissions)] @@ -143,6 +144,11 @@ [InlineData((byte)VsrOperation.DeletePersonalAccessToken)] [InlineData((byte)VsrOperation.JoinConsumerGroup)] [InlineData((byte)VsrOperation.LeaveConsumerGroup)] + [InlineData((byte)VsrOperation.UpdateResource)] + [InlineData((byte)VsrOperation.UpdateStreamResolved)] + [InlineData((byte)VsrOperation.UpdateTopicResolved)] + [InlineData((byte)VsrOperation.UpdateUserResolved)] + [InlineData((byte)VsrOperation.RewrapStreamKeys)] public void IsMetadata_CoversTheWholeControlPlane(byte operation) { Assert.True(((VsrOperation)operation).IsMetadata());
diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrProtocolDriftTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrProtocolDriftTests.cs index 6f18062..79954ea 100644 --- a/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrProtocolDriftTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrProtocolDriftTests.cs
@@ -317,6 +317,51 @@ } /// <summary> + /// Pins <see cref="VsrOperations.RequestOperation" /> against <c>Operation::request_operation</c>. A reply names + /// the operation the server committed, which prepare may have rewritten, so a rewrite missing here rejects + /// every successful reply to the request it came from as a mismatch. + /// </summary> + [Fact] + public void RequestOperations_MatchTheRustRewrites() + { + var source = ReadRustSource(OperationPath); + var start = source.IndexOf("fn request_operation(", StringComparison.Ordinal); + Assert.True(start >= 0, "Operation::request_operation is gone from the Rust source."); + var end = source.IndexOf("operation => *operation", start, StringComparison.Ordinal); + Assert.True(end >= 0, "Operation::request_operation no longer ends with an identity arm."); + + var rewrites = new Dictionary<string, string>(); + foreach (Match arm in Regex.Matches(source[start..end], + @"((?:Self::\w+\s*\|\s*)*Self::\w+)\s*=>\s*\{?\s*Self::(\w+)")) + { + foreach (var rewritten in Regex.Matches(arm.Groups[1].Value, @"Self::(\w+)").Select(match => match.Groups[1].Value)) + { + rewrites[rewritten] = arm.Groups[2].Value; + } + } + + Assert.NotEmpty(rewrites); + var mismatches = rewrites + .Where(rewrite => !Enum.TryParse<VsrOperation>(rewrite.Key, out _) + || !Enum.TryParse<VsrOperation>(rewrite.Value, out _)) + .Select(rewrite => $"{rewrite.Key} => {rewrite.Value}: absent from .NET") + .ToList(); + foreach (VsrOperation operation in Enum.GetValues<VsrOperation>()) + { + var expected = rewrites.TryGetValue(operation.ToString(), out var target) + && Enum.TryParse<VsrOperation>(target, out var parsed) + ? parsed + : operation; + if (operation.RequestOperation() != expected) + { + mismatches.Add($"{operation}: rust {expected}, .NET {operation.RequestOperation()}"); + } + } + + Assert.Empty(mismatches); + } + + /// <summary> /// Pins <see cref="VsrOperations.IsResultFramed" /> against <c>Operation::is_result_framed</c>. The Rust /// side composes it from <c>is_metadata</c> plus its own partition-plane list, so pinning /// <c>is_metadata</c> alone leaves the second half free to drift. An operation that gains result framing
diff --git a/foreign/csharp/README.md b/foreign/csharp/README.md index 9de45a3..4ba424d 100644 --- a/foreign/csharp/README.md +++ b/foreign/csharp/README.md
@@ -28,7 +28,7 @@ from the same checkout. The SDK targets .NET 8 and .NET 10; repository examples require .NET 10. `0.9.0` includes the independent message and consumer-offset durability options. -This checkout uses binary protocol 0.11.1. Servers and applicable SDKs must deploy together; mixed versions and +This checkout uses binary protocol 0.12.0. Servers and applicable SDKs must deploy together. Mixed versions and rolling upgrades are unsupported. This checkout requires fresh storage and refuses older data directories without modification. `--fresh` deletes existing data in the configured directory. See the [server recovery guide](../../core/server/README.md#upgrade-recovery). @@ -59,6 +59,7 @@ ```c# using System.Buffers; +using System.Security.Cryptography; using System.Text; using System.Text.Json; using Apache.Iggy; @@ -66,6 +67,7 @@ using Apache.Iggy.Consumers; using Apache.Iggy.Contracts; using Apache.Iggy.Contracts.Auth; +using Apache.Iggy.Encryption; using Apache.Iggy.Enums; using Apache.Iggy.Factory; using Apache.Iggy.Headers; @@ -445,6 +447,16 @@ await client.CreateStreamAsync("my-stream"); ``` +A stream can carry its own data encryption key, exactly 32 random bytes. The key travels inside the create request, so use TLS when the network is not trusted. The server stores it wrapped under its own key and never returns it: reads report only where the key came from and its fingerprints. When the server has encryption enabled and no key is given, it generates one for the stream. The key cannot change after creation. + +```c# +var key = RandomNumberGenerator.GetBytes(EncryptionKeys.KEY_LENGTH); +var stream = await client.CreateStreamAsync("payments", new StreamOptions { EncryptionKey = key }); + +Console.WriteLine(stream?.EncryptionKeySource); // Stream +Console.WriteLine(stream?.EncryptionKeyFingerprint); // equals EncryptionKeys.Fingerprint(key) +``` + You can reference streams by either numeric ID or name: ```c# @@ -454,7 +466,7 @@ ### Creating Topics -Every stream contains topics for organizing messages: +Every stream contains topics for organizing messages. Settings a topic takes at creation go in `TopicOptions`, and every one left null keeps the server default: ```c# var streamId = Identifier.String("my-stream"); @@ -463,12 +475,33 @@ streamId, name: "my-topic", partitionsCount: 3, - compressionAlgorithm: CompressionAlgorithm.None, - messageExpiry: TimeSpan.MaxValue, // never expire - maxTopicSize: 1024 * 1024 * 1024 // 1 GiB + options: new TopicOptions + { + CompressionAlgorithm = CompressionAlgorithm.None, + MessageExpiry = TimeSpan.MaxValue, // never expire + MaxTopicSize = 1024 * 1024 * 1024 // 1 GiB + } ); ``` +### Updating Streams, Topics and Users + +An update names only what changes. Every property left null keeps the stored value, and `Reset` returns a key to its default: + +```c# +await client.UpdateStreamAsync(streamId, new StreamUpdate { Name = "my-stream-v2" }); + +await client.UpdateTopicAsync(streamId, Identifier.String("my-topic"), new TopicUpdate +{ + MessageExpiry = TimeSpan.FromDays(7), + Reset = new HashSet<TopicOptionKey> { TopicOptionKey.MaxTopicSize } +}); + +await client.UpdateUserAsync(Identifier.String("alice"), new UserUpdate { Status = UserStatus.Inactive }); +``` + +Settings fixed at creation, such as compression, cannot be updated. `DescribeOptionsAsync(OptionsScope.Topic)` lists every key the server knows with its kind and whether it can be updated or reset. For a key newer than this SDK, set it through `Options` and reset it through `RawReset`, and the server decides whether the change is allowed. + Note: Stream and topic names use hyphens instead of spaces. Iggy automatically replaces spaces with hyphens. ## Publishing Messages
diff --git a/foreign/go/README.md b/foreign/go/README.md index c0e0f1b..5ac54e5 100644 --- a/foreign/go/README.md +++ b/foreign/go/README.md
@@ -41,11 +41,60 @@ IGGY_TCP_ADDRESS=127.0.0.1:8090 \ IGGY_HTTP_ENABLED=false IGGY_QUIC_ENABLED=false IGGY_WEBSOCKET_ENABLED=false \ IGGY_ROOT_USERNAME=iggy IGGY_ROOT_PASSWORD=iggy \ +IGGY_ENCRYPTION_ENABLED=true IGGY_ENCRYPTION_KEY="$(head -c 32 /dev/urandom | base64)" \ target/debug/iggy-server ``` QUIC, WebSocket and HTTP are enabled by default on ports 8080, 8092 and 3000. Disable the ones you do not need so they cannot race with another process. +The encryption variables enable at-rest encryption with a server key. Leave +them out for a plaintext server. The stream option end-to-end tests need them. + +## Resource options + +Streams, topics and users carry options from one server catalog. `CreateStream`, `CreateTopic` and `CreateUser` take typed option structs, and `UpdateStream`, `UpdateTopic` and `UpdateUser` take sparse patches: a nil field keeps its stored value, `Reset` returns a key to its default. `DescribeOptions` lists the catalog the server serves. `Raw`, `Options` and `RawReset` carry keys newer than this build, and the server decides whether such a patch is valid. + +```go +key := make([]byte, iggcon.StreamEncryptionKeyLength) +if _, err := rand.Read(key); err != nil { + log.Fatal(err) +} +stream, err := cli.CreateStream(ctx, "orders", &iggcon.StreamOptions{EncryptionKey: key}) +if err != nil { + log.Fatal(err) +} +source, _ := stream.EncryptionKeySource() +fingerprint, _ := stream.EncryptionKeyFingerprint() +log.Printf("key source %s, fingerprint %s", source, fingerprint) + +streamId, _ := iggcon.NewIdentifier(stream.Id) +partitions := uint32(3) +expiry := iggcon.Duration(time.Hour.Microseconds()) +durability := iggcon.DurabilityPersisted +topic, err := cli.CreateTopic(ctx, streamId, "events", &iggcon.TopicOptions{ + PartitionsCount: &partitions, + MessageExpiry: &expiry, + Durability: &durability, +}) +if err != nil { + log.Fatal(err) +} + +topicId, _ := iggcon.NewIdentifier(topic.Id) +size := uint64(10 << 30) +if err := cli.UpdateTopic(ctx, streamId, topicId, iggcon.TopicUpdate{ + MaxTopicSize: &size, + Reset: []iggcon.TopicOptionKey{iggcon.TopicOptionMessageExpiry}, +}); err != nil { + log.Fatal(err) +} +``` + +The sample is `Example_resourceOptions` in `client/example_resource_options_test.go`. + +A stream encryption key is exactly 32 random bytes. Never derive it from a password: the fingerprint reads would then be an offline guessing oracle. The key travels inside the create request over the client's transport, so use TLS when the network between client and server is not trusted. + +The server stores the key wrapped under its own key and never returns it. Reads report only its source and fingerprints. Without a key, a server with encryption enabled generates one for the stream, and a server without it keeps the stream plaintext and refuses a client key. ## Delivery semantics @@ -193,7 +242,9 @@ ``` The end-to-end suite runs against a server at the address in -`IGGY_TCP_ADDRESS` and skips when that variable is unset: +`IGGY_TCP_ADDRESS` and skips when that variable is unset. Start the server +with the encryption variables shown above, the stream option cases fail +without them: ```bash IGGY_TCP_ADDRESS=127.0.0.1:8090 go test ./tests
diff --git a/foreign/go/benchmarks/send_messages_benchmark_test.go b/foreign/go/benchmarks/send_messages_benchmark_test.go index 20322d5..5ed93be 100644 --- a/foreign/go/benchmarks/send_messages_benchmark_test.go +++ b/foreign/go/benchmarks/send_messages_benchmark_test.go
@@ -110,7 +110,7 @@ func ensureInfrastructureIsInitialized(cli iggcon.Client, streamId uint32) error { streamIdentifier, _ := iggcon.NewIdentifier(streamId) if _, streamErr := cli.GetStream(context.Background(), streamIdentifier); streamErr != nil { - _, streamErr = cli.CreateStream(context.Background(), "benchmark"+fmt.Sprint(streamId)) + _, streamErr = cli.CreateStream(context.Background(), "benchmark"+fmt.Sprint(streamId), nil) if streamErr != nil { panic(streamErr) } @@ -118,14 +118,20 @@ topicIdentifier, _ := iggcon.NewIdentifier(uint32(0)) if _, topicErr := cli.GetTopic(context.Background(), streamIdentifier, topicIdentifier); topicErr != nil { + partitions := uint32(1) + compression := iggcon.CompressionAlgorithmNone + expiry := iggcon.IggyExpiryServerDefault + maxTopicSize := uint64(1) _, topicErr = cli.CreateTopic( context.Background(), streamIdentifier, "benchmark", - 1, - iggcon.CompressionAlgorithmNone, - iggcon.IggyExpiryServerDefault, - 1, + &iggcon.TopicOptions{ + PartitionsCount: &partitions, + CompressionAlgorithm: &compression, + MessageExpiry: &expiry, + MaxTopicSize: &maxTopicSize, + }, ) if topicErr != nil {
diff --git a/foreign/go/binary_serialization/binary_response_deserializer.go b/foreign/go/binary_serialization/binary_response_deserializer.go index a7288b4..bde3b5a 100644 --- a/foreign/go/binary_serialization/binary_response_deserializer.go +++ b/foreign/go/binary_serialization/binary_response_deserializer.go
@@ -21,6 +21,7 @@ "encoding/binary" "errors" "fmt" + "math" "sort" "time" @@ -123,20 +124,28 @@ name := string(payload[position+33 : position+33+nameLength]) - options, optionsSize, err := deserializeOptions(payload, position+streamFixedSize+nameLength) + readBytes := streamFixedSize + nameLength + options, optionsSize, err := deserializeOptions(payload, position+readBytes) if err != nil { return iggcon.Stream{}, 0, fmt.Errorf("failed to read stream options: %w", err) } + readBytes += optionsSize + derivedOptions, derivedSize, err := deserializeOptions(payload, position+readBytes) + if err != nil { + return iggcon.Stream{}, 0, fmt.Errorf("failed to read stream derived options: %w", err) + } + readBytes += derivedSize return iggcon.Stream{ - Id: id, - TopicsCount: topicsCount, - Name: name, - SizeBytes: sizeBytes, - MessagesCount: messagesCount, - CreatedAt: createdAt, - Options: options, - }, streamFixedSize + nameLength + optionsSize, nil + Id: id, + TopicsCount: topicsCount, + Name: name, + SizeBytes: sizeBytes, + MessagesCount: messagesCount, + CreatedAt: createdAt, + Options: options, + DerivedOptions: derivedOptions, + }, readBytes, nil } // deserializeOptions reads a u32-length-prefixed options TLV block at @@ -277,14 +286,23 @@ } // optionSpecMinimumSize is the smallest catalog entry: a one-character name -// (a length byte plus at least one byte), a kind byte, and empty +// (a length byte plus at least one byte), kind and flags bytes, and empty // length-prefixed default and description. -const optionSpecMinimumSize = 2 + 1 + 4 + 4 +const optionSpecMinimumSize = 2 + 1 + 1 + 4 + 4 + +// Option flag bits of a DescribeOptions entry. Higher bits are ignored. +const ( + optionFlagUpdatable = 1 << 0 + optionFlagResettable = 1 << 1 + optionFlagSecret = 1 << 2 + optionFlagReadOnly = 1 << 3 + optionFlagCritical = 1 << 4 +) // DeserializeOptionSpecs reads a DescribeOptions response. // -// Wire format: [count:u32][ [key_len:u8][key][kind:u8][default_len:u32][default] -// [description_len:u32][description] ]* +// Wire format: [count:u32][ [key_len:u8][key][kind:u8][flags:u8] +// [default_len:u32][default][description_len:u32][description] ]* func DeserializeOptionSpecs(payload []byte) ([]iggcon.OptionSpec, error) { if len(payload) < 4 { return nil, fmt.Errorf( @@ -301,14 +319,15 @@ } keyLength := int(payload[position]) position++ - if len(payload)-position < keyLength+1 { + if len(payload)-position < keyLength+2 { return nil, fmt.Errorf("truncated option key at offset %d", position) } key := string(payload[position : position+keyLength]) position += keyLength kind := payload[position] - position++ + flags := payload[position+1] + position += 2 defaultValue, read, err := readLengthPrefixed(payload, position, "option default value") if err != nil { @@ -325,6 +344,11 @@ specs = append(specs, iggcon.OptionSpec{ Key: key, DefaultValue: iggcon.HeaderValue{Kind: iggcon.HeaderKind(kind), Value: defaultValue}, + Updatable: flags&optionFlagUpdatable != 0, + Resettable: flags&optionFlagResettable != 0, + Secret: flags&optionFlagSecret != 0, + ReadOnly: flags&optionFlagReadOnly != 0, + Critical: flags&optionFlagCritical != 0, Description: string(description), }) } @@ -394,9 +418,9 @@ } // topicFixedSize covers the fields before the name: -// id + created_at + partitions_count + message_expiry + compression + -// max_topic_size + size_bytes + messages_count + name_len. -const topicFixedSize = 4 + 8 + 4 + 8 + 1 + 8 + 8 + 8 + 1 // 50 bytes +// id + created_at + partitions_count + size_bytes + messages_count + name_len. +// Message expiry, compression and the size cap live in the options blocks. +const topicFixedSize = 4 + 8 + 4 + 8 + 8 + 1 // 33 bytes // topicMinimumSize is the smallest possible topic element: fixed fields, a // one-character name (the server rejects an empty one), and the two u32 @@ -431,21 +455,16 @@ topic.Id = binary.LittleEndian.Uint32(payload[position : position+4]) topic.CreatedAt = binary.LittleEndian.Uint64(payload[position+4 : position+12]) topic.PartitionsCount = binary.LittleEndian.Uint32(payload[position+12 : position+16]) - topic.MessageExpiry = iggcon.Duration(binary.LittleEndian.Uint64(payload[position+16 : position+24])) - topic.CompressionAlgorithm = payload[position+24] - topic.MaxTopicSize = binary.LittleEndian.Uint64(payload[position+25 : position+33]) - topic.Size = binary.LittleEndian.Uint64(payload[position+33 : position+41]) - topic.MessagesCount = binary.LittleEndian.Uint64(payload[position+41 : position+49]) - // Replication factor left the wire protocol together with the old fixed - // layout; every topic reports the single-copy default. + topic.Size = binary.LittleEndian.Uint64(payload[position+16 : position+24]) + topic.MessagesCount = binary.LittleEndian.Uint64(payload[position+24 : position+32]) - nameLength := int(payload[position+49]) + nameLength := int(payload[position+32]) if remaining < topicFixedSize+nameLength { return iggcon.Topic{}, 0, fmt.Errorf( "not enough data to read topic name: need %d bytes, got %d", topicFixedSize+nameLength, remaining) } - topic.Name = string(payload[position+50 : position+50+nameLength]) + topic.Name = string(payload[position+33 : position+33+nameLength]) readBytes := topicFixedSize + nameLength options, optionsSize, err := deserializeOptions(payload, position+readBytes) @@ -462,9 +481,45 @@ topic.DerivedOptions = derivedOptions readBytes += derivedSize + topic.MessageExpiry = iggcon.Duration(topicUint64Option(topic, iggcon.TopicOptionMessageExpiry, uint64(iggcon.IggyExpiryNeverExpire))) + topic.MaxTopicSize = topicUint64Option(topic, iggcon.TopicOptionMaxTopicSize, math.MaxUint64) + topic.CompressionAlgorithm = uint8(iggcon.CompressionAlgorithmNone) + if value, ok := topicOption(topic, iggcon.TopicOptionCompressionAlgorithm); ok && value.Kind == iggcon.String { + if algorithm, known := compressionAlgorithms[string(value.Value)]; known { + topic.CompressionAlgorithm = uint8(algorithm) + } + } + return topic, readBytes, nil } +// compressionAlgorithms maps the catalog spelling of compression_algorithm to +// its code. +var compressionAlgorithms = map[string]iggcon.CompressionAlgorithm{ + "none": iggcon.CompressionAlgorithmNone, + "gzip": iggcon.CompressionAlgorithmGzip, +} + +// topicOption looks a key up in both blocks: a key is explicit or derived, +// never both. +func topicOption(topic iggcon.Topic, key iggcon.TopicOptionKey) (iggcon.HeaderValue, bool) { + if value, ok := topic.Options[string(key)]; ok { + return value, true + } + value, ok := topic.DerivedOptions[string(key)] + return value, ok +} + +// topicUint64Option reads a Uint64 option, or the catalog default when the +// server reported none. +func topicUint64Option(topic iggcon.Topic, key iggcon.TopicOptionKey, fallback uint64) uint64 { + value, ok := topicOption(topic, key) + if !ok || value.Kind != iggcon.Uint64 || len(value.Value) != 8 { + return fallback + } + return binary.LittleEndian.Uint64(value.Value) +} + func DeserializePartition(payload []byte, position int) (iggcon.PartitionContract, int, error) { const partitionSize = 40 + iggcon.PartitionContextSize if position < 0 || position > len(payload) || len(payload)-position < partitionSize { @@ -590,11 +645,12 @@ } hasPermissions := payload[position] userInfo := iggcon.UserInfo{ - Id: response.Id, - CreatedAt: response.CreatedAt, - Username: response.Username, - Status: response.Status, - Options: response.Options, + Id: response.Id, + CreatedAt: response.CreatedAt, + Username: response.Username, + Status: response.Status, + Options: response.Options, + DerivedOptions: response.DerivedOptions, } if hasPermissions == 1 { permissionLength := binary.LittleEndian.Uint32(payload[position+1 : position+5]) @@ -726,13 +782,19 @@ return nil, 0, fmt.Errorf("failed to read user options: %w", err) } readBytes += optionsSize + derivedOptions, derivedSize, err := deserializeOptions(payload, position+readBytes) + if err != nil { + return nil, 0, fmt.Errorf("failed to read user derived options: %w", err) + } + readBytes += derivedSize return &iggcon.UserInfo{ - Id: id, - CreatedAt: createdAt, - Status: userStatus, - Username: username, - Options: options, + Id: id, + CreatedAt: createdAt, + Status: userStatus, + Username: username, + Options: options, + DerivedOptions: derivedOptions, }, readBytes, nil } @@ -818,7 +880,7 @@ func DeserializeAccessTokens(payload []byte) ([]iggcon.PersonalAccessTokenInfo, error) { if len(payload) == 0 { - return []iggcon.PersonalAccessTokenInfo{}, ierror.ErrEmptyMessagePayload + return []iggcon.PersonalAccessTokenInfo{}, nil } var result []iggcon.PersonalAccessTokenInfo @@ -841,9 +903,10 @@ var expiry *time.Time if len(expiryBytes) >= 8 { - unixMicroSeconds := binary.LittleEndian.Uint64(expiryBytes) - expiryTime := time.Unix(0, int64(unixMicroSeconds)) - expiry = &expiryTime + if unixMicroSeconds := binary.LittleEndian.Uint64(expiryBytes); unixMicroSeconds != 0 { + expiryTime := time.UnixMicro(int64(unixMicroSeconds)) + expiry = &expiryTime + } } readBytes := 1 + nameLength + 8
diff --git a/foreign/go/binary_serialization/binary_response_deserializer_test.go b/foreign/go/binary_serialization/binary_response_deserializer_test.go index afd3f52..ee3e13b 100644 --- a/foreign/go/binary_serialization/binary_response_deserializer_test.go +++ b/foreign/go/binary_serialization/binary_response_deserializer_test.go
@@ -22,6 +22,7 @@ "fmt" "strings" "testing" + "time" iggcon "github.com/apache/iggy/foreign/go/contracts" ) @@ -83,9 +84,14 @@ } func encodeStream(id uint32, createdAt uint64, topicsCount uint32, sizeBytes, messagesCount uint64, name string) []byte { - // Trailing 4 zero bytes: the u32 length prefix of an empty options block. + return encodeStreamWithOptions(id, createdAt, topicsCount, sizeBytes, messagesCount, name, nil, nil) +} + +// encodeStreamWithOptions appends the explicit and derived options blocks, +// each behind its u32 length prefix. +func encodeStreamWithOptions(id uint32, createdAt uint64, topicsCount uint32, sizeBytes, messagesCount uint64, name string, explicit, derived []byte) []byte { nameBytes := []byte(name) - buf := make([]byte, streamFixedSize+len(nameBytes)+4) + buf := make([]byte, streamFixedSize+len(nameBytes)) binary.LittleEndian.PutUint32(buf[0:4], id) binary.LittleEndian.PutUint64(buf[4:12], createdAt) binary.LittleEndian.PutUint32(buf[12:16], topicsCount) @@ -93,6 +99,10 @@ binary.LittleEndian.PutUint64(buf[24:32], messagesCount) buf[32] = byte(len(nameBytes)) copy(buf[33:], nameBytes) + for _, block := range [][]byte{explicit, derived} { + buf = binary.LittleEndian.AppendUint32(buf, uint32(len(block))) + buf = append(buf, block...) + } return buf } @@ -150,7 +160,7 @@ func TestDeserializeToStream_TruncatedOptions(t *testing.T) { full := encodeStream(1, 100, 0, 0, 0, "s") - for cut := 1; cut <= 4; cut++ { + for cut := 1; cut <= 8; cut++ { _, _, err := DeserializeToStream(full[:len(full)-cut], 0) if err == nil { t.Fatalf("expected error for options prefix cut by %d bytes, got nil", cut) @@ -163,9 +173,11 @@ Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte("future_key")}, Value: iggcon.HeaderValue{Kind: iggcon.String, Value: []byte("value")}, }}) - payload := encodeStream(7, 500, 0, 0, 0, "with-options") - binary.LittleEndian.PutUint32(payload[len(payload)-4:], uint32(len(options))) - payload = append(payload, options...) + derived := iggcon.GetHeadersBytes([]iggcon.HeaderEntry{{ + Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte("encryption_key_source")}, + Value: iggcon.HeaderValue{Kind: iggcon.String, Value: []byte("server")}, + }}) + payload := encodeStreamWithOptions(7, 500, 0, 0, 0, "with-options", options, derived) stream, readBytes, err := DeserializeToStream(payload, 0) if err != nil { @@ -181,6 +193,12 @@ if value.Kind != iggcon.String || string(value.Value) != "value" { t.Errorf("Options[future_key] = %+v, want String %q", value, "value") } + if _, explicit := stream.Options["encryption_key_source"]; explicit { + t.Errorf("Options = %v, a derived key must stay out of the explicit map", stream.Options) + } + if source, ok := stream.EncryptionKeySource(); !ok || source != iggcon.StreamEncryptionSourceServer { + t.Errorf("EncryptionKeySource = %q, %t, want %q from the derived block", source, ok, iggcon.StreamEncryptionSourceServer) + } } func TestReadLengthPrefixed_RejectsALengthPastTheBuffer(t *testing.T) { @@ -201,11 +219,17 @@ } func TestDeserializeToStream_RejectsAnOptionsLengthPastTheBuffer(t *testing.T) { - payload := encodeStream(1, 100, 0, 0, 0, "s") - binary.LittleEndian.PutUint32(payload[len(payload)-4:], 0xFFFFFFFF) + for _, prefix := range []string{"explicit", "derived"} { + payload := encodeStream(1, 100, 0, 0, 0, "s") + offset := len(payload) - 4 + if prefix == "explicit" { + offset -= 4 + } + binary.LittleEndian.PutUint32(payload[offset:], 0xFFFFFFFF) - if _, _, err := DeserializeToStream(payload, 0); err == nil { - t.Fatal("expected an error for an options length of 0xFFFFFFFF, got nil") + if _, _, err := DeserializeToStream(payload, 0); err == nil { + t.Fatalf("expected an error for a %s options length of 0xFFFFFFFF, got nil", prefix) + } } } @@ -334,3 +358,37 @@ t.Fatal("expected error for topics count with no elements, got nil") } } + +func TestDeserializeAccessTokens_ReadsExpiryAsMicroseconds(t *testing.T) { + const expiryMicros = 1_792_000_000_123_456 + appendToken := func(payload []byte, name string, expiry uint64) []byte { + payload = append(payload, byte(len(name))) + payload = append(payload, name...) + return binary.LittleEndian.AppendUint64(payload, expiry) + } + payload := appendToken(appendToken(nil, "never", 0), "expiring", expiryMicros) + + tokens, err := DeserializeAccessTokens(payload) + if err != nil { + t.Fatal(err) + } + if len(tokens) != 2 { + t.Fatalf("got %d tokens, want 2: %+v", len(tokens), tokens) + } + if tokens[0].Name != "never" || tokens[0].Expiry != nil { + t.Errorf("token without expiry decoded as %s expiring at %v, want never with no expiry", tokens[0].Name, tokens[0].Expiry) + } + if tokens[1].Name != "expiring" || tokens[1].Expiry == nil || !tokens[1].Expiry.Equal(time.UnixMicro(expiryMicros)) { + t.Errorf("token with expiry decoded as %s expiring at %v, want expiring at %v", tokens[1].Name, tokens[1].Expiry, time.UnixMicro(expiryMicros)) + } +} + +func TestDeserializeAccessTokens_EmptyPayloadIsAnEmptyList(t *testing.T) { + tokens, err := DeserializeAccessTokens(nil) + if err != nil { + t.Fatalf("a user without tokens got error %v, want an empty list", err) + } + if len(tokens) != 0 { + t.Fatalf("got %+v, want no tokens", tokens) + } +}
diff --git a/foreign/go/binary_serialization/vsr_response_deserializer_test.go b/foreign/go/binary_serialization/vsr_response_deserializer_test.go index 8d99254..e8c88dc 100644 --- a/foreign/go/binary_serialization/vsr_response_deserializer_test.go +++ b/foreign/go/binary_serialization/vsr_response_deserializer_test.go
@@ -20,6 +20,7 @@ import ( "encoding/binary" "encoding/hex" + "math" "testing" iggcon "github.com/apache/iggy/foreign/go/contracts" @@ -329,28 +330,32 @@ } func TestDeserializeToTopic_PinsTheFieldOrderOfTheWireLayout(t *testing.T) { - const nameLenOffset = 49 + const nameLenOffset = 32 name := "orders" - partitionsValue := binary.LittleEndian.AppendUint32(nil, 3) - explicitOptions := iggcon.GetHeadersBytes([]iggcon.HeaderEntry{{ - Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte("partitions_count")}, - Value: iggcon.HeaderValue{Kind: iggcon.Uint32, Value: partitionsValue}, - }}) + expiryValue := binary.LittleEndian.AppendUint64(nil, 60_000_000) + explicitOptions := iggcon.GetHeadersBytes([]iggcon.HeaderEntry{ + { + Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte("compression_algorithm")}, + Value: iggcon.HeaderValue{Kind: iggcon.String, Value: []byte("gzip")}, + }, + { + Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte("message_expiry")}, + Value: iggcon.HeaderValue{Kind: iggcon.Uint64, Value: expiryValue}, + }, + }) + sizeValue := binary.LittleEndian.AppendUint64(nil, 1<<30) derivedOptions := iggcon.GetHeadersBytes([]iggcon.HeaderEntry{{ - Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte("compression_algorithm")}, - Value: iggcon.HeaderValue{Kind: iggcon.String, Value: []byte("none")}, + Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte("max_topic_size")}, + Value: iggcon.HeaderValue{Kind: iggcon.Uint64, Value: sizeValue}, }}) payload := make([]byte, nameLenOffset+1+len(name)) binary.LittleEndian.PutUint32(payload[0:4], 11) binary.LittleEndian.PutUint64(payload[4:12], 1700000000) binary.LittleEndian.PutUint32(payload[12:16], 3) - binary.LittleEndian.PutUint64(payload[16:24], 60_000_000) - payload[24] = 2 - binary.LittleEndian.PutUint64(payload[25:33], 1<<30) - binary.LittleEndian.PutUint64(payload[33:41], 4096) - binary.LittleEndian.PutUint64(payload[41:49], 12) + binary.LittleEndian.PutUint64(payload[16:24], 4096) + binary.LittleEndian.PutUint64(payload[24:32], 12) payload[nameLenOffset] = byte(len(name)) copy(payload[nameLenOffset+1:], name) payload = binary.LittleEndian.AppendUint32(payload, uint32(len(explicitOptions))) @@ -364,25 +369,40 @@ assert.Equal(t, uint32(11), topic.Id) assert.Equal(t, uint64(1700000000), topic.CreatedAt) assert.Equal(t, uint32(3), topic.PartitionsCount) - assert.Equal(t, iggcon.Duration(60_000_000), topic.MessageExpiry, - "message expiry is the u64 at +16") - assert.Equal(t, uint8(2), topic.CompressionAlgorithm, - "compression is the u8 at +24") - assert.Equal(t, uint64(1<<30), topic.MaxTopicSize) - assert.Equal(t, uint64(4096), topic.Size) - assert.Equal(t, uint64(12), topic.MessagesCount) + assert.Equal(t, uint64(4096), topic.Size, "size is the u64 at +16") + assert.Equal(t, uint64(12), topic.MessagesCount, "messages count is the u64 at +24") assert.Equal(t, name, topic.Name) + assert.Equal(t, iggcon.Duration(60_000_000), topic.MessageExpiry, + "message expiry is read from the explicit block") + assert.Equal(t, uint8(iggcon.CompressionAlgorithmGzip), topic.CompressionAlgorithm) + assert.Equal(t, uint64(1<<30), topic.MaxTopicSize, "the size cap is read from the derived block") assert.Equal(t, map[string]iggcon.HeaderValue{ - "partitions_count": {Kind: iggcon.Uint32, Value: partitionsValue}, + "compression_algorithm": {Kind: iggcon.String, Value: []byte("gzip")}, + "message_expiry": {Kind: iggcon.Uint64, Value: expiryValue}, }, topic.Options) assert.Equal(t, map[string]iggcon.HeaderValue{ - "compression_algorithm": {Kind: iggcon.String, Value: []byte("none")}, + "max_topic_size": {Kind: iggcon.Uint64, Value: sizeValue}, }, topic.DerivedOptions) assert.Equal(t, len(payload), readBytes) } +func TestDeserializeToTopic_ResolvesMissingSettingsToTheCatalogDefaults(t *testing.T) { + const nameLenOffset = 32 + payload := make([]byte, nameLenOffset+2) + payload[nameLenOffset] = 1 + payload[nameLenOffset+1] = 't' + payload = binary.LittleEndian.AppendUint32(payload, 0) + payload = binary.LittleEndian.AppendUint32(payload, 0) + + topic, _, err := DeserializeToTopic(payload, 0) + require.NoError(t, err) + assert.Equal(t, iggcon.IggyExpiryNeverExpire, topic.MessageExpiry) + assert.Equal(t, uint64(math.MaxUint64), topic.MaxTopicSize) + assert.Equal(t, uint8(iggcon.CompressionAlgorithmNone), topic.CompressionAlgorithm) +} + func TestDeserializeToTopic_RejectsEveryTruncationOfTheOptionsBlocks(t *testing.T) { - const nameLenOffset = 49 + const nameLenOffset = 32 name := "t" fixed := make([]byte, nameLenOffset+1+len(name)) fixed[nameLenOffset] = byte(len(name)) @@ -407,7 +427,7 @@ } func TestDeserializeToTopic_RejectsAnOptionsLengthPastTheBuffer(t *testing.T) { - const nameLenOffset = 49 + const nameLenOffset = 32 name := "t" fixed := make([]byte, nameLenOffset+1+len(name)) fixed[nameLenOffset] = byte(len(name))
diff --git a/foreign/go/client/example_resource_options_test.go b/foreign/go/client/example_resource_options_test.go new file mode 100644 index 0000000..87a6230 --- /dev/null +++ b/foreign/go/client/example_resource_options_test.go
@@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package client_test + +import ( + "context" + "crypto/rand" + "log" + "time" + + "github.com/apache/iggy/foreign/go/client" + "github.com/apache/iggy/foreign/go/client/tcp" + iggcon "github.com/apache/iggy/foreign/go/contracts" +) + +// The README's resource options sample, kept here so it compiles. +func Example_resourceOptions() { + ctx := context.Background() + cli, err := client.NewIggyClient(client.WithTcp(tcp.WithServerAddress("127.0.0.1:8090"))) + if err != nil { + log.Fatal(err) + } + defer func() { _ = cli.Close() }() + if err := cli.Connect(ctx); err != nil { + log.Fatal(err) + } + if _, err := cli.LoginUser(ctx, "iggy", "iggy"); err != nil { + log.Fatal(err) + } + + key := make([]byte, iggcon.StreamEncryptionKeyLength) + if _, err := rand.Read(key); err != nil { + log.Fatal(err) + } + stream, err := cli.CreateStream(ctx, "orders", &iggcon.StreamOptions{EncryptionKey: key}) + if err != nil { + log.Fatal(err) + } + source, _ := stream.EncryptionKeySource() + fingerprint, _ := stream.EncryptionKeyFingerprint() + log.Printf("key source %s, fingerprint %s", source, fingerprint) + + streamId, _ := iggcon.NewIdentifier(stream.Id) + partitions := uint32(3) + expiry := iggcon.Duration(time.Hour.Microseconds()) + durability := iggcon.DurabilityPersisted + topic, err := cli.CreateTopic(ctx, streamId, "events", &iggcon.TopicOptions{ + PartitionsCount: &partitions, + MessageExpiry: &expiry, + Durability: &durability, + }) + if err != nil { + log.Fatal(err) + } + + topicId, _ := iggcon.NewIdentifier(topic.Id) + size := uint64(10 << 30) + if err := cli.UpdateTopic(ctx, streamId, topicId, iggcon.TopicUpdate{ + MaxTopicSize: &size, + Reset: []iggcon.TopicOptionKey{iggcon.TopicOptionMessageExpiry}, + }); err != nil { + log.Fatal(err) + } + + specs, err := cli.DescribeOptions(ctx, iggcon.OptionsScopeTopic) + if err != nil { + log.Fatal(err) + } + for _, spec := range specs { + log.Printf("%s updatable=%t resettable=%t", spec.Key, spec.Updatable, spec.Resettable) + } +}
diff --git a/foreign/go/client/tcp/tcp_access_token_management.go b/foreign/go/client/tcp/tcp_access_token_management.go index 61ba168..55ed7a3 100644 --- a/foreign/go/client/tcp/tcp_access_token_management.go +++ b/foreign/go/client/tcp/tcp_access_token_management.go
@@ -25,7 +25,7 @@ "github.com/apache/iggy/foreign/go/internal/command" ) -func (c *IggyTcpClient) CreatePersonalAccessToken(ctx context.Context, name string, expiry uint32) (*iggcon.RawPersonalAccessToken, error) { +func (c *IggyTcpClient) CreatePersonalAccessToken(ctx context.Context, name string, expiry iggcon.Duration) (*iggcon.RawPersonalAccessToken, error) { buffer, err := c.do(ctx, &command.CreatePersonalAccessToken{ Name: name, Expiry: expiry,
diff --git a/foreign/go/client/tcp/tcp_core_test.go b/foreign/go/client/tcp/tcp_core_test.go index 2332aa1..afb384a 100644 --- a/foreign/go/client/tcp/tcp_core_test.go +++ b/foreign/go/client/tcp/tcp_core_test.go
@@ -865,7 +865,7 @@ func topicDetailsBody(t *testing.T, partitionsCount uint32) []byte { t.Helper() - const nameLenOffset = 49 + const nameLenOffset = 32 name := "orders" body := make([]byte, nameLenOffset+1+len(name)+4+4) binary.LittleEndian.PutUint32(body[0:4], 1)
diff --git a/foreign/go/client/tcp/tcp_link.go b/foreign/go/client/tcp/tcp_link.go index 9755141..775e06c 100644 --- a/foreign/go/client/tcp/tcp_link.go +++ b/foreign/go/client/tcp/tcp_link.go
@@ -83,7 +83,8 @@ return linkReply{err: ierror.ErrTransientNotAccepted}, false default: } - if l.pending[id] != nil { + // A select picks a ready case at random, so a cancelled caller must be refused before its frame is queued. + if l.pending[id] != nil || ctx.Err() != nil { l.mu.Unlock() <-l.capacity return linkReply{err: ierror.ErrTransientNotAccepted}, false
diff --git a/foreign/go/client/tcp/tcp_link_test.go b/foreign/go/client/tcp/tcp_link_test.go index 36df922..075a900 100644 --- a/foreign/go/client/tcp/tcp_link_test.go +++ b/foreign/go/client/tcp/tcp_link_test.go
@@ -20,6 +20,7 @@ import ( "context" "encoding/binary" + "io" "net" "sync" "sync/atomic" @@ -75,6 +76,22 @@ } } +func TestTCPLinkCancelledCallerNeverSendsItsFrame(t *testing.T) { + client, server := net.Pipe() + link := newTCPLink(client, "pipe") + t.Cleanup(func() { link.close(ierror.ErrClientShutdown); _ = server.Close(); link.done.Wait() }) + go func() { _, _ = io.Copy(io.Discard, server) }() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + for id := uint64(1); id <= 100; id++ { + frame := linkTestFrame(id) + reply, sent := link.exchange(ctx, frame) + frame.release() + require.False(t, sent, "a cancelled caller sent request %d", id) + require.ErrorIs(t, reply.err, ierror.ErrTransientNotAccepted) + } +} + func TestPendingRegistryBlocksOnlyItsLaneAndPinsQueuedBytesAfterAbandon(t *testing.T) { registry := newOperationRegistry(1, vsr.HeaderSize) ticket, err := registry.enter(context.Background(), laneKey{incarnation: 1})
diff --git a/foreign/go/client/tcp/tcp_poll_routing_test.go b/foreign/go/client/tcp/tcp_poll_routing_test.go index 50c0b4f..68bcc60 100644 --- a/foreign/go/client/tcp/tcp_poll_routing_test.go +++ b/foreign/go/client/tcp/tcp_poll_routing_test.go
@@ -634,7 +634,7 @@ _, err = pollPrimaryPartition(context.Background(), fixture.client, 0) require.NoError(t, err) newName := "renamed" - require.NoError(t, fixture.client.UpdateUser(context.Background(), numericIdentifier(t, 7), &newName, nil)) + require.NoError(t, fixture.client.UpdateUser(context.Background(), numericIdentifier(t, 7), iggcon.UserUpdate{Username: &newName})) named, err := iggcon.NewIdentifier(newName) require.NoError(t, err) require.NoError(t, fixture.client.ChangePassword(context.Background(), named, "manual-secret", "changed-secret")) @@ -656,7 +656,7 @@ func TestPrimaryPoll_ConfiguredCredentialsFollowSuccessfulSelfUpdates(t *testing.T) { fixture := newPrimaryPollFixture(t, nil, nil) newName := "renamed" - require.NoError(t, fixture.client.UpdateUser(context.Background(), numericIdentifier(t, 7), &newName, nil)) + require.NoError(t, fixture.client.UpdateUser(context.Background(), numericIdentifier(t, 7), iggcon.UserUpdate{Username: &newName})) name, err := iggcon.NewIdentifier(newName) require.NoError(t, err) require.NoError(t, fixture.client.ChangePassword(context.Background(), name, "secret", "changed-secret"))
diff --git a/foreign/go/client/tcp/tcp_session_credentials_test.go b/foreign/go/client/tcp/tcp_session_credentials_test.go index 217eb4f..f3fb786 100644 --- a/foreign/go/client/tcp/tcp_session_credentials_test.go +++ b/foreign/go/client/tcp/tcp_session_credentials_test.go
@@ -66,11 +66,53 @@ return credentialChange{ request: &command.UpdateUser{UserID: user, Username: &username}, typed: func(ctx context.Context, client *IggyTcpClient) error { - return client.UpdateUser(ctx, user, &username, nil) + return client.UpdateUser(ctx, user, iggcon.UserUpdate{Username: &username}) }, } } +// otherRename is a patch that names something other than the signed-in user. +func otherRename(t *testing.T, name string) credentialChange { + t.Helper() + newName := renamedUsername + switch name { + case "other_user": + user := numericIdentifier(t, signedInUserID+1) + return credentialChange{ + request: &command.UpdateUser{UserID: user, Username: &newName}, + typed: func(ctx context.Context, client *IggyTcpClient) error { + return client.UpdateUser(ctx, user, iggcon.UserUpdate{Username: &newName}) + }, + } + case "status_only": + user := numericIdentifier(t, signedInUserID) + status := iggcon.Active + return credentialChange{ + request: &command.UpdateUser{UserID: user, Status: &status}, + typed: func(ctx context.Context, client *IggyTcpClient) error { + return client.UpdateUser(ctx, user, iggcon.UserUpdate{Status: &status}) + }, + } + case "stream": + stream := numericIdentifier(t, signedInUserID) + return credentialChange{ + request: &command.UpdateStream{StreamId: stream, Name: &newName}, + typed: func(ctx context.Context, client *IggyTcpClient) error { + return client.UpdateStream(ctx, stream, iggcon.StreamUpdate{Name: &newName}) + }, + } + default: + stream := numericIdentifier(t, signedInUserID) + topic := numericIdentifier(t, signedInUserID) + return credentialChange{ + request: &command.UpdateTopic{StreamId: stream, TopicId: topic, Name: &newName}, + typed: func(ctx context.Context, client *IggyTcpClient) error { + return client.UpdateTopic(ctx, stream, topic, iggcon.TopicUpdate{Name: &newName}) + }, + } + } +} + // send makes the change through its typed method, or as a raw request. A call // that ends uncertain is resolved, so the result is the proved outcome. func (change credentialChange) send(t *testing.T, client *IggyTcpClient, raw bool) error { @@ -110,12 +152,12 @@ return clusterMetadataFrame(t, 0, server.address()) case read.code() == uint32(command.PingCode) && evict.CompareAndSwap(true, false): return evictionFrame(vsr.EvictionSessionError, 0, 0) - case read.operation() == vsr.OperationChangePassword || read.operation() == vsr.OperationUpdateUser: + case read.operation() == vsr.OperationChangePassword || read.operation() == vsr.OperationUpdateResource: mtx.Lock() attempt := attempts[read.requestID()] attempts[read.requestID()]++ mtx.Unlock() - return change(attempt, read) + return resolvedReply(change(attempt, read), read) default: return replyFrame(read.operation(), nil) } @@ -161,8 +203,24 @@ require.Equal(t, []string{username, password}, names, "the username and password of the new sign-in") } +// resolvedReply stamps a committed patch reply with the operation prepare +// rewrites it to, as a server does: 73 for a topic, 72 for a stream, 74 for a +// user, chosen by the scope byte that leads the request body. +func resolvedReply(reply []byte, read request) []byte { + if read.operation() != vsr.OperationUpdateResource || reply[220] != 1 || len(read.payload) == 0 { + return reply + } + resolved := map[byte]vsr.Operation{ + byte(iggcon.OptionsScopeTopic): vsr.OperationUpdateTopicResolved, + byte(iggcon.OptionsScopeStream): vsr.OperationUpdateStreamResolved, + byte(iggcon.OptionsScopeUser): vsr.OperationUpdateUserResolved, + }[read.payload[0]] + reply[replyFrameOffsetOperation] = byte(resolved) + return reply +} + func credentialChanges(reads []request) int { - return len(requestsFor(reads, vsr.OperationChangePassword)) + len(requestsFor(reads, vsr.OperationUpdateUser)) + return len(requestsFor(reads, vsr.OperationChangePassword)) + len(requestsFor(reads, vsr.OperationUpdateResource)) } func TestCredentialChange_CommittedChangeIsTheNextSignIn(t *testing.T) { @@ -201,6 +259,52 @@ } } +func TestCredentialChange_GenericUserPatch(t *testing.T) { + t.Run("given_logged_in_user_rename_when_reconnecting_should_keep_new_username", func(t *testing.T) { + for _, raw := range []bool{false, true} { + for _, autoLogin := range []bool{false, true} { + t.Run(fmt.Sprintf("self_rename/raw_%t/auto_login_%t", raw, autoLogin), func(t *testing.T) { + server, evict := credentialServer(t, func(_ int, read request) []byte { + return committedReply(read, 0) + }) + client := signedInClient(t, server, autoLogin) + + require.NoError(t, rename(t).send(t, client, raw)) + + require.Len(t, requestsFor(server.recorded(), vsr.OperationUpdateResource), 1, + "a rename travels as the generic patch") + requireSignInAgain(t, client, server, evict, renamedUsername, signInPassword) + }) + + t.Run(fmt.Sprintf("rejected_rename/raw_%t/auto_login_%t", raw, autoLogin), func(t *testing.T) { + server, evict := credentialServer(t, func(_ int, read request) []byte { + return committedReply(read, ierror.UserAlreadyExistsCode) + }) + client := signedInClient(t, server, autoLogin) + + err := rename(t).send(t, client, raw) + + require.ErrorIs(t, err, ierror.ErrUserAlreadyExists) + requireSignInAgain(t, client, server, evict, signInUsername, signInPassword) + }) + + for _, name := range []string{"other_user", "status_only", "stream", "topic"} { + t.Run(fmt.Sprintf("%s/raw_%t/auto_login_%t", name, raw, autoLogin), func(t *testing.T) { + server, evict := credentialServer(t, func(_ int, read request) []byte { + return committedReply(read, 0) + }) + client := signedInClient(t, server, autoLogin) + + require.NoError(t, otherRename(t, name).send(t, client, raw)) + + requireSignInAgain(t, client, server, evict, signInUsername, signInPassword) + }) + } + } + } + }) +} + func TestCredentialChange_CommittedRejectionKeepsTheNextSignIn(t *testing.T) { for _, raw := range []bool{false, true} { for _, resolved := range []bool{false, true} {
diff --git a/foreign/go/client/tcp/tcp_stream_management.go b/foreign/go/client/tcp/tcp_stream_management.go index 3426c96..89a7295 100644 --- a/foreign/go/client/tcp/tcp_stream_management.go +++ b/foreign/go/client/tcp/tcp_stream_management.go
@@ -54,11 +54,15 @@ return stream, nil } -func (c *IggyTcpClient) CreateStream(ctx context.Context, name string) (*iggcon.StreamDetails, error) { +func (c *IggyTcpClient) CreateStream(ctx context.Context, name string, options *iggcon.StreamOptions) (*iggcon.StreamDetails, error) { if len(name) == 0 || MaxStringLength < len(name) { return nil, ierror.ErrInvalidStreamName } - buffer, err := c.do(ctx, &command.CreateStream{Name: name}) + request := &command.CreateStream{Name: name} + if options != nil { + request.Options = *options + } + buffer, err := c.do(ctx, request) if err != nil { return nil, err } @@ -70,11 +74,14 @@ return stream, err } -func (c *IggyTcpClient) UpdateStream(ctx context.Context, streamId iggcon.Identifier, name string) error { - if len(name) > MaxStringLength || len(name) == 0 { - return ierror.ErrInvalidStreamName - } - _, err := c.do(ctx, &command.UpdateStream{StreamId: streamId, Name: name}) +func (c *IggyTcpClient) UpdateStream(ctx context.Context, streamId iggcon.Identifier, update iggcon.StreamUpdate) error { + _, err := c.do(ctx, &command.UpdateStream{ + StreamId: streamId, + Name: update.Name, + Reset: update.Reset, + RawReset: update.RawReset, + Options: update.Options, + }) return err }
diff --git a/foreign/go/client/tcp/tcp_stream_management_test.go b/foreign/go/client/tcp/tcp_stream_management_test.go index f46033b..c21442b 100644 --- a/foreign/go/client/tcp/tcp_stream_management_test.go +++ b/foreign/go/client/tcp/tcp_stream_management_test.go
@@ -23,6 +23,7 @@ "strings" "testing" + iggcon "github.com/apache/iggy/foreign/go/contracts" ierror "github.com/apache/iggy/foreign/go/errors" "github.com/apache/iggy/foreign/go/internal/vsr" "github.com/stretchr/testify/assert" @@ -31,7 +32,7 @@ // streamDetailsBody builds a stream reply with no topics: // [id u32][created_at u64][topics u32][size u64][messages u64][name_len u8][name] -// [options_len u32 = 0]. +// [options_len u32 = 0][derived_options_len u32 = 0]. func streamDetailsBody(id uint32, name string) []byte { body := binary.LittleEndian.AppendUint32(nil, id) body = binary.LittleEndian.AppendUint64(body, 1700000000) @@ -40,6 +41,7 @@ body = binary.LittleEndian.AppendUint64(body, 0) body = append(body, byte(len(name))) body = append(body, name...) + body = binary.LittleEndian.AppendUint32(body, 0) return binary.LittleEndian.AppendUint32(body, 0) } @@ -50,7 +52,7 @@ append(resultSection(), streamDetailsBody(9, "orders")...)) }) - stream, err := client.CreateStream(context.Background(), "orders") + stream, err := client.CreateStream(context.Background(), "orders", nil) require.NoError(t, err) assert.Equal(t, uint32(9), stream.Id) assert.Equal(t, "orders", stream.Name) @@ -63,12 +65,27 @@ }) for _, name := range []string{"", strings.Repeat("x", MaxStringLength+1)} { - _, err := client.CreateStream(context.Background(), name) + _, err := client.CreateStream(context.Background(), name, nil) assert.ErrorIs(t, err, ierror.ErrInvalidStreamName, "name %q", name) } assert.Empty(t, server.recorded()) } +func TestCreateStream_RejectsAnInvalidKeyWithoutWriting(t *testing.T) { + t.Run("given_invalid_key_when_encoding_should_reject_before_network_io", func(t *testing.T) { + client, serverConn := newPipeClient(t) + server := serve(serverConn, func(_ int, _ request) []byte { + return replyFrame(vsr.OperationCreateStream, resultSection()) + }) + + for _, key := range [][]byte{{}, make([]byte, 31), make([]byte, 33)} { + _, err := client.CreateStream(context.Background(), "orders", &iggcon.StreamOptions{EncryptionKey: key}) + assert.ErrorIs(t, err, ierror.ErrInvalidEncryptionKey, "%d bytes", len(key)) + } + assert.Empty(t, server.recorded()) + }) +} + func TestGetStream_ReportsAMissingStreamAsATypedError(t *testing.T) { client, serverConn := newPipeClient(t) serve(serverConn, func(_ int, _ request) []byte { @@ -96,24 +113,31 @@ func TestUpdateStream_RejectsAnInvalidNameWithoutWriting(t *testing.T) { client, serverConn := newPipeClient(t) server := serve(serverConn, func(_ int, _ request) []byte { - return replyFrame(vsr.OperationUpdateStream, resultSection()) + return replyFrame(vsr.OperationUpdateStreamResolved, resultSection()) }) for _, name := range []string{"", strings.Repeat("x", MaxStringLength+1)} { - err := client.UpdateStream(context.Background(), numericIdentifier(t, 1), name) + err := client.UpdateStream(context.Background(), numericIdentifier(t, 1), iggcon.StreamUpdate{Name: &name}) assert.ErrorIs(t, err, ierror.ErrInvalidStreamName, "name %q", name) } + err := client.UpdateStream(context.Background(), numericIdentifier(t, 1), + iggcon.StreamUpdate{Reset: []iggcon.StreamOptionKey{iggcon.StreamOptionName}}) + assert.ErrorIs(t, err, ierror.ErrOptionNotResettable) assert.Empty(t, server.recorded()) } func TestUpdateStream_AcceptsACommittedRename(t *testing.T) { client, serverConn := newPipeClient(t) - serve(serverConn, func(_ int, _ request) []byte { - return replyFrame(vsr.OperationUpdateStream, resultSection()) + server := serve(serverConn, func(_ int, read request) []byte { + reply := committedReply(read, 0) + reply[replyFrameOffsetOperation] = byte(vsr.OperationUpdateStreamResolved) + return reply }) + name := "renamed" assert.NoError(t, client.UpdateStream( - context.Background(), numericIdentifier(t, 1), "renamed")) + context.Background(), numericIdentifier(t, 1), iggcon.StreamUpdate{Name: &name})) + assert.Len(t, requestsFor(server.recorded(), vsr.OperationUpdateResource), 1) } func TestCreatePartitions_InvalidatesTheCachedPartitionCount(t *testing.T) {
diff --git a/foreign/go/client/tcp/tcp_topic_management.go b/foreign/go/client/tcp/tcp_topic_management.go index f2d678b..1c741de 100644 --- a/foreign/go/client/tcp/tcp_topic_management.go +++ b/foreign/go/client/tcp/tcp_topic_management.go
@@ -56,27 +56,19 @@ ctx context.Context, streamId iggcon.Identifier, name string, - partitionsCount uint32, - compressionAlgorithm iggcon.CompressionAlgorithm, - messageExpiry iggcon.Duration, - maxTopicSize uint64, - options ...iggcon.HeaderEntry, + options *iggcon.TopicOptions, ) (*iggcon.TopicDetails, error) { if len(name) == 0 || len(name) > MaxStringLength { return nil, ierror.ErrInvalidTopicName } - if partitionsCount > MaxPartitionCount { + request := &command.CreateTopic{StreamId: streamId, Name: name} + if options != nil { + request.Options = *options + } + if request.Options.PartitionsCount != nil && *request.Options.PartitionsCount > MaxPartitionCount { return nil, ierror.ErrTooManyPartitions } - buffer, err := c.do(ctx, &command.CreateTopic{ - StreamId: streamId, - Name: name, - PartitionsCount: partitionsCount, - CompressionAlgorithm: compressionAlgorithm, - MessageExpiry: messageExpiry, - MaxTopicSize: maxTopicSize, - Options: options, - }) + buffer, err := c.do(ctx, request) if err != nil { return nil, err } @@ -88,23 +80,17 @@ ctx context.Context, streamId iggcon.Identifier, topicId iggcon.Identifier, - name string, - compressionAlgorithm iggcon.CompressionAlgorithm, - messageExpiry iggcon.Duration, - maxTopicSize uint64, - options ...iggcon.HeaderEntry, + update iggcon.TopicUpdate, ) error { - if len(name) == 0 || len(name) > MaxStringLength { - return ierror.ErrInvalidTopicName - } _, err := c.do(ctx, &command.UpdateTopic{ - StreamId: streamId, - TopicId: topicId, - CompressionAlgorithm: compressionAlgorithm, - MessageExpiry: messageExpiry, - MaxTopicSize: maxTopicSize, - Name: name, - Options: options, + StreamId: streamId, + TopicId: topicId, + Name: update.Name, + MessageExpiry: update.MessageExpiry, + MaxTopicSize: update.MaxTopicSize, + Reset: update.Reset, + RawReset: update.RawReset, + Options: update.Options, }) return err }
diff --git a/foreign/go/client/tcp/tcp_topic_management_test.go b/foreign/go/client/tcp/tcp_topic_management_test.go index 93a531c..1ec24b1 100644 --- a/foreign/go/client/tcp/tcp_topic_management_test.go +++ b/foreign/go/client/tcp/tcp_topic_management_test.go
@@ -37,8 +37,9 @@ append(resultSection(), topicDetailsBody(t, 3)...)) }) + partitions := uint32(3) topic, err := client.CreateTopic(context.Background(), - numericIdentifier(t, 1), "orders", 3, 0, iggcon.Duration(0), 0) + numericIdentifier(t, 1), "orders", &iggcon.TopicOptions{PartitionsCount: &partitions}) require.NoError(t, err) assert.Equal(t, uint32(3), topic.PartitionsCount) assert.Equal(t, "orders", topic.Name) @@ -67,7 +68,7 @@ for _, test := range tests { t.Run(test.name, func(t *testing.T) { _, err := client.CreateTopic(context.Background(), streamId, - test.topicName, test.partitions, 0, iggcon.Duration(0), 0) + test.topicName, &iggcon.TopicOptions{PartitionsCount: &test.partitions}) assert.ErrorIs(t, err, test.want) }) } @@ -77,27 +78,46 @@ func TestUpdateTopic_RejectsInvalidArgumentsWithoutWriting(t *testing.T) { client, serverConn := newPipeClient(t) server := serve(serverConn, func(_ int, _ request) []byte { - return replyFrame(vsr.OperationUpdateTopic, resultSection()) + return replyFrame(vsr.OperationUpdateTopicResolved, resultSection()) }) streamId := numericIdentifier(t, 1) topicId := numericIdentifier(t, 2) - err := client.UpdateTopic(context.Background(), streamId, topicId, - "", 0, iggcon.Duration(0), 0) - assert.ErrorIs(t, err, ierror.ErrInvalidTopicName) + empty := "" + serverDefault := iggcon.IggyExpiryServerDefault + tests := []struct { + name string + update iggcon.TopicUpdate + want error + }{ + {name: "empty name", update: iggcon.TopicUpdate{Name: &empty}, want: ierror.ErrInvalidTopicName}, + {name: "nothing to change", update: iggcon.TopicUpdate{}, want: ierror.ErrInvalidCommand}, + {name: "sentinel only", update: iggcon.TopicUpdate{MessageExpiry: &serverDefault}, want: ierror.ErrInvalidCommand}, + {name: "create-only reset", update: iggcon.TopicUpdate{Reset: []iggcon.TopicOptionKey{iggcon.TopicOptionSegmentSize}}, want: ierror.ErrOptionNotUpdatable}, + {name: "unknown reset", update: iggcon.TopicUpdate{Reset: []iggcon.TopicOptionKey{"not_a_key"}}, want: ierror.ErrUnsupportedOptionKey}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := client.UpdateTopic(context.Background(), streamId, topicId, test.update) + assert.ErrorIs(t, err, test.want) + }) + } assert.Empty(t, server.recorded()) } func TestUpdateTopic_AcceptsACommittedUpdate(t *testing.T) { client, serverConn := newPipeClient(t) - serve(serverConn, func(_ int, _ request) []byte { - return replyFrame(vsr.OperationUpdateTopic, resultSection()) + serve(serverConn, func(_ int, read request) []byte { + reply := committedReply(read, 0) + reply[replyFrameOffsetOperation] = byte(vsr.OperationUpdateTopicResolved) + return reply }) + name := "renamed" assert.NoError(t, client.UpdateTopic(context.Background(), numericIdentifier(t, 1), numericIdentifier(t, 2), - "renamed", 0, iggcon.Duration(0), 0)) + iggcon.TopicUpdate{Name: &name, Reset: []iggcon.TopicOptionKey{iggcon.TopicOptionMaxTopicSize}})) } func TestGetTopics_DecodesTheRoster(t *testing.T) {
diff --git a/foreign/go/client/tcp/tcp_user_management.go b/foreign/go/client/tcp/tcp_user_management.go index c99e2c3..0afcc09 100644 --- a/foreign/go/client/tcp/tcp_user_management.go +++ b/foreign/go/client/tcp/tcp_user_management.go
@@ -26,9 +26,6 @@ "github.com/apache/iggy/foreign/go/internal/command" ) -// usernamePresent flags the optional username of an UpdateUser body. -const usernamePresent = 1 - func (c *IggyTcpClient) GetUser(ctx context.Context, identifier iggcon.Identifier) (*iggcon.UserInfoDetails, error) { buffer, err := c.do(ctx, &command.GetUser{Id: identifier}) if err != nil { @@ -50,13 +47,24 @@ return binaryserialization.DeserializeUsers(buffer) } -func (c *IggyTcpClient) CreateUser(ctx context.Context, username string, password string, status iggcon.UserStatus, permissions *iggcon.Permissions) (*iggcon.UserInfoDetails, error) { - buffer, err := c.do(ctx, &command.CreateUser{ +func (c *IggyTcpClient) CreateUser( + ctx context.Context, + username string, + password string, + status iggcon.UserStatus, + permissions *iggcon.Permissions, + options *iggcon.UserOptions, +) (*iggcon.UserInfoDetails, error) { + request := &command.CreateUser{ Username: username, Password: password, Status: status, Permissions: permissions, - }) + } + if options != nil { + request.Options = *options + } + buffer, err := c.do(ctx, request) if err != nil { return nil, err } @@ -67,11 +75,14 @@ return userInfo, nil } -func (c *IggyTcpClient) UpdateUser(ctx context.Context, userID iggcon.Identifier, username *string, status *iggcon.UserStatus) error { +func (c *IggyTcpClient) UpdateUser(ctx context.Context, userID iggcon.Identifier, update iggcon.UserUpdate) error { _, err := c.do(ctx, &command.UpdateUser{ UserID: userID, - Username: username, - Status: status, + Username: update.Username, + Status: update.Status, + Reset: update.Reset, + RawReset: update.RawReset, + Options: update.Options, }) return err } @@ -105,29 +116,63 @@ // typed calls, raw sends and replays follow one rule. A body that does // not decode changes nothing and never fails the committed call. func (c *IggyTcpClient) refreshSignInLocked(epoch *signInEpoch, code command.Code, body []byte) { - if (code != command.ChangePasswordCode && code != command.UpdateUserCode) || len(body) < fieldPrefixSize { - return - } - end := fieldPrefixSize + int(body[1]) - if len(body) < end { - return - } - user, rest := string(body[:end]), body[end:] - if code == command.ChangePasswordCode { + switch code { + case command.ChangePasswordCode: + user, rest, ok := splitIdentifier(body) + if !ok { + return + } if _, rest, ok := splitName(rest); ok { if password, _, ok := splitName(rest); ok { c.refreshCredentialsLocked(epoch, user, nil, &password) } } - return - } - if len(rest) > 0 && rest[0] == usernamePresent { - if username, _, ok := splitName(rest[1:]); ok { + case command.UpdateResourceCode: + if len(body) == 0 || body[0] != byte(iggcon.OptionsScopeUser) { + return + } + if user, username, ok := patchedUsername(body[1:]); ok { c.refreshCredentialsLocked(epoch, user, &username, nil) } } } +// patchedUsername reads the user identifier and the username a user-scope +// UpdateResource body sets: [user_id][reset_count][reset keys][options]. +func patchedUsername(body []byte) (string, string, bool) { + user, rest, ok := splitIdentifier(body) + if !ok || len(rest) == 0 { + return "", "", false + } + resetCount := int(rest[0]) + rest = rest[1:] + for range resetCount { + if _, rest, ok = splitName(rest); !ok { + return "", "", false + } + } + options, err := iggcon.DeserializeHeaders(rest) + if err != nil { + return "", "", false + } + for _, entry := range options { + if entry.Key.Kind == iggcon.String && string(entry.Key.Value) == string(iggcon.UserOptionUsername) && entry.Value.Kind == iggcon.String { + return user, string(entry.Value.Value), true + } + } + return "", "", false +} + +// splitIdentifier splits a [kind][u8 length][value] identifier off data and +// returns it in its wire form. +func splitIdentifier(data []byte) (string, []byte, bool) { + if len(data) < fieldPrefixSize || len(data) < fieldPrefixSize+int(data[1]) { + return "", nil, false + } + end := fieldPrefixSize + int(data[1]) + return string(data[:end]), data[end:], true +} + // refreshCredentialsLocked updates each stored login that user names. Only the // signed-in user has a known numeric ID, so a change by ID reaches the // configured login only when it is the signed-in user.
diff --git a/foreign/go/contracts/client.go b/foreign/go/contracts/client.go index bb63706..53f6d71 100644 --- a/foreign/go/contracts/client.go +++ b/foreign/go/contracts/client.go
@@ -54,13 +54,19 @@ // Authentication is required, and the permission to read the streams. GetStreams(ctx context.Context) ([]Stream, error) - // CreateStream create a new stream. + // CreateStream create a new stream. Nil options select the defaults. // Authentication is required, and the permission to manage the streams. - CreateStream(ctx context.Context, name string) (*StreamDetails, error) + // + // An invalid StreamOptions.EncryptionKey is refused before anything is + // sent. A valid one travels inside this request, so use TLS on an + // untrusted network. The server never returns the key: the returned + // stream reports only its source and fingerprints. + CreateStream(ctx context.Context, name string, options *StreamOptions) (*StreamDetails, error) - // UpdateStream update a stream by unique ID or name. + // UpdateStream patch a stream by unique ID or name: set the keys update + // names, leave every other key alone. // Authentication is required, and the permission to manage the streams. - UpdateStream(ctx context.Context, streamId Identifier, name string) error + UpdateStream(ctx context.Context, streamId Identifier, update StreamUpdate) error // DeleteStream delete a stream by unique ID or name. // Authentication is required, and the permission to manage the streams. @@ -74,42 +80,37 @@ // Authentication is required, and the permission to read the topics. GetTopics(ctx context.Context, streamId Identifier) ([]Topic, error) - // DescribeOptions get the option catalog for a resource scope: every key its - // create command accepts, with the kind, default and bounds of each. + // DescribeOptions get the option catalog for a resource scope: every key, + // with its kind, default and bounds, and whether an update may set or + // reset it. // Authentication is required. DescribeOptions(ctx context.Context, scope OptionsScope) ([]OptionSpec, error) - // CreateTopic create a new topic. + // CreateTopic create a new topic. Nil options select the defaults. // Authentication is required, and the permission to manage the topics. // - // options carries topic option keys with no parameter of their own, for a key - // the server catalog gained after this build shipped; call DescribeOptions to - // see which keys a server accepts. A parameter above wins on collision. + // TopicOptions.Raw carries keys the server catalog gained after this build + // shipped. Call DescribeOptions to see which keys a server accepts. CreateTopic( ctx context.Context, streamId Identifier, name string, - partitionsCount uint32, - compressionAlgorithm CompressionAlgorithm, - messageExpiry Duration, - maxTopicSize uint64, - options ...HeaderEntry, + options *TopicOptions, ) (*TopicDetails, error) - // UpdateTopic update a topic by unique ID or name. + // UpdateTopic patch a topic by unique ID or name: set the keys update + // names, reset the keys it lists, leave every other key alone. // Authentication is required, and the permission to manage the topics. // - // options carries keys with no parameter of their own. The server refuses any - // key an update may not change, by name. + // A patch that changes nothing is refused before anything is sent, and so + // is a patch of typed fields that names a key it may not change. A patch + // with RawReset names or Options entries is checked by the server, whose + // catalog may be newer than this build. UpdateTopic( ctx context.Context, streamId Identifier, topicId Identifier, - name string, - compressionAlgorithm CompressionAlgorithm, - messageExpiry Duration, - maxTopicSize uint64, - options ...HeaderEntry, + update TopicUpdate, ) error // DeleteTopic delete a topic by unique ID or name. @@ -326,7 +327,7 @@ // Authentication is required, and the permission to read the users. GetUsers(ctx context.Context) ([]UserInfo, error) - // CreateUser create a new user. + // CreateUser create a new user. Nil options select the defaults. // Authentication is required, and the permission to manage the users. CreateUser( ctx context.Context, @@ -334,15 +335,16 @@ password string, status UserStatus, permissions *Permissions, + options *UserOptions, ) (*UserInfoDetails, error) - // UpdateUser update a user by unique ID or username. + // UpdateUser patch a user by unique ID or username: set the keys update + // names, leave every other key alone. // Authentication is required, and the permission to manage the users. UpdateUser( ctx context.Context, userID Identifier, - username *string, - status *UserStatus, + update UserUpdate, ) error // UpdatePermissions update the permissions of a user by unique ID or username. @@ -363,7 +365,8 @@ DeleteUser(ctx context.Context, identifier Identifier) error // CreatePersonalAccessToken create a new personal access token for the currently authenticated user. - CreatePersonalAccessToken(ctx context.Context, name string, expiry uint32) (*RawPersonalAccessToken, error) + // The expiry is the token lifetime in microseconds. IggyExpiryServerDefault and IggyExpiryNeverExpire create a token that never expires. + CreatePersonalAccessToken(ctx context.Context, name string, expiry Duration) (*RawPersonalAccessToken, error) // DeletePersonalAccessToken delete a personal access token of the currently authenticated user by unique token name. DeletePersonalAccessToken(ctx context.Context, name string) error
diff --git a/foreign/go/contracts/option_diagnostics.go b/foreign/go/contracts/option_diagnostics.go new file mode 100644 index 0000000..2b23bab --- /dev/null +++ b/foreign/go/contracts/option_diagnostics.go
@@ -0,0 +1,150 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package iggcon + +import ( + "bytes" + "encoding/json" + "fmt" + "reflect" + "strings" +) + +// redactedValue stands in for a value diagnostics never show. +const redactedValue = "<redacted>" + +// redactedEntry is a raw options entry with its value hidden. +type redactedEntry struct { + Key string `json:"key"` + Value string `json:"value"` +} + +// Format writes the options with the key and every raw value redacted. +func (o StreamOptions) Format(state fmt.State, _ rune) { formatRedacted(state, o) } + +// MarshalJSON writes the options with the key and every raw value redacted. +func (o StreamOptions) MarshalJSON() ([]byte, error) { return marshalRedacted(o) } + +// Format writes the options with every raw value redacted. +func (o TopicOptions) Format(state fmt.State, _ rune) { formatRedacted(state, o) } + +// MarshalJSON writes the options with every raw value redacted. +func (o TopicOptions) MarshalJSON() ([]byte, error) { return marshalRedacted(o) } + +// Format writes the options with every raw value redacted. +func (o UserOptions) Format(state fmt.State, _ rune) { formatRedacted(state, o) } + +// MarshalJSON writes the options with every raw value redacted. +func (o UserOptions) MarshalJSON() ([]byte, error) { return marshalRedacted(o) } + +// Format writes the patch with every raw value redacted. +func (u StreamUpdate) Format(state fmt.State, _ rune) { formatRedacted(state, u) } + +// MarshalJSON writes the patch with every raw value redacted. +func (u StreamUpdate) MarshalJSON() ([]byte, error) { return marshalRedacted(u) } + +// Format writes the patch with every raw value redacted. +func (u TopicUpdate) Format(state fmt.State, _ rune) { formatRedacted(state, u) } + +// MarshalJSON writes the patch with every raw value redacted. +func (u TopicUpdate) MarshalJSON() ([]byte, error) { return marshalRedacted(u) } + +// Format writes the patch with every raw value redacted. +func (u UserUpdate) Format(state fmt.State, _ rune) { formatRedacted(state, u) } + +// MarshalJSON writes the patch with every raw value redacted. +func (u UserUpdate) MarshalJSON() ([]byte, error) { return marshalRedacted(u) } + +func (e redactedEntry) String() string { return e.Key + ":" + e.Value } + +// formatRedacted prints every exported field, so a new field shows up without +// an edit here, for every verb. +func formatRedacted(state fmt.State, options any) { + fields := redactedFields(options) + printed := make([]string, 0, len(fields)) + for _, field := range fields { + value := field.value + if value == nil { + value = "<nil>" + } + printed = append(printed, fmt.Sprintf("%s:%v", field.name, value)) + } + _, _ = fmt.Fprintf(state, "%s{%s}", reflect.TypeOf(options).Name(), strings.Join(printed, " ")) +} + +func marshalRedacted(options any) ([]byte, error) { + var buffer bytes.Buffer + buffer.WriteByte('{') + for index, field := range redactedFields(options) { + if index > 0 { + buffer.WriteByte(',') + } + name, err := json.Marshal(field.name) + if err != nil { + return nil, err + } + value, err := json.Marshal(field.value) + if err != nil { + return nil, err + } + buffer.Write(name) + buffer.WriteByte(':') + buffer.Write(value) + } + buffer.WriteByte('}') + return buffer.Bytes(), nil +} + +type redactedField struct { + name string + value any +} + +// redactedFields hides byte slices, which hold key material, and every raw +// entry value, because this build cannot tell which newer keys are secret. +func redactedFields(options any) []redactedField { + value := reflect.ValueOf(options) + fields := make([]redactedField, 0, value.NumField()) + for index := range value.NumField() { + if !value.Type().Field(index).IsExported() { + continue + } + field := value.Field(index) + var shown any + switch typed := field.Interface().(type) { + case []byte: + if typed != nil { + shown = redactedValue + } + case []HeaderEntry: + entries := make([]redactedEntry, 0, len(typed)) + for _, entry := range typed { + entries = append(entries, redactedEntry{Key: string(entry.Key.Value), Value: redactedValue}) + } + shown = entries + default: + if field.Kind() != reflect.Pointer { + shown = field.Interface() + } else if !field.IsNil() { + shown = field.Elem().Interface() + } + } + fields = append(fields, redactedField{name: value.Type().Field(index).Name, value: shown}) + } + return fields +}
diff --git a/foreign/go/contracts/options.go b/foreign/go/contracts/options.go index ef256a0..80580c6 100644 --- a/foreign/go/contracts/options.go +++ b/foreign/go/contracts/options.go
@@ -53,17 +53,29 @@ // OptionSpec is one entry of a resource's option catalog. // -// This is the discovery surface for the option keys a create command accepts: a -// key outside the catalog is refused at create, and the binary transports carry -// only the error code back, so there is no other way to learn which keys a -// given server knows. +// This is the discovery surface for the option keys a resource accepts: a key +// outside the catalog is refused, and the binary transports carry only the +// error code back, so there is no other way to learn which keys a given server +// knows and what an update may do with each. type OptionSpec struct { - // Key is the option key the create command accepts. + // Key is the option key. Key string // DefaultValue is the key's default, carrying the kind the server encodes // it under. Empty for a key with no default. Reuses HeaderValue because // options ride the user-headers codec. DefaultValue HeaderValue + // Updatable means an update may set the key. + Updatable bool + // Resettable means an update may reset the key to its default. + Resettable bool + // Secret means the key is write-only: accepted, never returned. + Secret bool + // ReadOnly means the server computes the key and returns it on reads, and + // never accepts it. + ReadOnly bool + // Critical means a server that does not understand the key must not serve + // the resource. + Critical bool // Description says what the option does, including the bounds its value is // checked against. Description string
diff --git a/foreign/go/contracts/resource_options_test.go b/foreign/go/contracts/resource_options_test.go new file mode 100644 index 0000000..cb49dce --- /dev/null +++ b/foreign/go/contracts/resource_options_test.go
@@ -0,0 +1,988 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package iggcon_test + +import ( + "bytes" + "encoding/base64" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "reflect" + "slices" + "strconv" + "strings" + "testing" + + binaryserialization "github.com/apache/iggy/foreign/go/binary_serialization" + iggcon "github.com/apache/iggy/foreign/go/contracts" + ierror "github.com/apache/iggy/foreign/go/errors" + "github.com/apache/iggy/foreign/go/internal/command" + "github.com/apache/iggy/foreign/go/internal/vsr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestResourceOptions_SharedVectors(t *testing.T) { + fixture := loadFixture(t) + + t.Run("given_shared_vectors_when_encoding_should_match_server_contract", func(t *testing.T) { + t.Run("constants", func(t *testing.T) { checkConstants(t, fixture.Constants) }) + t.Run("catalog", func(t *testing.T) { checkCatalog(t, fixture.Catalog) }) + t.Run("typed_surface", func(t *testing.T) { checkTypedSurface(t, fixture.Catalog) }) + t.Run("fingerprint", func(t *testing.T) { checkFingerprint(t, fixture.StreamKey) }) + + for _, vector := range fixture.DescribeOptions { + t.Run("describe_options/"+vector.Scope, func(t *testing.T) { + checkDescribeOptions(t, fixture.Constants, vector) + }) + } + for _, vector := range fixture.UpdateResource { + t.Run("update_resource/"+vector.Case, func(t *testing.T) { + body, err := updateCommand(t, vector.Scope, vector.Target, vector.Input).MarshalBinary() + require.NoError(t, err) + assert.Equal(t, vector.BodyHex, hex.EncodeToString(body)) + }) + } + for _, vector := range fixture.UpdateResourceErrors { + t.Run("update_resource_errors/"+vector.Case, func(t *testing.T) { + _, err := updateCommand(t, vector.Scope, vector.Target, vector.Input).MarshalBinary() + requireIggyError(t, vector.Error, err) + }) + } + for _, vector := range fixture.CreateStream { + t.Run("create_stream/"+vector.Case, func(t *testing.T) { + body, err := createStreamCommand(t, vector.Input).MarshalBinary() + require.NoError(t, err) + assert.Equal(t, vector.BodyHex, hex.EncodeToString(body)) + }) + } + for _, vector := range fixture.CreateTopic { + t.Run("create_topic/"+vector.Case, func(t *testing.T) { + body, err := createTopicCommand(t, vector.Input).MarshalBinary() + require.NoError(t, err) + assert.Equal(t, vector.BodyHex, hex.EncodeToString(body)) + }) + } + for _, vector := range fixture.CreateUser { + t.Run("create_user/"+vector.Case, func(t *testing.T) { + body, err := createUserCommand(t, vector.Input).MarshalBinary() + require.NoError(t, err) + assert.Equal(t, vector.BodyHex, hex.EncodeToString(body)) + }) + } + for _, vector := range fixture.StreamResponses { + t.Run("stream_responses/"+vector.Case, func(t *testing.T) { + checkStreamResponse(t, fixture.Constants, vector) + }) + } + for _, vector := range fixture.TopicResponses { + t.Run("topic_responses/"+vector.Case, func(t *testing.T) { + checkTopicResponse(t, fixture.Constants, vector) + }) + } + for _, vector := range fixture.UserResponses { + t.Run("user_responses/"+vector.Case, func(t *testing.T) { + checkUserResponse(t, fixture.Constants, vector) + }) + } + }) +} + +func TestResourceOptions_Diagnostics(t *testing.T) { + t.Run("given_raw_entry_of_a_key_newer_than_this_build_when_printing_should_redact_its_value", func(t *testing.T) { + secret := "future-secret-value" + raw := []iggcon.HeaderEntry{{ + Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte("future_secret")}, + Value: iggcon.HeaderValue{Kind: iggcon.String, Value: []byte(secret)}, + }} + name := "orders" + values := []any{ + iggcon.StreamOptions{Raw: raw}, + iggcon.TopicOptions{Raw: raw}, + iggcon.UserOptions{Raw: raw}, + iggcon.StreamUpdate{Name: &name, Options: raw}, + iggcon.TopicUpdate{Name: &name, Options: raw}, + iggcon.UserUpdate{Username: &name, Options: raw}, + } + for _, value := range values { + printed := fmt.Sprintf("%+v", value) + assert.NotContains(t, printed, secret, "%T", value) + assert.Contains(t, printed, "future_secret", "%T", value) + encoded, err := json.Marshal(value) + require.NoError(t, err) + assert.NotContains(t, string(encoded), secret, "JSON of %T", value) + assert.Contains(t, string(encoded), "future_secret", "JSON of %T", value) + } + }) + + t.Run("given_typed_fields_when_printing_should_show_every_field", func(t *testing.T) { + size := uint64(4096) + durability := iggcon.DurabilityPersisted + printed := fmt.Sprintf("%v", iggcon.TopicOptions{MaxTopicSize: &size, Durability: &durability}) + assert.Contains(t, printed, "MaxTopicSize:4096") + assert.Contains(t, printed, "Durability:persisted") + assert.Contains(t, printed, "SegmentSize:<nil>") + }) +} + +func TestResourceOptions_InvalidKeys(t *testing.T) { + fixture := loadFixture(t) + + t.Run("given_invalid_key_when_encoding_should_reject_before_network_io", func(t *testing.T) { + for _, vector := range fixture.StreamKey.InvalidKeys { + t.Run(vector.Case, func(t *testing.T) { + if vector.Input.RawHex != "" { + raw := decodeHex(t, vector.Input.RawHex) + _, err := iggcon.EncryptionKeyFingerprint(raw) + requireIggyError(t, vector.Error, err) + _, err = iggcon.EncryptionKeyOption(raw) + requireIggyError(t, vector.Error, err) + _, err = (&command.CreateStream{Name: "orders", Options: iggcon.StreamOptions{EncryptionKey: raw}}).MarshalBinary() + requireIggyError(t, vector.Error, err) + return + } + _, err := iggcon.DecodeEncryptionKey(vector.Input.Base64) + requireIggyError(t, vector.Error, err) + }) + } + + t.Run("empty_key_is_not_absent", func(t *testing.T) { + _, err := (&command.CreateStream{Name: "orders", Options: iggcon.StreamOptions{EncryptionKey: []byte{}}}).MarshalBinary() + assert.ErrorIs(t, err, ierror.ErrInvalidEncryptionKey) + }) + + t.Run("exactly_32_bytes_are_accepted", func(t *testing.T) { + key := decodeHex(t, fixture.StreamKey.KeyHex) + decoded, err := iggcon.DecodeEncryptionKey(fixture.StreamKey.KeyBase64) + require.NoError(t, err) + assert.Equal(t, key, decoded) + entry, err := iggcon.EncryptionKeyOption(key) + require.NoError(t, err) + assert.Equal(t, iggcon.HeaderEntry{ + Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte("encryption_key")}, + Value: iggcon.HeaderValue{Kind: iggcon.Raw, Value: key}, + }, entry) + _, err = (&command.CreateStream{Name: "orders", Options: iggcon.StreamOptions{EncryptionKey: key}}).MarshalBinary() + require.NoError(t, err) + }) + + t.Run("diagnostics_never_show_the_key", func(t *testing.T) { + key := decodeHex(t, fixture.StreamKey.KeyHex) + options := iggcon.StreamOptions{ + EncryptionKey: key, + Raw: []iggcon.HeaderEntry{{Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte("encryption_key")}, Value: iggcon.HeaderValue{Kind: iggcon.Raw, Value: key}}}, + } + request := &command.CreateStream{Name: "orders", Options: options} + secrets := []string{fixture.StreamKey.KeyHex, fixture.StreamKey.KeyBase64, hex.EncodeToString(key), string(key)} + for _, verb := range []string{"%v", "%+v", "%#v", "%s", "%x", "%d", "%q"} { + for _, value := range []any{options, &options, request, *request} { + printed := fmt.Sprintf(verb, value) + for _, secret := range secrets { + assert.NotContains(t, printed, secret, "%s of %T", verb, value) + } + } + } + for _, value := range []any{options, &options, request} { + encoded, err := json.Marshal(value) + require.NoError(t, err) + for _, secret := range secrets { + assert.NotContains(t, string(encoded), secret, "JSON of %T", value) + } + } + var logged bytes.Buffer + slog.New(slog.NewJSONHandler(&logged, nil)).Info("create", "options", options, "request", request) + slog.New(slog.NewTextHandler(&logged, nil)).Info("create", "options", options, "request", request) + for _, secret := range secrets { + assert.NotContains(t, logged.String(), secret, "slog output") + } + short := key[:31] + _, err := (&command.CreateStream{Name: "orders", Options: iggcon.StreamOptions{EncryptionKey: short}}).MarshalBinary() + require.Error(t, err) + assert.NotContains(t, err.Error(), hex.EncodeToString(short)) + }) + }) +} + +type fixtureFile struct { + SchemaVersion int `json:"schema_version"` + Constants fixtureConstants `json:"constants"` + Catalog []catalogRow `json:"catalog"` + DescribeOptions []describeVector `json:"describe_options"` + StreamKey streamKeyVector `json:"stream_key"` + UpdateResource []updateVector `json:"update_resource"` + UpdateResourceErrors []updateErrorVector `json:"update_resource_errors"` + CreateStream []createVector `json:"create_stream"` + CreateTopic []createVector `json:"create_topic"` + CreateUser []createVector `json:"create_user"` + StreamResponses []streamResponseVector `json:"stream_responses"` + TopicResponses []topicResponseVector `json:"topic_responses"` + UserResponses []userResponseVector `json:"user_responses"` +} + +type fixtureConstants struct { + ProtocolVersion protocolVersion `json:"protocol_version"` + CommandCodes map[string]commandCode `json:"command_codes"` + ErrorCodes map[string]uint32 `json:"error_codes"` + RetiredCommandCodes map[string]uint32 `json:"retired_command_codes"` + Operations map[string]operationEntry `json:"operations"` + RetiredOperations map[string]uint8 `json:"retired_operations"` + ScopeCodes map[string]uint8 `json:"scope_codes"` + HeaderKindCodes map[string]int `json:"header_kind_codes"` + OptionFlags map[string]uint8 `json:"option_flags"` + StreamKeyLength int `json:"stream_key_length"` + NeverReturnedKeys []string `json:"never_returned_keys"` +} + +type protocolVersion struct { + Packed uint32 `json:"packed"` + Text string `json:"text"` +} + +type commandCode struct { + Code uint32 `json:"code"` + Operation *uint8 `json:"operation"` +} + +type operationEntry struct { + Value uint8 `json:"value"` + Internal bool `json:"internal"` + RequestOperation uint8 `json:"request_operation"` +} + +type catalogRow struct { + Key string `json:"key"` + Scopes []string `json:"scopes"` + Mutability string `json:"mutability"` + Visibility string `json:"visibility"` + Storage string `json:"storage"` + Flags uint8 `json:"flags"` + Described bool `json:"described"` +} + +type describeVector struct { + Scope string `json:"scope"` + RequestHex string `json:"request_hex"` + ResponseHex string `json:"response_hex"` + Entries []describeEntry `json:"entries"` +} + +type describeEntry struct { + Key string `json:"key"` + Kind string `json:"kind"` + Flags uint8 `json:"flags"` + Default *string `json:"default"` +} + +type streamKeyVector struct { + KeyHex string `json:"key_hex"` + KeyBase64 string `json:"key_base64"` + Fingerprint string `json:"fingerprint"` + InvalidKeys []invalidKey `json:"invalid_keys"` +} + +type invalidKey struct { + Case string `json:"case"` + Input struct { + RawHex string `json:"raw_hex"` + Base64 string `json:"base64"` + } `json:"input"` + Error fixtureIggyCode `json:"error"` +} + +type fixtureIggyCode struct { + Name string `json:"name"` + Code uint32 `json:"code"` +} + +type updateVector struct { + Case string `json:"case"` + Scope string `json:"scope"` + Target map[string]json.RawMessage `json:"target"` + Input map[string]json.RawMessage `json:"input"` + BodyHex string `json:"body_hex"` +} + +type updateErrorVector struct { + Case string `json:"case"` + Scope string `json:"scope"` + Target map[string]json.RawMessage `json:"target"` + Input map[string]json.RawMessage `json:"input"` + Error fixtureIggyCode `json:"error"` +} + +type createVector struct { + Case string `json:"case"` + Input map[string]json.RawMessage `json:"input"` + BodyHex string `json:"body_hex"` +} + +type optionView struct { + Value string `json:"value"` + Kind string `json:"kind"` + Explicit bool `json:"explicit"` +} + +type streamResponseVector struct { + Case string `json:"case"` + ResponseHex string `json:"response_hex"` + Expected struct { + Id uint32 `json:"id"` + CreatedAt uint64 `json:"created_at"` + Name string `json:"name"` + TopicsCount uint32 `json:"topics_count"` + SizeBytes uint64 `json:"size_bytes"` + MessagesCount uint64 `json:"messages_count"` + Options map[string]optionView `json:"options"` + EncryptionKeySource string `json:"encryption_key_source"` + EncryptionKeyFingerprint *string `json:"encryption_key_fingerprint"` + EncryptionKekFingerprints []string `json:"encryption_kek_fingerprints"` + } `json:"expected"` +} + +type topicResponseVector struct { + Case string `json:"case"` + ResponseHex string `json:"response_hex"` + Expected struct { + Id uint32 `json:"id"` + CreatedAt uint64 `json:"created_at"` + Name string `json:"name"` + PartitionsCount uint32 `json:"partitions_count"` + SizeBytes uint64 `json:"size_bytes"` + MessagesCount uint64 `json:"messages_count"` + Options map[string]optionView `json:"options"` + CompressionAlgorithm string `json:"compression_algorithm"` + MessageExpiry string `json:"message_expiry"` + MaxTopicSize string `json:"max_topic_size"` + } `json:"expected"` +} + +type userResponseVector struct { + Case string `json:"case"` + ResponseHex string `json:"response_hex"` + Expected struct { + Id uint32 `json:"id"` + CreatedAt uint64 `json:"created_at"` + Status string `json:"status"` + Username string `json:"username"` + Options map[string]optionView `json:"options"` + } `json:"expected"` +} + +func loadFixture(t *testing.T) fixtureFile { + t.Helper() + content, err := os.ReadFile(filepath.Join("testdata", "resource_options.json")) + require.NoError(t, err) + var fixture fixtureFile + require.NoError(t, json.Unmarshal(content, &fixture)) + require.Equal(t, 1, fixture.SchemaVersion) + return fixture +} + +func checkConstants(t *testing.T, constants fixtureConstants) { + assert.Equal(t, constants.ProtocolVersion.Packed, vsr.ProtocolVersion, constants.ProtocolVersion.Text) + assert.Equal(t, uint32(ierror.CannotModifyRootUserCode), constants.ErrorCodes["cannot_modify_root_user"]) + + codes := map[string]command.Code{ + "create_stream": command.CreateStreamCode, + "create_topic": command.CreateTopicCode, + "create_user": command.CreateUserCode, + "describe_options": command.DescribeOptionsCode, + "get_stream": command.GetStreamCode, + "get_streams": command.GetStreamsCode, + "get_topic": command.GetTopicCode, + "get_topics": command.GetTopicsCode, + "get_user": command.GetUserCode, + "get_users": command.GetUsersCode, + "update_resource": command.UpdateResourceCode, + } + require.Len(t, codes, len(constants.CommandCodes)) + for name, want := range constants.CommandCodes { + code, ok := codes[name] + require.True(t, ok, "the SDK has no command code for %s", name) + assert.Equal(t, want.Code, uint32(code), name) + if want.Operation == nil { + assert.Equal(t, vsr.OperationNonReplicated, vsr.OperationForCode(want.Code), name) + continue + } + assert.Equal(t, vsr.Operation(*want.Operation), vsr.OperationForCode(want.Code), name) + } + for name, code := range constants.RetiredCommandCodes { + _, replicated := vsr.ReplicatedOperation(code) + assert.False(t, replicated, "retired %s code %d", name, code) + } + + operations := map[string]vsr.Operation{ + "rewrap_stream_keys": vsr.OperationRewrapStreamKeys, + "update_resource": vsr.OperationUpdateResource, + "update_stream_resolved": vsr.OperationUpdateStreamResolved, + "update_topic_resolved": vsr.OperationUpdateTopicResolved, + "update_user_resolved": vsr.OperationUpdateUserResolved, + } + require.Len(t, operations, len(constants.Operations)) + for name, want := range constants.Operations { + operation := operations[name] + assert.Equal(t, want.Value, uint8(operation), name) + assert.Equal(t, want.Internal, vsr.IsInternal(operation), name) + assert.True(t, vsr.IsKnownOperation(operation), name) + assert.True(t, vsr.ReplyOperationMatches(vsr.Operation(want.RequestOperation), operation), name) + } + for name, value := range constants.RetiredOperations { + assert.False(t, vsr.IsKnownOperation(vsr.Operation(value)), "retired %s operation %d", name, value) + } + + assert.Equal(t, map[string]uint8{ + "topic": uint8(iggcon.OptionsScopeTopic), + "stream": uint8(iggcon.OptionsScopeStream), + "user": uint8(iggcon.OptionsScopeUser), + }, constants.ScopeCodes) + for name, code := range constants.HeaderKindCodes { + assert.Equal(t, code, int(headerKind(t, name)), name) + } + assert.Equal(t, constants.StreamKeyLength, iggcon.StreamEncryptionKeyLength) +} + +// checkCatalog proves the typed key constants name exactly the described keys +// of each scope, and that a reset of any catalog key fails before sending with +// the category server admission would return. +func checkCatalog(t *testing.T, catalog []catalogRow) { + typed := map[string][]string{ + "stream": { + string(iggcon.StreamOptionName), + string(iggcon.StreamOptionEncryptionKey), + string(iggcon.StreamOptionEncryptionKeySource), + string(iggcon.StreamOptionEncryptionKeyFingerprint), + string(iggcon.StreamOptionEncryptionKekFingerprint), + }, + "topic": { + string(iggcon.TopicOptionName), + string(iggcon.TopicOptionCompressionAlgorithm), + string(iggcon.TopicOptionMessageExpiry), + string(iggcon.TopicOptionMaxTopicSize), + string(iggcon.TopicOptionSegmentSize), + string(iggcon.TopicOptionDurability), + string(iggcon.TopicOptionConsumerOffsetDurability), + string(iggcon.TopicOptionPartitionResizePolicy), + string(iggcon.TopicOptionMessagesRequiredToSave), + string(iggcon.TopicOptionSizeOfMessagesRequiredToSave), + string(iggcon.TopicOptionPreallocateSegments), + }, + "user": { + string(iggcon.UserOptionUsername), + string(iggcon.UserOptionStatus), + }, + } + described := map[string][]string{} + for _, row := range catalog { + if !row.Described { + continue + } + for _, scope := range row.Scopes { + described[scope] = append(described[scope], row.Key) + } + } + for scope, keys := range typed { + assert.ElementsMatch(t, described[scope], keys, scope) + } + + targets := map[string]map[string]json.RawMessage{ + "stream": {"stream_id": json.RawMessage("1")}, + "topic": {"stream_id": json.RawMessage("1"), "topic_id": json.RawMessage("2")}, + "user": {"user_id": json.RawMessage("1")}, + } + for _, row := range catalog { + for scope, target := range targets { + t.Run(scope+"/reset_"+row.Key, func(t *testing.T) { + input := map[string]json.RawMessage{"reset": json.RawMessage(`["` + row.Key + `"]`)} + _, err := updateCommand(t, scope, target, input).MarshalBinary() + switch { + case !slices.Contains(row.Scopes, scope): + assert.ErrorIs(t, err, ierror.ErrUnsupportedOptionKey) + case row.Visibility == "internal" || row.Visibility == "read_only": + assert.ErrorIs(t, err, ierror.ErrReadOnlyOption) + case row.Mutability != "updatable": + assert.ErrorIs(t, err, ierror.ErrOptionNotUpdatable) + case row.Flags&2 == 0: + assert.ErrorIs(t, err, ierror.ErrOptionNotResettable) + default: + assert.NoError(t, err) + } + }) + } + } +} + +// checkTypedSurface proves the typed API follows each catalog row: a create +// field for every client-settable map key, an update field for every +// updatable key, a reset constant for every resettable key and an accessor +// for every read-only key. A row whose mutability changes fails here until +// the typed API follows it. +func checkTypedSurface(t *testing.T, catalog []catalogRow) { + type surface struct { + create, update, reset, accessor string + } + typed := map[string]map[string]surface{ + "stream": { + "name": {update: "Name"}, + "encryption_key": {create: "EncryptionKey"}, + "encryption_key_source": {accessor: "EncryptionKeySource"}, + "encryption_key_fingerprint": {accessor: "EncryptionKeyFingerprint"}, + "encryption_kek_fingerprint": {accessor: "EncryptionKekFingerprints"}, + }, + "topic": { + "name": {update: "Name"}, + "compression_algorithm": {create: "CompressionAlgorithm"}, + "message_expiry": {create: "MessageExpiry", update: "MessageExpiry", reset: string(iggcon.TopicOptionMessageExpiry)}, + "max_topic_size": {create: "MaxTopicSize", update: "MaxTopicSize", reset: string(iggcon.TopicOptionMaxTopicSize)}, + "segment_size": {create: "SegmentSize"}, + "durability": {create: "Durability"}, + "consumer_offset_durability": {create: "ConsumerOffsetDurability"}, + "partition_resize_policy": {create: "PartitionResizePolicy"}, + "messages_required_to_save": {create: "MessagesRequiredToSave"}, + "size_of_messages_required_to_save": {create: "SizeOfMessagesRequiredToSave"}, + "preallocate_segments": {create: "PreallocateSegments"}, + }, + "user": { + "username": {update: "Username"}, + "status": {update: "Status"}, + }, + } + createTypes := map[string]reflect.Type{ + "stream": reflect.TypeFor[iggcon.StreamOptions](), + "topic": reflect.TypeFor[iggcon.TopicOptions](), + "user": reflect.TypeFor[iggcon.UserOptions](), + } + updateTypes := map[string]reflect.Type{ + "stream": reflect.TypeFor[iggcon.StreamUpdate](), + "topic": reflect.TypeFor[iggcon.TopicUpdate](), + "user": reflect.TypeFor[iggcon.UserUpdate](), + } + accessorTypes := map[string]reflect.Type{"stream": reflect.TypeFor[iggcon.Stream]()} + for _, row := range catalog { + if row.Visibility == "internal" { + continue + } + for _, scope := range row.Scopes { + name := scope + "/" + row.Key + expected, known := typed[scope][row.Key] + if !assert.True(t, known, "%s has no typed surface", name) { + continue + } + clientSettable := row.Storage == "map" && row.Mutability != "read_only" + assert.Equal(t, clientSettable, expected.create != "", "%s create field", name) + if expected.create != "" { + _, found := createTypes[scope].FieldByName(expected.create) + assert.True(t, found, "%s create field %s", name, expected.create) + } + assert.Equal(t, row.Mutability == "updatable", expected.update != "", "%s update field", name) + if expected.update != "" { + _, found := updateTypes[scope].FieldByName(expected.update) + assert.True(t, found, "%s update field %s", name, expected.update) + } + assert.Equal(t, row.Flags&2 != 0, expected.reset != "", "%s reset constant", name) + if expected.reset != "" { + assert.Equal(t, row.Key, expected.reset, "%s reset constant", name) + } + assert.Equal(t, row.Mutability == "read_only", expected.accessor != "", "%s accessor", name) + if expected.accessor != "" { + _, found := accessorTypes[scope].MethodByName(expected.accessor) + assert.True(t, found, "%s accessor %s", name, expected.accessor) + } + } + } + for scope, keys := range typed { + for key := range keys { + assert.True(t, slices.ContainsFunc(catalog, func(row catalogRow) bool { + return row.Key == key && slices.Contains(row.Scopes, scope) + }), "%s/%s is not in the catalog", scope, key) + } + } +} + +func checkFingerprint(t *testing.T, vector streamKeyVector) { + key := decodeHex(t, vector.KeyHex) + fingerprint, err := iggcon.EncryptionKeyFingerprint(key) + require.NoError(t, err) + assert.Equal(t, vector.Fingerprint, fingerprint) + + decoded, err := iggcon.DecodeEncryptionKey(vector.KeyBase64) + require.NoError(t, err) + fingerprint, err = iggcon.EncryptionKeyFingerprint(decoded) + require.NoError(t, err) + assert.Equal(t, vector.Fingerprint, fingerprint) +} + +func checkDescribeOptions(t *testing.T, constants fixtureConstants, vector describeVector) { + request, err := (&command.DescribeOptions{Scope: optionsScope(t, vector.Scope)}).MarshalBinary() + require.NoError(t, err) + assert.Equal(t, vector.RequestHex, hex.EncodeToString(request)) + + specs, err := binaryserialization.DeserializeOptionSpecs(decodeHex(t, vector.ResponseHex)) + require.NoError(t, err) + got := make([]describeEntry, 0, len(specs)) + for _, spec := range specs { + var flags uint8 + for name, set := range map[string]bool{ + "updatable": spec.Updatable, + "resettable": spec.Resettable, + "secret": spec.Secret, + "read_only": spec.ReadOnly, + "critical": spec.Critical, + } { + if set { + flags |= constants.OptionFlags[name] + } + } + var defaultText *string + if len(spec.DefaultValue.Value) > 0 { + text := valueText(t, spec.DefaultValue) + defaultText = &text + } + assert.NotEmpty(t, spec.Description, spec.Key) + got = append(got, describeEntry{Key: spec.Key, Kind: kindName(t, spec.DefaultValue.Kind), Flags: flags, Default: defaultText}) + } + assert.Equal(t, vector.Entries, got) +} + +func checkStreamResponse(t *testing.T, constants fixtureConstants, vector streamResponseVector) { + response := decodeHex(t, vector.ResponseHex) + details, err := binaryserialization.DeserializeStream(response) + require.NoError(t, err) + streams, err := binaryserialization.DeserializeStreams(response) + require.NoError(t, err) + require.Len(t, streams, 1) + assert.Equal(t, details.Stream, streams[0]) + + stream := details.Stream + expected := vector.Expected + assert.Equal(t, expected.Id, stream.Id) + assert.Equal(t, expected.CreatedAt, stream.CreatedAt) + assert.Equal(t, expected.Name, stream.Name) + assert.Equal(t, expected.TopicsCount, stream.TopicsCount) + assert.Equal(t, expected.SizeBytes, stream.SizeBytes) + assert.Equal(t, expected.MessagesCount, stream.MessagesCount) + assert.Equal(t, expected.Options, optionViews(t, stream.Options, stream.DerivedOptions)) + requireNoSecret(t, constants, stream.Options, stream.DerivedOptions) + + source, ok := stream.EncryptionKeySource() + assert.True(t, ok) + assert.Equal(t, iggcon.StreamEncryptionSource(expected.EncryptionKeySource), source) + fingerprint, ok := details.EncryptionKeyFingerprint() + if expected.EncryptionKeyFingerprint == nil { + assert.False(t, ok) + } else { + assert.True(t, ok) + assert.Equal(t, *expected.EncryptionKeyFingerprint, fingerprint) + } + if len(expected.EncryptionKekFingerprints) == 0 { + assert.Empty(t, stream.EncryptionKekFingerprints()) + } else { + assert.Equal(t, expected.EncryptionKekFingerprints, stream.EncryptionKekFingerprints()) + } +} + +func checkTopicResponse(t *testing.T, constants fixtureConstants, vector topicResponseVector) { + details, err := binaryserialization.DeserializeTopic(decodeHex(t, vector.ResponseHex)) + require.NoError(t, err) + topic := details.Topic + expected := vector.Expected + assert.Equal(t, expected.Id, topic.Id) + assert.Equal(t, expected.CreatedAt, topic.CreatedAt) + assert.Equal(t, expected.Name, topic.Name) + assert.Equal(t, expected.PartitionsCount, topic.PartitionsCount) + assert.Equal(t, expected.SizeBytes, topic.Size) + assert.Equal(t, expected.MessagesCount, topic.MessagesCount) + assert.Equal(t, expected.Options, optionViews(t, topic.Options, topic.DerivedOptions)) + requireNoSecret(t, constants, topic.Options, topic.DerivedOptions) + + compression := map[string]iggcon.CompressionAlgorithm{"none": iggcon.CompressionAlgorithmNone, "gzip": iggcon.CompressionAlgorithmGzip} + assert.Equal(t, uint8(compression[expected.CompressionAlgorithm]), topic.CompressionAlgorithm) + assert.Equal(t, parseUint64(t, expected.MessageExpiry), uint64(topic.MessageExpiry)) + assert.Equal(t, parseUint64(t, expected.MaxTopicSize), topic.MaxTopicSize) +} + +func checkUserResponse(t *testing.T, constants fixtureConstants, vector userResponseVector) { + details, err := binaryserialization.DeserializeUser(decodeHex(t, vector.ResponseHex)) + require.NoError(t, err) + user := details.UserInfo + expected := vector.Expected + assert.Equal(t, expected.Id, user.Id) + assert.Equal(t, expected.CreatedAt, user.CreatedAt) + assert.Equal(t, userStatus(t, expected.Status), user.Status) + assert.Equal(t, expected.Username, user.Username) + assert.Equal(t, expected.Options, optionViews(t, user.Options, user.DerivedOptions)) + requireNoSecret(t, constants, user.Options, user.DerivedOptions) + assert.Nil(t, details.Permissions) +} + +func updateCommand(t *testing.T, scope string, target, input map[string]json.RawMessage) command.Command { + t.Helper() + var reset []string + if raw, ok := input["reset"]; ok { + require.NoError(t, json.Unmarshal(raw, &reset)) + } + switch scope { + case "topic": + update := &command.UpdateTopic{ + StreamId: identifier(t, target["stream_id"]), + TopicId: identifier(t, target["topic_id"]), + Name: optionalText(t, input["name"]), + } + if raw, ok := input["message_expiry"]; ok { + expiry := iggcon.Duration(parseUint64(t, text(t, raw))) + update.MessageExpiry = &expiry + } + if raw, ok := input["max_topic_size"]; ok { + size := parseUint64(t, text(t, raw)) + update.MaxTopicSize = &size + } + for _, key := range reset { + update.Reset = append(update.Reset, iggcon.TopicOptionKey(key)) + } + return update + case "stream": + update := &command.UpdateStream{ + StreamId: identifier(t, target["stream_id"]), + Name: optionalText(t, input["name"]), + } + for _, key := range reset { + update.Reset = append(update.Reset, iggcon.StreamOptionKey(key)) + } + return update + case "user": + update := &command.UpdateUser{ + UserID: identifier(t, target["user_id"]), + Username: optionalText(t, input["username"]), + } + if raw, ok := input["status"]; ok { + status := userStatus(t, text(t, raw)) + update.Status = &status + } + for _, key := range reset { + update.Reset = append(update.Reset, iggcon.UserOptionKey(key)) + } + return update + default: + require.FailNow(t, "unknown scope", scope) + return nil + } +} + +func createStreamCommand(t *testing.T, input map[string]json.RawMessage) *command.CreateStream { + t.Helper() + request := &command.CreateStream{Name: text(t, input["name"])} + if raw, ok := input["encryption_key"]; ok { + var key struct { + RawHex string `json:"raw_hex"` + } + require.NoError(t, json.Unmarshal(raw, &key)) + request.Options.EncryptionKey = decodeHex(t, key.RawHex) + } + if raw, ok := input["raw"]; ok { + request.Options.Raw = rawEntries(t, raw) + } + return request +} + +func createTopicCommand(t *testing.T, input map[string]json.RawMessage) *command.CreateTopic { + t.Helper() + partitions := uint32(parseUint64(t, string(input["partitions_count"]))) + compression := map[string]iggcon.CompressionAlgorithm{"none": iggcon.CompressionAlgorithmNone, "gzip": iggcon.CompressionAlgorithmGzip}[text(t, input["compression_algorithm"])] + expiry := iggcon.Duration(parseUint64(t, text(t, input["message_expiry"]))) + maxTopicSize := parseUint64(t, text(t, input["max_topic_size"])) + segmentSize := parseUint64(t, text(t, input["segment_size"])) + durability := iggcon.Durability(text(t, input["durability"])) + consumerOffsetDurability := iggcon.Durability(text(t, input["consumer_offset_durability"])) + return &command.CreateTopic{ + StreamId: identifier(t, input["stream_id"]), + Name: text(t, input["name"]), + Options: iggcon.TopicOptions{ + PartitionsCount: &partitions, + CompressionAlgorithm: &compression, + MessageExpiry: &expiry, + MaxTopicSize: &maxTopicSize, + SegmentSize: &segmentSize, + Durability: &durability, + ConsumerOffsetDurability: &consumerOffsetDurability, + }, + } +} + +func createUserCommand(t *testing.T, input map[string]json.RawMessage) *command.CreateUser { + t.Helper() + request := &command.CreateUser{ + Username: text(t, input["username"]), + Password: text(t, input["password"]), + Status: userStatus(t, text(t, input["status"])), + } + if raw, ok := input["raw"]; ok { + request.Options.Raw = rawEntries(t, raw) + } + return request +} + +func rawEntries(t *testing.T, raw json.RawMessage) []iggcon.HeaderEntry { + t.Helper() + var values map[string]string + require.NoError(t, json.Unmarshal(raw, &values)) + entries := make([]iggcon.HeaderEntry, 0, len(values)) + for key, value := range values { + entries = append(entries, iggcon.HeaderEntry{ + Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte(key)}, + Value: iggcon.HeaderValue{Kind: iggcon.String, Value: []byte(value)}, + }) + } + return entries +} + +// optionViews merges the explicit and derived maps into the fixture's HTTP +// JSON shape, so a whole map compares at once. +func optionViews(t *testing.T, explicit, derived map[string]iggcon.HeaderValue) map[string]optionView { + t.Helper() + views := make(map[string]optionView, len(explicit)+len(derived)) + for provenance, options := range map[bool]map[string]iggcon.HeaderValue{true: explicit, false: derived} { + for key, value := range options { + _, duplicate := views[key] + require.False(t, duplicate, "%s is both explicit and derived", key) + views[key] = optionView{Value: valueText(t, value), Kind: kindName(t, value.Kind), Explicit: provenance} + } + } + return views +} + +func requireNoSecret(t *testing.T, constants fixtureConstants, maps ...map[string]iggcon.HeaderValue) { + t.Helper() + for _, options := range maps { + for _, key := range constants.NeverReturnedKeys { + assert.NotContains(t, options, key) + } + } +} + +func valueText(t *testing.T, value iggcon.HeaderValue) string { + t.Helper() + switch value.Kind { + case iggcon.String: + return string(value.Value) + case iggcon.Bool: + require.Len(t, value.Value, 1) + return strconv.FormatBool(value.Value[0] == 1) + case iggcon.Uint32: + require.Len(t, value.Value, 4) + return strconv.FormatUint(uint64(binary.LittleEndian.Uint32(value.Value)), 10) + case iggcon.Uint64: + require.Len(t, value.Value, 8) + return strconv.FormatUint(binary.LittleEndian.Uint64(value.Value), 10) + case iggcon.Raw: + return base64.StdEncoding.EncodeToString(value.Value) + default: + require.FailNow(t, "unexpected option kind", "%d", value.Kind) + return "" + } +} + +func kindName(t *testing.T, kind iggcon.HeaderKind) string { + t.Helper() + names := map[iggcon.HeaderKind]string{iggcon.Raw: "raw", iggcon.String: "string", iggcon.Bool: "bool", iggcon.Uint32: "uint32", iggcon.Uint64: "uint64"} + name, ok := names[kind] + require.True(t, ok, "unexpected kind %d", kind) + return name +} + +func headerKind(t *testing.T, name string) iggcon.HeaderKind { + t.Helper() + kinds := map[string]iggcon.HeaderKind{"raw": iggcon.Raw, "string": iggcon.String, "bool": iggcon.Bool, "uint32": iggcon.Uint32, "uint64": iggcon.Uint64} + kind, ok := kinds[name] + require.True(t, ok, "unexpected kind %s", name) + return kind +} + +func optionsScope(t *testing.T, name string) iggcon.OptionsScope { + t.Helper() + scopes := map[string]iggcon.OptionsScope{"topic": iggcon.OptionsScopeTopic, "stream": iggcon.OptionsScopeStream, "user": iggcon.OptionsScopeUser} + scope, ok := scopes[name] + require.True(t, ok, "unexpected scope %s", name) + return scope +} + +func userStatus(t *testing.T, name string) iggcon.UserStatus { + t.Helper() + statuses := map[string]iggcon.UserStatus{"active": iggcon.Active, "inactive": iggcon.Inactive} + status, ok := statuses[name] + require.True(t, ok, "unexpected status %s", name) + return status +} + +// identifier reads a fixture target: a number is a numeric id, a string a +// named one. +func identifier(t *testing.T, raw json.RawMessage) iggcon.Identifier { + t.Helper() + var id iggcon.Identifier + var err error + if strings.HasPrefix(string(raw), `"`) { + id, err = iggcon.NewIdentifier(text(t, raw)) + } else { + id, err = iggcon.NewIdentifier(uint32(parseUint64(t, string(raw)))) + } + require.NoError(t, err) + return id +} + +func optionalText(t *testing.T, raw json.RawMessage) *string { + t.Helper() + if raw == nil { + return nil + } + value := text(t, raw) + return &value +} + +func text(t *testing.T, raw json.RawMessage) string { + t.Helper() + var value string + require.NoError(t, json.Unmarshal(raw, &value)) + return value +} + +func parseUint64(t *testing.T, value string) uint64 { + t.Helper() + parsed, err := strconv.ParseUint(value, 10, 64) + require.NoError(t, err) + return parsed +} + +func decodeHex(t *testing.T, value string) []byte { + t.Helper() + decoded, err := hex.DecodeString(value) + require.NoError(t, err) + return decoded +} + +func requireIggyError(t *testing.T, want fixtureIggyCode, err error) { + t.Helper() + require.Error(t, err) + var iggyError ierror.IggyError + require.True(t, errors.As(err, &iggyError), "%v is not an IggyError", err) + assert.Equal(t, want.Code, uint32(iggyError.Code()), "%s, got %v", want.Name, err) + assert.ErrorIs(t, err, ierror.FromCode(ierror.Code(want.Code))) +}
diff --git a/foreign/go/contracts/stream.go b/foreign/go/contracts/stream.go index 03091a5..b607b16 100644 --- a/foreign/go/contracts/stream.go +++ b/foreign/go/contracts/stream.go
@@ -17,19 +17,25 @@ package iggcon +import "strings" + type CreateStreamRequest struct { StreamId int `json:"streamId"` Name string `json:"name"` } type Stream struct { - Id uint32 `json:"id"` - Name string `json:"name"` - SizeBytes uint64 `json:"sizeBytes"` - CreatedAt uint64 `json:"createdAt"` - MessagesCount uint64 `json:"messagesCount"` - TopicsCount uint32 `json:"topicsCount"` - Options map[string]HeaderValue `json:"options,omitempty"` + Id uint32 `json:"id"` + Name string `json:"name"` + SizeBytes uint64 `json:"sizeBytes"` + CreatedAt uint64 `json:"createdAt"` + MessagesCount uint64 `json:"messagesCount"` + TopicsCount uint32 `json:"topicsCount"` + // Options are the keys the client set. + Options map[string]HeaderValue `json:"options,omitempty"` + // DerivedOptions are the values the server filled or computes, such as the + // encryption key projections. + DerivedOptions map[string]HeaderValue `json:"derivedOptions,omitempty"` } type StreamDetails struct { @@ -40,3 +46,36 @@ type GetStreamRequest struct { StreamID Identifier } + +// EncryptionKeySource reports who chose the stream's at-rest key, false when +// the server reported no source. +func (s Stream) EncryptionKeySource() (StreamEncryptionSource, bool) { + source, ok := s.textOption(StreamOptionEncryptionKeySource) + return StreamEncryptionSource(source), ok +} + +// EncryptionKeyFingerprint reports the fingerprint of the stream's key, false +// for a plaintext stream. EncryptionKeyFingerprint of a key computes the same. +func (s Stream) EncryptionKeyFingerprint() (string, bool) { + return s.textOption(StreamOptionEncryptionKeyFingerprint) +} + +// EncryptionKekFingerprints reports the fingerprints of the server keys that +// wrap the stream's key: one outside a key rotation, none for a plaintext +// stream. +func (s Stream) EncryptionKekFingerprints() []string { + joined, ok := s.textOption(StreamOptionEncryptionKekFingerprint) + if !ok || joined == "" { + return nil + } + return strings.Split(joined, ",") +} + +func (s Stream) textOption(key StreamOptionKey) (string, bool) { + for _, options := range []map[string]HeaderValue{s.DerivedOptions, s.Options} { + if value, ok := options[string(key)]; ok && value.Kind == String { + return string(value.Value), true + } + } + return "", false +}
diff --git a/foreign/go/contracts/stream_options.go b/foreign/go/contracts/stream_options.go new file mode 100644 index 0000000..89be815 --- /dev/null +++ b/foreign/go/contracts/stream_options.go
@@ -0,0 +1,138 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package iggcon + +import ( + "crypto/sha256" + "encoding/base64" + "encoding/hex" + + ierror "github.com/apache/iggy/foreign/go/errors" +) + +// StreamOptionKey names a stream option of the server catalog. +type StreamOptionKey string + +const ( + // StreamOptionName is the stream's name, set by CreateStream and changed + // by UpdateStream. + StreamOptionName StreamOptionKey = "name" + // StreamOptionEncryptionKey is the stream's own data key: create-only and + // write-only. + StreamOptionEncryptionKey StreamOptionKey = "encryption_key" + // StreamOptionEncryptionKeySource is read-only: who chose the stream's key. + StreamOptionEncryptionKeySource StreamOptionKey = "encryption_key_source" + // StreamOptionEncryptionKeyFingerprint is read-only: the fingerprint of the + // stream's key. + StreamOptionEncryptionKeyFingerprint StreamOptionKey = "encryption_key_fingerprint" + // StreamOptionEncryptionKekFingerprint is read-only: the comma-joined + // fingerprints of the server keys that wrap the stream's key. + StreamOptionEncryptionKekFingerprint StreamOptionKey = "encryption_kek_fingerprint" +) + +// StreamEncryptionKeyLength is the size of a stream data key in bytes. +const StreamEncryptionKeyLength = 32 + +// encryptionKeyFingerprintLabel prefixes the key in the fingerprint digest. +const encryptionKeyFingerprintLabel = "iggy/stream-key-fingerprint/v1" + +// encryptionKeyFingerprintBytes is how much of the digest a fingerprint keeps. +const encryptionKeyFingerprintBytes = 8 + +// StreamEncryptionSource says who chose a stream's at-rest key. +type StreamEncryptionSource string + +const ( + // StreamEncryptionSourceStream is a key the client sent at create. + StreamEncryptionSourceStream StreamEncryptionSource = "stream" + // StreamEncryptionSourceServer is a key the server generated at create. + StreamEncryptionSourceServer StreamEncryptionSource = "server" + // StreamEncryptionSourceNone is a plaintext stream. + StreamEncryptionSourceNone StreamEncryptionSource = "none" +) + +// StreamOptions are the options CreateStream may carry. A nil *StreamOptions +// selects the defaults. +// +// Formatting a StreamOptions with any fmt verb, or encoding it as JSON, +// redacts the key and every Raw value. +type StreamOptions struct { + // EncryptionKey is the stream's own data key, exactly 32 random bytes. + // Nil follows the server: it generates a key when its encryption is + // enabled and leaves the stream plaintext otherwise. The key travels inside + // the create request over the client's transport, so TLS is the caller's + // job. The server stores the key wrapped and never returns it. Reads + // report only its source and fingerprints. Never derive it from a + // password: the fingerprint would then be an offline guessing oracle. + EncryptionKey []byte + // Raw carries keys newer than this build. A typed field wins on collision. + Raw []HeaderEntry +} + +// StreamUpdate is a patch of a stream's options. Absent keys keep their +// stored values. +type StreamUpdate struct { + // Name renames the stream when set. + Name *string + // Reset names keys to return to their defaults. No stream key is + // resettable, so a non-empty list is refused before anything is sent. + Reset []StreamOptionKey + // RawReset names keys to reset that this build may not know. The server + // decides whether each one may be reset, and a patch that carries any raw + // name or Options entry is checked by the server alone. + RawReset []string + // Options carries updatable keys newer than this build. A typed field wins + // on collision. + Options []HeaderEntry +} + +// EncryptionKeyOption builds the encryption_key entry for a raw options list. +// StreamOptions.EncryptionKey is the typed way to set it. +func EncryptionKeyOption(key []byte) (HeaderEntry, error) { + if len(key) != StreamEncryptionKeyLength { + return HeaderEntry{}, ierror.ErrInvalidEncryptionKey + } + return HeaderEntry{ + Key: HeaderKey{Kind: String, Value: []byte(StreamOptionEncryptionKey)}, + Value: HeaderValue{Kind: Raw, Value: key}, + }, nil +} + +// DecodeEncryptionKey decodes a stream key from canonical standard base64 +// with padding. Any other spelling, whitespace included, and any length but +// 32 bytes is refused, so one key has exactly one text form. +func DecodeEncryptionKey(encoded string) ([]byte, error) { + key, err := base64.StdEncoding.Strict().DecodeString(encoded) + if err != nil || len(key) != StreamEncryptionKeyLength || base64.StdEncoding.EncodeToString(key) != encoded { + return nil, ierror.ErrInvalidEncryptionKey + } + return key, nil +} + +// EncryptionKeyFingerprint computes the fingerprint a stream created with key +// reports, 16 lowercase hex characters, so a caller can check which key a +// stream uses without the server returning key bytes. +func EncryptionKeyFingerprint(key []byte) (string, error) { + if len(key) != StreamEncryptionKeyLength { + return "", ierror.ErrInvalidEncryptionKey + } + digest := sha256.New() + digest.Write([]byte(encryptionKeyFingerprintLabel)) + digest.Write(key) + return hex.EncodeToString(digest.Sum(nil)[:encryptionKeyFingerprintBytes]), nil +}
diff --git a/foreign/go/contracts/testdata/resource_options.json b/foreign/go/contracts/testdata/resource_options.json new file mode 100644 index 0000000..c5cc2ac --- /dev/null +++ b/foreign/go/contracts/testdata/resource_options.json
@@ -0,0 +1,1619 @@ +{ + "schema_version": 1, + "generated_by": "core/common/examples/resource_options_fixtures.rs", + "conventions": { + "byte_strings": "Every *_hex field is lowercase hex without separators.", + "errors": "name is the snake_case IggyError name and code its numeric code.", + "max_topic_size": "Bytes. 18446744073709551615 means unlimited.", + "message_expiry": "Microseconds. 18446744073709551615 means never expire.", + "never_returned_keys": "No response, decoded projection or accessor may contain these keys or their values.", + "option_maps": "Option maps use the HTTP JSON form: value as reversible text, kind, and explicit provenance. Request blocks mark every entry explicit.", + "options_block_order": "Options block entries are encoded in ascending key byte order.", + "request_bodies": "body_hex and update_resource_hex are command payloads without framing. constants.command_codes maps each command to its code and VSR operation.", + "reset_order": "The typed encoders and the HTTP handler write reset keys in ascending key byte order.", + "stream_state": "state is the stored server state a stream response was projected from. SDKs need only response_hex and expected.", + "u64_values": "u64 option values and typed u64 inputs are decimal strings, so values above 2^53 keep their precision." + }, + "constants": { + "protocol_version": { + "packed": 12288, + "text": "0.12.0", + "major": 0, + "minor": 12, + "patch": 0 + }, + "protocol_version_min": { + "packed": 12288, + "text": "0.12.0", + "major": 0, + "minor": 12, + "patch": 0 + }, + "command_codes": { + "create_stream": { + "code": 202, + "operation": 128 + }, + "create_topic": { + "code": 302, + "operation": 132 + }, + "create_user": { + "code": 33, + "operation": 141 + }, + "describe_options": { + "code": 13, + "operation": null + }, + "get_stream": { + "code": 200, + "operation": null + }, + "get_streams": { + "code": 201, + "operation": null + }, + "get_topic": { + "code": 300, + "operation": null + }, + "get_topics": { + "code": 301, + "operation": null + }, + "get_user": { + "code": 31, + "operation": null + }, + "get_users": { + "code": 32, + "operation": null + }, + "update_resource": { + "code": 16, + "operation": 150 + } + }, + "error_codes": { + "cannot_modify_root_user": 4108 + }, + "retired_command_codes": { + "update_stream": 204, + "update_topic": 304, + "update_user": 35 + }, + "operations": { + "rewrap_stream_keys": { + "value": 75, + "internal": true, + "request_operation": 75 + }, + "update_resource": { + "value": 150, + "internal": false, + "request_operation": 150 + }, + "update_stream_resolved": { + "value": 72, + "internal": true, + "request_operation": 150 + }, + "update_topic_resolved": { + "value": 73, + "internal": true, + "request_operation": 150 + }, + "update_user_resolved": { + "value": 74, + "internal": true, + "request_operation": 150 + } + }, + "retired_operations": { + "update_stream": 129, + "update_topic": 133, + "update_user": 142 + }, + "scope_codes": { + "stream": 2, + "topic": 1, + "user": 3 + }, + "header_kind_codes": { + "raw": 1, + "string": 2, + "bool": 3, + "uint32": 11, + "uint64": 12 + }, + "option_flags": { + "critical": 16, + "read_only": 8, + "resettable": 2, + "secret": 4, + "updatable": 1 + }, + "user_status_codes": { + "active": 1, + "inactive": 2 + }, + "stream_key_length": 32, + "never_returned_keys": [ + "encryption_key", + "encryption_key_wrapped", + "critical_keys" + ] + }, + "catalog": [ + { + "key": "name", + "scopes": [ + "stream", + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "column", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 1, + "described": true, + "default": null + }, + { + "key": "username", + "scopes": [ + "user" + ], + "kind": "string", + "kind_code": 2, + "storage": "column", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 1, + "described": true, + "default": null + }, + { + "key": "status", + "scopes": [ + "user" + ], + "kind": "string", + "kind_code": 2, + "storage": "column", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 1, + "described": true, + "default": null + }, + { + "key": "compression_algorithm", + "scopes": [ + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "none" + }, + { + "key": "message_expiry", + "scopes": [ + "topic" + ], + "kind": "uint64", + "kind_code": 12, + "storage": "map", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 3, + "described": true, + "default": "18446744073709551615" + }, + { + "key": "max_topic_size", + "scopes": [ + "topic" + ], + "kind": "uint64", + "kind_code": 12, + "storage": "map", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 3, + "described": true, + "default": "18446744073709551615" + }, + { + "key": "segment_size", + "scopes": [ + "topic" + ], + "kind": "uint64", + "kind_code": 12, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "1073741824" + }, + { + "key": "durability", + "scopes": [ + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "replicated" + }, + { + "key": "consumer_offset_durability", + "scopes": [ + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "replicated" + }, + { + "key": "partition_resize_policy", + "scopes": [ + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "mutable" + }, + { + "key": "messages_required_to_save", + "scopes": [ + "topic" + ], + "kind": "uint32", + "kind_code": 11, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "1024" + }, + { + "key": "size_of_messages_required_to_save", + "scopes": [ + "topic" + ], + "kind": "uint64", + "kind_code": 12, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "1048576" + }, + { + "key": "preallocate_segments", + "scopes": [ + "topic" + ], + "kind": "bool", + "kind_code": 3, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "false" + }, + { + "key": "encryption_key", + "scopes": [ + "stream" + ], + "kind": "raw", + "kind_code": 1, + "storage": "map", + "mutability": "create_only", + "visibility": "secret", + "critical": true, + "flags": 20, + "described": true, + "default": null + }, + { + "key": "encryption_key_wrapped", + "scopes": [ + "stream" + ], + "kind": "raw", + "kind_code": 1, + "storage": "map", + "mutability": "create_only", + "visibility": "internal", + "critical": true, + "flags": 16, + "described": false, + "default": null + }, + { + "key": "encryption_key_source", + "scopes": [ + "stream" + ], + "kind": "string", + "kind_code": 2, + "storage": "computed", + "mutability": "read_only", + "visibility": "read_only", + "critical": false, + "flags": 8, + "described": true, + "default": null + }, + { + "key": "encryption_key_fingerprint", + "scopes": [ + "stream" + ], + "kind": "string", + "kind_code": 2, + "storage": "computed", + "mutability": "read_only", + "visibility": "read_only", + "critical": false, + "flags": 8, + "described": true, + "default": null + }, + { + "key": "encryption_kek_fingerprint", + "scopes": [ + "stream" + ], + "kind": "string", + "kind_code": 2, + "storage": "computed", + "mutability": "read_only", + "visibility": "read_only", + "critical": false, + "flags": 8, + "described": true, + "default": null + }, + { + "key": "critical_keys", + "scopes": [ + "stream", + "topic", + "user" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "internal", + "critical": false, + "flags": 0, + "described": false, + "default": null + } + ], + "describe_options": [ + { + "scope": "topic", + "request_hex": "01", + "response_hex": "0b000000046e616d650201000000007e0000005265736f75726365206e616d652c203120746f203235352062797465732c20756e697175652077697468696e2069747320706172656e742e20536574206279207468652063726561746520636f6d6d616e642773206e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e15636f6d7072657373696f6e5f616c676f726974686d0200040000006e6f6e6550000000436f6d7072657373696f6e20616c676f726974686d20286e6f6e652c20677a6970292e2053746f72656420616e64207265706f727465642c206e6f74206170706c69656420746f206d657373616765730e6d6573736167655f6578706972790c0308000000ffffffffffffffff430000004d6573736167652065787069727920696e206d6963726f7365636f6e64732c206f7220612068756d616e74696d6520737472696e672028652e672e20372064617973290e6d61785f746f7069635f73697a650c0308000000ffffffffffffffff9d000000546f7069632d77696465207365616c65642d7365676d656e742073697a65206361702c2073706c6974206163726f737320616c6c20706172746974696f6e732c20696e206279746573206f72206120627974652d73697a6520737472696e672028652e672e203120476942293b2066696e6974652076616c756573206d757374206265206174206c6561737420746865207365676d656e742073697a650c7365676d656e745f73697a650c000800000000000040000000006c0000005365676d656e742073697a6520696e2062797465732c206f72206120627974652d73697a6520737472696e672028652e672e20313238204d6942293b2061203531322d62797465206d756c7469706c652077697468696e20313034383537362e2e3d313037333734313832340a6475726162696c69747902000a0000007265706c6963617465643e0100004d65737361676520636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20412073696e676c65746f6e2071756f72756d20686173206f6e6520636f70792e20496e207265706c6963617465642067726f7570732c20706572736973746564206d65737361676573207573652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c2057414c207265636c616d6174696f6e2e205468652066756c6c20626f64792073697a65207374696c6c20636f756e747320616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e1a636f6e73756d65725f6f66667365745f6475726162696c69747902000a0000007265706c6963617465646e0100004578706c69636974206f666673657420636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20506f6c6c206175746f2d636f6d6d69742072656d61696e73206173796e6368726f6e6f75732e20496e207265706c6963617465642067726f7570732c20706572736973746564206f66667365747320616c736f20656e61626c652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c207265636c616d6174696f6e2c206576656e2077697468207265706c696361746564206d657373616765206475726162696c6974792e2046756c6c20626f64792073697a657320636f756e7420616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e17706172746974696f6e5f726573697a655f706f6c6963790200070000006d757461626c65c8000000496d6d757461626c65206166746572206372656174696f6e3a206d757461626c65207065726d69747320706172746974696f6e206164646974696f6e2f64656c6574696f6e20776974686f7574206120626f756e646564206375746f766572206f72207065722d6b6579206f72646572696e67206163726f7373206c61796f7574733b2066697865642072656a6563747320626f74682e20526574656e74696f6e20616e6420746f7069632064656c6574696f6e2072656d61696e2064657374727563746976652e196d657373616765735f72657175697265645f746f5f736176650b000400000000040000870000004f7264696e617279206d6573736167652d636f756e7420666c75736820747269676765723b20312e2e3d31363737373231362e2052657175697265642070657273697374656e63652c2063617061636974792070726573737572652c20616e64206c6966656379636c65206f7065726174696f6e732063616e20666c757368206561726c6965722173697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c0008000000000010000000000083000000466c75736820746865206a6f75726e616c206f6e636520697420686f6c64732074686973206d616e792062797465732c206f72206120627974652d73697a6520737472696e673b20776869636865766572207468726573686f6c6420747269707320666972737420666c75736865732e204174206d6f7374203130373337343138323414707265616c6c6f636174655f7365676d656e7473030001000000001201000052657175657374207365676d656e745f73697a65206279746573206f66206469736b207265736572766174696f6e207768656e2065616368207365676d656e74206f70656e732e20556e737570706f72746564206f72206661696c6564207265736572766174696f6e732066616c6c206261636b20746f206f7264696e61727920616c6c6f636174696f6e20776974682061207761726e696e672e205265736572766174696f6e2072756e7320696e6c696e65206f6e20746865206f776e696e672073686172642e207365676d656e745f73697a65202a20706172746974696f6e735f636f756e74206973206361707065642061742036383731393437363733362062797465732070657220637265617465", + "entries": [ + { + "key": "name", + "kind": "string", + "flags": 1, + "default": null + }, + { + "key": "compression_algorithm", + "kind": "string", + "flags": 0, + "default": "none" + }, + { + "key": "message_expiry", + "kind": "uint64", + "flags": 3, + "default": "18446744073709551615" + }, + { + "key": "max_topic_size", + "kind": "uint64", + "flags": 3, + "default": "18446744073709551615" + }, + { + "key": "segment_size", + "kind": "uint64", + "flags": 0, + "default": "1073741824" + }, + { + "key": "durability", + "kind": "string", + "flags": 0, + "default": "replicated" + }, + { + "key": "consumer_offset_durability", + "kind": "string", + "flags": 0, + "default": "replicated" + }, + { + "key": "partition_resize_policy", + "kind": "string", + "flags": 0, + "default": "mutable" + }, + { + "key": "messages_required_to_save", + "kind": "uint32", + "flags": 0, + "default": "1024" + }, + { + "key": "size_of_messages_required_to_save", + "kind": "uint64", + "flags": 0, + "default": "1048576" + }, + { + "key": "preallocate_segments", + "kind": "bool", + "flags": 0, + "default": "false" + } + ] + }, + { + "scope": "stream", + "request_hex": "02", + "response_hex": "05000000046e616d650201000000007e0000005265736f75726365206e616d652c203120746f203235352062797465732c20756e697175652077697468696e2069747320706172656e742e20536574206279207468652063726561746520636f6d6d616e642773206e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e0e656e6372797074696f6e5f6b6579011400000000a30100005468652073747265616d2773206f776e206461746120656e6372797074696f6e206b65793a2033322072616e646f6d2062797465732c20726177206f72207374616e6461726420706164646564206261736536342e204372656174652d6f6e6c7920616e642077726974652d6f6e6c793a20746865207365727665722073746f726573206974207772617070656420756e6465722061206b657920646572697665642066726f6d207468652073657276657220656e6372797074696f6e206b657920616e64206e657665722072657475726e732069742e205265717569726573207468652073657276657220656e6372797074696f6e206b657920746f20626520636f6e666967757265642e20576974686f75742069742c206120736572766572207769746820656e6372797074696f6e20656e61626c65642067656e6572617465732061206b657920666f72207468652073747265616d2e20546865206b65792074726176656c7320696e20746865206372656174652072657175657374206f76657220746865207472616e73706f72742074686520636c69656e7420757365732e15656e6372797074696f6e5f6b65795f736f757263650208000000009f00000057686f2063686f7365207468652073747265616d27732061742d72657374206b65793a2073747265616d207768656e2074686520636c69656e742073656e742069742c20736572766572207768656e20746865207365727665722067656e6572617465642069742c206e6f6e65207768656e207468652073747265616d2773206d65737361676573206172652073746f72656420756e656e637279707465641a656e6372797074696f6e5f6b65795f66696e6765727072696e740208000000009f0000005468652066697273742038206279746573206f66205348412d323536206f76657220746865206c6162656c20696767792f73747265616d2d6b65792d66696e6765727072696e742f763120616e64207468652073747265616d2773206b65792c206173203136206c6f776572636173652068657820636861726163746572732e20416273656e74207768656e2074686520736f75726365206973206e6f6e651a656e6372797074696f6e5f6b656b5f66696e6765727072696e74020800000000b800000046696e6765727072696e7473206f6620746865206b65792d656e6372797074696f6e206b65797320746861742077726170207468652073747265616d2773206b65792c20636f6d6d612d7365706172617465643a206f6e65206f7574736964652061206b657920726f746174696f6e2c206f6e652070657220636f6e6669677572656420736572766572206b657920647572696e672069742e20416273656e74207768656e2074686520736f75726365206973206e6f6e65", + "entries": [ + { + "key": "name", + "kind": "string", + "flags": 1, + "default": null + }, + { + "key": "encryption_key", + "kind": "raw", + "flags": 20, + "default": null + }, + { + "key": "encryption_key_source", + "kind": "string", + "flags": 8, + "default": null + }, + { + "key": "encryption_key_fingerprint", + "kind": "string", + "flags": 8, + "default": null + }, + { + "key": "encryption_kek_fingerprint", + "kind": "string", + "flags": 8, + "default": null + } + ] + }, + { + "scope": "user", + "request_hex": "03", + "response_hex": "0200000008757365726e616d650201000000006b00000055736572206e616d652c203320746f2035302062797465732c20756e697175652e20536574206279207468652063726561746520636f6d6d616e64277320757365726e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e067374617475730201000000008f00000055736572207374617475732c20616374697665206f7220696e6163746976652e20536574206279207468652063726561746520636f6d6d616e64277320737461747573206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e20446561637469766174696e672061207573657220656e6473206974732073657373696f6e732e", + "entries": [ + { + "key": "username", + "kind": "string", + "flags": 1, + "default": null + }, + { + "key": "status", + "kind": "string", + "flags": 1, + "default": null + } + ] + } + ], + "stream_key": { + "key_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "key_base64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", + "fingerprint_label": "iggy/stream-key-fingerprint/v1", + "fingerprint": "8441e38538cbfa57", + "invalid_keys": [ + { + "case": "raw_31_bytes", + "input": { + "raw_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + }, + { + "case": "raw_33_bytes", + "input": { + "raw_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + }, + { + "case": "base64_with_whitespace", + "input": { + "base64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=\n" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + }, + { + "case": "base64_missing_padding", + "input": { + "base64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + }, + { + "case": "base64_noncanonical_trailing_bits", + "input": { + "base64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh9=" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + } + ] + }, + "update_resource": [ + { + "case": "topic_set_expiry_without_name_reset_max_topic_size", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "message_expiry": "604800000000", + "reset": [ + "max_topic_size" + ] + }, + "wire_set": { + "message_expiry": { + "value": "604800000000", + "kind": "uint64", + "explicit": true + } + }, + "wire_reset": [ + "max_topic_size" + ], + "body_hex": "01010401000000010402000000010e6d61785f746f7069635f73697a65020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000" + }, + { + "case": "topic_rename_only_named_identifiers", + "scope": "topic", + "target": { + "stream_id": "orders", + "topic_id": "events" + }, + "input": { + "name": "events-v2" + }, + "wire_set": { + "name": { + "value": "events-v2", + "kind": "string", + "explicit": true + } + }, + "wire_reset": [], + "body_hex": "0102066f726465727302066576656e74730002040000006e616d6502090000006576656e74732d7632" + }, + { + "case": "topic_rename_set_expiry_reset_max_topic_size", + "scope": "topic", + "target": { + "stream_id": 7, + "topic_id": "events" + }, + "input": { + "name": "events-v3", + "message_expiry": "86400000000", + "reset": [ + "max_topic_size" + ] + }, + "wire_set": { + "message_expiry": { + "value": "86400000000", + "kind": "uint64", + "explicit": true + }, + "name": { + "value": "events-v3", + "kind": "string", + "explicit": true + } + }, + "wire_reset": [ + "max_topic_size" + ], + "body_hex": "0101040700000002066576656e7473010e6d61785f746f7069635f73697a65020e0000006d6573736167655f6578706972790c080000000060d71d1400000002040000006e616d6502090000006576656e74732d7633" + }, + { + "case": "topic_reset_both_defaults", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "reset": [ + "max_topic_size", + "message_expiry" + ] + }, + "wire_set": {}, + "wire_reset": [ + "max_topic_size", + "message_expiry" + ], + "body_hex": "01010401000000010402000000020e6d61785f746f7069635f73697a650e6d6573736167655f657870697279" + }, + { + "case": "topic_max_topic_size_above_2_pow_53", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "max_topic_size": "9007199254740993" + }, + "wire_set": { + "max_topic_size": { + "value": "9007199254740993", + "kind": "uint64", + "explicit": true + } + }, + "wire_reset": [], + "body_hex": "0101040100000001040200000000020e0000006d61785f746f7069635f73697a650c080000000100000000002000" + }, + { + "case": "stream_rename", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "input": { + "name": "payments-v2" + }, + "wire_set": { + "name": { + "value": "payments-v2", + "kind": "string", + "explicit": true + } + }, + "wire_reset": [], + "body_hex": "020104030000000002040000006e616d65020b0000007061796d656e74732d7632" + }, + { + "case": "user_username_and_status", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "username": "alice2", + "status": "inactive" + }, + "wire_set": { + "status": { + "value": "inactive", + "kind": "string", + "explicit": true + }, + "username": { + "value": "alice2", + "kind": "string", + "explicit": true + } + }, + "wire_reset": [], + "body_hex": "030104050000000002060000007374617475730208000000696e6163746976650208000000757365726e616d650206000000616c69636532" + } + ], + "update_resource_errors": [ + { + "case": "topic_empty_update", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": {}, + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_set_and_reset_same_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "message_expiry": "604800000000", + "reset": [ + "message_expiry" + ] + }, + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_reset_create_only_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "reset": [ + "compression_algorithm" + ] + }, + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "stream_reset_name", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "input": { + "reset": [ + "name" + ] + }, + "error": { + "name": "option_not_resettable", + "code": 4104 + } + }, + { + "case": "user_reset_status", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "reset": [ + "status" + ] + }, + "error": { + "name": "option_not_resettable", + "code": 4104 + } + }, + { + "case": "stream_reset_read_only_key", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "input": { + "reset": [ + "encryption_key_source" + ] + }, + "error": { + "name": "read_only_option", + "code": 4105 + } + }, + { + "case": "user_reset_internal_key", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "reset": [ + "critical_keys" + ] + }, + "error": { + "name": "read_only_option", + "code": 4105 + } + }, + { + "case": "topic_empty_name", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "name": "" + }, + "error": { + "name": "invalid_topic_name", + "code": 2014 + } + }, + { + "case": "topic_empty_name_and_create_only_reset", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "name": "", + "reset": [ + "segment_size" + ] + }, + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "stream_empty_name", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "input": { + "name": "" + }, + "error": { + "name": "invalid_stream_name", + "code": 1013 + } + }, + { + "case": "user_short_username", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "username": "a" + }, + "error": { + "name": "invalid_username", + "code": 43 + } + }, + { + "case": "user_short_username_and_status_reset", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "username": "a", + "reset": [ + "status" + ] + }, + "error": { + "name": "option_not_resettable", + "code": 4104 + } + } + ], + "create_stream": [ + { + "case": "stream_without_key", + "input": { + "name": "orders" + }, + "wire_options": {}, + "body_hex": "066f7264657273", + "http_body": "{\"name\":\"orders\"}", + "admission_error": null + }, + { + "case": "stream_with_key", + "input": { + "name": "orders", + "encryption_key": { + "raw_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f" + } + }, + "wire_options": { + "encryption_key": { + "value": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", + "kind": "raw", + "explicit": true + } + }, + "body_hex": "066f7264657273020e000000656e6372797074696f6e5f6b65790120000000000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "http_body": "{\"name\":\"orders\",\"options\":{\"encryption_key\":\"AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=\"}}", + "admission_error": null + }, + { + "case": "stream_with_raw_extra_key", + "input": { + "name": "orders", + "raw": { + "future_option": "enabled" + } + }, + "wire_options": { + "future_option": { + "value": "enabled", + "kind": "string", + "explicit": true + } + }, + "body_hex": "066f7264657273020d0000006675747572655f6f7074696f6e0207000000656e61626c6564", + "http_body": "{\"name\":\"orders\",\"options\":{\"future_option\":\"enabled\"}}", + "admission_error": { + "name": "unsupported_option_key", + "code": 4041 + } + } + ], + "create_topic": [ + { + "case": "topic_with_options", + "input": { + "stream_id": 1, + "name": "events", + "partitions_count": 3, + "compression_algorithm": "gzip", + "message_expiry": "604800000000", + "max_topic_size": "9007199254740993", + "segment_size": "134217728", + "durability": "persisted", + "consumer_offset_durability": "replicated" + }, + "wire_options": { + "compression_algorithm": { + "value": "gzip", + "kind": "string", + "explicit": true + }, + "consumer_offset_durability": { + "value": "replicated", + "kind": "string", + "explicit": true + }, + "durability": { + "value": "persisted", + "kind": "string", + "explicit": true + }, + "max_topic_size": { + "value": "9007199254740993", + "kind": "uint64", + "explicit": true + }, + "message_expiry": { + "value": "604800000000", + "kind": "uint64", + "explicit": true + }, + "segment_size": { + "value": "134217728", + "kind": "uint64", + "explicit": true + } + }, + "body_hex": "01040100000003000000066576656e74730215000000636f6d7072657373696f6e5f616c676f726974686d0204000000677a6970021a000000636f6e73756d65725f6f66667365745f6475726162696c697479020a0000007265706c696361746564020a0000006475726162696c6974790209000000706572736973746564020e0000006d61785f746f7069635f73697a650c080000000100000000002000020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000020c0000007365676d656e745f73697a650c080000000000000800000000", + "http_body": "{\"partitions_count\":3,\"name\":\"events\",\"options\":{\"compression_algorithm\":\"gzip\",\"consumer_offset_durability\":\"replicated\",\"durability\":\"persisted\",\"max_topic_size\":\"9007199254740993\",\"message_expiry\":604800000000,\"segment_size\":134217728}}", + "admission_error": null + } + ], + "create_user": [ + { + "case": "user_with_empty_options", + "input": { + "username": "alice", + "password": "s3cret-pass", + "status": "active" + }, + "wire_options": {}, + "body_hex": "05616c6963650b7333637265742d706173730100", + "http_body": "{\"username\":\"alice\",\"password\":\"s3cret-pass\",\"status\":\"active\",\"permissions\":null}", + "admission_error": null + } + ], + "http_update": [ + { + "case": "topic_flat_keys", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"name\":\"events-v2\",\"message_expiry\":\"7 days\"}", + "update_resource_hex": "0101040100000001040200000000020e0000006d6573736167655f657870697279020600000037206461797302040000006e616d6502090000006576656e74732d7632", + "outcome": "accepted", + "error": null + }, + { + "case": "topic_reset_array", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":604800000000,\"reset\":[\"max_topic_size\"]}", + "update_resource_hex": "01010401000000010402000000010e6d61785f746f7069635f73697a65020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000", + "outcome": "accepted", + "error": null + }, + { + "case": "topic_null_ignored", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"name\":null,\"max_topic_size\":null,\"message_expiry\":\"1 day\"}", + "update_resource_hex": "0101040100000001040200000000020e0000006d6573736167655f65787069727902050000003120646179", + "outcome": "accepted", + "error": null + }, + { + "case": "topic_u64_above_2_pow_53_as_string", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"max_topic_size\":\"9007199254740993\"}", + "update_resource_hex": "0101040100000001040200000000020e0000006d61785f746f7069635f73697a65021000000039303037313939323534373430393933", + "outcome": "accepted", + "error": null + }, + { + "case": "stream_rename", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "body": "{\"name\":\"payments-v2\"}", + "update_resource_hex": "020104030000000002040000006e616d65020b0000007061796d656e74732d7632", + "outcome": "accepted", + "error": null + }, + { + "case": "user_username_and_status", + "scope": "user", + "target": { + "user_id": 5 + }, + "body": "{\"username\":\"alice2\",\"status\":\"inactive\"}", + "update_resource_hex": "030104050000000002060000007374617475730208000000696e6163746976650208000000757365726e616d650206000000616c69636532", + "outcome": "accepted", + "error": null + }, + { + "case": "topic_duplicate_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":\"1 day\",\"message_expiry\":\"2 days\"}", + "update_resource_hex": null, + "outcome": "json_refused", + "error": null + }, + { + "case": "topic_nested_object", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":{\"value\":\"1 day\"}}", + "update_resource_hex": null, + "outcome": "json_refused", + "error": null + }, + { + "case": "topic_null_only", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"max_topic_size\":null}", + "update_resource_hex": "0101040100000001040200000000", + "outcome": "decode_refused", + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_zero_sentinel_only", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":0}", + "update_resource_hex": "0101040100000001040200000000020e0000006d6573736167655f6578706972790c080000000000000000000000", + "outcome": "admission_refused", + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_reset_names_set_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":\"1 day\",\"reset\":[\"message_expiry\"]}", + "update_resource_hex": "01010401000000010402000000010e6d6573736167655f657870697279020e0000006d6573736167655f65787069727902050000003120646179", + "outcome": "decode_refused", + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_unknown_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"retention\":\"1 day\"}", + "update_resource_hex": "01010401000000010402000000000209000000726574656e74696f6e02050000003120646179", + "outcome": "admission_refused", + "error": { + "name": "unsupported_option_key", + "code": 4041 + } + }, + { + "case": "topic_set_create_only_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"compression_algorithm\":\"gzip\"}", + "update_resource_hex": "01010401000000010402000000000215000000636f6d7072657373696f6e5f616c676f726974686d0204000000677a6970", + "outcome": "admission_refused", + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "topic_reset_create_only_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"reset\":[\"compression_algorithm\"]}", + "update_resource_hex": "010104010000000104020000000115636f6d7072657373696f6e5f616c676f726974686d", + "outcome": "admission_refused", + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "stream_reset_name", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "body": "{\"reset\":[\"name\"]}", + "update_resource_hex": "0201040300000001046e616d65", + "outcome": "admission_refused", + "error": { + "name": "option_not_resettable", + "code": 4104 + } + }, + { + "case": "topic_name_as_number", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"name\":5}", + "update_resource_hex": "010104010000000104020000000002040000006e616d650c080000000500000000000000", + "outcome": "admission_refused", + "error": { + "name": "invalid_topic_name", + "code": 2014 + } + }, + { + "case": "stream_empty_name", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "body": "{\"name\":\"\"}", + "update_resource_hex": "020104030000000002040000006e616d65030100000000", + "outcome": "admission_refused", + "error": { + "name": "invalid_stream_name", + "code": 1013 + } + }, + { + "case": "user_status_as_bool", + "scope": "user", + "target": { + "user_id": 5 + }, + "body": "{\"status\":true}", + "update_resource_hex": "03010405000000000206000000737461747573030100000001", + "outcome": "admission_refused", + "error": { + "name": "invalid_user_status", + "code": 45 + } + }, + { + "case": "topic_create_only_key_as_bool", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"segment_size\":true}", + "update_resource_hex": "0101040100000001040200000000020c0000007365676d656e745f73697a65030100000001", + "outcome": "admission_refused", + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "stream_read_only_key_as_number", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "body": "{\"encryption_key_source\":1}", + "update_resource_hex": "02010403000000000215000000656e6372797074696f6e5f6b65795f736f757263650c080000000100000000000000", + "outcome": "admission_refused", + "error": { + "name": "read_only_option", + "code": 4105 + } + }, + { + "case": "topic_invalid_value", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":\"soon\"}", + "update_resource_hex": "0101040100000001040200000000020e0000006d6573736167655f6578706972790204000000736f6f6e", + "outcome": "admission_refused", + "error": { + "name": "invalid_option_value", + "code": 4042 + } + } + ], + "stream_responses": [ + { + "case": "stream_source_none", + "state": { + "encryption_key_source": "none", + "stored_extra": {} + }, + "response_hex": "0100000000e090663c1306000000000000100000000000000c000000000000000c706c61696e2d73747265616d00000000230000000215000000656e6372797074696f6e5f6b65795f736f7572636502040000006e6f6e65", + "expected": { + "id": 1, + "created_at": 1710000000000000, + "name": "plain-stream", + "topics_count": 0, + "size_bytes": 4096, + "messages_count": 12, + "options": { + "encryption_key_source": { + "value": "none", + "kind": "string", + "explicit": false + } + }, + "encryption_key_source": "none", + "encryption_key_fingerprint": null, + "encryption_kek_fingerprints": [] + } + }, + { + "case": "stream_source_stream_with_unknown_explicit_key", + "state": { + "encryption_key_source": "stream", + "data_key": { + "raw_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f" + }, + "kek_config_keys": [ + "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=" + ], + "stored_extra": { + "future_option": { + "value": "enabled", + "kind": "string", + "explicit": true + } + } + }, + "response_hex": "0200000000e090663c1306000000000000100000000000000c0000000000000011636c69656e742d6b65792d73747265616d1e000000020d0000006675747572655f6f7074696f6e0207000000656e61626c65648d000000021a000000656e6372797074696f6e5f6b656b5f66696e6765727072696e74021000000033396534663066393563333639336337021a000000656e6372797074696f6e5f6b65795f66696e6765727072696e740210000000383434316533383533386362666135370215000000656e6372797074696f6e5f6b65795f736f75726365020600000073747265616d", + "expected": { + "id": 2, + "created_at": 1710000000000000, + "name": "client-key-stream", + "topics_count": 0, + "size_bytes": 4096, + "messages_count": 12, + "options": { + "encryption_kek_fingerprint": { + "value": "39e4f0f95c3693c7", + "kind": "string", + "explicit": false + }, + "encryption_key_fingerprint": { + "value": "8441e38538cbfa57", + "kind": "string", + "explicit": false + }, + "encryption_key_source": { + "value": "stream", + "kind": "string", + "explicit": false + }, + "future_option": { + "value": "enabled", + "kind": "string", + "explicit": true + } + }, + "encryption_key_source": "stream", + "encryption_key_fingerprint": "8441e38538cbfa57", + "encryption_kek_fingerprints": [ + "39e4f0f95c3693c7" + ] + } + }, + { + "case": "stream_source_server_two_keks", + "state": { + "encryption_key_source": "server", + "data_key": { + "raw_hex": "202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f" + }, + "kek_config_keys": [ + "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=", + "AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA=" + ], + "stored_extra": {} + }, + "response_hex": "0300000000e090663c1306000000000000100000000000000c00000000000000117365727665722d6b65792d73747265616d000000009e000000021a000000656e6372797074696f6e5f6b656b5f66696e6765727072696e740221000000333965346630663935633336393363372c64353564343739383239626332393336021a000000656e6372797074696f6e5f6b65795f66696e6765727072696e740210000000306563616666316261323931666530640215000000656e6372797074696f6e5f6b65795f736f757263650206000000736572766572", + "expected": { + "id": 3, + "created_at": 1710000000000000, + "name": "server-key-stream", + "topics_count": 0, + "size_bytes": 4096, + "messages_count": 12, + "options": { + "encryption_kek_fingerprint": { + "value": "39e4f0f95c3693c7,d55d479829bc2936", + "kind": "string", + "explicit": false + }, + "encryption_key_fingerprint": { + "value": "0ecaff1ba291fe0d", + "kind": "string", + "explicit": false + }, + "encryption_key_source": { + "value": "server", + "kind": "string", + "explicit": false + } + }, + "encryption_key_source": "server", + "encryption_key_fingerprint": "0ecaff1ba291fe0d", + "encryption_kek_fingerprints": [ + "39e4f0f95c3693c7", + "d55d479829bc2936" + ] + } + } + ], + "topic_responses": [ + { + "case": "topic_explicit_and_derived", + "response_hex": "0400000000e090663c1306000000000000080000000000000700000000000000066576656e7473430000000215000000636f6d7072657373696f6e5f616c676f726974686d0204000000677a6970020e0000006d6573736167655f6578706972790c0800000000a0e3d08c0000002b010000021a000000636f6e73756d65725f6f66667365745f6475726162696c697479020a0000007265706c696361746564020a0000006475726162696c697479020a0000007265706c696361746564020e0000006d61785f746f7069635f73697a650c08000000ffffffffffffffff02190000006d657373616765735f72657175697265645f746f5f736176650b04000000000400000217000000706172746974696f6e5f726573697a655f706f6c69637902070000006d757461626c650214000000707265616c6c6f636174655f7365676d656e7473030100000000020c0000007365676d656e745f73697a650c080000000000004000000000022100000073697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c080000000000100000000000", + "expected": { + "id": 4, + "created_at": 1710000000000000, + "name": "events", + "partitions_count": 0, + "size_bytes": 2048, + "messages_count": 7, + "options": { + "compression_algorithm": { + "value": "gzip", + "kind": "string", + "explicit": true + }, + "consumer_offset_durability": { + "value": "replicated", + "kind": "string", + "explicit": false + }, + "durability": { + "value": "replicated", + "kind": "string", + "explicit": false + }, + "max_topic_size": { + "value": "18446744073709551615", + "kind": "uint64", + "explicit": false + }, + "message_expiry": { + "value": "604800000000", + "kind": "uint64", + "explicit": true + }, + "messages_required_to_save": { + "value": "1024", + "kind": "uint32", + "explicit": false + }, + "partition_resize_policy": { + "value": "mutable", + "kind": "string", + "explicit": false + }, + "preallocate_segments": { + "value": "false", + "kind": "bool", + "explicit": false + }, + "segment_size": { + "value": "1073741824", + "kind": "uint64", + "explicit": false + }, + "size_of_messages_required_to_save": { + "value": "1048576", + "kind": "uint64", + "explicit": false + } + }, + "compression_algorithm": "gzip", + "message_expiry": "604800000000", + "max_topic_size": "18446744073709551615" + } + } + ], + "user_responses": [ + { + "case": "user_empty_blocks", + "response_hex": "0500000000e090663c1306000105616c696365000000000000000000000000", + "expected": { + "id": 5, + "created_at": 1710000000000000, + "status": "active", + "username": "alice", + "options": {} + } + }, + { + "case": "user_populated_blocks", + "response_hex": "0600000000e090663c1306000203626f621e000000020d0000006675747572655f6f7074696f6e0207000000656e61626c65641e000000020c0000006675747572655f71756f74610c08000000010000000000200000000000", + "expected": { + "id": 6, + "created_at": 1710000000000000, + "status": "inactive", + "username": "bob", + "options": { + "future_option": { + "value": "enabled", + "kind": "string", + "explicit": true + }, + "future_quota": { + "value": "9007199254740993", + "kind": "uint64", + "explicit": false + } + } + } + } + ] +}
diff --git a/foreign/go/contracts/topic_options.go b/foreign/go/contracts/topic_options.go index 7f1069b..f1d417f 100644 --- a/foreign/go/contracts/topic_options.go +++ b/foreign/go/contracts/topic_options.go
@@ -19,28 +19,91 @@ import "encoding/binary" -// Topic option keys that CreateTopic has no parameter of its own for. -// -// The constructors below are the spelling to use for them: create admission -// refuses a key outside the server's catalog by name, and refuses a value whose -// kind is neither the kind the catalog gives that key nor a String that key -// parses. DescribeOptions enumerates the catalog one server serves, with the -// bounds each value is checked against. Every key here is create-time only, so -// UpdateTopic refuses it by name. +// TopicOptionKey names a topic option of the server catalog. +type TopicOptionKey string + +// Topic option keys. Only TopicOptionName, TopicOptionMessageExpiry and +// TopicOptionMaxTopicSize are updatable, and only the last two resettable. +// Every other key is create-only, so an update refuses it by name. +// DescribeOptions enumerates the catalog one server serves, with the bounds +// each value is checked against. const ( - topicOptionSegmentSize = "segment_size" - topicOptionDurability = "durability" - topicOptionConsumerOffsetDurability = "consumer_offset_durability" - topicOptionMessagesRequiredToSave = "messages_required_to_save" - topicOptionSizeOfMessagesRequiredToSave = "size_of_messages_required_to_save" - topicOptionPreallocateSegments = "preallocate_segments" + TopicOptionName TopicOptionKey = "name" + TopicOptionCompressionAlgorithm TopicOptionKey = "compression_algorithm" + TopicOptionMessageExpiry TopicOptionKey = "message_expiry" + TopicOptionMaxTopicSize TopicOptionKey = "max_topic_size" + TopicOptionSegmentSize TopicOptionKey = "segment_size" + TopicOptionDurability TopicOptionKey = "durability" + TopicOptionConsumerOffsetDurability TopicOptionKey = "consumer_offset_durability" + TopicOptionPartitionResizePolicy TopicOptionKey = "partition_resize_policy" + TopicOptionMessagesRequiredToSave TopicOptionKey = "messages_required_to_save" + TopicOptionSizeOfMessagesRequiredToSave TopicOptionKey = "size_of_messages_required_to_save" + TopicOptionPreallocateSegments TopicOptionKey = "preallocate_segments" ) +// TopicOptions are the options CreateTopic may carry. A nil *TopicOptions +// selects the defaults, and so does a nil field. +// +// The constructors below build the same entries for Raw, the escape hatch +// for keys newer than this build. A typed field wins on collision. +type TopicOptions struct { + // PartitionsCount is how many partitions the topic starts with, 1 when + // nil. It is an argument of the create, not a stored option. + PartitionsCount *uint32 + // CompressionAlgorithm is a create-only label the server stores and + // reports. CompressionAlgorithmNone and the default leave it derived. + CompressionAlgorithm *CompressionAlgorithm + // MessageExpiry in microseconds. IggyExpiryServerDefault leaves it derived. + MessageExpiry *Duration + // MaxTopicSize in bytes, math.MaxUint64 for unlimited. Zero leaves it + // derived. + MaxTopicSize *uint64 + // SegmentSize, see SegmentSizeOption. + SegmentSize *uint64 + // Durability of messages, replicated when nil. + Durability *Durability + // ConsumerOffsetDurability of explicit offset writes, replicated when nil. + ConsumerOffsetDurability *Durability + // PartitionResizePolicy, mutable when nil. + PartitionResizePolicy *PartitionResizePolicy + // MessagesRequiredToSave, see MessagesRequiredToSaveOption. + MessagesRequiredToSave *uint32 + // SizeOfMessagesRequiredToSave, see SizeOfMessagesRequiredToSaveOption. + SizeOfMessagesRequiredToSave *uint64 + // PreallocateSegments, see PreallocateSegmentsOption. + PreallocateSegments *bool + // Raw carries keys newer than this build. + Raw []HeaderEntry +} + +// TopicUpdate is a patch of a topic's options. Absent keys keep their stored +// values. Create-only settings, compression included, have no field here. +type TopicUpdate struct { + // Name renames the topic when set. + Name *string + // MessageExpiry sets the expiry in microseconds. IggyExpiryServerDefault + // changes nothing, Reset returns the key to its default. + MessageExpiry *Duration + // MaxTopicSize sets the size cap in bytes. Zero changes nothing, Reset + // returns the key to its default. + MaxTopicSize *uint64 + // Reset names keys to return to their defaults: TopicOptionMessageExpiry + // or TopicOptionMaxTopicSize. + Reset []TopicOptionKey + // RawReset names keys to reset that this build may not know. The server + // decides whether each one may be reset, and a patch that carries any raw + // name or Options entry is checked by the server alone. + RawReset []string + // Options carries updatable keys newer than this build. A typed field wins + // on collision. + Options []HeaderEntry +} + // SegmentSizeOption sets how large a partition segment grows before it rotates. // The server takes a 512-byte multiple inside the bounds its catalog reports. // Zero is not refused: it leaves the key to the server default. func SegmentSizeOption(bytes uint64) HeaderEntry { - return uint64Option(topicOptionSegmentSize, bytes) + return uint64Option(TopicOptionSegmentSize, bytes) } type Durability string @@ -51,35 +114,56 @@ ) func DurabilityOption(value Durability) HeaderEntry { - return HeaderEntry{Key: HeaderKey{Kind: String, Value: []byte(topicOptionDurability)}, Value: HeaderValue{Kind: String, Value: []byte(value)}} + return stringOption(TopicOptionDurability, string(value)) } func ConsumerOffsetDurabilityOption(value Durability) HeaderEntry { - return HeaderEntry{Key: HeaderKey{Kind: String, Value: []byte(topicOptionConsumerOffsetDurability)}, Value: HeaderValue{Kind: String, Value: []byte(value)}} + return stringOption(TopicOptionConsumerOffsetDurability, string(value)) +} + +// PartitionResizePolicy says whether a topic may add and delete partitions. +type PartitionResizePolicy string + +const ( + PartitionResizePolicyMutable PartitionResizePolicy = "mutable" + PartitionResizePolicyFixed PartitionResizePolicy = "fixed" +) + +// PartitionResizePolicyOption fixes the policy at create. A fixed topic keeps +// the partitions it was created with. +func PartitionResizePolicyOption(value PartitionResizePolicy) HeaderEntry { + return stringOption(TopicOptionPartitionResizePolicy, string(value)) } // MessagesRequiredToSaveOption flushes the journal once it holds this many // messages. Zero is refused. Pairs with SizeOfMessagesRequiredToSaveOption: // whichever threshold trips first flushes. func MessagesRequiredToSaveOption(messages uint32) HeaderEntry { - return uint32Option(topicOptionMessagesRequiredToSave, messages) + return uint32Option(TopicOptionMessagesRequiredToSave, messages) } // SizeOfMessagesRequiredToSaveOption flushes the journal once it holds this many // bytes. A threshold above the largest a segment may be is refused, since it // could never trip. Zero leaves the key to the server default. func SizeOfMessagesRequiredToSaveOption(bytes uint64) HeaderEntry { - return uint64Option(topicOptionSizeOfMessagesRequiredToSave, bytes) + return uint64Option(TopicOptionSizeOfMessagesRequiredToSave, bytes) } // PreallocateSegmentsOption reserves each segment's bytes on disk up front. A // topic reserves segment_size for every partition it has, and the server refuses // a create whose product crosses its preallocation cap. func PreallocateSegmentsOption(enabled bool) HeaderEntry { - return boolOption(topicOptionPreallocateSegments, enabled) + return boolOption(TopicOptionPreallocateSegments, enabled) } -func uint64Option(key string, value uint64) HeaderEntry { +func stringOption(key TopicOptionKey, value string) HeaderEntry { + return HeaderEntry{ + Key: HeaderKey{Kind: String, Value: []byte(key)}, + Value: HeaderValue{Kind: String, Value: []byte(value)}, + } +} + +func uint64Option(key TopicOptionKey, value uint64) HeaderEntry { buf := make([]byte, 8) binary.LittleEndian.PutUint64(buf, value) return HeaderEntry{ @@ -88,7 +172,7 @@ } } -func uint32Option(key string, value uint32) HeaderEntry { +func uint32Option(key TopicOptionKey, value uint32) HeaderEntry { buf := make([]byte, 4) binary.LittleEndian.PutUint32(buf, value) return HeaderEntry{ @@ -97,7 +181,7 @@ } } -func boolOption(key string, value bool) HeaderEntry { +func boolOption(key TopicOptionKey, value bool) HeaderEntry { encoded := byte(0) if value { encoded = 1
diff --git a/foreign/go/contracts/topics.go b/foreign/go/contracts/topics.go index 65fed72..81bdb9d 100644 --- a/foreign/go/contracts/topics.go +++ b/foreign/go/contracts/topics.go
@@ -39,10 +39,14 @@ } type Topic struct { - Id uint32 `json:"id"` - CreatedAt uint64 `json:"createdAt"` - Name string `json:"name"` - Size uint64 `json:"size"` + Id uint32 `json:"id"` + CreatedAt uint64 `json:"createdAt"` + Name string `json:"name"` + Size uint64 `json:"size"` + // MessageExpiry, CompressionAlgorithm and MaxTopicSize are read from the + // options, or are the catalog default when the server reported none or a + // value this build cannot read, such as an algorithm a newer server added. + // The server's own value stays in Options or DerivedOptions. MessageExpiry Duration `json:"messageExpiry"` CompressionAlgorithm uint8 `json:"compressionAlgorithm"` MaxTopicSize uint64 `json:"maxTopicSize"`
diff --git a/foreign/go/contracts/user_options.go b/foreign/go/contracts/user_options.go new file mode 100644 index 0000000..baa146f --- /dev/null +++ b/foreign/go/contracts/user_options.go
@@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package iggcon + +// UserOptionKey names a user option of the server catalog. +type UserOptionKey string + +const ( + // UserOptionUsername is the user's name, set by CreateUser and changed by + // UpdateUser. + UserOptionUsername UserOptionKey = "username" + // UserOptionStatus is the user's status, set by CreateUser and changed by + // UpdateUser. + UserOptionStatus UserOptionKey = "status" +) + +// UserOptions are the options CreateUser may carry. No user option exists +// yet, so the value is empty unless Raw names a key newer than this build. A +// nil *UserOptions selects the defaults. +type UserOptions struct { + // Raw carries keys newer than this build. + Raw []HeaderEntry +} + +// UserUpdate is a patch of a user's options. Absent keys keep their stored +// values. +type UserUpdate struct { + // Username renames the user when set. A rename of the signed-in user also + // becomes the username the client signs in with after a reconnect. + Username *string + // Status changes the user's status when set. Deactivating a user ends its + // sessions. + Status *UserStatus + // Reset names keys to return to their defaults. No user key is resettable, + // so a non-empty list is refused before anything is sent. + Reset []UserOptionKey + // RawReset names keys to reset that this build may not know. The server + // decides whether each one may be reset, and a patch that carries any raw + // name or Options entry is checked by the server alone. + RawReset []string + // Options carries updatable keys newer than this build. A typed field wins + // on collision. + Options []HeaderEntry +}
diff --git a/foreign/go/contracts/users.go b/foreign/go/contracts/users.go index 8dbafe7..4f79638 100644 --- a/foreign/go/contracts/users.go +++ b/foreign/go/contracts/users.go
@@ -20,11 +20,14 @@ import "encoding/binary" type UserInfo struct { - Id uint32 `json:"Id"` - CreatedAt uint64 `json:"CreatedAt"` - Status UserStatus `json:"Status"` - Username string `json:"Username"` - Options map[string]HeaderValue `json:"Options,omitempty"` + Id uint32 `json:"Id"` + CreatedAt uint64 `json:"CreatedAt"` + Status UserStatus `json:"Status"` + Username string `json:"Username"` + // Options are the keys the client set. + Options map[string]HeaderValue `json:"Options,omitempty"` + // DerivedOptions are the values the server filled. + DerivedOptions map[string]HeaderValue `json:"DerivedOptions,omitempty"` } type UserInfoDetails struct {
diff --git a/foreign/go/contracts/version.go b/foreign/go/contracts/version.go index fa851c7..ab6b6ff 100644 --- a/foreign/go/contracts/version.go +++ b/foreign/go/contracts/version.go
@@ -17,4 +17,4 @@ package iggcon -const Version = "0.9.1-edge.1" +const Version = "0.9.1-edge.2"
diff --git a/foreign/go/errors/errors.yaml b/foreign/go/errors/errors.yaml index 677336c..5ef461a 100644 --- a/foreign/go/errors/errors.yaml +++ b/foreign/go/errors/errors.yaml
@@ -1097,6 +1097,38 @@ type: uint64 - name: Expected type: uint64 +- name: OptionNotUpdatable + code: 4103 + format: "option cannot be updated after creation: %s" + fields: + - name: Key + type: string +- name: OptionNotResettable + code: 4104 + format: "option cannot be reset: %s" + fields: + - name: Key + type: string +- name: ReadOnlyOption + code: 4105 + format: "option is read-only: %s" + fields: + - name: Key + type: string +- name: StreamEncryptionUnavailable + code: 4106 + format: "stream encryption key requires a server encryption key" + fields: [] +- name: UnsupportedCriticalOption + code: 4107 + format: "unsupported critical option: %s" + fields: + - name: Key + type: string +- name: CannotModifyRootUser + code: 4108 + format: "cannot modify root user" + fields: [] - name: ConsumerGroupIdNotFound code: 5000 format: "consumer group with id: %d for topic with id: %d was not found."
diff --git a/foreign/go/errors/errors_gen.go b/foreign/go/errors/errors_gen.go index d400e7c..c865689 100644 --- a/foreign/go/errors/errors_gen.go +++ b/foreign/go/errors/errors_gen.go
@@ -2272,6 +2272,78 @@ return ok } +type OptionNotUpdatable struct { + Key string +} + +func (e OptionNotUpdatable) Error() string { + return fmt.Sprintf("option cannot be updated after creation: %s", e.Key) +} +func (e OptionNotUpdatable) Code() Code { return 4103 } +func (e OptionNotUpdatable) Is(target error) bool { + _, ok := target.(OptionNotUpdatable) + return ok +} + +type OptionNotResettable struct { + Key string +} + +func (e OptionNotResettable) Error() string { + return fmt.Sprintf("option cannot be reset: %s", e.Key) +} +func (e OptionNotResettable) Code() Code { return 4104 } +func (e OptionNotResettable) Is(target error) bool { + _, ok := target.(OptionNotResettable) + return ok +} + +type ReadOnlyOption struct { + Key string +} + +func (e ReadOnlyOption) Error() string { + return fmt.Sprintf("option is read-only: %s", e.Key) +} +func (e ReadOnlyOption) Code() Code { return 4105 } +func (e ReadOnlyOption) Is(target error) bool { + _, ok := target.(ReadOnlyOption) + return ok +} + +type StreamEncryptionUnavailable struct{} + +func (e StreamEncryptionUnavailable) Error() string { + return "stream encryption key requires a server encryption key" +} +func (e StreamEncryptionUnavailable) Code() Code { return 4106 } +func (e StreamEncryptionUnavailable) Is(target error) bool { + _, ok := target.(StreamEncryptionUnavailable) + return ok +} + +type UnsupportedCriticalOption struct { + Key string +} + +func (e UnsupportedCriticalOption) Error() string { + return fmt.Sprintf("unsupported critical option: %s", e.Key) +} +func (e UnsupportedCriticalOption) Code() Code { return 4107 } +func (e UnsupportedCriticalOption) Is(target error) bool { + _, ok := target.(UnsupportedCriticalOption) + return ok +} + +type CannotModifyRootUser struct{} + +func (e CannotModifyRootUser) Error() string { return "cannot modify root user" } +func (e CannotModifyRootUser) Code() Code { return 4108 } +func (e CannotModifyRootUser) Is(target error) bool { + _, ok := target.(CannotModifyRootUser) + return ok +} + type ConsumerGroupIdNotFound struct { GroupId uint32 TopicId uint32 @@ -3003,6 +3075,12 @@ ErrInvalidOffset = InvalidOffset{} ErrInvalidReservedField = InvalidReservedField{} ErrSegmentSizeMismatchAtOpen = SegmentSizeMismatchAtOpen{} + ErrOptionNotUpdatable = OptionNotUpdatable{} + ErrOptionNotResettable = OptionNotResettable{} + ErrReadOnlyOption = ReadOnlyOption{} + ErrStreamEncryptionUnavailable = StreamEncryptionUnavailable{} + ErrUnsupportedCriticalOption = UnsupportedCriticalOption{} + ErrCannotModifyRootUser = CannotModifyRootUser{} ErrConsumerGroupIdNotFound = ConsumerGroupIdNotFound{} ErrInvalidConsumerGroupId = InvalidConsumerGroupId{} ErrConsumerGroupNameNotFound = ConsumerGroupNameNotFound{} @@ -3260,6 +3338,12 @@ InvalidOffsetCode Code = 4100 InvalidReservedFieldCode Code = 4101 SegmentSizeMismatchAtOpenCode Code = 4102 + OptionNotUpdatableCode Code = 4103 + OptionNotResettableCode Code = 4104 + ReadOnlyOptionCode Code = 4105 + StreamEncryptionUnavailableCode Code = 4106 + UnsupportedCriticalOptionCode Code = 4107 + CannotModifyRootUserCode Code = 4108 ConsumerGroupIdNotFoundCode Code = 5000 InvalidConsumerGroupIdCode Code = 5002 ConsumerGroupNameNotFoundCode Code = 5003 @@ -3719,6 +3803,18 @@ return "InvalidReservedField" case SegmentSizeMismatchAtOpenCode: return "SegmentSizeMismatchAtOpen" + case OptionNotUpdatableCode: + return "OptionNotUpdatable" + case OptionNotResettableCode: + return "OptionNotResettable" + case ReadOnlyOptionCode: + return "ReadOnlyOption" + case StreamEncryptionUnavailableCode: + return "StreamEncryptionUnavailable" + case UnsupportedCriticalOptionCode: + return "UnsupportedCriticalOption" + case CannotModifyRootUserCode: + return "CannotModifyRootUser" case ConsumerGroupIdNotFoundCode: return "ConsumerGroupIdNotFound" case InvalidConsumerGroupIdCode: @@ -4230,6 +4326,18 @@ return ErrInvalidReservedField case SegmentSizeMismatchAtOpenCode: return ErrSegmentSizeMismatchAtOpen + case OptionNotUpdatableCode: + return ErrOptionNotUpdatable + case OptionNotResettableCode: + return ErrOptionNotResettable + case ReadOnlyOptionCode: + return ErrReadOnlyOption + case StreamEncryptionUnavailableCode: + return ErrStreamEncryptionUnavailable + case UnsupportedCriticalOptionCode: + return ErrUnsupportedCriticalOption + case CannotModifyRootUserCode: + return ErrCannotModifyRootUser case ConsumerGroupIdNotFoundCode: return ErrConsumerGroupIdNotFound case InvalidConsumerGroupIdCode:
diff --git a/foreign/go/internal/command/access_token.go b/foreign/go/internal/command/access_token.go index 68260a6..bf56d1f 100644 --- a/foreign/go/internal/command/access_token.go +++ b/foreign/go/internal/command/access_token.go
@@ -17,11 +17,15 @@ package command -import "encoding/binary" +import ( + "encoding/binary" + + iggcon "github.com/apache/iggy/foreign/go/contracts" +) type CreatePersonalAccessToken struct { - Name string `json:"Name"` - Expiry uint32 `json:"Expiry"` + Name string `json:"Name"` + Expiry iggcon.Duration `json:"Expiry"` } func (c *CreatePersonalAccessToken) Code() Code { @@ -33,7 +37,7 @@ bytes := make([]byte, length) bytes[0] = byte(len(c.Name)) copy(bytes[1:], c.Name) - binary.LittleEndian.PutUint32(bytes[len(bytes)-4:], c.Expiry) + binary.LittleEndian.PutUint64(bytes[1+len(c.Name):], uint64(c.Expiry)) return bytes, nil }
diff --git a/foreign/go/internal/command/access_token_test.go b/foreign/go/internal/command/access_token_test.go new file mode 100644 index 0000000..56f4c43 --- /dev/null +++ b/foreign/go/internal/command/access_token_test.go
@@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package command + +import ( + "bytes" + "testing" + + iggcon "github.com/apache/iggy/foreign/go/contracts" +) + +func TestCreatePersonalAccessTokenMarshalBinary(t *testing.T) { + tests := []struct { + name string + expiry iggcon.Duration + want []byte + }{ + {"server default", iggcon.IggyExpiryServerDefault, []byte{5, 't', 'o', 'k', 'e', 'n', 0, 0, 0, 0, 0, 0, 0, 0}}, + {"little endian", 0x01020304, []byte{5, 't', 'o', 'k', 'e', 'n', 4, 3, 2, 1, 0, 0, 0, 0}}, + {"past 32 bits", 0x0102030405060708, []byte{5, 't', 'o', 'k', 'e', 'n', 8, 7, 6, 5, 4, 3, 2, 1}}, + {"never expire", iggcon.IggyExpiryNeverExpire, []byte{5, 't', 'o', 'k', 'e', 'n', 255, 255, 255, 255, 255, 255, 255, 255}}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := (&CreatePersonalAccessToken{Name: "token", Expiry: test.expiry}).MarshalBinary() + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, test.want) { + t.Fatalf("CreatePersonalAccessToken body = % x, want % x", got, test.want) + } + }) + } +}
diff --git a/foreign/go/internal/command/code.go b/foreign/go/internal/command/code.go index 32153e1..45f2611 100644 --- a/foreign/go/internal/command/code.go +++ b/foreign/go/internal/command/code.go
@@ -26,6 +26,7 @@ GetClusterMetadataCode Code = 12 DescribeOptionsCode Code = 13 BindSessionCode Code = 15 + UpdateResourceCode Code = 16 GetMeCode Code = 20 GetClientCode Code = 21 GetClientsCode Code = 22 @@ -33,7 +34,6 @@ GetUsersCode Code = 32 CreateUserCode Code = 33 DeleteUserCode Code = 34 - UpdateUserCode Code = 35 UpdatePermissionsCode Code = 36 ChangePasswordCode Code = 37 LoginUserCode Code = 38 @@ -46,7 +46,7 @@ LoginRegisterWithPATCode Code = 45 PollMessagesCode Code = 100 SendMessagesCode Code = 101 - // 103, 104, 123 and 124 are retired and must never be reused. + // 35, 103, 104, 123, 124, 204 and 304 are retired and must never be reused. GetPartitionRouteCode Code = 105 GetOffsetCode Code = 120 StoreOffsetCode Code = 121 @@ -55,12 +55,10 @@ GetStreamsCode Code = 201 CreateStreamCode Code = 202 DeleteStreamCode Code = 203 - UpdateStreamCode Code = 204 GetTopicCode Code = 300 GetTopicsCode Code = 301 CreateTopicCode Code = 302 DeleteTopicCode Code = 303 - UpdateTopicCode Code = 304 CreatePartitionsCode Code = 402 DeletePartitionsCode Code = 403 DeleteSegmentsCode Code = 503
diff --git a/foreign/go/internal/command/stream.go b/foreign/go/internal/command/stream.go index 2102966..5a573a5 100644 --- a/foreign/go/internal/command/stream.go +++ b/foreign/go/internal/command/stream.go
@@ -17,7 +17,10 @@ package command -import iggcon "github.com/apache/iggy/foreign/go/contracts" +import ( + iggcon "github.com/apache/iggy/foreign/go/contracts" + ierror "github.com/apache/iggy/foreign/go/errors" +) const ( nameLengthOffset = 0 @@ -25,19 +28,62 @@ ) type CreateStream struct { - Name string + Name string + Options iggcon.StreamOptions } func (request *CreateStream) Code() Code { return CreateStreamCode } +// MarshalBinary refuses an invalid key, typed or raw, so it never leaves the +// client. func (request *CreateStream) MarshalBinary() ([]byte, error) { + var typed []iggcon.HeaderEntry + if request.Options.EncryptionKey != nil { + entry, err := iggcon.EncryptionKeyOption(request.Options.EncryptionKey) + if err != nil { + return nil, err + } + typed = append(typed, entry) + } + options, err := mergeOptions(typed, request.Options.Raw) + if err != nil { + return nil, err + } + for _, entry := range options { + if err := validateEncryptionKeyEntry(entry); err != nil { + return nil, err + } + } + optionsBytes, err := encodeOptions(options) + if err != nil { + return nil, err + } nameLength := len(request.Name) - serialized := make([]byte, payloadOffset+nameLength) + serialized := make([]byte, payloadOffset+nameLength, payloadOffset+nameLength+len(optionsBytes)) serialized[nameLengthOffset] = byte(nameLength) copy(serialized[payloadOffset:], request.Name) - return serialized, nil + return append(serialized, optionsBytes...), nil +} + +// validateEncryptionKeyEntry checks an encryption_key entry in either form the +// server takes: 32 raw bytes, or their canonical base64 text. +func validateEncryptionKeyEntry(entry iggcon.HeaderEntry) error { + if string(entry.Key.Value) != string(iggcon.StreamOptionEncryptionKey) { + return nil + } + switch entry.Value.Kind { + case iggcon.Raw: + if len(entry.Value.Value) == iggcon.StreamEncryptionKeyLength { + return nil + } + case iggcon.String: + if _, err := iggcon.DecodeEncryptionKey(string(entry.Value.Value)); err == nil { + return nil + } + } + return ierror.ErrInvalidEncryptionKey } type GetStream struct { @@ -62,34 +108,29 @@ return []byte{}, nil } +// UpdateStream patches a stream through the generic UpdateResource command. type UpdateStream struct { - StreamId iggcon.Identifier `json:"streamId"` - Name string `json:"name"` - // Options carries the trailing options block. Streams have no catalog - // keys yet, so the server rejects every key; the block is the extension - // point for the first one. - Options []iggcon.HeaderEntry `json:"-"` + StreamId iggcon.Identifier + Name *string + Reset []iggcon.StreamOptionKey + RawReset []string + Options []iggcon.HeaderEntry } func (u *UpdateStream) Code() Code { - return UpdateStreamCode + return UpdateResourceCode } func (u *UpdateStream) MarshalBinary() ([]byte, error) { - streamIdBytes, err := u.StreamId.MarshalBinary() - if err != nil { - return nil, err + var typed []iggcon.HeaderEntry + if u.Name != nil { + typed = append(typed, textOption(string(iggcon.StreamOptionName), *u.Name)) } - nameLength := len(u.Name) - optionsBytes := iggcon.GetHeadersBytes(u.Options) - bytes := make([]byte, len(streamIdBytes)+1+nameLength+len(optionsBytes)) - copy(bytes[0:len(streamIdBytes)], streamIdBytes) - position := len(streamIdBytes) - bytes[position] = byte(nameLength) - position++ - position += copy(bytes[position:], u.Name) - copy(bytes[position:], optionsBytes) - return bytes, nil + reset := make([]string, 0, len(u.Reset)) + for _, key := range u.Reset { + reset = append(reset, string(key)) + } + return encodeUpdateResource(iggcon.OptionsScopeStream, []iggcon.Identifier{u.StreamId}, typed, u.Options, reset, u.RawReset) } type DeleteStream struct {
diff --git a/foreign/go/internal/command/stream_test.go b/foreign/go/internal/command/stream_test.go index b3859fe..98cd4c3 100644 --- a/foreign/go/internal/command/stream_test.go +++ b/foreign/go/internal/command/stream_test.go
@@ -58,9 +58,10 @@ func TestSerialize_UpdateStream(t *testing.T) { streamId, _ := iggcon.NewIdentifier("stream") + name := "update_stream" request := UpdateStream{ StreamId: streamId, - Name: "update_stream", + Name: &name, } serialized1, err := request.MarshalBinary() @@ -69,14 +70,21 @@ } expected := []byte{ + 0x02, // Scope (stream) 0x02, // StreamId Kind (StringId) - 0x06, // StreamId Length (2) + 0x06, // StreamId Length (6) 0x73, 0x74, 0x72, 0x65, 0x61, 0x6D, // StreamId Value ("stream") - 0x0D, // Name Length (13) - 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5F, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6D, // Name ("update_stream") + 0x00, // Reset count + 0x02, 0x04, 0x00, 0x00, 0x00, // key kind (String), key length (4) + 0x6E, 0x61, 0x6D, 0x65, // "name" + 0x02, 0x0D, 0x00, 0x00, 0x00, // value kind (String), value length (13) + 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5F, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6D, // "update_stream" } if !bytes.Equal(serialized1, expected) { t.Errorf("Test case 1 failed. \nExpected:\t%v\nGot:\t\t%v", expected, serialized1) } + if request.Code() != UpdateResourceCode { + t.Errorf("Code = %d, want the generic update %d", request.Code(), UpdateResourceCode) + } }
diff --git a/foreign/go/internal/command/topic.go b/foreign/go/internal/command/topic.go index 8455ef4..7eaf7a4 100644 --- a/foreign/go/internal/command/topic.go +++ b/foreign/go/internal/command/topic.go
@@ -19,28 +19,20 @@ import ( "encoding/binary" - "fmt" + "slices" "github.com/apache/iggy/foreign/go/contracts" + ierror "github.com/apache/iggy/foreign/go/errors" ) -const ( - topicOptionCompressionAlgorithm = "compression_algorithm" - topicOptionMessageExpiry = "message_expiry" - topicOptionMaxTopicSize = "max_topic_size" -) +// defaultPartitionsCount is what a topic starts with when the options name +// no count, as in the other SDKs. +const defaultPartitionsCount = 1 type CreateTopic struct { - StreamId iggcon.Identifier `json:"streamId"` - PartitionsCount uint32 `json:"partitionsCount"` - CompressionAlgorithm iggcon.CompressionAlgorithm `json:"compressionAlgorithm"` - MessageExpiry iggcon.Duration `json:"messageExpiry"` - MaxTopicSize uint64 `json:"maxTopicSize"` - Name string `json:"name"` - // Options carries keys with no field above, for a key the server catalog - // gained after this build shipped. Entries reuse HeaderEntry because options - // ride the user-headers codec. A field above wins on collision. - Options []iggcon.HeaderEntry `json:"-"` + StreamId iggcon.Identifier + Name string + Options iggcon.TopicOptions } func (t *CreateTopic) Code() Code { @@ -57,11 +49,18 @@ if err != nil { return nil, err } - optionsBytes := iggcon.GetHeadersBytes(options) + optionsBytes, err := encodeOptions(options) + if err != nil { + return nil, err + } + partitionsCount := uint32(defaultPartitionsCount) + if t.Options.PartitionsCount != nil { + partitionsCount = *t.Options.PartitionsCount + } bytes := make([]byte, 0, len(streamIdBytes)+4+1+len(nameBytes)+len(optionsBytes)) bytes = append(bytes, streamIdBytes...) - bytes = binary.LittleEndian.AppendUint32(bytes, t.PartitionsCount) + bytes = binary.LittleEndian.AppendUint32(bytes, partitionsCount) bytes = append(bytes, byte(len(nameBytes))) bytes = append(bytes, nameBytes...) bytes = append(bytes, optionsBytes...) @@ -72,86 +71,71 @@ // options builds the trailing options block. partitions_count is not an // option: it fills the command's own fixed field. Keys carrying the // server-default sentinel (expiry 0, size 0, compression none) are omitted so -// the server derives them and returns them as derived entries. +// the server derives them and returns them as derived entries. Both +// durability keys are always sent, replicated unless named otherwise. func (t *CreateTopic) options() ([]iggcon.HeaderEntry, error) { - var options []iggcon.HeaderEntry - compression, err := t.CompressionAlgorithm.OptionValue() + typed, err := t.typedOptions() if err != nil { return nil, err } - if compression != "" { - options = append(options, stringOption(topicOptionCompressionAlgorithm, compression)) - } - if t.MessageExpiry != 0 { - options = append(options, uint64Option(topicOptionMessageExpiry, uint64(t.MessageExpiry))) - } - if t.MaxTopicSize != 0 { - options = append(options, uint64Option(topicOptionMaxTopicSize, t.MaxTopicSize)) - } - merged, err := mergeOptions(options, t.Options) + merged, err := mergeOptions(typed, t.Options.Raw) if err != nil { return nil, err } - for _, key := range []string{"durability", "consumer_offset_durability"} { - found := false - for _, entry := range merged { - if string(entry.Key.Value) == key { - found = true - if entry.Value.Kind != iggcon.String || (string(entry.Value.Value) != string(iggcon.DurabilityReplicated) && string(entry.Value.Value) != string(iggcon.DurabilityPersisted)) { - return nil, fmt.Errorf("invalid %s", key) - } - } - } - if !found { - merged = append(merged, stringOption(key, string(iggcon.DurabilityReplicated))) - } - } - return merged, nil -} - -// mergeOptions appends the caller's entries to the ones the typed fields -// produced, dropping any that would duplicate a typed key. -// -// The block must not carry a key twice: wire validation refuses the whole -// request for a duplicate. The typed field wins because it is the specific -// argument, mirroring how the Rust SDK inserts typed values after raw ones. -func mergeOptions(typed, extra []iggcon.HeaderEntry) ([]iggcon.HeaderEntry, error) { - if len(extra) == 0 { - return typed, nil - } - seen := make(map[string]struct{}, len(typed)+len(extra)) - for _, entry := range typed { - seen[string(entry.Key.Value)] = struct{}{} - } - merged := typed - for _, entry := range extra { - if entry.Key.Kind != iggcon.String { - return nil, fmt.Errorf("option key kind %d is not a string", entry.Key.Kind) - } - key := string(entry.Key.Value) - if _, duplicate := seen[key]; duplicate { + for _, key := range []iggcon.TopicOptionKey{iggcon.TopicOptionDurability, iggcon.TopicOptionConsumerOffsetDurability} { + index := slices.IndexFunc(merged, func(entry iggcon.HeaderEntry) bool { return string(entry.Key.Value) == string(key) }) + if index < 0 { + merged = append(merged, stringOption(string(key), string(iggcon.DurabilityReplicated))) continue } - seen[key] = struct{}{} - merged = append(merged, entry) + value := merged[index].Value + if value.Kind != iggcon.String || string(value.Value) != string(iggcon.DurabilityReplicated) && string(value.Value) != string(iggcon.DurabilityPersisted) { + return nil, ierror.InvalidOptionValue{Key: string(key)} + } } return merged, nil } -func uint64Option(key string, value uint64) iggcon.HeaderEntry { - buf := make([]byte, 8) - binary.LittleEndian.PutUint64(buf, value) - return iggcon.HeaderEntry{ - Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte(key)}, - Value: iggcon.HeaderValue{Kind: iggcon.Uint64, Value: buf}, +func (t *CreateTopic) typedOptions() ([]iggcon.HeaderEntry, error) { + options := t.Options + var typed []iggcon.HeaderEntry + if options.CompressionAlgorithm != nil { + compression, err := options.CompressionAlgorithm.OptionValue() + if err != nil { + return nil, ierror.InvalidOptionValue{Key: string(iggcon.TopicOptionCompressionAlgorithm)} + } + if compression != "" { + typed = append(typed, stringOption(string(iggcon.TopicOptionCompressionAlgorithm), compression)) + } } -} - -func stringOption(key string, value string) iggcon.HeaderEntry { - return iggcon.HeaderEntry{ - Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte(key)}, - Value: iggcon.HeaderValue{Kind: iggcon.String, Value: []byte(value)}, + if options.MessageExpiry != nil && *options.MessageExpiry != iggcon.IggyExpiryServerDefault { + typed = append(typed, uint64Option(string(iggcon.TopicOptionMessageExpiry), uint64(*options.MessageExpiry))) } + if options.MaxTopicSize != nil && *options.MaxTopicSize != 0 { + typed = append(typed, uint64Option(string(iggcon.TopicOptionMaxTopicSize), *options.MaxTopicSize)) + } + if options.SegmentSize != nil { + typed = append(typed, iggcon.SegmentSizeOption(*options.SegmentSize)) + } + if options.Durability != nil { + typed = append(typed, iggcon.DurabilityOption(*options.Durability)) + } + if options.ConsumerOffsetDurability != nil { + typed = append(typed, iggcon.ConsumerOffsetDurabilityOption(*options.ConsumerOffsetDurability)) + } + if options.PartitionResizePolicy != nil { + typed = append(typed, iggcon.PartitionResizePolicyOption(*options.PartitionResizePolicy)) + } + if options.MessagesRequiredToSave != nil { + typed = append(typed, iggcon.MessagesRequiredToSaveOption(*options.MessagesRequiredToSave)) + } + if options.SizeOfMessagesRequiredToSave != nil { + typed = append(typed, iggcon.SizeOfMessagesRequiredToSaveOption(*options.SizeOfMessagesRequiredToSave)) + } + if options.PreallocateSegments != nil { + typed = append(typed, iggcon.PreallocateSegmentsOption(*options.PreallocateSegments)) + } + return typed, nil } type GetTopic struct { @@ -192,73 +176,39 @@ return iggcon.MarshalIdentifiers(d.StreamId, d.TopicId) } +// UpdateTopic patches a topic through the generic UpdateResource command. +// Create-only settings, compression included, have no field here. type UpdateTopic struct { - StreamId iggcon.Identifier `json:"streamId"` - TopicId iggcon.Identifier `json:"topicId"` - Name string `json:"name"` - // Options carries keys with no field below. The server refuses any key an - // update may not change, by name. - Options []iggcon.HeaderEntry `json:"-"` - // Settings ride the options block. A zero value means "leave it alone", - // so a rename does not silently reset the rest. - CompressionAlgorithm iggcon.CompressionAlgorithm `json:"compressionAlgorithm"` - MessageExpiry iggcon.Duration `json:"messageExpiry"` - MaxTopicSize uint64 `json:"maxTopicSize"` + StreamId iggcon.Identifier + TopicId iggcon.Identifier + Name *string + MessageExpiry *iggcon.Duration + MaxTopicSize *uint64 + Reset []iggcon.TopicOptionKey + RawReset []string + Options []iggcon.HeaderEntry } func (u *UpdateTopic) Code() Code { - return UpdateTopicCode + return UpdateResourceCode } -// options builds the trailing options block. Only keys an update may change are -// allowed; the server rejects the create-time knobs by name. A zero value means -// the caller did not set the key, so it is omitted and the server leaves the -// topic's current value alone. -func (u *UpdateTopic) options() ([]iggcon.HeaderEntry, error) { - var options []iggcon.HeaderEntry - compression, err := u.CompressionAlgorithm.OptionValue() - if err != nil { - return nil, err - } - if compression != "" { - options = append(options, stringOption(topicOptionCompressionAlgorithm, compression)) - } - if u.MessageExpiry != 0 { - options = append(options, uint64Option(topicOptionMessageExpiry, uint64(u.MessageExpiry))) - } - if u.MaxTopicSize != 0 { - options = append(options, uint64Option(topicOptionMaxTopicSize, u.MaxTopicSize)) - } - return mergeOptions(options, u.Options) -} - +// MarshalBinary encodes the patch. A zero expiry or size is the server-default +// sentinel and changes nothing, Reset returns a key to its default. func (u *UpdateTopic) MarshalBinary() ([]byte, error) { - streamIdBytes, err := u.StreamId.MarshalBinary() - if err != nil { - return nil, err + var typed []iggcon.HeaderEntry + if u.Name != nil { + typed = append(typed, textOption(string(iggcon.TopicOptionName), *u.Name)) } - topicIdBytes, err := u.TopicId.MarshalBinary() - if err != nil { - return nil, err + if u.MessageExpiry != nil && *u.MessageExpiry != iggcon.IggyExpiryServerDefault { + typed = append(typed, uint64Option(string(iggcon.TopicOptionMessageExpiry), uint64(*u.MessageExpiry))) } - - options, err := u.options() - if err != nil { - return nil, err + if u.MaxTopicSize != nil && *u.MaxTopicSize != 0 { + typed = append(typed, uint64Option(string(iggcon.TopicOptionMaxTopicSize), *u.MaxTopicSize)) } - optionsBytes := iggcon.GetHeadersBytes(options) - buffer := make([]byte, 1+len(streamIdBytes)+len(topicIdBytes)+len(u.Name)+len(optionsBytes)) - - offset := 0 - - offset += copy(buffer[offset:], streamIdBytes) - offset += copy(buffer[offset:], topicIdBytes) - - buffer[offset] = uint8(len(u.Name)) - offset++ - - offset += copy(buffer[offset:], u.Name) - copy(buffer[offset:], optionsBytes) - - return buffer, nil + reset := make([]string, 0, len(u.Reset)) + for _, key := range u.Reset { + reset = append(reset, string(key)) + } + return encodeUpdateResource(iggcon.OptionsScopeTopic, []iggcon.Identifier{u.StreamId, u.TopicId}, typed, u.Options, reset, u.RawReset) }
diff --git a/foreign/go/internal/command/topic_test.go b/foreign/go/internal/command/topic_test.go index 2c450a0..eb542e8 100644 --- a/foreign/go/internal/command/topic_test.go +++ b/foreign/go/internal/command/topic_test.go
@@ -27,11 +27,12 @@ func TestSerialize_CreateTopic_ServerDefaults(t *testing.T) { streamId, _ := iggcon.NewIdentifier("stream") + partitions := uint32(2) + compression := iggcon.CompressionAlgorithmNone request := CreateTopic{ - StreamId: streamId, - Name: "topic", - PartitionsCount: 2, - CompressionAlgorithm: iggcon.CompressionAlgorithmNone, + StreamId: streamId, + Name: "topic", + Options: iggcon.TopicOptions{PartitionsCount: &partitions, CompressionAlgorithm: &compression}, } serialized, err := request.MarshalBinary() @@ -46,10 +47,11 @@ 0x02, 0x00, 0x00, 0x00, // PartitionsCount (2) 0x05, // Name Length (5) 0x74, 0x6F, 0x70, 0x69, 0x63, // Name ("topic") - 2, 10, 0, 0, 0, 'd', 'u', 'r', 'a', 'b', 'i', 'l', 'i', 't', 'y', - 2, 10, 0, 0, 0, 'r', 'e', 'p', 'l', 'i', 'c', 'a', 't', 'e', 'd', + // Options travel in ascending key order. 2, 26, 0, 0, 0, 'c', 'o', 'n', 's', 'u', 'm', 'e', 'r', '_', 'o', 'f', 'f', 's', 'e', 't', '_', 'd', 'u', 'r', 'a', 'b', 'i', 'l', 'i', 't', 'y', 2, 10, 0, 0, 0, 'r', 'e', 'p', 'l', 'i', 'c', 'a', 't', 'e', 'd', + 2, 10, 0, 0, 0, 'd', 'u', 'r', 'a', 'b', 'i', 'l', 'i', 't', 'y', + 2, 10, 0, 0, 0, 'r', 'e', 'p', 'l', 'i', 'c', 'a', 't', 'e', 'd', } if !bytes.Equal(serialized, expected) { @@ -59,13 +61,19 @@ func TestSerialize_CreateTopic_NonDefaultsBecomeOptions(t *testing.T) { streamId, _ := iggcon.NewIdentifier(uint32(1)) + partitions := uint32(4) + compression := iggcon.CompressionAlgorithmGzip + expiry := 100 * iggcon.Microsecond + maxTopicSize := uint64(1 << 30) request := CreateTopic{ - StreamId: streamId, - Name: "topic", - PartitionsCount: 4, - CompressionAlgorithm: iggcon.CompressionAlgorithmGzip, - MessageExpiry: 100 * iggcon.Microsecond, - MaxTopicSize: 1 << 30, + StreamId: streamId, + Name: "topic", + Options: iggcon.TopicOptions{ + PartitionsCount: &partitions, + CompressionAlgorithm: &compression, + MessageExpiry: &expiry, + MaxTopicSize: &maxTopicSize, + }, } serialized, err := request.MarshalBinary() @@ -99,15 +107,15 @@ if _, found := byKey["partitions_count"]; found { t.Error("partitions_count rides the fixed field, not the options block") } - compression := byKey[topicOptionCompressionAlgorithm] - if compression.Kind != iggcon.String || string(compression.Value) != "gzip" { - t.Errorf("compression_algorithm = %+v, want String %q", compression, "gzip") + compressionValue := byKey[string(iggcon.TopicOptionCompressionAlgorithm)] + if compressionValue.Kind != iggcon.String || string(compressionValue.Value) != "gzip" { + t.Errorf("compression_algorithm = %+v, want String %q", compressionValue, "gzip") } - expiry := byKey[topicOptionMessageExpiry] - if expiry.Kind != iggcon.Uint64 || binary.LittleEndian.Uint64(expiry.Value) != 100 { - t.Errorf("message_expiry = %+v, want Uint64 100", expiry) + expiryValue := byKey[string(iggcon.TopicOptionMessageExpiry)] + if expiryValue.Kind != iggcon.Uint64 || binary.LittleEndian.Uint64(expiryValue.Value) != 100 { + t.Errorf("message_expiry = %+v, want Uint64 100", expiryValue) } - maxSize := byKey[topicOptionMaxTopicSize] + maxSize := byKey[string(iggcon.TopicOptionMaxTopicSize)] if maxSize.Kind != iggcon.Uint64 || binary.LittleEndian.Uint64(maxSize.Value) != 1<<30 { t.Errorf("max_topic_size = %+v, want Uint64 %d", maxSize, 1<<30) } @@ -116,12 +124,15 @@ func TestSerialize_UpdateTopic(t *testing.T) { streamId, _ := iggcon.NewIdentifier("stream") topicId, _ := iggcon.NewIdentifier(uint32(1)) + name := "update_topic" + expiry := 100 * iggcon.Microsecond + maxTopicSize := uint64(100) request := UpdateTopic{ StreamId: streamId, TopicId: topicId, - Name: "update_topic", - MessageExpiry: 100 * iggcon.Microsecond, - MaxTopicSize: 100, + Name: &name, + MessageExpiry: &expiry, + MaxTopicSize: &maxTopicSize, } serialized1, err := request.MarshalBinary() @@ -130,23 +141,27 @@ } expected := []byte{ + 0x01, // Scope (topic) 0x02, // StreamId Kind (StringId) - 0x06, // StreamId Length (2) + 0x06, // StreamId Length (6) 0x73, 0x74, 0x72, 0x65, 0x61, 0x6D, // StreamId Value ("stream") 0x01, // TopicId Kind (NumericId) 0x04, // TopicId Length (4) 0x01, 0x00, 0x00, 0x00, // TopicId Value (1) - 0x0C, // Name Length (12) - 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5F, 0x74, 0x6F, 0x70, 0x69, 0x63, // Name ("update_topic") - // Settings ride the options block; compression is None so it is omitted. - 0x02, 0x0E, 0x00, 0x00, 0x00, // key kind (String), key length (14) - 0x6D, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x5F, 0x65, 0x78, 0x70, 0x69, 0x72, 0x79, // "message_expiry" - 0x0C, 0x08, 0x00, 0x00, 0x00, // value kind (Uint64), value length (8) - 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 100 + 0x00, // Reset count + // The name and the settings ride the options block in ascending key order. 0x02, 0x0E, 0x00, 0x00, 0x00, // key kind (String), key length (14) 0x6D, 0x61, 0x78, 0x5F, 0x74, 0x6F, 0x70, 0x69, 0x63, 0x5F, 0x73, 0x69, 0x7A, 0x65, // "max_topic_size" 0x0C, 0x08, 0x00, 0x00, 0x00, // value kind (Uint64), value length (8) 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 100 + 0x02, 0x0E, 0x00, 0x00, 0x00, // key kind (String), key length (14) + 0x6D, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x5F, 0x65, 0x78, 0x70, 0x69, 0x72, 0x79, // "message_expiry" + 0x0C, 0x08, 0x00, 0x00, 0x00, // value kind (Uint64), value length (8) + 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 100 + 0x02, 0x04, 0x00, 0x00, 0x00, // key kind (String), key length (4) + 0x6E, 0x61, 0x6D, 0x65, // "name" + 0x02, 0x0C, 0x00, 0x00, 0x00, // value kind (String), value length (12) + 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5F, 0x74, 0x6F, 0x70, 0x69, 0x63, // "update_topic" } if !bytes.Equal(serialized1, expected) { @@ -159,12 +174,11 @@ if err != nil { t.Fatalf("unexpected error: %v", err) } + maxTopicSize := uint64(4096) command := CreateTopic{ - StreamId: streamId, - Name: "t", - PartitionsCount: 1, - MaxTopicSize: 4096, - Options: []iggcon.HeaderEntry{ + StreamId: streamId, + Name: "t", + Options: iggcon.TopicOptions{MaxTopicSize: &maxTopicSize, Raw: []iggcon.HeaderEntry{ { Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte("durability")}, Value: iggcon.HeaderValue{Kind: iggcon.String, Value: []byte("persisted")}, @@ -175,7 +189,7 @@ Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte("max_topic_size")}, Value: iggcon.HeaderValue{Kind: iggcon.String, Value: []byte("1 GiB")}, }, - }, + }}, } options, err := command.options() @@ -205,16 +219,15 @@ t.Fatalf("unexpected error: %v", err) } command := CreateTopic{ - StreamId: streamId, - Name: "topic", - PartitionsCount: 1, - Options: []iggcon.HeaderEntry{ + StreamId: streamId, + Name: "topic", + Options: iggcon.TopicOptions{Raw: []iggcon.HeaderEntry{ iggcon.SegmentSizeOption(1 << 20), iggcon.DurabilityOption(iggcon.DurabilityPersisted), iggcon.MessagesRequiredToSaveOption(7), iggcon.SizeOfMessagesRequiredToSaveOption(4096), iggcon.PreallocateSegmentsOption(false), - }, + }}, } serialized, err := command.MarshalBinary() @@ -271,13 +284,12 @@ t.Fatalf("unexpected error: %v", err) } command := CreateTopic{ - StreamId: streamId, - Name: "t", - PartitionsCount: 1, - Options: []iggcon.HeaderEntry{{ + StreamId: streamId, + Name: "t", + Options: iggcon.TopicOptions{Raw: []iggcon.HeaderEntry{{ Key: iggcon.HeaderKey{Kind: iggcon.Uint32, Value: []byte{1, 0, 0, 0}}, Value: iggcon.HeaderValue{Kind: iggcon.String, Value: []byte("x")}, - }}, + }}}, } if _, err := command.MarshalBinary(); err == nil {
diff --git a/foreign/go/internal/command/update_resource.go b/foreign/go/internal/command/update_resource.go new file mode 100644 index 0000000..670e649 --- /dev/null +++ b/foreign/go/internal/command/update_resource.go
@@ -0,0 +1,318 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package command + +import ( + "bytes" + "encoding/binary" + "fmt" + "slices" + "unicode/utf8" + + iggcon "github.com/apache/iggy/foreign/go/contracts" + ierror "github.com/apache/iggy/foreign/go/errors" +) + +const ( + // maxOptions and maxOptionsBytes mirror the server's options block bounds. + maxOptions = 1024 + maxOptionsBytes = 100_000 + // maxOptionField bounds every key and value of the header codec. + maxOptionField = 255 + // maxResetKeys is what the one-byte reset count carries. + maxResetKeys = 255 + // minUsernameLength and maxUsernameLength mirror the server's username bounds. + minUsernameLength = 3 + maxUsernameLength = 50 + // keyPreviewLength caps how much of a refused key an error repeats. + keyPreviewLength = 64 +) + +// optionRule is what this build knows about one catalog key, enough to refuse +// a reset with the category server admission would return. +type optionRule struct { + key string + scopes []iggcon.OptionsScope + updatable bool + resettable bool + // readOnly covers the internal keys too: a client may send neither. + readOnly bool +} + +// optionCatalog lists the server catalog in its own order. +var optionCatalog = []optionRule{ + {key: string(iggcon.TopicOptionName), scopes: []iggcon.OptionsScope{iggcon.OptionsScopeStream, iggcon.OptionsScopeTopic}, updatable: true}, + {key: string(iggcon.UserOptionUsername), scopes: []iggcon.OptionsScope{iggcon.OptionsScopeUser}, updatable: true}, + {key: string(iggcon.UserOptionStatus), scopes: []iggcon.OptionsScope{iggcon.OptionsScopeUser}, updatable: true}, + {key: string(iggcon.TopicOptionCompressionAlgorithm), scopes: []iggcon.OptionsScope{iggcon.OptionsScopeTopic}}, + {key: string(iggcon.TopicOptionMessageExpiry), scopes: []iggcon.OptionsScope{iggcon.OptionsScopeTopic}, updatable: true, resettable: true}, + {key: string(iggcon.TopicOptionMaxTopicSize), scopes: []iggcon.OptionsScope{iggcon.OptionsScopeTopic}, updatable: true, resettable: true}, + {key: string(iggcon.TopicOptionSegmentSize), scopes: []iggcon.OptionsScope{iggcon.OptionsScopeTopic}}, + {key: string(iggcon.TopicOptionDurability), scopes: []iggcon.OptionsScope{iggcon.OptionsScopeTopic}}, + {key: string(iggcon.TopicOptionConsumerOffsetDurability), scopes: []iggcon.OptionsScope{iggcon.OptionsScopeTopic}}, + {key: string(iggcon.TopicOptionPartitionResizePolicy), scopes: []iggcon.OptionsScope{iggcon.OptionsScopeTopic}}, + {key: string(iggcon.TopicOptionMessagesRequiredToSave), scopes: []iggcon.OptionsScope{iggcon.OptionsScopeTopic}}, + {key: string(iggcon.TopicOptionSizeOfMessagesRequiredToSave), scopes: []iggcon.OptionsScope{iggcon.OptionsScopeTopic}}, + {key: string(iggcon.TopicOptionPreallocateSegments), scopes: []iggcon.OptionsScope{iggcon.OptionsScopeTopic}}, + {key: string(iggcon.StreamOptionEncryptionKey), scopes: []iggcon.OptionsScope{iggcon.OptionsScopeStream}}, + {key: "encryption_key_wrapped", scopes: []iggcon.OptionsScope{iggcon.OptionsScopeStream}, readOnly: true}, + {key: string(iggcon.StreamOptionEncryptionKeySource), scopes: []iggcon.OptionsScope{iggcon.OptionsScopeStream}, readOnly: true}, + {key: string(iggcon.StreamOptionEncryptionKeyFingerprint), scopes: []iggcon.OptionsScope{iggcon.OptionsScopeStream}, readOnly: true}, + {key: string(iggcon.StreamOptionEncryptionKekFingerprint), scopes: []iggcon.OptionsScope{iggcon.OptionsScopeStream}, readOnly: true}, + {key: "critical_keys", scopes: []iggcon.OptionsScope{iggcon.OptionsScopeStream, iggcon.OptionsScopeTopic, iggcon.OptionsScopeUser}, readOnly: true}, +} + +// encodeUpdateResource builds an UpdateResource body: +// [scope:u8][target identifiers][reset_count:u8][reset_key:u8 length + bytes]*[options to end]. +// +// typed entries come from the update's own fields and win over extra entries +// with the same key. A typed-only patch the server would refuse is refused +// here first, with the same error. A patch with extra entries or raw reset +// names gets only the structural checks, and the server decides the rest. +func encodeUpdateResource( + scope iggcon.OptionsScope, + targets []iggcon.Identifier, + typed, extra []iggcon.HeaderEntry, + reset, rawReset []string, +) ([]byte, error) { + set, err := mergeOptions(typed, extra) + if err != nil { + return nil, err + } + reset, err = validatePatch(scope, set, append(slices.Clip(reset), rawReset...), len(extra) > 0 || len(rawReset) > 0) + if err != nil { + return nil, err + } + options, err := encodeOptions(set) + if err != nil { + return nil, err + } + + body := []byte{byte(scope)} + for _, target := range targets { + if body, err = target.AppendBinary(body); err != nil { + return nil, err + } + } + body = append(body, byte(len(reset))) + for _, key := range reset { + body = append(body, byte(len(key))) + body = append(body, key...) + } + return append(body, options...), nil +} + +// validatePatch returns the reset names deduplicated in ascending order, the +// order the server's own encoders write. A typed-only patch gets server +// admission's update checks in the server's order. A raw patch gets only the +// structural ones, because the server's catalog may know keys or flags this +// build does not. +func validatePatch(scope iggcon.OptionsScope, set []iggcon.HeaderEntry, reset []string, raw bool) ([]string, error) { + names := slices.Compact(slices.Sorted(slices.Values(reset))) + if len(set) == 0 && len(names) == 0 { + return nil, ierror.ErrInvalidCommand + } + for _, entry := range set { + if slices.Contains(names, string(entry.Key.Value)) { + return nil, ierror.ErrInvalidCommand + } + } + if !raw { + if err := admitUpdate(scope, set, names); err != nil { + return nil, err + } + } + for _, name := range names { + if len(name) == 0 || len(name) > maxOptionField { + return nil, ierror.UnsupportedOptionKey{Key: keyPreview(name)} + } + } + if len(names) > maxResetKeys { + // The catalog is far smaller than the limit, so such a list always + // names a key admission would refuse. + for _, name := range names { + if _, known := catalogRule(scope, name); !known { + return nil, ierror.UnsupportedOptionKey{Key: keyPreview(name)} + } + } + } + return names, nil +} + +// admitUpdate mirrors server admission for a patch of typed fields: reset key +// categories first, then the column values. +func admitUpdate(scope iggcon.OptionsScope, set []iggcon.HeaderEntry, names []string) error { + rules := make([]optionRule, 0, len(names)) + for _, name := range names { + rule, known := catalogRule(scope, name) + if !known { + return ierror.UnsupportedOptionKey{Key: keyPreview(name)} + } + rules = append(rules, rule) + } + for _, rule := range rules { + if rule.readOnly { + return ierror.ReadOnlyOption{Key: rule.key} + } + } + for _, rule := range rules { + if !rule.updatable { + return ierror.OptionNotUpdatable{Key: rule.key} + } + } + for _, rule := range rules { + if !rule.resettable { + return ierror.OptionNotResettable{Key: rule.key} + } + } + for _, entry := range slices.SortedFunc(slices.Values(set), func(left, right iggcon.HeaderEntry) int { + return bytes.Compare(left.Key.Value, right.Key.Value) + }) { + if err := admitColumn(scope, entry); err != nil { + return err + } + } + return nil +} + +// admitColumn applies the server's bounds to a name or username value. +func admitColumn(scope iggcon.OptionsScope, entry iggcon.HeaderEntry) error { + text := entry.Value.Value + isText := entry.Value.Kind == iggcon.String + switch string(entry.Key.Value) { + case string(iggcon.TopicOptionName): + if !isText || len(text) == 0 || len(text) > maxOptionField { + if scope == iggcon.OptionsScopeStream { + return ierror.ErrInvalidStreamName + } + return ierror.ErrInvalidTopicName + } + case string(iggcon.UserOptionUsername): + if !isText || len(text) < minUsernameLength || len(text) > maxUsernameLength { + return ierror.ErrInvalidUsername + } + } + return nil +} + +// catalogRule finds the rule of a key that applies to scope. +func catalogRule(scope iggcon.OptionsScope, name string) (optionRule, bool) { + index := slices.IndexFunc(optionCatalog, func(rule optionRule) bool { + return rule.key == name && slices.Contains(rule.scopes, scope) + }) + if index < 0 { + return optionRule{}, false + } + return optionCatalog[index], true +} + +// mergeOptions appends the caller's entries to the ones the typed fields +// produced, dropping any that would duplicate a typed key. +// +// The block must not carry a key twice: wire validation refuses the whole +// request for a duplicate. The typed field wins because it is the specific +// argument, mirroring how the Rust SDK inserts typed values after raw ones. +func mergeOptions(typed, extra []iggcon.HeaderEntry) ([]iggcon.HeaderEntry, error) { + if len(extra) == 0 { + return typed, nil + } + seen := make(map[string]struct{}, len(typed)+len(extra)) + for _, entry := range typed { + seen[string(entry.Key.Value)] = struct{}{} + } + merged := slices.Clip(typed) + for _, entry := range extra { + if entry.Key.Kind != iggcon.String { + return nil, ierror.UnsupportedOptionKey{Key: keyPreview(string(entry.Key.Value))} + } + if _, duplicate := seen[string(entry.Key.Value)]; duplicate { + continue + } + merged = append(merged, entry) + } + return merged, nil +} + +// encodeOptions encodes an options block in ascending key order, after the +// checks the server's decoder applies to it. +func encodeOptions(entries []iggcon.HeaderEntry) ([]byte, error) { + if len(entries) > maxOptions { + return nil, ierror.OptionsBlockTooLarge{Details: fmt.Sprintf("%d entries, at most %d", len(entries), maxOptions)} + } + sorted := slices.Clone(entries) + slices.SortFunc(sorted, func(left, right iggcon.HeaderEntry) int { + return bytes.Compare(left.Key.Value, right.Key.Value) + }) + size := 0 + for index, entry := range sorted { + key := entry.Key.Value + if entry.Key.Kind != iggcon.String || len(key) == 0 || len(key) > maxOptionField || !utf8.Valid(key) { + return nil, ierror.UnsupportedOptionKey{Key: keyPreview(string(key))} + } + if index > 0 && bytes.Equal(sorted[index-1].Key.Value, key) { + return nil, ierror.ErrInvalidCommand + } + value := entry.Value + expected := value.Kind.ExpectedSize() + if value.Kind <= 0 || value.Kind > 255 || len(value.Value) == 0 || len(value.Value) > maxOptionField || + expected != -1 && len(value.Value) != expected { + return nil, ierror.InvalidOptionValue{Key: keyPreview(string(key))} + } + size += 1 + 4 + len(key) + 1 + 4 + len(value.Value) + } + if size > maxOptionsBytes { + return nil, ierror.OptionsBlockTooLarge{Details: fmt.Sprintf("%d bytes, at most %d", size, maxOptionsBytes)} + } + return iggcon.GetHeadersBytes(sorted), nil +} + +// textOption builds a String entry for a column key. Text no header value +// holds travels as a Bool, a kind the column never accepts, so admission +// reports the column's own error after every check that comes before it. +func textOption(key string, value string) iggcon.HeaderEntry { + if len(value) == 0 || len(value) > maxOptionField { + return iggcon.HeaderEntry{ + Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte(key)}, + Value: iggcon.HeaderValue{Kind: iggcon.Bool, Value: []byte{0}}, + } + } + return stringOption(key, value) +} + +func stringOption(key string, value string) iggcon.HeaderEntry { + return iggcon.HeaderEntry{ + Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte(key)}, + Value: iggcon.HeaderValue{Kind: iggcon.String, Value: []byte(value)}, + } +} + +func uint64Option(key string, value uint64) iggcon.HeaderEntry { + return iggcon.HeaderEntry{ + Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte(key)}, + Value: iggcon.HeaderValue{Kind: iggcon.Uint64, Value: binary.LittleEndian.AppendUint64(nil, value)}, + } +} + +// keyPreview bounds how much of a refused, caller-sized key an error repeats. +func keyPreview(key string) string { + runes := []rune(key) + if len(runes) > keyPreviewLength { + return string(runes[:keyPreviewLength]) + } + return key +}
diff --git a/foreign/go/internal/command/update_resource_test.go b/foreign/go/internal/command/update_resource_test.go new file mode 100644 index 0000000..8172914 --- /dev/null +++ b/foreign/go/internal/command/update_resource_test.go
@@ -0,0 +1,181 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package command + +import ( + "bytes" + "errors" + "fmt" + "strings" + "testing" + + iggcon "github.com/apache/iggy/foreign/go/contracts" + ierror "github.com/apache/iggy/foreign/go/errors" +) + +func TestUpdateResource_RawPatches(t *testing.T) { + streamId, _ := iggcon.NewIdentifier(uint32(1)) + topicId, _ := iggcon.NewIdentifier(uint32(2)) + + t.Run("given_raw_set_key_and_create_only_reset_when_patching_should_send_it_for_the_server_to_decide", func(t *testing.T) { + update := &UpdateTopic{ + StreamId: streamId, + TopicId: topicId, + Reset: []iggcon.TopicOptionKey{iggcon.TopicOptionDurability}, + Options: []iggcon.HeaderEntry{rawEntry("retention_class", "cold")}, + } + body, err := update.MarshalBinary() + if err != nil { + t.Fatalf("raw patch refused locally: %v", err) + } + if !bytes.Contains(body, []byte("\x0adurability")) || !bytes.Contains(body, []byte("retention_class")) { + t.Fatalf("body lacks the reset or the raw key: %x", body) + } + }) + + t.Run("given_raw_reset_when_patching_should_send_it_for_the_server_to_decide", func(t *testing.T) { + update := &UpdateStream{StreamId: streamId, RawReset: []string{"retention_class", "retention_class"}} + body, err := update.MarshalBinary() + if err != nil { + t.Fatalf("raw reset refused locally: %v", err) + } + want := append([]byte{byte(iggcon.OptionsScopeStream)}, mustIdentifier(t, streamId)...) + want = append(want, 1, byte(len("retention_class"))) + want = append(want, "retention_class"...) + want = append(want, iggcon.GetHeadersBytes(nil)...) + if !bytes.Equal(body, want) { + t.Fatalf("body %x, want %x", body, want) + } + }) + + t.Run("given_raw_reset_also_set_when_patching_should_refuse_invalid_command", func(t *testing.T) { + update := &UpdateTopic{ + StreamId: streamId, + TopicId: topicId, + RawReset: []string{"retention_class"}, + Options: []iggcon.HeaderEntry{rawEntry("retention_class", "cold")}, + } + _, err := update.MarshalBinary() + if !errors.Is(err, ierror.ErrInvalidCommand) { + t.Fatalf("got %v, want invalid command", err) + } + }) + + t.Run("given_more_resets_than_the_wire_carries_when_patching_should_refuse_unsupported_key", func(t *testing.T) { + reset := make([]string, 0, 256) + for index := range 256 { + reset = append(reset, fmt.Sprintf("future_%03d", index)) + } + _, err := (&UpdateStream{StreamId: streamId, RawReset: reset}).MarshalBinary() + if !errors.Is(err, ierror.ErrUnsupportedOptionKey) { + t.Fatalf("got %v, want unsupported option key", err) + } + }) + + t.Run("given_reset_name_the_wire_cannot_carry_when_patching_should_refuse_with_a_bounded_preview", func(t *testing.T) { + for _, name := range []string{"", strings.Repeat("k", 256)} { + _, err := (&UpdateStream{StreamId: streamId, RawReset: []string{name}}).MarshalBinary() + var unsupported ierror.UnsupportedOptionKey + if !errors.As(err, &unsupported) || len(unsupported.Key) > keyPreviewLength { + t.Fatalf("name of %d bytes: got %v", len(name), err) + } + } + }) +} + +func TestUpdateResource_OptionsBlockBounds(t *testing.T) { + t.Run("given_block_at_the_byte_limit_when_encoding_should_accept_it_and_refuse_one_byte_more", func(t *testing.T) { + // 200 entries of 10 framing bytes, a 245-byte key and a 245-byte value. + entries := make([]iggcon.HeaderEntry, 0, 200) + for index := range 200 { + key := fmt.Sprintf("%03d", index) + strings.Repeat("k", 242) + entries = append(entries, rawEntry(key, strings.Repeat("v", 245))) + } + if _, err := encodeOptions(entries); err != nil { + t.Fatalf("block of exactly %d bytes refused: %v", maxOptionsBytes, err) + } + entries[0].Value.Value = append(entries[0].Value.Value, 'v') + if _, err := encodeOptions(entries); !errors.Is(err, ierror.ErrOptionsBlockTooLarge) { + t.Fatalf("got %v, want options block too large", err) + } + }) + + t.Run("given_more_entries_than_the_limit_when_encoding_should_refuse_block_too_large", func(t *testing.T) { + entries := make([]iggcon.HeaderEntry, 0, maxOptions+1) + for index := range maxOptions + 1 { + entries = append(entries, rawEntry(fmt.Sprintf("key_%04d", index), "v")) + } + if _, err := encodeOptions(entries[:maxOptions]); err != nil { + t.Fatalf("block of %d entries refused: %v", maxOptions, err) + } + if _, err := encodeOptions(entries); !errors.Is(err, ierror.ErrOptionsBlockTooLarge) { + t.Fatalf("got %v, want options block too large", err) + } + }) + + t.Run("given_malformed_entry_when_encoding_should_refuse_with_the_server_category", func(t *testing.T) { + cases := []struct { + name string + entry iggcon.HeaderEntry + want error + }{ + {"empty_key", rawEntry("", "v"), ierror.ErrUnsupportedOptionKey}, + {"key_over_255_bytes", rawEntry(strings.Repeat("k", 256), "v"), ierror.ErrUnsupportedOptionKey}, + {"key_not_utf8", rawEntry("\xff\xfe", "v"), ierror.ErrUnsupportedOptionKey}, + {"empty_value", rawEntry("future", ""), ierror.ErrInvalidOptionValue}, + {"value_over_255_bytes", rawEntry("future", strings.Repeat("v", 256)), ierror.ErrInvalidOptionValue}, + {"fixed_kind_with_wrong_size", iggcon.HeaderEntry{ + Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte("future")}, + Value: iggcon.HeaderValue{Kind: iggcon.Uint64, Value: []byte{1, 2, 3, 4}}, + }, ierror.ErrInvalidOptionValue}, + } + for _, testCase := range cases { + _, err := encodeOptions([]iggcon.HeaderEntry{testCase.entry}) + if !errors.Is(err, testCase.want) { + t.Errorf("%s: got %v, want %v", testCase.name, err, testCase.want) + } + var unsupported ierror.UnsupportedOptionKey + if errors.As(err, &unsupported) && len(unsupported.Key) > keyPreviewLength { + t.Errorf("%s: error repeats %d bytes of the key", testCase.name, len(unsupported.Key)) + } + } + }) + + t.Run("given_raw_entries_with_the_same_key_when_encoding_should_refuse_invalid_command", func(t *testing.T) { + _, err := encodeOptions([]iggcon.HeaderEntry{rawEntry("future", "a"), rawEntry("future", "b")}) + if !errors.Is(err, ierror.ErrInvalidCommand) { + t.Fatalf("got %v, want invalid command", err) + } + }) +} + +func rawEntry(key, value string) iggcon.HeaderEntry { + return iggcon.HeaderEntry{ + Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte(key)}, + Value: iggcon.HeaderValue{Kind: iggcon.String, Value: []byte(value)}, + } +} + +func mustIdentifier(t *testing.T, identifier iggcon.Identifier) []byte { + t.Helper() + encoded, err := identifier.AppendBinary(nil) + if err != nil { + t.Fatalf("identifier: %v", err) + } + return encoded +}
diff --git a/foreign/go/internal/command/update_user.go b/foreign/go/internal/command/update_user.go index 452b4db..c12c724 100644 --- a/foreign/go/internal/command/update_user.go +++ b/foreign/go/internal/command/update_user.go
@@ -17,80 +17,52 @@ package command -import "github.com/apache/iggy/foreign/go/contracts" +import ( + "github.com/apache/iggy/foreign/go/contracts" + ierror "github.com/apache/iggy/foreign/go/errors" +) +// UpdateUser patches a user through the generic UpdateResource command. type UpdateUser struct { - UserID iggcon.Identifier `json:"-"` - Username *string `json:"username"` - Status *iggcon.UserStatus `json:"userStatus"` - // Options carries the trailing options block. Users have no catalog keys - // yet, so the server rejects every key; the block is the extension point - // for the first one. - Options []iggcon.HeaderEntry `json:"-"` + UserID iggcon.Identifier + Username *string + Status *iggcon.UserStatus + Reset []iggcon.UserOptionKey + RawReset []string + Options []iggcon.HeaderEntry } func (u *UpdateUser) Code() Code { - return UpdateUserCode + return UpdateResourceCode } func (u *UpdateUser) MarshalBinary() ([]byte, error) { - userIdBytes, err := u.UserID.MarshalBinary() - if err != nil { - return nil, err - } - username := "" + var typed []iggcon.HeaderEntry if u.Username != nil { - username = *u.Username - } - - // Both presence flags are always written; only the payload behind a flag is - // conditional. The options block decodes to the end of the payload, so a - // byte of slack here is a stray zero-kind entry the server rejects. - length := len(userIdBytes) + 2 - if len(username) != 0 { - length += 1 + len(username) + typed = append(typed, textOption(string(iggcon.UserOptionUsername), *u.Username)) } if u.Status != nil { - length++ - } - - optionsBytes := iggcon.GetHeadersBytes(u.Options) - bytes := make([]byte, length+len(optionsBytes)) - position := 0 - - copy(bytes[position:position+len(userIdBytes)], userIdBytes) - position += len(userIdBytes) - - if len(username) != 0 { - bytes[position] = 1 - position++ - bytes[position] = byte(len(username)) - position++ - copy(bytes[position:position+len(username)], username) - position += len(username) - } else { - bytes[position] = 0 - position++ - } - - if u.Status != nil { - bytes[position] = 1 - position++ - statusByte := byte(0) - switch *u.Status { - case iggcon.Active: - statusByte = 1 - case iggcon.Inactive: - statusByte = 2 + status, err := userStatusName(*u.Status) + if err != nil { + return nil, err } - bytes[position] = statusByte - position++ - } else { - bytes[position] = 0 - position++ + typed = append(typed, stringOption(string(iggcon.UserOptionStatus), status)) } + reset := make([]string, 0, len(u.Reset)) + for _, key := range u.Reset { + reset = append(reset, string(key)) + } + return encodeUpdateResource(iggcon.OptionsScopeUser, []iggcon.Identifier{u.UserID}, typed, u.Options, reset, u.RawReset) +} - copy(bytes[position:], optionsBytes) - - return bytes, nil +// userStatusName is the status as the catalog spells it in an update. +func userStatusName(status iggcon.UserStatus) (string, error) { + switch status { + case iggcon.Active: + return "active", nil + case iggcon.Inactive: + return "inactive", nil + default: + return "", ierror.ErrInvalidUserStatus + } }
diff --git a/foreign/go/internal/command/update_user_test.go b/foreign/go/internal/command/update_user_test.go index 5cead96..dd76f7c 100644 --- a/foreign/go/internal/command/update_user_test.go +++ b/foreign/go/internal/command/update_user_test.go
@@ -18,9 +18,12 @@ package command import ( + "errors" + "slices" "testing" iggcon "github.com/apache/iggy/foreign/go/contracts" + ierror "github.com/apache/iggy/foreign/go/errors" ) // The options block decodes to the end of the payload, so any slack the sizing @@ -41,12 +44,11 @@ name string username *string status *iggcon.UserStatus - want int + want []string }{ - {"both set", &username, &status, len(userIDBytes) + 2 + len(username) + 2}, - {"username only", &username, nil, len(userIDBytes) + 2 + len(username) + 1}, - {"status only", nil, &status, len(userIDBytes) + 1 + 2}, - {"neither set", nil, nil, len(userIDBytes) + 2}, + {"both set", &username, &status, []string{"status", "username"}}, + {"username only", &username, nil, []string{"username"}}, + {"status only", nil, &status, []string{"status"}}, } for _, test := range tests { command := UpdateUser{UserID: userID, Username: test.username, Status: test.status} @@ -54,9 +56,33 @@ if err != nil { t.Fatalf("%s: unexpected error: %v", test.name, err) } - if len(payload) != test.want { - t.Errorf("%s: payload length = %d, want %d", test.name, len(payload), test.want) + prefix := 1 + len(userIDBytes) + 1 + if len(payload) < prefix || payload[0] != byte(iggcon.OptionsScopeUser) || payload[prefix-1] != 0 { + t.Fatalf("%s: payload %v does not lead with the user scope, id and no resets", test.name, payload) } + entries, err := iggcon.DeserializeHeaders(payload[prefix:]) + if err != nil { + t.Fatalf("%s: options block does not decode to the end: %v", test.name, err) + } + var keys []string + for _, entry := range entries { + keys = append(keys, string(entry.Key.Value)) + } + if !slices.Equal(keys, test.want) { + t.Errorf("%s: keys = %v, want %v", test.name, keys, test.want) + } + } +} + +func TestUpdateUser_EmptyPatchIsRefused(t *testing.T) { + userID, err := iggcon.NewIdentifier(uint32(7)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + command := UpdateUser{UserID: userID} + + if _, err := command.MarshalBinary(); !errors.Is(err, ierror.ErrInvalidCommand) { + t.Errorf("err = %v, want InvalidCommand", err) } } @@ -65,7 +91,8 @@ if err != nil { t.Fatalf("unexpected error: %v", err) } - command := UpdateUser{UserID: userID} + status := iggcon.Inactive + command := UpdateUser{UserID: userID, Status: &status} if _, err = command.MarshalBinary(); err != nil { t.Fatalf("unexpected error: %v", err)
diff --git a/foreign/go/internal/command/user.go b/foreign/go/internal/command/user.go index a0470d9..ed4f6c7 100644 --- a/foreign/go/internal/command/user.go +++ b/foreign/go/internal/command/user.go
@@ -28,6 +28,7 @@ Password string `json:"Password"` Status iggcon.UserStatus `json:"Status"` Permissions *iggcon.Permissions `json:"Permissions,omitempty"` + Options iggcon.UserOptions `json:"-"` } func (c *CreateUser) Code() Code { @@ -35,12 +36,16 @@ } func (c *CreateUser) MarshalBinary() ([]byte, error) { + optionsBytes, err := encodeOptions(c.Options.Raw) + if err != nil { + return nil, err + } capacity := 4 + len(c.Username) + len(c.Password) if c.Permissions != nil { capacity += 4 + c.Permissions.Size() } - bytes := make([]byte, capacity) + bytes := make([]byte, capacity, capacity+len(optionsBytes)) position := 0 bytes[position] = byte(len(c.Username)) @@ -77,7 +82,7 @@ bytes[position] = byte(0) } - return bytes, nil + return append(bytes, optionsBytes...), nil } type GetUser struct {
diff --git a/foreign/go/internal/vsr/envelope.go b/foreign/go/internal/vsr/envelope.go index fdc0fc4..8ace3a8 100644 --- a/foreign/go/internal/vsr/envelope.go +++ b/foreign/go/internal/vsr/envelope.go
@@ -18,9 +18,9 @@ package vsr import ( + "crypto/rand" "encoding/binary" ierror "github.com/apache/iggy/foreign/go/errors" - "github.com/zeebo/xxh3" ) // EncodeRequest returns the complete request frame for a command code and its @@ -86,7 +86,15 @@ } EncodeRequestHeader((*[HeaderSize]byte)(frame[:HeaderSize]), fields) if operation != OperationNonReplicated && operation < OperationSendMessages { - binary.LittleEndian.PutUint64(frame[requestOffsetChecksum:], xxh3.Hash(frame[HeaderSize:])) + for { + if _, err := rand.Read(frame[requestOffsetChecksum : requestOffsetChecksum+16]); err != nil { + return err + } + if binary.LittleEndian.Uint64(frame[requestOffsetChecksum:]) != 0 || + binary.LittleEndian.Uint64(frame[requestOffsetChecksum+8:]) != 0 { + break + } + } } return nil }
diff --git a/foreign/go/internal/vsr/envelope_test.go b/foreign/go/internal/vsr/envelope_test.go index 89b5f0a..30f6279 100644 --- a/foreign/go/internal/vsr/envelope_test.go +++ b/foreign/go/internal/vsr/envelope_test.go
@@ -25,7 +25,6 @@ "github.com/apache/iggy/foreign/go/internal/command" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/zeebo/xxh3" ) // boundSession returns a session already registered and bound to fence 100. @@ -81,11 +80,16 @@ assert.Equal(t, uint64(2), session.CurrentRequestID()) } -func TestEncodeRequest_ChecksumsDeleteSegmentsPayload(t *testing.T) { +func TestEncodeRequest_StampsRandomIdentifierForDeleteSegments(t *testing.T) { payload := []byte{1, 2, 3} - frame, err := EncodeRequest(boundSession(t), uint32(command.DeleteSegmentsCode), payload) + session := boundSession(t) + frame, err := EncodeRequest(session, uint32(command.DeleteSegmentsCode), payload) require.NoError(t, err) - require.Equal(t, xxh3.Hash(payload), binary.LittleEndian.Uint64(frame[requestOffsetChecksum:])) + another, err := EncodeRequest(session, uint32(command.DeleteSegmentsCode), payload) + require.NoError(t, err) + require.NotEqual(t, [16]byte{}, [16]byte(frame[requestOffsetChecksum:requestOffsetChecksum+16])) + require.NotEqual(t, frame[requestOffsetChecksum:requestOffsetChecksum+16], + another[requestOffsetChecksum:requestOffsetChecksum+16]) } func TestEncodeRequest_CarriesTheCodeOfANonReplicatedCommand(t *testing.T) {
diff --git a/foreign/go/internal/vsr/operation.go b/foreign/go/internal/vsr/operation.go index 7ca4b1e..6c277ec 100644 --- a/foreign/go/internal/vsr/operation.go +++ b/foreign/go/internal/vsr/operation.go
@@ -38,12 +38,15 @@ OperationTruncatePartition Operation = 68 OperationFinalizeSession Operation = 70 OperationCompleteLifecycle Operation = 71 + OperationUpdateStreamResolved Operation = 72 + OperationUpdateTopicResolved Operation = 73 + OperationUpdateUserResolved Operation = 74 + OperationRewrapStreamKeys Operation = 75 + // 129, 133 and 142 were the per-resource updates. Never reuse them. OperationCreateStream Operation = 128 - OperationUpdateStream Operation = 129 OperationDeleteStream Operation = 130 OperationCreateTopic Operation = 132 - OperationUpdateTopic Operation = 133 OperationDeleteTopic Operation = 134 OperationCreatePartitions Operation = 136 OperationDeletePartitions Operation = 137 @@ -51,7 +54,6 @@ OperationCreateConsumerGroup Operation = 139 OperationDeleteConsumerGroup Operation = 140 OperationCreateUser Operation = 141 - OperationUpdateUser Operation = 142 OperationDeleteUser Operation = 143 OperationChangePassword Operation = 144 OperationUpdatePermissions Operation = 145 @@ -59,6 +61,7 @@ OperationDeletePersonalAccessToken Operation = 147 OperationJoinConsumerGroup Operation = 148 OperationLeaveConsumerGroup Operation = 149 + OperationUpdateResource Operation = 150 OperationSendMessages Operation = 160 OperationStoreConsumerOffset Operation = 161 @@ -89,11 +92,13 @@ OperationTruncatePartition, OperationFinalizeSession, OperationCompleteLifecycle, + OperationUpdateStreamResolved, + OperationUpdateTopicResolved, + OperationUpdateUserResolved, + OperationRewrapStreamKeys, OperationCreateStream, - OperationUpdateStream, OperationDeleteStream, OperationCreateTopic, - OperationUpdateTopic, OperationDeleteTopic, OperationCreatePartitions, OperationDeletePartitions, @@ -101,7 +106,6 @@ OperationCreateConsumerGroup, OperationDeleteConsumerGroup, OperationCreateUser, - OperationUpdateUser, OperationDeleteUser, OperationChangePassword, OperationUpdatePermissions, @@ -109,6 +113,7 @@ OperationDeletePersonalAccessToken, OperationJoinConsumerGroup, OperationLeaveConsumerGroup, + OperationUpdateResource, OperationSendMessages, OperationStoreConsumerOffset, OperationDeleteConsumerOffset, @@ -127,7 +132,6 @@ var replicatedOperation = map[uint32]Operation{ uint32(command.CreateUserCode): OperationCreateUser, uint32(command.DeleteUserCode): OperationDeleteUser, - uint32(command.UpdateUserCode): OperationUpdateUser, uint32(command.UpdatePermissionsCode): OperationUpdatePermissions, uint32(command.ChangePasswordCode): OperationChangePassword, uint32(command.CreateAccessTokenCode): OperationCreatePersonalAccessToken, @@ -137,10 +141,8 @@ uint32(command.DeleteConsumerOffsetCode): OperationDeleteConsumerOffset, uint32(command.CreateStreamCode): OperationCreateStream, uint32(command.DeleteStreamCode): OperationDeleteStream, - uint32(command.UpdateStreamCode): OperationUpdateStream, uint32(command.CreateTopicCode): OperationCreateTopic, uint32(command.DeleteTopicCode): OperationDeleteTopic, - uint32(command.UpdateTopicCode): OperationUpdateTopic, uint32(command.CreatePartitionsCode): OperationCreatePartitions, uint32(command.DeletePartitionsCode): OperationDeletePartitions, uint32(command.DeleteSegmentsCode): OperationDeleteSegments, @@ -148,6 +150,7 @@ uint32(command.DeleteGroupCode): OperationDeleteConsumerGroup, uint32(command.JoinGroupCode): OperationJoinConsumerGroup, uint32(command.LeaveGroupCode): OperationLeaveConsumerGroup, + uint32(command.UpdateResourceCode): OperationUpdateResource, } func newOperationSet(operations []Operation) map[Operation]struct{} { @@ -208,7 +211,7 @@ if operation == OperationDeleteSegments { return false } - return operation >= metadataBandStart && operation <= OperationLeaveConsumerGroup + return operation >= metadataBandStart && operation <= OperationUpdateResource } // IsResultFramed reports whether the reply body leads with a committed result
diff --git a/foreign/go/internal/vsr/operation_test.go b/foreign/go/internal/vsr/operation_test.go index 8a10bfd..1412884 100644 --- a/foreign/go/internal/vsr/operation_test.go +++ b/foreign/go/internal/vsr/operation_test.go
@@ -33,7 +33,6 @@ }{ {code: command.CreateUserCode, want: OperationCreateUser}, {code: command.DeleteUserCode, want: OperationDeleteUser}, - {code: command.UpdateUserCode, want: OperationUpdateUser}, {code: command.UpdatePermissionsCode, want: OperationUpdatePermissions}, {code: command.ChangePasswordCode, want: OperationChangePassword}, {code: command.CreateAccessTokenCode, want: OperationCreatePersonalAccessToken}, @@ -43,10 +42,8 @@ {code: command.DeleteConsumerOffsetCode, want: OperationDeleteConsumerOffset}, {code: command.CreateStreamCode, want: OperationCreateStream}, {code: command.DeleteStreamCode, want: OperationDeleteStream}, - {code: command.UpdateStreamCode, want: OperationUpdateStream}, {code: command.CreateTopicCode, want: OperationCreateTopic}, {code: command.DeleteTopicCode, want: OperationDeleteTopic}, - {code: command.UpdateTopicCode, want: OperationUpdateTopic}, {code: command.CreatePartitionsCode, want: OperationCreatePartitions}, {code: command.DeletePartitionsCode, want: OperationDeletePartitions}, {code: command.DeleteSegmentsCode, want: OperationDeleteSegments}, @@ -54,6 +51,7 @@ {code: command.DeleteGroupCode, want: OperationDeleteConsumerGroup}, {code: command.JoinGroupCode, want: OperationJoinConsumerGroup}, {code: command.LeaveGroupCode, want: OperationLeaveConsumerGroup}, + {code: command.UpdateResourceCode, want: OperationUpdateResource}, } require.Len(t, tests, len(replicatedOperation), "every replicated entry is covered") for _, test := range tests { @@ -85,6 +83,9 @@ command.SyncGroupCode, command.LoginUserCode, command.LoginWithAccessTokenCode, + 35, + 204, + 304, } for _, code := range codes { assert.Equal(t, OperationNonReplicated, OperationForCode(uint32(code)), "code %d", code) @@ -110,7 +111,7 @@ assert.True(t, IsKnownOperation(operation), "operation %d", operation) } - undeclared := []Operation{4, 63, 69, 127, 131, 135, 150, 159, 163, 164, 165, 255} + undeclared := []Operation{4, 63, 69, 76, 127, 129, 131, 133, 135, 142, 151, 159, 163, 164, 165, 255} for _, operation := range undeclared { assert.False(t, IsKnownOperation(operation), "operation %d", operation) } @@ -119,6 +120,9 @@ func TestIsInternal_CoversTheInternalBandOnly(t *testing.T) { assert.True(t, IsInternal(OperationCreateTopicWithAssignments)) assert.True(t, IsInternal(OperationTruncatePartition)) + assert.True(t, IsInternal(OperationUpdateTopicResolved)) + assert.True(t, IsInternal(OperationRewrapStreamKeys)) + assert.False(t, IsInternal(OperationUpdateResource)) assert.False(t, IsInternal(OperationLogout)) assert.False(t, IsInternal(OperationCreateStream)) assert.False(t, IsInternal(OperationSendMessages)) @@ -135,6 +139,9 @@ OperationCreateConsumerGroup, OperationCreatePersonalAccessToken, OperationLeaveConsumerGroup, + OperationUpdateResource, + OperationUpdateStreamResolved, + OperationUpdateUserResolved, } for _, operation := range metadata { assert.True(t, IsMetadata(operation), "operation %d", operation) @@ -161,6 +168,8 @@ framed := []Operation{ OperationCreateStream, OperationLeaveConsumerGroup, + OperationUpdateResource, + OperationUpdateTopicResolved, OperationStoreConsumerOffset, OperationDeleteConsumerOffset, }
diff --git a/foreign/go/internal/vsr/protocol_parity_test.go b/foreign/go/internal/vsr/protocol_parity_test.go index a01b064..e932b9f 100644 --- a/foreign/go/internal/vsr/protocol_parity_test.go +++ b/foreign/go/internal/vsr/protocol_parity_test.go
@@ -63,11 +63,13 @@ "CompleteConsumerGroupRevocation": OperationCompleteConsumerGroupRevocation, "TruncatePartition": OperationTruncatePartition, "FinalizeSession": OperationFinalizeSession, + "UpdateStreamResolved": OperationUpdateStreamResolved, + "UpdateTopicResolved": OperationUpdateTopicResolved, + "UpdateUserResolved": OperationUpdateUserResolved, + "RewrapStreamKeys": OperationRewrapStreamKeys, "CreateStream": OperationCreateStream, - "UpdateStream": OperationUpdateStream, "DeleteStream": OperationDeleteStream, "CreateTopic": OperationCreateTopic, - "UpdateTopic": OperationUpdateTopic, "DeleteTopic": OperationDeleteTopic, "CreatePartitions": OperationCreatePartitions, "DeletePartitions": OperationDeletePartitions, @@ -75,7 +77,6 @@ "CreateConsumerGroup": OperationCreateConsumerGroup, "DeleteConsumerGroup": OperationDeleteConsumerGroup, "CreateUser": OperationCreateUser, - "UpdateUser": OperationUpdateUser, "DeleteUser": OperationDeleteUser, "ChangePassword": OperationChangePassword, "UpdatePermissions": OperationUpdatePermissions, @@ -83,6 +84,7 @@ "DeletePersonalAccessToken": OperationDeletePersonalAccessToken, "JoinConsumerGroup": OperationJoinConsumerGroup, "LeaveConsumerGroup": OperationLeaveConsumerGroup, + "UpdateResource": OperationUpdateResource, "SendMessages": OperationSendMessages, "StoreConsumerOffset": OperationStoreConsumerOffset, "DeleteConsumerOffset": OperationDeleteConsumerOffset,
diff --git a/foreign/go/internal/vsr/register.go b/foreign/go/internal/vsr/register.go index 922eecc..6f995c3 100644 --- a/foreign/go/internal/vsr/register.go +++ b/foreign/go/internal/vsr/register.go
@@ -25,10 +25,10 @@ ) // ProtocolVersion is the packed semver of the wire contract this codec -// implements, 0.11.1 per core/binary_protocol/src/version.rs. Update it with +// implements, 0.12.0 per core/binary_protocol/src/version.rs. Update it with // the Rust constant during stable release preparation. Edge builds sharing // this number are not guaranteed to have compatible layouts. -var ProtocolVersion = packProtocolVersion(0, 11, 1) +var ProtocolVersion = packProtocolVersion(0, 12, 0) // packProtocolVersion packs a semver into the ten-bits-per-field layout the // register handshake carries.
diff --git a/foreign/go/internal/vsr/register_test.go b/foreign/go/internal/vsr/register_test.go index 2b7ff99..dd93a36 100644 --- a/foreign/go/internal/vsr/register_test.go +++ b/foreign/go/internal/vsr/register_test.go
@@ -36,10 +36,10 @@ } func TestProtocolVersion_PacksTheWireContractSemver(t *testing.T) { - assert.Equal(t, uint32(11265), ProtocolVersion) + assert.Equal(t, uint32(12288), ProtocolVersion) assert.Equal(t, uint32(0), ProtocolVersion>>20) - assert.Equal(t, uint32(11), (ProtocolVersion>>10)&0x3FF) - assert.Equal(t, uint32(1), ProtocolVersion&0x3FF) + assert.Equal(t, uint32(12), (ProtocolVersion>>10)&0x3FF) + assert.Equal(t, uint32(0), ProtocolVersion&0x3FF) } func TestSerializeLoginRegister_EncodesTheDeclaredLayout(t *testing.T) {
diff --git a/foreign/go/internal/vsr/reply.go b/foreign/go/internal/vsr/reply.go index 311b1ab..6246903 100644 --- a/foreign/go/internal/vsr/reply.go +++ b/foreign/go/internal/vsr/reply.go
@@ -80,7 +80,9 @@ return request == reply || request == OperationCreateTopic && reply == OperationCreateTopicWithAssignments || request == OperationCreatePartitions && reply == OperationCreatePartitionsWithAssignments || - request == OperationDeleteSegments && reply == OperationTruncatePartition + request == OperationDeleteSegments && reply == OperationTruncatePartition || + request == OperationUpdateResource && (reply == OperationUpdateStreamResolved || + reply == OperationUpdateTopicResolved || reply == OperationUpdateUserResolved) } // Result section layout: [count u32][{index u32, result u32} x count].
diff --git a/foreign/go/internal/vsr/reply_test.go b/foreign/go/internal/vsr/reply_test.go index 94712c0..2d0fbd1 100644 --- a/foreign/go/internal/vsr/reply_test.go +++ b/foreign/go/internal/vsr/reply_test.go
@@ -168,7 +168,7 @@ } func TestDecodeReplyFor_RejectsAnUndeclaredReplyOperation(t *testing.T) { - for _, operation := range []Operation{4, 63, 150, 163, 255} { + for _, operation := range []Operation{4, 63, 129, 133, 142, 151, 163, 255} { request, reply := exchange(OperationNonReplicated, 0, 0) reply[replyOffsetOperation] = byte(operation) @@ -375,6 +375,45 @@ require.False(t, outcome.Committed) } +func TestDecodeReplyFor_ResolvedUpdate(t *testing.T) { + t.Run("given_resolved_update_reply_when_correlating_should_accept_only_matching_request_identity", func(t *testing.T) { + var request [HeaderSize]byte + EncodeRequestHeader(&request, RequestFields{Size: HeaderSize, Client: ClientID{Lo: 7, Hi: 9}, Request: 42, Operation: OperationUpdateResource, Session: 3}) + request[requestOffsetChecksum] = 11 + body := resultSection() + for _, resolved := range []Operation{OperationUpdateStreamResolved, OperationUpdateTopicResolved, OperationUpdateUserResolved} { + header := replyHeader(resolved, 0, len(body)) + header[replyOffsetResult] = committedResult + copy(header[replyOffsetClient:replyOffsetClient+16], request[requestOffsetClient:requestOffsetClient+16]) + copy(header[replyOffsetChecksum:replyOffsetChecksum+16], request[requestOffsetChecksum:requestOffsetChecksum+16]) + binary.LittleEndian.PutUint64(header[replyOffsetRequest:], 42) + binary.LittleEndian.PutUint64(header[replyOffsetOp:], 17) + binary.LittleEndian.PutUint64(header[replyOffsetCommit:], 17) + + outcome, err := DecodeReplyFor(&request, header, body) + require.NoError(t, err, "operation %d", resolved) + assert.True(t, outcome.Committed, "operation %d", resolved) + assert.NoError(t, outcome.Err, "operation %d", resolved) + + for name, mismatch := range map[string]func(*[HeaderSize]byte){ + "client": func(reply *[HeaderSize]byte) { reply[replyOffsetClient] ^= 1 }, + "checksum": func(reply *[HeaderSize]byte) { reply[replyOffsetChecksum] ^= 1 }, + "operation": func(reply *[HeaderSize]byte) { reply[replyOffsetOperation] = byte(OperationCreateTopicWithAssignments) }, + "rewrap": func(reply *[HeaderSize]byte) { reply[replyOffsetOperation] = byte(OperationRewrapStreamKeys) }, + } { + unrelated := *header + mismatch(&unrelated) + _, err := DecodeReplyFor(&request, &unrelated, body) + assert.ErrorIs(t, err, ierror.ErrInvalidCommand, "operation %d, mismatched %s", resolved, name) + } + + assert.False(t, ReplyOperationMatches(resolved, OperationUpdateResource), "the rewrite is directed") + assert.False(t, ReplyOperationMatches(OperationCreateStream, resolved)) + } + assert.False(t, ReplyOperationMatches(OperationUpdateResource, OperationRewrapStreamKeys)) + }) +} + func TestReplyOperationMatchesOnlyCanonicalDirectedRewrites(t *testing.T) { for _, pair := range [][2]Operation{{OperationCreateTopic, OperationCreateTopicWithAssignments}, {OperationCreatePartitions, OperationCreatePartitionsWithAssignments}, {OperationDeleteSegments, OperationTruncatePartition}} { require.True(t, ReplyOperationMatches(pair[0], pair[1]))
diff --git a/foreign/go/tests/e2e_helpers_test.go b/foreign/go/tests/e2e_helpers_test.go index 412123f..2c8c3fd 100644 --- a/foreign/go/tests/e2e_helpers_test.go +++ b/foreign/go/tests/e2e_helpers_test.go
@@ -24,10 +24,14 @@ // IGGY_TCP_ADDRESS=127.0.0.1:8090 \ // IGGY_HTTP_ENABLED=false IGGY_QUIC_ENABLED=false IGGY_WEBSOCKET_ENABLED=false \ // IGGY_ROOT_USERNAME=iggy IGGY_ROOT_PASSWORD=iggy \ +// IGGY_ENCRYPTION_ENABLED=true IGGY_ENCRYPTION_KEY=<base64 of 32 random bytes> \ // target/debug/iggy-server // // IGGY_TCP_ADDRESS=127.0.0.1:8090 go test ./tests // +// The stream option cases need the encryption settings: without them a keyless +// stream reports no generated key and a stream key is refused. +// // Set IGGY_TCP_TLS_ENABLED=true to run the TLS cases against a server started // with IGGY_TCP_TLS_ENABLED=true and the certificate pair in core/certs. package tests_test @@ -139,14 +143,13 @@ name = name[:255] } - stream, err := connected.CreateStream(ctx, name) + stream, err := connected.CreateStream(ctx, name, nil) require.NoError(t, err) streamId, err := iggcon.NewIdentifier(stream.Id) require.NoError(t, err) t.Cleanup(func() { _ = connected.DeleteStream(context.Background(), streamId) }) - topic, err := connected.CreateTopic(ctx, streamId, name, partitionsCount, - iggcon.CompressionAlgorithmNone, iggcon.Duration(0), 0) + topic, err := connected.CreateTopic(ctx, streamId, name, &iggcon.TopicOptions{PartitionsCount: &partitionsCount}) require.NoError(t, err) topicId, err := iggcon.NewIdentifier(topic.Id) require.NoError(t, err)
diff --git a/foreign/go/tests/e2e_personal_access_token_test.go b/foreign/go/tests/e2e_personal_access_token_test.go new file mode 100644 index 0000000..f859861 --- /dev/null +++ b/foreign/go/tests/e2e_personal_access_token_test.go
@@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package tests_test + +import ( + "context" + "testing" + "time" + + iggcon "github.com/apache/iggy/foreign/go/contracts" + ierror "github.com/apache/iggy/foreign/go/errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + tokenLifetime = 2 * time.Second + tokenExpiryBudget = 15 * time.Second + tokenExpiryInterval = 100 * time.Millisecond +) + +func TestE2E_PersonalAccessTokenLivesAsLongAsRequested(t *testing.T) { + owner := connect(t) + ctx := context.Background() + name := uniqueName("pat") + + before := time.Now() + token, err := owner.CreatePersonalAccessToken(ctx, name, iggcon.Duration(tokenLifetime.Microseconds())) + require.NoError(t, err) + after := time.Now() + t.Cleanup(func() { _ = owner.DeletePersonalAccessToken(context.Background(), name) }) + + listed := findPersonalAccessToken(t, owner, name) + require.NotNil(t, listed.Expiry, "a token created with a lifetime lists its expiry") + assert.WithinRange(t, *listed.Expiry, + before.Add(tokenLifetime).Add(-time.Second), after.Add(tokenLifetime).Add(time.Second)) + + _, err = newClient(t).LoginWithPersonalAccessToken(ctx, token.Token) + require.NoError(t, err, "the token signs in before its lifetime ends") + + deadline := time.Now().Add(tokenExpiryBudget) + for { + _, err = newClient(t).LoginWithPersonalAccessToken(ctx, token.Token) + if err != nil { + break + } + if time.Now().After(deadline) { + require.Failf(t, "the token outlived its lifetime", + "a token created for %v still signs in after %v", tokenLifetime, tokenExpiryBudget) + } + time.Sleep(tokenExpiryInterval) + } + assert.False(t, time.Now().Before(before.Add(tokenLifetime)), "the token is refused only once its lifetime ends") +} + +func TestE2E_PersonalAccessTokenPastTheClockIsRefused(t *testing.T) { + owner := connect(t) + ctx := context.Background() + name := uniqueName("pat") + + _, err := owner.CreatePersonalAccessToken(ctx, name, iggcon.IggyExpiryNeverExpire-1) + require.ErrorIs(t, err, ierror.ErrInvalidPersonalAccessTokenExpiry) + + tokens, err := owner.GetPersonalAccessTokens(ctx) + require.NoError(t, err, "the server keeps serving after the refusal") + for _, token := range tokens { + assert.NotEqual(t, name, token.Name, "a refused token is not stored") + } +} + +func findPersonalAccessToken(t *testing.T, connected iggcon.Client, name string) iggcon.PersonalAccessTokenInfo { + t.Helper() + + tokens, err := connected.GetPersonalAccessTokens(context.Background()) + require.NoError(t, err) + for _, token := range tokens { + if token.Name == name { + return token + } + } + require.Failf(t, "token not listed", "no token named %s in %+v", name, tokens) + return iggcon.PersonalAccessTokenInfo{} +}
diff --git a/foreign/go/tests/e2e_resource_options_test.go b/foreign/go/tests/e2e_resource_options_test.go new file mode 100644 index 0000000..fb4f1f1 --- /dev/null +++ b/foreign/go/tests/e2e_resource_options_test.go
@@ -0,0 +1,301 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package tests_test + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/base64" + "fmt" + "math" + "testing" + "time" + + iggcon "github.com/apache/iggy/foreign/go/contracts" + ierror "github.com/apache/iggy/foreign/go/errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestE2E_StreamKeyProjection(t *testing.T) { + t.Run("given_stream_key_when_creating_should_read_matching_projection", func(t *testing.T) { + connected := connect(t) + ctx := context.Background() + key := randomStreamKey(t) + fingerprint, err := iggcon.EncryptionKeyFingerprint(key) + require.NoError(t, err) + + created, err := connected.CreateStream(ctx, uniqueName("keyed"), &iggcon.StreamOptions{EncryptionKey: key}) + require.NoError(t, err) + streamId := scratchStream(t, connected, created.Id) + + fetched, err := connected.GetStream(ctx, streamId) + require.NoError(t, err) + streams, err := connected.GetStreams(ctx) + require.NoError(t, err) + listed := findStream(t, streams, created.Id) + + for source, stream := range map[string]iggcon.Stream{"create": created.Stream, "get": fetched.Stream, "list": listed} { + requireProjection(t, source, stream, iggcon.StreamEncryptionSourceStream) + got, ok := stream.EncryptionKeyFingerprint() + require.True(t, ok, source) + assert.Equal(t, fingerprint, got, source) + requireNoKeyMaterial(t, source, stream, key) + } + assert.Equal(t, created.EncryptionKekFingerprints(), fetched.EncryptionKekFingerprints()) + }) +} + +func TestE2E_KeylessStream(t *testing.T) { + t.Run("given_keyless_stream_when_reading_should_report_server_generated_key", func(t *testing.T) { + connected := connect(t) + ctx := context.Background() + + created, err := connected.CreateStream(ctx, uniqueName("keyless"), nil) + require.NoError(t, err) + streamId := scratchStream(t, connected, created.Id) + + fetched, err := connected.GetStream(ctx, streamId) + require.NoError(t, err) + for source, stream := range map[string]iggcon.Stream{"create": created.Stream, "get": fetched.Stream} { + requireProjection(t, source, stream, iggcon.StreamEncryptionSourceServer) + fingerprint, ok := stream.EncryptionKeyFingerprint() + require.True(t, ok, source) + assert.Len(t, fingerprint, 16, source) + requireNoKeyMaterial(t, source, stream, nil) + } + }) +} + +func TestE2E_StreamRename(t *testing.T) { + t.Run("given_stream_rename_when_updating_should_preserve_encryption_projection", func(t *testing.T) { + connected := connect(t) + ctx := context.Background() + key := randomStreamKey(t) + fingerprint, err := iggcon.EncryptionKeyFingerprint(key) + require.NoError(t, err) + + created, err := connected.CreateStream(ctx, uniqueName("rename"), &iggcon.StreamOptions{EncryptionKey: key}) + require.NoError(t, err) + streamId := scratchStream(t, connected, created.Id) + partitions := uint32(1) + topic, err := connected.CreateTopic(ctx, streamId, "events", &iggcon.TopicOptions{PartitionsCount: &partitions}) + require.NoError(t, err) + topicId, err := iggcon.NewIdentifier(topic.Id) + require.NoError(t, err) + payload := []byte("written before the rename") + message, err := iggcon.NewIggyMessage(payload) + require.NoError(t, err) + _, err = connected.SendMessages(ctx, streamId, topicId, iggcon.PartitionId(0), []iggcon.IggyMessage{message}) + require.NoError(t, err) + + newName := uniqueName("renamed") + require.NoError(t, connected.UpdateStream(ctx, streamId, iggcon.StreamUpdate{Name: &newName})) + + renamed, err := connected.GetStream(ctx, streamId) + require.NoError(t, err) + assert.Equal(t, created.Id, renamed.Id) + assert.Equal(t, newName, renamed.Name) + requireProjection(t, "renamed", renamed.Stream, iggcon.StreamEncryptionSourceStream) + got, ok := renamed.EncryptionKeyFingerprint() + require.True(t, ok) + assert.Equal(t, fingerprint, got) + assert.Equal(t, created.EncryptionKekFingerprints(), renamed.EncryptionKekFingerprints()) + + partitionId := uint32(0) + polled, err := connected.PollMessages(ctx, streamId, topicId, + iggcon.DefaultConsumer(), iggcon.OffsetPollingStrategy(0), 1, false, &partitionId) + require.NoError(t, err) + require.Len(t, polled.Messages, 1) + assert.Equal(t, payload, polled.Messages[0].Payload) + + err = connected.UpdateStream(ctx, streamId, iggcon.StreamUpdate{Reset: []iggcon.StreamOptionKey{iggcon.StreamOptionEncryptionKey}}) + assert.ErrorIs(t, err, ierror.ErrOptionNotUpdatable) + }) +} + +func TestE2E_TopicReset(t *testing.T) { + t.Run("given_topic_reset_when_updating_should_preserve_unmentioned_values", func(t *testing.T) { + connected := connect(t) + ctx := context.Background() + created, err := connected.CreateStream(ctx, uniqueName("topic-reset"), nil) + require.NoError(t, err) + streamId := scratchStream(t, connected, created.Id) + + partitions := uint32(1) + compression := iggcon.CompressionAlgorithmGzip + expiry := iggcon.Duration(3_600_000_000) + maxTopicSize := uint64(1 << 30) + name := uniqueName("topic") + topic, err := connected.CreateTopic(ctx, streamId, name, &iggcon.TopicOptions{ + PartitionsCount: &partitions, + CompressionAlgorithm: &compression, + MessageExpiry: &expiry, + MaxTopicSize: &maxTopicSize, + }) + require.NoError(t, err) + topicId, err := iggcon.NewIdentifier(topic.Id) + require.NoError(t, err) + + newExpiry := iggcon.Duration(7_200_000_000) + require.NoError(t, connected.UpdateTopic(ctx, streamId, topicId, iggcon.TopicUpdate{ + MessageExpiry: &newExpiry, + Reset: []iggcon.TopicOptionKey{iggcon.TopicOptionMaxTopicSize}, + })) + + updated, err := connected.GetTopic(ctx, streamId, topicId) + require.NoError(t, err) + assert.Equal(t, name, updated.Name) + assert.Equal(t, newExpiry, updated.MessageExpiry) + assert.Contains(t, updated.Options, string(iggcon.TopicOptionMessageExpiry)) + assert.Equal(t, uint64(math.MaxUint64), updated.MaxTopicSize) + assert.NotContains(t, updated.Options, string(iggcon.TopicOptionMaxTopicSize)) + assert.Contains(t, updated.DerivedOptions, string(iggcon.TopicOptionMaxTopicSize)) + assert.Equal(t, uint8(iggcon.CompressionAlgorithmGzip), updated.CompressionAlgorithm) + assert.Equal(t, iggcon.HeaderValue{Kind: iggcon.String, Value: []byte("gzip")}, updated.Options[string(iggcon.TopicOptionCompressionAlgorithm)]) + + err = connected.UpdateTopic(ctx, streamId, topicId, iggcon.TopicUpdate{Options: []iggcon.HeaderEntry{{ + Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte(iggcon.TopicOptionCompressionAlgorithm)}, + Value: iggcon.HeaderValue{Kind: iggcon.String, Value: []byte("none")}, + }}}) + assert.ErrorIs(t, err, ierror.ErrOptionNotUpdatable) + err = connected.UpdateTopic(ctx, streamId, topicId, iggcon.TopicUpdate{ + Reset: []iggcon.TopicOptionKey{iggcon.TopicOptionCompressionAlgorithm}, + }) + assert.ErrorIs(t, err, ierror.ErrOptionNotUpdatable) + + unchanged, err := connected.GetTopic(ctx, streamId, topicId) + require.NoError(t, err) + assert.Equal(t, updated.Options, unchanged.Options) + assert.Equal(t, updated.DerivedOptions, unchanged.DerivedOptions) + }) +} + +func TestE2E_UserOptions(t *testing.T) { + t.Run("given_user_options_when_creating_and_updating_should_preserve_user_behavior", func(t *testing.T) { + connected := connect(t) + ctx := context.Background() + + first := scratchUser(t, connected, uniqueUsername("first")) + second := scratchUser(t, connected, uniqueUsername("second")) + assert.Empty(t, first.Options) + assert.Empty(t, first.DerivedOptions) + + renamed := uniqueUsername("renamed") + inactive := iggcon.Inactive + firstId, err := iggcon.NewIdentifier(first.Id) + require.NoError(t, err) + require.NoError(t, connected.UpdateUser(ctx, firstId, iggcon.UserUpdate{Username: &renamed, Status: &inactive})) + + updated, err := connected.GetUser(ctx, firstId) + require.NoError(t, err) + assert.Equal(t, renamed, updated.Username) + assert.Equal(t, iggcon.Inactive, updated.Status) + assert.Empty(t, updated.Options) + assert.Empty(t, updated.DerivedOptions) + + secondId, err := iggcon.NewIdentifier(second.Id) + require.NoError(t, err) + err = connected.UpdateUser(ctx, secondId, iggcon.UserUpdate{Username: &renamed, Status: &inactive}) + require.Error(t, err, "a taken username is refused") + unchanged, err := connected.GetUser(ctx, secondId) + require.NoError(t, err) + assert.Equal(t, second.Username, unchanged.Username) + assert.Equal(t, iggcon.Active, unchanged.Status, "a refused patch applies nothing") + + err = connected.UpdateUser(ctx, secondId, iggcon.UserUpdate{Reset: []iggcon.UserOptionKey{iggcon.UserOptionStatus}}) + assert.ErrorIs(t, err, ierror.ErrOptionNotResettable) + err = connected.UpdateUser(ctx, secondId, iggcon.UserUpdate{}) + assert.ErrorIs(t, err, ierror.ErrInvalidCommand) + }) +} + +func randomStreamKey(t *testing.T) []byte { + t.Helper() + key := make([]byte, iggcon.StreamEncryptionKeyLength) + _, err := rand.Read(key) + require.NoError(t, err) + return key +} + +func uniqueName(prefix string) string { + return fmt.Sprintf("go-e2e-%s-%d", prefix, time.Now().UnixNano()) +} + +// uniqueUsername stays inside the 50-byte username bound. +func uniqueUsername(prefix string) string { + return fmt.Sprintf("%s-%d", prefix, time.Now().UnixNano()) +} + +func scratchStream(t *testing.T, connected iggcon.Client, id uint32) iggcon.Identifier { + t.Helper() + streamId, err := iggcon.NewIdentifier(id) + require.NoError(t, err) + t.Cleanup(func() { _ = connected.DeleteStream(context.Background(), streamId) }) + return streamId +} + +func scratchUser(t *testing.T, connected iggcon.Client, username string) *iggcon.UserInfoDetails { + t.Helper() + user, err := connected.CreateUser(context.Background(), username, "user-password", iggcon.Active, nil, &iggcon.UserOptions{}) + require.NoError(t, err) + userId, err := iggcon.NewIdentifier(user.Id) + require.NoError(t, err) + t.Cleanup(func() { _ = connected.DeleteUser(context.Background(), userId) }) + return user +} + +func findStream(t *testing.T, streams []iggcon.Stream, id uint32) iggcon.Stream { + t.Helper() + for _, stream := range streams { + if stream.Id == id { + return stream + } + } + require.FailNow(t, "the stream is not listed", "id %d", id) + return iggcon.Stream{} +} + +func requireProjection(t *testing.T, source string, stream iggcon.Stream, want iggcon.StreamEncryptionSource) { + t.Helper() + got, ok := stream.EncryptionKeySource() + require.True(t, ok, source) + assert.Equal(t, want, got, source) + keks := stream.EncryptionKekFingerprints() + require.NotEmpty(t, keks, source) + for _, kek := range keks { + assert.Len(t, kek, 16, source) + } +} + +func requireNoKeyMaterial(t *testing.T, source string, stream iggcon.Stream, key []byte) { + t.Helper() + for _, options := range []map[string]iggcon.HeaderValue{stream.Options, stream.DerivedOptions} { + for _, secret := range []string{"encryption_key", "encryption_key_wrapped", "critical_keys"} { + assert.NotContains(t, options, secret, source) + } + if key == nil { + continue + } + for name, value := range options { + assert.False(t, bytes.Contains(value.Value, key), "%s: %s carries the key", source, name) + assert.NotContains(t, string(value.Value), base64.StdEncoding.EncodeToString(key), "%s: %s", source, name) + } + } +}
diff --git a/foreign/go/tests/e2e_test.go b/foreign/go/tests/e2e_test.go index 92dbb85..b68244b 100644 --- a/foreign/go/tests/e2e_test.go +++ b/foreign/go/tests/e2e_test.go
@@ -470,25 +470,32 @@ assert.NotEmpty(t, byKey["segment_size"].Description) assert.Equal(t, iggcon.Uint64, byKey["segment_size"].DefaultValue.Kind) - // Scopes with no catalog keys answer with an empty list rather than failing. + // Every scope has a catalog, and each entry says what an update may do. streamSpecs, err := connected.DescribeOptions(ctx, iggcon.OptionsScopeStream) require.NoError(t, err) - assert.Empty(t, streamSpecs) + streamByKey := map[string]iggcon.OptionSpec{} + for _, spec := range streamSpecs { + streamByKey[spec.Key] = spec + } + require.Contains(t, streamByKey, "encryption_key") + assert.True(t, streamByKey["encryption_key"].Secret) + assert.False(t, streamByKey["encryption_key"].Updatable) + assert.True(t, streamByKey["name"].Updatable) + assert.True(t, byKey["message_expiry"].Resettable) + assert.False(t, byKey["compression_algorithm"].Updatable) name := fmt.Sprintf("go-e2e-options-%d", time.Now().UnixNano()) - stream, err := connected.CreateStream(ctx, name) + stream, err := connected.CreateStream(ctx, name, nil) require.NoError(t, err) streamId, err := iggcon.NewIdentifier(stream.Id) require.NoError(t, err) t.Cleanup(func() { _ = connected.DeleteStream(context.Background(), streamId) }) - // A key with no parameter of its own rides the variadic options block. - created, err := connected.CreateTopic(ctx, streamId, name, 1, - iggcon.CompressionAlgorithmNone, iggcon.Duration(0), 0, - iggcon.HeaderEntry{ - Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte("durability")}, - Value: iggcon.HeaderValue{Kind: iggcon.String, Value: []byte("persisted")}, - }) + // A key with no field of its own rides the raw options. + created, err := connected.CreateTopic(ctx, streamId, name, &iggcon.TopicOptions{Raw: []iggcon.HeaderEntry{{ + Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte("durability")}, + Value: iggcon.HeaderValue{Kind: iggcon.String, Value: []byte("persisted")}, + }}}) require.NoError(t, err) topicId, err := iggcon.NewIdentifier(created.Id) @@ -508,12 +515,10 @@ assert.NotContains(t, topic.DerivedOptions, "durability") // A key outside the catalog is refused by name. - _, err = connected.CreateTopic(ctx, streamId, name+"-bad", 1, - iggcon.CompressionAlgorithmNone, iggcon.Duration(0), 0, - iggcon.HeaderEntry{ - Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte("not_a_real_option")}, - Value: iggcon.HeaderValue{Kind: iggcon.String, Value: []byte("1")}, - }) + _, err = connected.CreateTopic(ctx, streamId, name+"-bad", &iggcon.TopicOptions{Raw: []iggcon.HeaderEntry{{ + Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte("not_a_real_option")}, + Value: iggcon.HeaderValue{Kind: iggcon.String, Value: []byte("1")}, + }}}) require.Error(t, err) } @@ -547,14 +552,13 @@ } name := fmt.Sprintf("go-e2e-typed-options-%d", time.Now().UnixNano()) - stream, err := connected.CreateStream(ctx, name) + stream, err := connected.CreateStream(ctx, name, nil) require.NoError(t, err) streamId, err := iggcon.NewIdentifier(stream.Id) require.NoError(t, err) t.Cleanup(func() { _ = connected.DeleteStream(context.Background(), streamId) }) - created, err := connected.CreateTopic(ctx, streamId, name, 1, - iggcon.CompressionAlgorithmNone, iggcon.Duration(0), 0, typed...) + created, err := connected.CreateTopic(ctx, streamId, name, &iggcon.TopicOptions{Raw: typed}) require.NoError(t, err) topicId, err := iggcon.NewIdentifier(created.Id) @@ -574,21 +578,22 @@ } // A flush threshold of zero messages can never trip, so it is refused. - _, err = connected.CreateTopic(ctx, streamId, name+"-zero-flush", 1, - iggcon.CompressionAlgorithmNone, iggcon.Duration(0), 0, - iggcon.MessagesRequiredToSaveOption(0)) + _, err = connected.CreateTopic(ctx, streamId, name+"-zero-flush", &iggcon.TopicOptions{ + Raw: []iggcon.HeaderEntry{iggcon.MessagesRequiredToSaveOption(0)}, + }) require.Error(t, err) // A segment size off the 512-byte grid is refused, but zero is not: it // leaves the key derived. - _, err = connected.CreateTopic(ctx, streamId, name+"-odd-segment", 1, - iggcon.CompressionAlgorithmNone, iggcon.Duration(0), 0, - iggcon.SegmentSizeOption(1024*1024+1)) + _, err = connected.CreateTopic(ctx, streamId, name+"-odd-segment", &iggcon.TopicOptions{ + Raw: []iggcon.HeaderEntry{iggcon.SegmentSizeOption(1024*1024 + 1)}, + }) require.Error(t, err) zeroSegment := iggcon.SegmentSizeOption(0) - derived, err := connected.CreateTopic(ctx, streamId, name+"-zero-segment", 1, - iggcon.CompressionAlgorithmNone, iggcon.Duration(0), 0, zeroSegment) + derived, err := connected.CreateTopic(ctx, streamId, name+"-zero-segment", &iggcon.TopicOptions{ + Raw: []iggcon.HeaderEntry{zeroSegment}, + }) require.NoError(t, err) segmentSizeKey := string(zeroSegment.Key.Value) assert.NotContains(t, derived.Options, segmentSizeKey)
diff --git a/foreign/java/README.md b/foreign/java/README.md index dfb137d..864fcfc 100644 --- a/foreign/java/README.md +++ b/foreign/java/README.md
@@ -38,7 +38,7 @@ ### Snapshot Versions -Development builds of the next release carry the version `0.9.1-SNAPSHOT`. Get them from the ASF snapshot repository: +Development builds of the next release carry the version `0.9.2-SNAPSHOT`. Get them from the ASF snapshot repository: **Gradle:** @@ -51,7 +51,7 @@ } dependencies { - implementation 'org.apache.iggy:iggy:0.9.1-SNAPSHOT' + implementation 'org.apache.iggy:iggy:0.9.2-SNAPSHOT' } ``` @@ -72,7 +72,7 @@ <dependency> <groupId>org.apache.iggy</groupId> <artifactId>iggy</artifactId> - <version>0.9.1-SNAPSHOT</version> + <version>0.9.2-SNAPSHOT</version> </dependency> </dependencies> ``` @@ -304,6 +304,46 @@ info.getUserAgent(); // User-Agent string for HTTP ``` +### Resource Options + +Streams, topics and users carry typed options. Create calls take an options value, updates send only the keys they change, and `reset` returns an updatable key to its server default. + +```java +import org.apache.iggy.identifier.StreamId; +import org.apache.iggy.identifier.TopicId; +import org.apache.iggy.stream.StreamOptions; +import org.apache.iggy.system.OptionsScope; +import org.apache.iggy.topic.CompressionAlgorithm; +import org.apache.iggy.topic.TopicOptionKey; +import org.apache.iggy.topic.TopicOptions; +import org.apache.iggy.topic.TopicUpdate; + +import java.math.BigInteger; +import java.security.SecureRandom; + +byte[] key = new byte[32]; +new SecureRandom().nextBytes(key); +var stream = client.streams().createStream("payments", StreamOptions.builder().encryptionKey(key).build()); +stream.encryptionKeyFingerprint(); // the key itself is never returned + +var streamId = StreamId.of(stream.id()); +var topic = client.topics().createTopic(streamId, "orders", 1L, TopicOptions.builder() + .compressionAlgorithm(CompressionAlgorithm.Gzip) + .messageExpiry(BigInteger.valueOf(3_600_000_000L)) + .build()); + +client.topics().updateTopic(streamId, TopicId.of(topic.id()), TopicUpdate.builder() + .name("orders-v2") + .reset(TopicOptionKey.MESSAGE_EXPIRY) + .build()); + +client.system().describeOptions(OptionsScope.Topic); // keys, kinds, defaults and flags +``` + +The stream key is 32 random bytes. It travels inside the create request over the client's transport, so protecting it on the network is the job of TLS. The server stores it wrapped under its own key and never returns it, reads report only its source and fingerprints. Without a key, a server with encryption enabled generates one for the stream. + +The typed builders refuse an update the server would refuse for its shape before anything is sent, with the server's error code. `raw(key, value)` and `rawReset(key)` reach keys this build has no field for, and the server decides whether they are allowed. + ## Exception Handling The SDK's custom exception types inherit from `IggyException`. When you join a failed future, the cause can come back inside a `CompletionException`. The HTTP client's `close()` method declares `IOException`. Handle these two boundaries as well as the specific SDK errors.
diff --git a/foreign/java/bench/src/main/java/org/apache/iggy/bench/common/provision/ResourceProvisioner.java b/foreign/java/bench/src/main/java/org/apache/iggy/bench/common/provision/ResourceProvisioner.java index 8fdae2b..01761c2 100644 --- a/foreign/java/bench/src/main/java/org/apache/iggy/bench/common/provision/ResourceProvisioner.java +++ b/foreign/java/bench/src/main/java/org/apache/iggy/bench/common/provision/ResourceProvisioner.java
@@ -25,7 +25,7 @@ import org.apache.iggy.bench.models.common.provision.ProvisionedResources; import org.apache.iggy.client.blocking.tcp.IggyTcpClient; import org.apache.iggy.identifier.StreamId; -import org.apache.iggy.topic.CompressionAlgorithm; +import org.apache.iggy.topic.TopicOptions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -82,11 +82,12 @@ client.topics() .createTopic( StreamId.of(streamName), + topicNames.get(0), 1L, - CompressionAlgorithm.None, - BigInteger.valueOf(pinnedProducerCliArgs.messageExpiry()), - BigInteger.valueOf(pinnedProducerCliArgs.maxTopicSize()), - topicNames.get(0)); + TopicOptions.builder() + .messageExpiry(BigInteger.valueOf(pinnedProducerCliArgs.messageExpiry())) + .maxTopicSize(BigInteger.valueOf(pinnedProducerCliArgs.maxTopicSize())) + .build()); } } } catch (RuntimeException exception) {
diff --git a/foreign/java/external-processors/iggy-connector-pinot/src/test/java/org/apache/iggy/connector/pinot/IggyPinotIntegrationTest.java b/foreign/java/external-processors/iggy-connector-pinot/src/test/java/org/apache/iggy/connector/pinot/IggyPinotIntegrationTest.java index a65dc36..a2658c9 100644 --- a/foreign/java/external-processors/iggy-connector-pinot/src/test/java/org/apache/iggy/connector/pinot/IggyPinotIntegrationTest.java +++ b/foreign/java/external-processors/iggy-connector-pinot/src/test/java/org/apache/iggy/connector/pinot/IggyPinotIntegrationTest.java
@@ -29,7 +29,6 @@ import org.apache.iggy.identifier.TopicId; import org.apache.iggy.message.Message; import org.apache.iggy.message.Partitioning; -import org.apache.iggy.topic.CompressionAlgorithm; import org.apache.pinot.spi.exception.QueryErrorCode; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -42,7 +41,6 @@ import org.testcontainers.utility.MountableFile; import java.io.IOException; -import java.math.BigInteger; import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; @@ -294,9 +292,7 @@ private static void createIggyResources() { iggyClient.streams().createStream(STREAM_NAME); StreamId streamId = StreamId.of(STREAM_NAME); - iggyClient - .topics() - .createTopic(streamId, 2L, CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, TOPIC_NAME); + iggyClient.topics().createTopic(streamId, TOPIC_NAME, 2L); iggyClient.consumerGroups().createConsumerGroup(streamId, TopicId.of(TOPIC_NAME), CONSUMER_GROUP_NAME); }
diff --git a/foreign/java/gradle.properties b/foreign/java/gradle.properties index d6e8e6b..d36a136 100644 --- a/foreign/java/gradle.properties +++ b/foreign/java/gradle.properties
@@ -15,5 +15,5 @@ # specific language governing permissions and limitations # under the License. -version=0.9.1-SNAPSHOT +version=0.9.2-SNAPSHOT group=org.apache.iggy
diff --git a/foreign/java/java-sdk/build.gradle.kts b/foreign/java/java-sdk/build.gradle.kts index 6e7e048..2679f97 100644 --- a/foreign/java/java-sdk/build.gradle.kts +++ b/foreign/java/java-sdk/build.gradle.kts
@@ -54,7 +54,7 @@ testImplementation(libs.testcontainers.junit) testImplementation(platform(libs.junit.bom)) testImplementation(libs.bundles.testing) - testRuntimeOnly(libs.logback.classic) + testImplementation(libs.logback.classic) testRuntimeOnly(libs.netty.dns.macos) { artifact { classifier = "osx-aarch_64" } } }
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/StreamsClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/StreamsClient.java index c55e0ba..9e00b00 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/StreamsClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/StreamsClient.java
@@ -20,10 +20,14 @@ package org.apache.iggy.client.async; import org.apache.iggy.identifier.StreamId; +import org.apache.iggy.message.HeaderValue; import org.apache.iggy.stream.StreamBase; import org.apache.iggy.stream.StreamDetails; +import org.apache.iggy.stream.StreamOptions; +import org.apache.iggy.stream.StreamUpdate; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -88,40 +92,68 @@ CompletableFuture<List<StreamBase>> getStreams(); /** - * Creates a new stream with the given name. + * Creates a new stream with the given name and no options. * * <p>The stream ID is assigned by the server. Stream names must be unique across - * the server. + * the server. A server with encryption enabled generates the stream's own data key. * * @param name the name for the new stream * @return a {@link CompletableFuture} that completes with the created {@link StreamDetails} * @throws org.apache.iggy.exception.IggyException if a stream with the same name * already exists */ - CompletableFuture<StreamDetails> createStream(String name); + default CompletableFuture<StreamDetails> createStream(String name) { + return createStream(name, StreamOptions.empty()); + } /** - * Updates the name of an existing stream identified by its numeric ID. + * Creates a new stream with typed options, such as its own encryption key of 32 random bytes. + * + * <p>The key travels inside the request over this client's transport, so protecting it on the + * network is the job of TLS. The server never returns it, only its source and fingerprints. + * + * @param name the name for the new stream + * @param options the stream's options + * @return a {@link CompletableFuture} that completes with the created {@link StreamDetails} + */ + default CompletableFuture<StreamDetails> createStream(String name, StreamOptions options) { + return createStream(name, options.entries()); + } + + /** + * Creates a new stream with options keyed by option name, for keys newer than this build. + * + * <p>The server refuses a key outside its catalog by name. Prefer {@link StreamOptions} for the + * keys this build knows. + * + * @param name the name for the new stream + * @param options option values keyed by option name + * @return a {@link CompletableFuture} that completes with the created {@link StreamDetails} + */ + CompletableFuture<StreamDetails> createStream(String name, Map<String, HeaderValue> options); + + /** + * Updates an existing stream identified by its numeric ID. * * <p>This is a convenience overload that wraps the numeric ID into a {@link StreamId}. * * @param streamId the numeric stream identifier - * @param name the new name for the stream + * @param update what to change, every key left out keeps its value * @return a {@link CompletableFuture} that completes when the update is done */ - default CompletableFuture<Void> updateStream(Long streamId, String name) { - return updateStream(StreamId.of(streamId), name); + default CompletableFuture<Void> updateStream(Long streamId, StreamUpdate update) { + return updateStream(StreamId.of(streamId), update); } /** - * Updates the name of an existing stream. + * Updates an existing stream: renames it. Its encryption key cannot change or be reset. * * @param streamId the stream identifier (numeric or string-based) - * @param name the new name for the stream + * @param update what to change, every key left out keeps its value * @return a {@link CompletableFuture} that completes when the update is done * @throws org.apache.iggy.exception.IggyException if the stream does not exist */ - CompletableFuture<Void> updateStream(StreamId streamId, String name); + CompletableFuture<Void> updateStream(StreamId streamId, StreamUpdate update); /** * Deletes a stream and all of its topics, partitions, and messages by numeric ID.
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/TopicsClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/TopicsClient.java index 711bdb7..a251778 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/TopicsClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/TopicsClient.java
@@ -22,11 +22,11 @@ import org.apache.iggy.identifier.StreamId; import org.apache.iggy.identifier.TopicId; import org.apache.iggy.message.HeaderValue; -import org.apache.iggy.topic.CompressionAlgorithm; import org.apache.iggy.topic.Topic; import org.apache.iggy.topic.TopicDetails; +import org.apache.iggy.topic.TopicOptions; +import org.apache.iggy.topic.TopicUpdate; -import java.math.BigInteger; import java.util.List; import java.util.Map; import java.util.Optional; @@ -36,17 +36,15 @@ * Async client interface for topic management operations. * * <p>Topics exist within streams and contain one or more partitions that hold the actual - * messages. Each topic has configurable properties including compression, message expiry, - * maximum size, and replication factor. + * messages. Each topic has settings such as message expiry and maximum size, set through + * {@link TopicOptions} at create and changed through {@link TopicUpdate}. * * <p>Usage example: * <pre>{@code * TopicsClient topics = client.topics(); * - * // Create a topic with 3 partitions and no message expiry - * topics.createTopic( - * StreamId.of(1L), 3L, CompressionAlgorithm.none(), - * BigInteger.ZERO, BigInteger.ZERO, "events") + * // Create a topic with 3 partitions and the server's default settings + * topics.createTopic(StreamId.of(1L), "events", 3) * .thenAccept(details -> System.out.println("Topic created: " + details.name())); * * // List all topics in a stream @@ -80,99 +78,61 @@ CompletableFuture<List<Topic>> getTopics(StreamId streamId); /** - * Creates a new topic within a stream. + * Creates a new topic within a stream with the server's default settings. * - * <p>The topic is created with the specified number of partitions and configuration. - * Partition count cannot be changed after creation, but additional partitions can be - * added via the partitions API. - * - * @param streamId the stream identifier to create the topic in - * @param partitionsCount the initial number of partitions (must be at least 1) - * @param compressionAlgorithm the compression algorithm for stored messages - * (e.g., {@link CompressionAlgorithm#None}) - * @param messageExpiry message expiry time in microseconds; {@link BigInteger#ZERO} - * means messages never expire - * @param maxTopicSize maximum topic size in bytes; {@link BigInteger#ZERO} - * means unlimited - * the server default is used - * @param name the topic name (must be unique within the stream) + * @param streamId the stream identifier to create the topic in + * @param name the topic name (must be unique within the stream) + * @param partitionsCount the initial number of partitions * @return a {@link CompletableFuture} that completes with the created {@link TopicDetails} * @throws org.apache.iggy.exception.IggyException if the stream does not exist or a * topic with the same name already exists */ - default CompletableFuture<TopicDetails> createTopic( - StreamId streamId, - Long partitionsCount, - CompressionAlgorithm compressionAlgorithm, - BigInteger messageExpiry, - BigInteger maxTopicSize, - String name) { + default CompletableFuture<TopicDetails> createTopic(StreamId streamId, String name, long partitionsCount) { return createTopic( - streamId, partitionsCount, compressionAlgorithm, messageExpiry, maxTopicSize, name, Map.of()); + streamId, name, partitionsCount, TopicOptions.builder().build()); } /** - * Creates a topic, carrying option keys that have no parameter of their own. + * Creates a new topic within a stream with typed settings. * - * <p>{@code options} is keyed by option name and reuses {@link HeaderValue} because options ride - * the user-headers codec. It reaches keys the server catalog gained after this build shipped; a - * parameter above wins on collision, and a key outside the catalog is refused by name. Call - * {@code describeOptions} on the server to see which keys it accepts. + * <p>Partition count is an argument rather than a setting: additional partitions can be + * added later via the partitions API. * - * @param options option keys with no parameter of their own + * @param streamId the stream identifier to create the topic in + * @param name the topic name (must be unique within the stream) + * @param partitionsCount the initial number of partitions + * @param options the topic's settings, unset keys resolve to the server defaults + * @return a {@link CompletableFuture} that completes with the created {@link TopicDetails} + */ + default CompletableFuture<TopicDetails> createTopic( + StreamId streamId, String name, long partitionsCount, TopicOptions options) { + return createTopic(streamId, name, partitionsCount, options.entries()); + } + + /** + * Creates a topic with settings keyed by option name, for keys newer than this build. + * + * <p>{@code options} reuses {@link HeaderValue} because options ride the user-headers codec. A + * key outside the catalog is refused by name. Call {@code describeOptions} on the server to see + * which keys it accepts. + * + * @param options option values keyed by option name * @return a {@link CompletableFuture} that completes with the created {@link TopicDetails} */ CompletableFuture<TopicDetails> createTopic( - StreamId streamId, - Long partitionsCount, - CompressionAlgorithm compressionAlgorithm, - BigInteger messageExpiry, - BigInteger maxTopicSize, - String name, - Map<String, HeaderValue> options); + StreamId streamId, String name, long partitionsCount, Map<String, HeaderValue> options); /** - * Updates the configuration of an existing topic. + * Updates an existing topic: its name, message expiry or size cap, or resets the last two + * to their defaults. Every key left out keeps its value. Partition count is not affected. * - * <p>This allows changing the topic's name, compression, expiry, size limit, and - * replication factor. Partition count is not affected by this operation. - * - * @param streamId the stream identifier containing the topic - * @param topicId the topic identifier to update - * @param compressionAlgorithm the new compression algorithm - * @param messageExpiry the new message expiry in microseconds - * @param maxTopicSize the new maximum topic size in bytes - * @param name the new topic name + * @param streamId the stream identifier containing the topic + * @param topicId the topic identifier to update + * @param update what to change * @return a {@link CompletableFuture} that completes when the update is done * @throws org.apache.iggy.exception.IggyException if the topic does not exist */ - default CompletableFuture<Void> updateTopic( - StreamId streamId, - TopicId topicId, - CompressionAlgorithm compressionAlgorithm, - BigInteger messageExpiry, - BigInteger maxTopicSize, - String name) { - return updateTopic(streamId, topicId, compressionAlgorithm, messageExpiry, maxTopicSize, name, Map.of()); - } - - /** - * Updates a topic, carrying option keys that have no parameter of their own. - * - * <p>The server refuses any key an update may not change, by name. A key left out keeps its - * current value. - * - * @param options option keys with no parameter of their own - * @return a {@link CompletableFuture} that completes when the update is done - */ - CompletableFuture<Void> updateTopic( - StreamId streamId, - TopicId topicId, - CompressionAlgorithm compressionAlgorithm, - BigInteger messageExpiry, - BigInteger maxTopicSize, - String name, - Map<String, HeaderValue> options); + CompletableFuture<Void> updateTopic(StreamId streamId, TopicId topicId, TopicUpdate update); /** * Deletes a topic and all of its partitions and messages.
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/UsersClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/UsersClient.java index 4ccbce1..d027642 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/UsersClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/UsersClient.java
@@ -24,7 +24,9 @@ import org.apache.iggy.user.Permissions; import org.apache.iggy.user.UserInfo; import org.apache.iggy.user.UserInfoDetails; +import org.apache.iggy.user.UserOptions; import org.apache.iggy.user.UserStatus; +import org.apache.iggy.user.UserUpdate; import java.util.List; import java.util.Optional; @@ -85,16 +87,31 @@ CompletableFuture<List<UserInfo>> getUsers(); /** + * Create a user with the details provided and no options. + * + * @see #createUser(String, String, UserStatus, Optional, UserOptions) + */ + default CompletableFuture<UserInfoDetails> createUser( + String username, String password, UserStatus status, Optional<Permissions> permissions) { + return createUser(username, password, status, permissions, UserOptions.empty()); + } + + /** * Create a user with the details provided. * * @param username The username of the new user. * @param password The password of the new user. * @param status The status of the new user. * @param permissions An optional set of permissions for the new user. + * @param options The user's options. No user key exists yet, so these carry only raw entries. * @return A {@link CompletableFuture} that completes with a {@link UserInfoDetails} that gives information about the newly created user. */ CompletableFuture<UserInfoDetails> createUser( - String username, String password, UserStatus status, Optional<Permissions> permissions); + String username, + String password, + UserStatus status, + Optional<Permissions> permissions, + UserOptions options); /** * Delete the user with the given ID. @@ -114,22 +131,24 @@ CompletableFuture<Void> deleteUser(UserId userId); /** - * Update the user identified by the given userId, setting (if provided) their username and or status. + * Update the user identified by the given userId. * - * @see #updateUser(UserId, Optional, Optional) + * @see #updateUser(UserId, UserUpdate) */ - default CompletableFuture<Void> updateUser(Long userId, Optional<String> username, Optional<UserStatus> status) { - return updateUser(UserId.of(userId), username, status); + default CompletableFuture<Void> updateUser(Long userId, UserUpdate update) { + return updateUser(UserId.of(userId), update); } /** - * Update the user identified by the given userId, setting (if provided) their username and or status. + * Update the user identified by the given userId: its username, its status, or both. Every key left + * out keeps its value. Renaming the signed-in user also moves the credentials this client signs back + * in with after a reconnect. + * * @param userId The ID of the user to update. - * @param username The new username of the user, or an empty optional if no update is required. - * @param status The new status of the user, or an empty optional if no update is required. + * @param update What to change. * @return A {@link CompletableFuture} that completes but yields no value. */ - CompletableFuture<Void> updateUser(UserId userId, Optional<String> username, Optional<UserStatus> status); + CompletableFuture<Void> updateUser(UserId userId, UserUpdate update); /** * Update the permissions of the user identified by the provided userId.
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/SessionLogin.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/SessionLogin.java index 3094b8c..5693507 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/SessionLogin.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/SessionLogin.java
@@ -26,6 +26,7 @@ import org.apache.iggy.client.async.tcp.vsr.VsrHeaders; import org.apache.iggy.identifier.UserId; import org.apache.iggy.serde.CommandCode; +import org.apache.iggy.system.OptionsScope; import java.nio.charset.StandardCharsets; import java.util.Optional; @@ -36,9 +37,6 @@ * none of them signs in with an old copy. */ final class SessionLogin { - // Flags the optional username of an UpdateUser body. - private static final int USERNAME_PRESENT = 1; - private int commandCode; private byte[] payload; private long version; @@ -82,13 +80,17 @@ @SuppressWarnings({"checkstyle:CyclomaticComplexity", "checkstyle:NPathComplexity"}) synchronized Optional<Change> refresh(int requestCode, ByteBuf frame, long userId) { boolean passwordChange = requestCode == CommandCode.User.CHANGE_PASSWORD.getValue(); - if ((!passwordChange && requestCode != CommandCode.User.UPDATE.getValue()) + if ((!passwordChange && requestCode != CommandCode.System.UPDATE_RESOURCE.getValue()) || payload == null || commandCode != CommandCode.User.LOGIN.getValue()) { return Optional.empty(); } ByteBuf change = frame.slice( frame.readerIndex() + VsrHeaders.HEADER_SIZE, frame.readableBytes() - VsrHeaders.HEADER_SIZE); + // A stream or topic patch never touches credentials, even when it renames. + if (!passwordChange && (!change.isReadable() || change.readUnsignedByte() != OptionsScope.User.asCode())) { + return Optional.empty(); + } ByteBuf current = Unpooled.wrappedBuffer(payload); String oldUsername = current.readCharSequence(current.readUnsignedByte(), StandardCharsets.UTF_8) .toString(); @@ -101,8 +103,8 @@ Optional<String> password = Optional.empty(); if (passwordChange) { password = readField(change).flatMap(currentPassword -> readField(change)); - } else if (change.isReadable() && change.readUnsignedByte() == USERNAME_PRESENT) { - username = readField(change); + } else { + username = UserPatchCredentials.patchedUsername(change); } if (username.isEmpty() && password.isEmpty()) { return Optional.empty();
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/StreamsTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/StreamsTcpClient.java index 4e03cbb..6e108d2 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/StreamsTcpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/StreamsTcpClient.java
@@ -22,13 +22,16 @@ import io.netty.buffer.Unpooled; import org.apache.iggy.client.async.StreamsClient; import org.apache.iggy.identifier.StreamId; +import org.apache.iggy.message.HeaderValue; import org.apache.iggy.serde.BytesSerializer; import org.apache.iggy.serde.CommandCode; import org.apache.iggy.stream.StreamBase; import org.apache.iggy.stream.StreamDetails; +import org.apache.iggy.stream.StreamUpdate; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.function.Supplier; @@ -63,8 +66,8 @@ } @Override - public CompletableFuture<StreamDetails> createStream(String name) { - var payload = BytesSerializer.toBytes(name, "name"); + public CompletableFuture<StreamDetails> createStream(String name, Map<String, HeaderValue> options) { + var payload = BytesSerializer.toCreateStream(name, options); return AsyncTcpConnection.mapResponse( connection().send(CommandCode.Stream.CREATE.getValue(), payload), response -> { @@ -105,14 +108,10 @@ } @Override - public CompletableFuture<Void> updateStream(StreamId streamId, String name) { - var payload = toBytes(streamId); - payload.writeBytes(BytesSerializer.toBytes(name, "name")); - // No trailing options block: streams have no catalog keys yet and the - // server reads an absent block as empty. Settings will ride one here, - // as topics do. + public CompletableFuture<Void> updateStream(StreamId streamId, StreamUpdate update) { + var payload = BytesSerializer.toUpdateResource(streamId, update); - return connection().sendAndRelease(CommandCode.Stream.UPDATE, payload); + return connection().sendAndRelease(CommandCode.System.UPDATE_RESOURCE, payload); } @Override
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/TopicsTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/TopicsTcpClient.java index 4ba9b9f..9e0c650 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/TopicsTcpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/TopicsTcpClient.java
@@ -19,23 +19,20 @@ package org.apache.iggy.client.async.tcp; -import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import org.apache.iggy.client.async.TopicsClient; import org.apache.iggy.identifier.StreamId; import org.apache.iggy.identifier.TopicId; -import org.apache.iggy.message.HeaderKey; import org.apache.iggy.message.HeaderValue; import org.apache.iggy.serde.BytesDeserializer; import org.apache.iggy.serde.BytesSerializer; import org.apache.iggy.serde.CommandCode; -import org.apache.iggy.topic.CompressionAlgorithm; import org.apache.iggy.topic.Topic; import org.apache.iggy.topic.TopicDetails; +import org.apache.iggy.topic.TopicOptions; +import org.apache.iggy.topic.TopicUpdate; -import java.math.BigInteger; import java.util.ArrayList; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -49,10 +46,6 @@ */ public class TopicsTcpClient implements TopicsClient { - private static final String COMPRESSION_ALGORITHM_OPTION = "compression_algorithm"; - private static final String MESSAGE_EXPIRY_OPTION = "message_expiry"; - private static final String MAX_TOPIC_SIZE_OPTION = "max_topic_size"; - private final Supplier<AsyncTcpConnection> connectionSupplier; private final Runnable topologyChanged; @@ -113,22 +106,9 @@ @Override public CompletableFuture<TopicDetails> createTopic( - StreamId streamId, - Long partitionsCount, - CompressionAlgorithm compressionAlgorithm, - BigInteger messageExpiry, - BigInteger maxTopicSize, - String name, - Map<String, HeaderValue> options) { - - var payload = createTopicPayload( - streamId, - partitionsCount, - compressionAlgorithm, - messageExpiry, - maxTopicSize, - name, - org.apache.iggy.topic.TopicOptions.withDurabilityDefaults(options)); + StreamId streamId, String name, long partitionsCount, Map<String, HeaderValue> options) { + var payload = BytesSerializer.toCreateTopic( + streamId, name, partitionsCount, TopicOptions.withDurabilityDefaults(options)); return AsyncTcpConnection.mapResponse( connection().send(CommandCode.Topic.CREATE.getValue(), payload), response -> { @@ -140,74 +120,11 @@ }); } - static ByteBuf createTopicPayload( - StreamId streamId, - Long partitionsCount, - CompressionAlgorithm compressionAlgorithm, - BigInteger messageExpiry, - BigInteger maxTopicSize, - String name, - Map<String, HeaderValue> options) { - // partitions_count is a fixed field of the command, never an option: - // admission consumes it to compute partition assignments. - var payload = Unpooled.buffer(); - payload.writeBytes(toBytes(streamId)); - payload.writeIntLE(partitionsCount.intValue()); - payload.writeBytes(BytesSerializer.toBytes(name, "name")); - payload.writeBytes(BytesSerializer.toBytes( - createTopicOptions(compressionAlgorithm, messageExpiry, maxTopicSize, options))); - return payload; - } - - private static Map<HeaderKey, HeaderValue> createTopicOptions( - CompressionAlgorithm compressionAlgorithm, - BigInteger messageExpiry, - BigInteger maxTopicSize, - Map<String, HeaderValue> extra) { - // Server-default sentinels (compression none, expiry 0, size 0) are - // omitted so the admitting server resolves them from its config. - Map<HeaderKey, HeaderValue> options = new LinkedHashMap<>(); - // Caller keys go in first so a parameter above overwrites one of them: - // the block must not carry a key twice, or the server refuses it whole. - extra.forEach((key, value) -> options.put(HeaderKey.fromString(key), value)); - if (compressionAlgorithm != CompressionAlgorithm.None) { - var compressionName = - switch (compressionAlgorithm) { - case None -> "none"; - case Gzip -> "gzip"; - }; - options.put(HeaderKey.fromString(COMPRESSION_ALGORITHM_OPTION), HeaderValue.fromString(compressionName)); - } - if (messageExpiry.signum() != 0) { - options.put(HeaderKey.fromString(MESSAGE_EXPIRY_OPTION), HeaderValue.fromUint64(messageExpiry)); - } - if (maxTopicSize.signum() != 0) { - options.put(HeaderKey.fromString(MAX_TOPIC_SIZE_OPTION), HeaderValue.fromUint64(maxTopicSize)); - } - return options; - } - @Override - public CompletableFuture<Void> updateTopic( - StreamId streamId, - TopicId topicId, - CompressionAlgorithm compressionAlgorithm, - BigInteger messageExpiry, - BigInteger maxTopicSize, - String name, - Map<String, HeaderValue> options) { + public CompletableFuture<Void> updateTopic(StreamId streamId, TopicId topicId, TopicUpdate update) { + var payload = BytesSerializer.toUpdateResource(streamId, topicId, update); - var payload = Unpooled.buffer(); - payload.writeBytes(toBytes(streamId)); - payload.writeBytes(toBytes(topicId)); - payload.writeBytes(BytesSerializer.toBytes(name, "name")); - // Settings ride the options block. A default value means the caller did - // not set the key, so it is omitted and the server leaves the topic's - // current value alone. - payload.writeBytes(BytesSerializer.toBytes( - createTopicOptions(compressionAlgorithm, messageExpiry, maxTopicSize, options))); - - return connection().sendAndRelease(CommandCode.Topic.UPDATE, payload); + return connection().sendAndRelease(CommandCode.System.UPDATE_RESOURCE, payload); } @Override
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/UserPatchCredentials.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/UserPatchCredentials.java new file mode 100644 index 0000000..a3e1a30 --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/UserPatchCredentials.java
@@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.client.async.tcp; + +import io.netty.buffer.ByteBuf; +import org.apache.iggy.message.HeaderKind; +import org.apache.iggy.user.UserOptionKey; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Optional; + +final class UserPatchCredentials { + + private static final byte[] USERNAME_KEY = UserOptionKey.USERNAME.value().getBytes(StandardCharsets.UTF_8); + + private UserPatchCredentials() {} + + static Optional<String> patchedUsername(ByteBuf patch) { + if (!skipResetKeys(patch)) { + return Optional.empty(); + } + while (patch.readableBytes() >= 2 * (1 + Integer.BYTES)) { + patch.skipBytes(1); + Optional<byte[]> key = readOptionField(patch); + if (key.isEmpty() || !patch.isReadable()) { + return Optional.empty(); + } + int valueKind = patch.readUnsignedByte(); + Optional<byte[]> value = readOptionField(patch); + if (value.isEmpty()) { + return Optional.empty(); + } + if (Arrays.equals(key.get(), USERNAME_KEY) && valueKind == HeaderKind.String.asCode()) { + return Optional.of(new String(value.get(), StandardCharsets.UTF_8)); + } + } + return Optional.empty(); + } + + private static boolean skipResetKeys(ByteBuf patch) { + if (!patch.isReadable()) { + return false; + } + int resetCount = patch.readUnsignedByte(); + for (int index = 0; index < resetCount; index++) { + if (!patch.isReadable()) { + return false; + } + int length = patch.readUnsignedByte(); + if (patch.readableBytes() < length) { + return false; + } + patch.skipBytes(length); + } + return true; + } + + private static Optional<byte[]> readOptionField(ByteBuf body) { + if (body.readableBytes() < Integer.BYTES) { + return Optional.empty(); + } + long length = body.readUnsignedIntLE(); + if (length > body.readableBytes()) { + return Optional.empty(); + } + byte[] field = new byte[(int) length]; + body.readBytes(field); + return Optional.of(field); + } +}
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/UsersTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/UsersTcpClient.java index e5119ca..d3f0f03 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/UsersTcpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/UsersTcpClient.java
@@ -23,12 +23,15 @@ import org.apache.iggy.client.async.UsersClient; import org.apache.iggy.identifier.UserId; import org.apache.iggy.serde.BytesDeserializer; +import org.apache.iggy.serde.BytesSerializer; import org.apache.iggy.serde.CommandCode; import org.apache.iggy.user.IdentityInfo; import org.apache.iggy.user.Permissions; import org.apache.iggy.user.UserInfo; import org.apache.iggy.user.UserInfoDetails; +import org.apache.iggy.user.UserOptions; import org.apache.iggy.user.UserStatus; +import org.apache.iggy.user.UserUpdate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,6 +40,10 @@ import java.util.concurrent.CompletableFuture; import java.util.function.Supplier; +import static org.apache.iggy.serde.BytesSerializer.MAX_PASSWORD_LENGTH; +import static org.apache.iggy.serde.BytesSerializer.MAX_USERNAME_LENGTH; +import static org.apache.iggy.serde.BytesSerializer.MIN_PASSWORD_LENGTH; +import static org.apache.iggy.serde.BytesSerializer.MIN_USERNAME_LENGTH; import static org.apache.iggy.serde.BytesSerializer.toBytes; /** @@ -46,16 +53,6 @@ public class UsersTcpClient implements UsersClient { private static final Logger log = LoggerFactory.getLogger(UsersTcpClient.class); - /** - * Credential bounds the server enforces, in UTF-8 bytes. Checked here so a bad value fails - * before the round trip instead of as an opaque server error. - */ - private static final int MIN_USERNAME_LENGTH = 3; - - private static final int MAX_USERNAME_LENGTH = 50; - private static final int MIN_PASSWORD_LENGTH = 3; - private static final int MAX_PASSWORD_LENGTH = 100; - private final Supplier<AsyncTcpConnection> connectionSupplier; private final LoginRoutingHook routingHook; @@ -86,19 +83,12 @@ @Override public CompletableFuture<UserInfoDetails> createUser( - String username, String password, UserStatus status, Optional<Permissions> permissions) { - var payload = Unpooled.buffer(); - payload.writeBytes(toBytes(username, "username", MIN_USERNAME_LENGTH, MAX_USERNAME_LENGTH)); - payload.writeBytes(toBytes(password, "password", MIN_PASSWORD_LENGTH, MAX_PASSWORD_LENGTH)); - payload.writeByte(status.asCode()); - permissions.ifPresentOrElse( - perms -> { - payload.writeByte(1); - var permissionBytes = toBytes(perms); - payload.writeIntLE(permissionBytes.readableBytes()); - payload.writeBytes(permissionBytes); - }, - () -> payload.writeByte(0)); + String username, + String password, + UserStatus status, + Optional<Permissions> permissions, + UserOptions options) { + var payload = BytesSerializer.toCreateUser(username, password, status, permissions, options.entries()); return connection().exchangeForEntity(CommandCode.User.CREATE, payload, BytesDeserializer::readUserInfoDetails); } @@ -110,25 +100,10 @@ } @Override - public CompletableFuture<Void> updateUser(UserId userId, Optional<String> username, Optional<UserStatus> status) { - var payload = toBytes(userId); - username.ifPresentOrElse( - un -> { - payload.writeByte(1); - payload.writeBytes(toBytes(un, "username", MIN_USERNAME_LENGTH, MAX_USERNAME_LENGTH)); - }, - () -> payload.writeByte(0)); - status.ifPresentOrElse( - s -> { - payload.writeByte(1); - payload.writeByte(s.asCode()); - }, - () -> payload.writeByte(0)); - // No trailing options block: users have no catalog keys yet and the - // server reads an absent block as empty. Settings will ride one here, - // as topics do. + public CompletableFuture<Void> updateUser(UserId userId, UserUpdate update) { + var payload = BytesSerializer.toUpdateResource(userId, update); - return connection().sendAndRelease(CommandCode.User.UPDATE, payload); + return connection().sendAndRelease(CommandCode.System.UPDATE_RESOURCE, payload); } @Override
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/ConsensusSession.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/ConsensusSession.java index ce6ab82..249d6d3 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/ConsensusSession.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/ConsensusSession.java
@@ -62,6 +62,8 @@ private long clientIdLow; private long clientIdHigh; + private long registerIdentifierLow; + private long registerIdentifierHigh; private Long session; private long requestCounter = 1; private long generation; @@ -453,6 +455,14 @@ return parent == null ? clientIdHigh : parent.clientIdHigh(); } + synchronized long registerIdentifierLow() { + return parent == null ? registerIdentifierLow : parent.registerIdentifierLow(); + } + + synchronized long registerIdentifierHigh() { + return parent == null ? registerIdentifierHigh : parent.registerIdentifierHigh(); + } + /** * Takes no monitor. A link that holds its session waits for the owner, so a caller that holds the owner and * resolves the owner of another link, for the lineage of a call that started there, must not wait for that link. @@ -723,6 +733,10 @@ private void regenerateClientId() { RANDOM.nextBytes(bindSecret); do { + registerIdentifierLow = RANDOM.nextLong(); + registerIdentifierHigh = RANDOM.nextLong(); + } while (registerIdentifierLow == 0 && registerIdentifierHigh == 0); + do { clientIdLow = RANDOM.nextLong(); // Keep generated IDs outside the server-only range. clientIdHigh = RANDOM.nextLong() & Long.MAX_VALUE;
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrLoginCodec.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrLoginCodec.java index 219e8fa..fb26fc5 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrLoginCodec.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrLoginCodec.java
@@ -45,7 +45,7 @@ * protocol version through its current version, including patch components. * Versions outside this range, including newer patches, are rejected. */ - static final int PROTOCOL_VERSION = (11 << 10) | 1; // 0.11.1 + static final int PROTOCOL_VERSION = 12 << 10; // 0.12.0 static final String SDK_NAME = "java-sdk";
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrOperation.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrOperation.java index 8b8e6b7..f5a59f7 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrOperation.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrOperation.java
@@ -41,12 +41,16 @@ public static final int TRUNCATE_PARTITION = 68; public static final int FINALIZE_SESSION = 70; public static final int COMPLETE_LIFECYCLE = 71; + public static final int UPDATE_STREAM_RESOLVED = 72; + public static final int UPDATE_TOPIC_RESOLVED = 73; + public static final int UPDATE_USER_RESOLVED = 74; + public static final int REWRAP_STREAM_KEYS = 75; public static final int CREATE_STREAM = 128; - public static final int UPDATE_STREAM = 129; + // 129 was UpdateStream. Never reuse it. public static final int DELETE_STREAM = 130; public static final int CREATE_TOPIC = 132; - public static final int UPDATE_TOPIC = 133; + // 133 was UpdateTopic. Never reuse it. public static final int DELETE_TOPIC = 134; public static final int CREATE_PARTITIONS = 136; public static final int DELETE_PARTITIONS = 137; @@ -54,7 +58,7 @@ public static final int CREATE_CONSUMER_GROUP = 139; public static final int DELETE_CONSUMER_GROUP = 140; public static final int CREATE_USER = 141; - public static final int UPDATE_USER = 142; + // 142 was UpdateUser. Never reuse it. public static final int DELETE_USER = 143; public static final int CHANGE_PASSWORD = 144; public static final int UPDATE_PERMISSIONS = 145; @@ -62,6 +66,7 @@ public static final int DELETE_PERSONAL_ACCESS_TOKEN = 147; public static final int JOIN_CONSUMER_GROUP = 148; public static final int LEAVE_CONSUMER_GROUP = 149; + public static final int UPDATE_RESOURCE = 150; public static final int SEND_MESSAGES = 160; public static final int STORE_CONSUMER_OFFSET = 161; @@ -82,8 +87,8 @@ */ private static final Map<Integer, Integer> REPLICATED_OPERATIONS = Map.ofEntries( Map.entry(33, CREATE_USER), + Map.entry(16, UPDATE_RESOURCE), Map.entry(34, DELETE_USER), - Map.entry(35, UPDATE_USER), Map.entry(36, UPDATE_PERMISSIONS), Map.entry(37, CHANGE_PASSWORD), Map.entry(42, CREATE_PERSONAL_ACCESS_TOKEN), @@ -93,10 +98,8 @@ Map.entry(122, DELETE_CONSUMER_OFFSET), Map.entry(202, CREATE_STREAM), Map.entry(203, DELETE_STREAM), - Map.entry(204, UPDATE_STREAM), Map.entry(302, CREATE_TOPIC), Map.entry(303, DELETE_TOPIC), - Map.entry(304, UPDATE_TOPIC), Map.entry(402, CREATE_PARTITIONS), Map.entry(403, DELETE_PARTITIONS), Map.entry(503, DELETE_SEGMENTS), @@ -112,9 +115,13 @@ static { KNOWN_OPERATIONS.set(RESERVED, LOGOUT + 1); KNOWN_OPERATIONS.set(CREATE_TOPIC_WITH_ASSIGNMENTS, TRUNCATE_PARTITION + 1); - KNOWN_OPERATIONS.set(CREATE_STREAM, DELETE_STREAM + 1); - KNOWN_OPERATIONS.set(CREATE_TOPIC, DELETE_TOPIC + 1); - KNOWN_OPERATIONS.set(CREATE_PARTITIONS, LEAVE_CONSUMER_GROUP + 1); + KNOWN_OPERATIONS.set(UPDATE_STREAM_RESOLVED, REWRAP_STREAM_KEYS + 1); + KNOWN_OPERATIONS.set(CREATE_STREAM); + KNOWN_OPERATIONS.set(DELETE_STREAM); + KNOWN_OPERATIONS.set(CREATE_TOPIC); + KNOWN_OPERATIONS.set(DELETE_TOPIC); + KNOWN_OPERATIONS.set(CREATE_PARTITIONS, CREATE_USER + 1); + KNOWN_OPERATIONS.set(DELETE_USER, UPDATE_RESOURCE + 1); KNOWN_OPERATIONS.set(SEND_MESSAGES); KNOWN_OPERATIONS.set(STORE_CONSUMER_OFFSET); KNOWN_OPERATIONS.set(DELETE_CONSUMER_OFFSET); @@ -149,7 +156,7 @@ if (operation == DELETE_SEGMENTS) { return false; } - return operation >= METADATA_START && operation <= LEAVE_CONSUMER_GROUP; + return operation >= METADATA_START && operation <= UPDATE_RESOURCE; } /** @@ -178,6 +185,7 @@ case CREATE_TOPIC_WITH_ASSIGNMENTS -> CREATE_TOPIC; case CREATE_PARTITIONS_WITH_ASSIGNMENTS -> CREATE_PARTITIONS; case TRUNCATE_PARTITION -> DELETE_SEGMENTS; + case UPDATE_STREAM_RESOLVED, UPDATE_TOPIC_RESOLVED, UPDATE_USER_RESOLVED -> UPDATE_RESOURCE; default -> operation; }; }
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrRequestEncoder.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrRequestEncoder.java index 10c0ab9..8d8d069 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrRequestEncoder.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrRequestEncoder.java
@@ -23,6 +23,9 @@ import io.netty.buffer.ByteBufAllocator; import org.apache.iggy.partition.PartitionContext; +import java.security.SecureRandom; +import java.util.Arrays; + /** * Encodes a (code, payload) command into a VSR request frame: * a 256-byte {@code RequestHeader} followed by the unchanged command payload. @@ -37,6 +40,8 @@ private static final int LOGIN_REGISTER_CODE = 40; private static final int LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE = 44; private static final int LOGIN_REGISTER_WITH_PAT_CODE = 45; + private static final SecureRandom RANDOM = new SecureRandom(); + private static final byte[] ZERO_IDENTIFIER = new byte[16]; private final ConsensusSession session; @@ -125,6 +130,7 @@ frame.setByte(VsrHeaders.COMMAND_OFFSET, VsrHeaders.COMMAND_REQUEST); frame.setLongLE(VsrHeaders.REQUEST_CLIENT_OFFSET, session.clientIdLow()); frame.setLongLE(VsrHeaders.REQUEST_CLIENT_OFFSET + 8, session.clientIdHigh()); + stampRequestIdentifier(frame, operation); frame.setLongLE(VsrHeaders.REQUEST_ID_OFFSET, requestId); frame.setByte(VsrHeaders.REQUEST_OPERATION_OFFSET, operation); frame.setLongLE(VsrHeaders.REQUEST_SESSION_OFFSET, sessionId); @@ -149,4 +155,17 @@ } } } + + private void stampRequestIdentifier(ByteBuf frame, int operation) { + if (operation == VsrOperation.REGISTER) { + frame.setLongLE(VsrHeaders.REQUEST_CHECKSUM_OFFSET, session.registerIdentifierLow()); + frame.setLongLE(VsrHeaders.REQUEST_CHECKSUM_OFFSET + 8, session.registerIdentifierHigh()); + } else if (operation != VsrOperation.NON_REPLICATED && operation < VsrOperation.SEND_MESSAGES) { + byte[] identifier = new byte[16]; + do { + RANDOM.nextBytes(identifier); + } while (Arrays.equals(identifier, ZERO_IDENTIFIER)); + frame.setBytes(VsrHeaders.REQUEST_CHECKSUM_OFFSET, identifier); + } + } }
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/StreamsClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/StreamsClient.java index 8a35f01..436ad45 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/StreamsClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/StreamsClient.java
@@ -20,10 +20,14 @@ package org.apache.iggy.client.blocking; import org.apache.iggy.identifier.StreamId; +import org.apache.iggy.message.HeaderValue; import org.apache.iggy.stream.StreamBase; import org.apache.iggy.stream.StreamDetails; +import org.apache.iggy.stream.StreamOptions; +import org.apache.iggy.stream.StreamUpdate; import java.util.List; +import java.util.Map; import java.util.Optional; public interface StreamsClient { @@ -36,13 +40,33 @@ List<StreamBase> getStreams(); - StreamDetails createStream(String name); - - default void updateStream(Long streamId, String name) { - updateStream(StreamId.of(streamId), name); + /** Creates a stream without options. A server with encryption enabled generates the stream's own data key. */ + default StreamDetails createStream(String name) { + return createStream(name, StreamOptions.empty()); } - void updateStream(StreamId streamId, String name); + /** + * Creates a stream with typed options, such as its own encryption key of 32 random bytes. + * + * <p>The key travels inside the request over this client's transport, so protecting it on the network + * is the job of TLS. The server never returns it, only its source and fingerprints. + */ + default StreamDetails createStream(String name, StreamOptions options) { + return createStream(name, options.entries()); + } + + /** + * Creates a stream with options keyed by option name, for keys newer than this build. The server + * refuses a key outside its catalog by name. + */ + StreamDetails createStream(String name, Map<String, HeaderValue> options); + + default void updateStream(Long streamId, StreamUpdate update) { + updateStream(StreamId.of(streamId), update); + } + + /** Renames a stream. Every key left out keeps its value. */ + void updateStream(StreamId streamId, StreamUpdate update); default void deleteStream(Long streamId) { deleteStream(StreamId.of(streamId));
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/TopicsClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/TopicsClient.java index 57c7bd8..19261d3 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/TopicsClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/TopicsClient.java
@@ -22,11 +22,11 @@ import org.apache.iggy.identifier.StreamId; import org.apache.iggy.identifier.TopicId; import org.apache.iggy.message.HeaderValue; -import org.apache.iggy.topic.CompressionAlgorithm; import org.apache.iggy.topic.Topic; import org.apache.iggy.topic.TopicDetails; +import org.apache.iggy.topic.TopicOptions; +import org.apache.iggy.topic.TopicUpdate; -import java.math.BigInteger; import java.util.List; import java.util.Map; import java.util.Optional; @@ -45,79 +45,42 @@ List<Topic> getTopics(StreamId streamId); - default TopicDetails createTopic( - Long streamId, - Long partitionsCount, - CompressionAlgorithm compressionAlgorithm, - BigInteger messageExpiry, - BigInteger maxTopicSize, - String name) { - return createTopic( - StreamId.of(streamId), partitionsCount, compressionAlgorithm, messageExpiry, maxTopicSize, name); + default TopicDetails createTopic(Long streamId, String name, long partitionsCount) { + return createTopic(StreamId.of(streamId), name, partitionsCount); } - default TopicDetails createTopic( - StreamId streamId, - Long partitionsCount, - CompressionAlgorithm compressionAlgorithm, - BigInteger messageExpiry, - BigInteger maxTopicSize, - String name) { + default TopicDetails createTopic(Long streamId, String name, long partitionsCount, TopicOptions options) { + return createTopic(StreamId.of(streamId), name, partitionsCount, options); + } + + /** Creates a topic with the server's default settings. */ + default TopicDetails createTopic(StreamId streamId, String name, long partitionsCount) { return createTopic( - streamId, partitionsCount, compressionAlgorithm, messageExpiry, maxTopicSize, name, Map.of()); + streamId, name, partitionsCount, TopicOptions.builder().build()); + } + + /** Creates a topic with typed settings. Unset keys resolve to the server defaults. */ + default TopicDetails createTopic(StreamId streamId, String name, long partitionsCount, TopicOptions options) { + return createTopic(streamId, name, partitionsCount, options.entries()); } /** - * Creates a topic, carrying option keys that have no parameter of their own. + * Creates a topic with settings keyed by option name, for keys newer than this build. * - * <p>{@code options} is keyed by option name and reuses {@link HeaderValue} because options ride - * the user-headers codec. It reaches keys the server catalog gained after this build shipped; a - * parameter above wins on collision, and a key outside the catalog is refused by name. + * <p>{@code options} reuses {@link HeaderValue} because options ride the user-headers codec. A key + * outside the catalog is refused by name. */ - TopicDetails createTopic( - StreamId streamId, - Long partitionsCount, - CompressionAlgorithm compressionAlgorithm, - BigInteger messageExpiry, - BigInteger maxTopicSize, - String name, - Map<String, HeaderValue> options); + TopicDetails createTopic(StreamId streamId, String name, long partitionsCount, Map<String, HeaderValue> options); - default void updateTopic( - Long streamId, - Long topicId, - CompressionAlgorithm compressionAlgorithm, - BigInteger messageExpiry, - BigInteger maxTopicSize, - String name) { - updateTopic( - StreamId.of(streamId), TopicId.of(topicId), compressionAlgorithm, messageExpiry, maxTopicSize, name); - } - - default void updateTopic( - StreamId streamId, - TopicId topicId, - CompressionAlgorithm compressionAlgorithm, - BigInteger messageExpiry, - BigInteger maxTopicSize, - String name) { - updateTopic(streamId, topicId, compressionAlgorithm, messageExpiry, maxTopicSize, name, Map.of()); + default void updateTopic(Long streamId, Long topicId, TopicUpdate update) { + updateTopic(StreamId.of(streamId), TopicId.of(topicId), update); } /** - * Updates a topic, carrying option keys that have no parameter of their own. - * - * <p>The server refuses any key an update may not change, by name. A key left out keeps its - * current value. + * Updates a topic's name, message expiry or size cap, or resets the last two to their defaults. Every + * key left out keeps its value. */ - void updateTopic( - StreamId streamId, - TopicId topicId, - CompressionAlgorithm compressionAlgorithm, - BigInteger messageExpiry, - BigInteger maxTopicSize, - String name, - Map<String, HeaderValue> options); + void updateTopic(StreamId streamId, TopicId topicId, TopicUpdate update); default void deleteTopic(Long streamId, Long topicId) { deleteTopic(StreamId.of(streamId), TopicId.of(topicId));
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/UsersClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/UsersClient.java index 01d559c..593b59a 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/UsersClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/UsersClient.java
@@ -24,7 +24,9 @@ import org.apache.iggy.user.Permissions; import org.apache.iggy.user.UserInfo; import org.apache.iggy.user.UserInfoDetails; +import org.apache.iggy.user.UserOptions; import org.apache.iggy.user.UserStatus; +import org.apache.iggy.user.UserUpdate; import java.util.List; import java.util.Optional; @@ -39,7 +41,17 @@ List<UserInfo> getUsers(); - UserInfoDetails createUser(String username, String password, UserStatus status, Optional<Permissions> permissions); + default UserInfoDetails createUser( + String username, String password, UserStatus status, Optional<Permissions> permissions) { + return createUser(username, password, status, permissions, UserOptions.empty()); + } + + UserInfoDetails createUser( + String username, + String password, + UserStatus status, + Optional<Permissions> permissions, + UserOptions options); default void deleteUser(Long userId) { deleteUser(UserId.of(userId)); @@ -47,11 +59,12 @@ void deleteUser(UserId userId); - default void updateUser(Long userId, Optional<String> username, Optional<UserStatus> status) { - updateUser(UserId.of(userId), username, status); + default void updateUser(Long userId, UserUpdate update) { + updateUser(UserId.of(userId), update); } - void updateUser(UserId userId, Optional<String> username, Optional<UserStatus> status); + /** Changes a user's username, status or both. Every key left out keeps its value. */ + void updateUser(UserId userId, UserUpdate update); default void updatePermissions(Long userId, Optional<Permissions> permissions) { updatePermissions(UserId.of(userId), permissions);
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/HttpOptionValue.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/HttpOptionValue.java new file mode 100644 index 0000000..7d5eb2e --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/HttpOptionValue.java
@@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.client.blocking.http; + +import org.apache.iggy.exception.IggyInvalidArgumentException; +import org.apache.iggy.exception.IggyMalformedResponseException; +import org.apache.iggy.message.HeaderKind; +import org.apache.iggy.message.HeaderValue; + +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * One option entry as REST renders it: the value as reversible text, its kind, and the provenance flag + * that the binary protocol carries as two separate blocks instead. + * + * <p>The text is UTF-8 for strings, {@code true} or {@code false}, base-10 integers, standard padded + * base64 for raw bytes, and {@code 0x} followed by the IEEE-754 bits in fixed-width lowercase hex for + * floats. Rebuilding each value in its kind lets a caller read the same {@link HeaderValue} on either + * transport. + */ +record HttpOptionValue(String value, HeaderKind kind, boolean explicit) { + + private static final String RESET_FIELD = "reset"; + private static final String FLOAT_PREFIX = "0x"; + + /** The entries of {@code options} whose provenance is {@code explicit}, rebuilt in their kinds. */ + static Map<String, HeaderValue> select(Map<String, HttpOptionValue> options, boolean explicit) { + if (options == null) { + return Map.of(); + } + Map<String, HeaderValue> selected = new LinkedHashMap<>(); + options.forEach((key, option) -> { + if (option.explicit() == explicit) { + selected.put(key, option.toHeaderValue(key)); + } + }); + return Map.copyOf(selected); + } + + /** + * Option values for a create body: strings, booleans, unsigned integers as JSON numbers, and raw bytes + * as standard padded base64. + */ + static Map<String, Object> requestValues(Map<String, HeaderValue> options) { + Map<String, Object> values = new LinkedHashMap<>(); + options.forEach((key, value) -> values.put(key, requestValue(key, value))); + return values; + } + + /** + * A {@code PUT} update body: one flat object of the keys to set, plus a {@code reset} array naming the + * keys to return to their defaults. + */ + static Map<String, Object> updateBody(Map<String, HeaderValue> set, List<String> reset) { + Map<String, Object> body = requestValues(set); + if (!reset.isEmpty()) { + body.put(RESET_FIELD, reset); + } + return body; + } + + private static Object requestValue(String key, HeaderValue value) { + return switch (value.kind()) { + case String -> value.asString(); + case Bool -> value.asBool(); + case Uint8 -> value.asUint8(); + case Uint16 -> value.asUint16(); + case Uint32 -> value.asUint32(); + case Uint64 -> value.asUint64(); + case Raw -> Base64.getEncoder().encodeToString(value.asRaw()); + default -> + throw new IggyInvalidArgumentException( + "Option '" + key + "' of kind " + value.kind() + " has no HTTP form"); + }; + } + + private HeaderValue toHeaderValue(String key) { + try { + return switch (kind) { + case Raw -> raw(); + case String -> new HeaderValue(kind, value.getBytes(StandardCharsets.UTF_8)); + case Bool -> bool(); + case Float32 -> floatBits(Integer.BYTES); + case Float64 -> floatBits(Long.BYTES); + default -> integer(); + }; + } catch (IllegalArgumentException malformed) { + throw new IggyMalformedResponseException("Option '" + key + "' value does not decode as " + + kind.name().toLowerCase(Locale.ROOT)); + } + } + + private HeaderValue raw() { + byte[] bytes = Base64.getDecoder().decode(value); + if (!Base64.getEncoder().encodeToString(bytes).equals(value)) { + throw new IllegalArgumentException("noncanonical base64"); + } + return new HeaderValue(kind, bytes); + } + + private HeaderValue bool() { + if (!"true".equals(value) && !"false".equals(value)) { + throw new IllegalArgumentException("not a boolean"); + } + return HeaderValue.fromBool("true".equals(value)); + } + + /** An integer kind: Int8 to Int128 are signed, Uint8 to Uint128 unsigned, widths doubling from one byte. */ + private HeaderValue integer() { + boolean signed = kind.asCode() <= HeaderKind.Int128.asCode(); + int first = signed ? HeaderKind.Int8.asCode() : HeaderKind.Uint8.asCode(); + int width = 1 << (kind.asCode() - first); + BigInteger number = new BigInteger(value); + if (!number.toString().equals(value)) { + throw new IllegalArgumentException("noncanonical integer"); + } + boolean fits = signed + ? number.bitLength() < width * Byte.SIZE + : number.signum() >= 0 && number.bitLength() <= width * Byte.SIZE; + if (!fits) { + throw new IllegalArgumentException("out of range"); + } + return new HeaderValue(kind, littleEndian(number, width)); + } + + private HeaderValue floatBits(int width) { + String hex = value.startsWith(FLOAT_PREFIX) ? value.substring(FLOAT_PREFIX.length()) : ""; + if (hex.length() != width * 2 || !hex.matches("[0-9a-f]+")) { + throw new IllegalArgumentException("not fixed-width float bits"); + } + return new HeaderValue(kind, littleEndian(new BigInteger(hex, 16), width)); + } + + private static byte[] littleEndian(BigInteger number, int width) { + byte[] bytes = new byte[width]; + for (int index = 0; index < width; index++) { + bytes[index] = (byte) number.shiftRight(Byte.SIZE * index).intValue(); + } + return bytes; + } +}
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/InternalHttpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/InternalHttpClient.java index fdfec00..47d650a 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/InternalHttpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/InternalHttpClient.java
@@ -217,7 +217,7 @@ private ClassicHttpRequest addRequestBody(ClassicRequestBuilder requestBuilder, Object body) { var encodedBody = objectMapper.writeValueAsString(body); - log.debug("Request body: {}", encodedBody); + log.debug("Request body: {} bytes", encodedBody.length()); return requestBuilder .setHeader("Content-Type", "application/json") .setEntity(encodedBody)
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/StreamsHttpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/StreamsHttpClient.java index 609eb1c..aea5fd2 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/StreamsHttpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/StreamsHttpClient.java
@@ -21,12 +21,15 @@ import org.apache.iggy.client.blocking.StreamsClient; import org.apache.iggy.identifier.StreamId; +import org.apache.iggy.message.HeaderValue; import org.apache.iggy.stream.StreamBase; import org.apache.iggy.stream.StreamDetails; +import org.apache.iggy.stream.StreamUpdate; import tools.jackson.core.type.TypeReference; import java.math.BigInteger; import java.util.List; +import java.util.Map; import java.util.Optional; class StreamsHttpClient implements StreamsClient { @@ -51,18 +54,21 @@ @Override public List<StreamBase> getStreams() { var request = httpClient.prepareGetRequest(STREAMS); - return httpClient.execute(request, new TypeReference<>() {}); + List<HttpStream> streams = httpClient.execute(request, new TypeReference<>() {}); + return streams.stream().map(HttpStream::toStreamBase).toList(); } @Override - public StreamDetails createStream(String name) { - var request = httpClient.preparePostRequest(STREAMS, new CreateStream(name)); - return httpClient.execute(request, new TypeReference<>() {}); + public StreamDetails createStream(String name, Map<String, HeaderValue> options) { + var request = + httpClient.preparePostRequest(STREAMS, new CreateStream(name, HttpOptionValue.requestValues(options))); + return httpClient.execute(request, HttpStreamDetails.class).toStreamDetails(); } @Override - public void updateStream(StreamId streamId, String name) { - var request = httpClient.preparePutRequest(STREAMS + "/" + streamId, new UpdateStream(name)); + public void updateStream(StreamId streamId, StreamUpdate update) { + var request = httpClient.preparePutRequest( + STREAMS + "/" + streamId, HttpOptionValue.updateBody(update.entries(), update.resetKeys())); httpClient.execute(request); } @@ -73,17 +79,38 @@ sendContexts.clear(); } - record CreateStream(String name) {} - - record UpdateStream(String name) {} + record CreateStream(String name, Map<String, Object> options) {} /** - * The REST shape of a stream with its topics. + * The REST shape of a stream. * - * <p>Only the nested topics need mapping: they carry options, which REST reports in one map with a - * per-entry provenance flag rather than as the two blocks the binary protocol sends. See - * {@link TopicsHttpClient.HttpTopic}. + * <p>REST reports every option in one map with a per-entry provenance flag rather than as the two + * blocks the binary protocol sends. Splitting here is what lets a caller read {@code options()} and + * {@code derivedOptions()} the same way on either transport. */ + record HttpStream( + Long id, + BigInteger createdAt, + String name, + String size, + BigInteger messagesCount, + Long topicsCount, + Map<String, HttpOptionValue> options) { + + StreamBase toStreamBase() { + return new StreamBase( + id, + createdAt, + name, + size, + messagesCount, + topicsCount, + HttpOptionValue.select(options, true), + HttpOptionValue.select(options, false)); + } + } + + /** The REST shape of a stream with its topics. See {@link HttpStream}. */ record HttpStreamDetails( Long id, BigInteger createdAt, @@ -91,7 +118,8 @@ String size, BigInteger messagesCount, Long topicsCount, - List<TopicsHttpClient.HttpTopic> topics) { + List<TopicsHttpClient.HttpTopic> topics, + Map<String, HttpOptionValue> options) { StreamDetails toStreamDetails() { return new StreamDetails( @@ -105,7 +133,9 @@ ? List.of() : topics.stream() .map(TopicsHttpClient.HttpTopic::toTopic) - .toList()); + .toList(), + HttpOptionValue.select(options, true), + HttpOptionValue.select(options, false)); } } }
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/SystemHttpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/SystemHttpClient.java index 1ac0c72..2e88c44 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/SystemHttpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/SystemHttpClient.java
@@ -90,13 +90,34 @@ } /** - * The REST shape of a catalog entry: the kind arrives as its lowercase name and the default as a - * JSON array, where the binary transport sends a kind code and raw bytes. Mapping it here keeps - * {@link OptionSpec} the one shape a caller sees on either transport. + * The REST shape of a catalog entry: the kind arrives as its lowercase name, the default as a JSON + * array and each flag as a boolean, where the binary transport sends a kind code, raw bytes and one + * flags byte. Mapping it here keeps {@link OptionSpec} the one shape a caller sees on either transport. */ - record HttpOptionSpec(String key, HeaderKind kind, byte[] defaultValue, String description) { + record HttpOptionSpec( + String key, + HeaderKind kind, + byte[] defaultValue, + String description, + boolean updatable, + boolean resettable, + boolean secret, + boolean readOnly, + boolean critical) { OptionSpec toOptionSpec() { - return new OptionSpec(key, new HeaderValue(kind, defaultValue), description); + return new OptionSpec(key, new HeaderValue(kind, defaultValue), description, flags()); + } + + private int flags() { + return flag(updatable, OptionSpec.FLAG_UPDATABLE) + | flag(resettable, OptionSpec.FLAG_RESETTABLE) + | flag(secret, OptionSpec.FLAG_SECRET) + | flag(readOnly, OptionSpec.FLAG_READ_ONLY) + | flag(critical, OptionSpec.FLAG_CRITICAL); + } + + private static int flag(boolean set, int bit) { + return set ? bit : 0; } } }
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/TopicsHttpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/TopicsHttpClient.java index 5a4369c..f68eefe 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/TopicsHttpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/TopicsHttpClient.java
@@ -24,13 +24,13 @@ import org.apache.iggy.identifier.TopicId; import org.apache.iggy.message.HeaderValue; import org.apache.iggy.partition.Partition; -import org.apache.iggy.topic.CompressionAlgorithm; import org.apache.iggy.topic.Topic; import org.apache.iggy.topic.TopicDetails; +import org.apache.iggy.topic.TopicOptions; +import org.apache.iggy.topic.TopicUpdate; import tools.jackson.core.type.TypeReference; import java.math.BigInteger; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -64,37 +64,21 @@ @Override public TopicDetails createTopic( - StreamId streamId, - Long partitionsCount, - CompressionAlgorithm compressionAlgorithm, - BigInteger messageExpiry, - BigInteger maxTopicSize, - String name, - Map<String, HeaderValue> options) { + StreamId streamId, String name, long partitionsCount, Map<String, HeaderValue> options) { var request = httpClient.preparePostRequest( STREAMS + "/" + streamId + TOPICS, new CreateTopic( partitionsCount, - compressionAlgorithm, - messageExpiry, - maxTopicSize, name, - toStringOptions(org.apache.iggy.topic.TopicOptions.withDurabilityDefaults(options)))); + HttpOptionValue.requestValues(TopicOptions.withDurabilityDefaults(options)))); return httpClient.execute(request, HttpTopicDetails.class).toTopicDetails(); } @Override - public void updateTopic( - StreamId streamId, - TopicId topicId, - CompressionAlgorithm compressionAlgorithm, - BigInteger messageExpiry, - BigInteger maxTopicSize, - String name, - Map<String, HeaderValue> options) { + public void updateTopic(StreamId streamId, TopicId topicId, TopicUpdate update) { var request = httpClient.preparePutRequest( STREAMS + "/" + streamId + TOPICS + "/" + topicId, - new UpdateTopic(compressionAlgorithm, messageExpiry, maxTopicSize, name, toStringOptions(options))); + HttpOptionValue.updateBody(update.entries(), update.resetKeys())); httpClient.execute(request); } @@ -105,53 +89,7 @@ sendContexts.clear(); } - /** - * Renders option values as the strings the REST body carries them in. - * - * <p>The binary transports send a typed TLV block, but the JSON body takes a plain string map - * that the server parses by the same rules a config file value goes through, so a typed value - * handed in here is rendered: sending a {@code Uint64} 134217728 and sending "134217728" land - * the same stored value. - */ - private static Map<String, String> toStringOptions(Map<String, HeaderValue> options) { - Map<String, String> rendered = new LinkedHashMap<>(); - options.forEach((key, value) -> rendered.put(key, value.toStringValue())); - return rendered; - } - - private static Map<String, HeaderValue> optionsOf(Map<String, HttpOptionValue> options, boolean explicit) { - if (options == null) { - return Map.of(); - } - Map<String, HeaderValue> selected = new LinkedHashMap<>(); - options.forEach((key, option) -> { - if (option.explicit() == explicit) { - selected.put(key, HeaderValue.fromString(option.value())); - } - }); - return Map.copyOf(selected); - } - - record CreateTopic( - Long partitionsCount, - CompressionAlgorithm compressionAlgorithm, - BigInteger messageExpiry, - BigInteger maxTopicSize, - String name, - Map<String, String> options) {} - - record UpdateTopic( - CompressionAlgorithm compressionAlgorithm, - BigInteger messageExpiry, - BigInteger maxTopicSize, - String name, - Map<String, String> options) {} - - /** - * One option entry as REST renders it: the value in the string form the create body takes, plus - * the provenance flag that the binary protocol carries as two separate blocks instead. - */ - record HttpOptionValue(String value, boolean explicit) {} + record CreateTopic(long partitionsCount, String name, Map<String, Object> options) {} /** * The REST shape of a topic. @@ -159,17 +97,13 @@ * <p>It differs from {@link Topic} in one place: REST reports every option in a single map with a * per-entry {@code explicit} flag, where the binary protocol sends an explicit block and a derived * block. Splitting here is what lets a caller read {@code options()} and {@code derivedOptions()} - * the same way on either transport. Values arrive in string form, so each is String-kinded rather - * than carrying the kind the server stored. + * the same way on either transport, each value in the kind the server stored. */ record HttpTopic( Long id, BigInteger createdAt, String name, String size, - BigInteger messageExpiry, - CompressionAlgorithm compressionAlgorithm, - BigInteger maxTopicSize, BigInteger messagesCount, Long partitionsCount, Map<String, HttpOptionValue> options) { @@ -180,13 +114,10 @@ createdAt, name, size, - messageExpiry, - compressionAlgorithm, - maxTopicSize, messagesCount, partitionsCount, - optionsOf(options, true), - optionsOf(options, false)); + HttpOptionValue.select(options, true), + HttpOptionValue.select(options, false)); } } @@ -196,9 +127,6 @@ BigInteger createdAt, String name, String size, - BigInteger messageExpiry, - CompressionAlgorithm compressionAlgorithm, - BigInteger maxTopicSize, BigInteger messagesCount, Long partitionsCount, List<Partition> partitions, @@ -210,14 +138,11 @@ createdAt, name, size, - messageExpiry, - compressionAlgorithm, - maxTopicSize, messagesCount, partitionsCount, partitions == null ? List.of() : partitions, - optionsOf(options, true), - optionsOf(options, false)); + HttpOptionValue.select(options, true), + HttpOptionValue.select(options, false)); } } }
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/UsersHttpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/UsersHttpClient.java index 6b5ed38..e62b1d5 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/UsersHttpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/UsersHttpClient.java
@@ -27,10 +27,14 @@ import org.apache.iggy.user.TokenInfo; import org.apache.iggy.user.UserInfo; import org.apache.iggy.user.UserInfoDetails; +import org.apache.iggy.user.UserOptions; import org.apache.iggy.user.UserStatus; +import org.apache.iggy.user.UserUpdate; import tools.jackson.core.type.TypeReference; +import java.math.BigInteger; import java.util.List; +import java.util.Map; import java.util.Optional; class UsersHttpClient implements UsersClient { @@ -45,20 +49,30 @@ @Override public Optional<UserInfoDetails> getUser(UserId userId) { var request = httpClient.prepareGetRequest(USERS + "/" + userId); - return httpClient.executeWithOptionalResponse(request, UserInfoDetails.class); + return httpClient + .executeWithOptionalResponse(request, HttpUserDetails.class) + .map(HttpUserDetails::toUserInfoDetails); } @Override public List<UserInfo> getUsers() { var request = httpClient.prepareGetRequest(USERS); - return httpClient.execute(request, new TypeReference<>() {}); + List<HttpUser> users = httpClient.execute(request, new TypeReference<>() {}); + return users.stream().map(HttpUser::toUserInfo).toList(); } @Override public UserInfoDetails createUser( - String username, String password, UserStatus status, Optional<Permissions> permissions) { - var request = httpClient.preparePostRequest(USERS, new CreateUser(username, password, status, permissions)); - return httpClient.execute(request, new TypeReference<>() {}); + String username, + String password, + UserStatus status, + Optional<Permissions> permissions, + UserOptions options) { + var request = httpClient.preparePostRequest( + USERS, + new CreateUser( + username, password, status, permissions, HttpOptionValue.requestValues(options.entries()))); + return httpClient.execute(request, HttpUserDetails.class).toUserInfoDetails(); } @Override @@ -68,8 +82,9 @@ } @Override - public void updateUser(UserId userId, Optional<String> username, Optional<UserStatus> status) { - var request = httpClient.preparePutRequest(USERS + "/" + userId, new UpdateUser(username, status)); + public void updateUser(UserId userId, UserUpdate update) { + var request = httpClient.preparePutRequest( + USERS + "/" + userId, HttpOptionValue.updateBody(update.entries(), update.resetKeys())); httpClient.execute(request); } @@ -106,11 +121,53 @@ record Login(String username, String password, String version, String context) {} - private record CreateUser(String username, String password, UserStatus status, Optional<Permissions> permissions) {} - - private record UpdateUser(Optional<String> username, Optional<UserStatus> status) {} + private record CreateUser( + String username, + String password, + UserStatus status, + Optional<Permissions> permissions, + Map<String, Object> options) {} private record UpdatePermissions(Optional<Permissions> permissions) {} private record ChangePassword(String currentPassword, String newPassword) {} + + /** + * The REST shape of a user: one option map with a per-entry provenance flag, split here into the two + * blocks the binary protocol sends. + */ + record HttpUser( + Long id, BigInteger createdAt, UserStatus status, String username, Map<String, HttpOptionValue> options) { + + UserInfo toUserInfo() { + return new UserInfo( + id, + createdAt, + status, + username, + HttpOptionValue.select(options, true), + HttpOptionValue.select(options, false)); + } + } + + /** The REST shape of a user with its permissions. See {@link HttpUser}. */ + record HttpUserDetails( + Long id, + BigInteger createdAt, + UserStatus status, + String username, + Optional<Permissions> permissions, + Map<String, HttpOptionValue> options) { + + UserInfoDetails toUserInfoDetails() { + return new UserInfoDetails( + id, + createdAt, + status, + username, + permissions, + HttpOptionValue.select(options, true), + HttpOptionValue.select(options, false)); + } + } }
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/StreamsTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/StreamsTcpClient.java index 8f7e35c..ab12356 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/StreamsTcpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/StreamsTcpClient.java
@@ -21,10 +21,13 @@ import org.apache.iggy.client.blocking.StreamsClient; import org.apache.iggy.identifier.StreamId; +import org.apache.iggy.message.HeaderValue; import org.apache.iggy.stream.StreamBase; import org.apache.iggy.stream.StreamDetails; +import org.apache.iggy.stream.StreamUpdate; import java.util.List; +import java.util.Map; import java.util.Optional; final class StreamsTcpClient implements StreamsClient { @@ -46,13 +49,13 @@ } @Override - public StreamDetails createStream(String name) { - return FutureUtil.resolve(delegate.createStream(name)); + public StreamDetails createStream(String name, Map<String, HeaderValue> options) { + return FutureUtil.resolve(delegate.createStream(name, options)); } @Override - public void updateStream(StreamId streamId, String name) { - FutureUtil.resolve(delegate.updateStream(streamId, name)); + public void updateStream(StreamId streamId, StreamUpdate update) { + FutureUtil.resolve(delegate.updateStream(streamId, update)); } @Override
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/TopicsTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/TopicsTcpClient.java index d81196c..042cd49 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/TopicsTcpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/TopicsTcpClient.java
@@ -23,11 +23,10 @@ import org.apache.iggy.identifier.StreamId; import org.apache.iggy.identifier.TopicId; import org.apache.iggy.message.HeaderValue; -import org.apache.iggy.topic.CompressionAlgorithm; import org.apache.iggy.topic.Topic; import org.apache.iggy.topic.TopicDetails; +import org.apache.iggy.topic.TopicUpdate; -import java.math.BigInteger; import java.util.List; import java.util.Map; import java.util.Optional; @@ -52,28 +51,13 @@ @Override public TopicDetails createTopic( - StreamId streamId, - Long partitionsCount, - CompressionAlgorithm compressionAlgorithm, - BigInteger messageExpiry, - BigInteger maxTopicSize, - String name, - Map<String, HeaderValue> options) { - return FutureUtil.resolve(delegate.createTopic( - streamId, partitionsCount, compressionAlgorithm, messageExpiry, maxTopicSize, name, options)); + StreamId streamId, String name, long partitionsCount, Map<String, HeaderValue> options) { + return FutureUtil.resolve(delegate.createTopic(streamId, name, partitionsCount, options)); } @Override - public void updateTopic( - StreamId streamId, - TopicId topicId, - CompressionAlgorithm compressionAlgorithm, - BigInteger messageExpiry, - BigInteger maxTopicSize, - String name, - Map<String, HeaderValue> options) { - FutureUtil.resolve(delegate.updateTopic( - streamId, topicId, compressionAlgorithm, messageExpiry, maxTopicSize, name, options)); + public void updateTopic(StreamId streamId, TopicId topicId, TopicUpdate update) { + FutureUtil.resolve(delegate.updateTopic(streamId, topicId, update)); } @Override
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/UsersTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/UsersTcpClient.java index 416885f..102ec82 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/UsersTcpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/UsersTcpClient.java
@@ -25,7 +25,9 @@ import org.apache.iggy.user.Permissions; import org.apache.iggy.user.UserInfo; import org.apache.iggy.user.UserInfoDetails; +import org.apache.iggy.user.UserOptions; import org.apache.iggy.user.UserStatus; +import org.apache.iggy.user.UserUpdate; import java.util.List; import java.util.Optional; @@ -50,8 +52,12 @@ @Override public UserInfoDetails createUser( - String username, String password, UserStatus status, Optional<Permissions> permissions) { - return FutureUtil.resolve(delegate.createUser(username, password, status, permissions)); + String username, + String password, + UserStatus status, + Optional<Permissions> permissions, + UserOptions options) { + return FutureUtil.resolve(delegate.createUser(username, password, status, permissions, options)); } @Override @@ -60,8 +66,8 @@ } @Override - public void updateUser(UserId userId, Optional<String> username, Optional<UserStatus> status) { - FutureUtil.resolve(delegate.updateUser(userId, username, status)); + public void updateUser(UserId userId, UserUpdate update) { + FutureUtil.resolve(delegate.updateUser(userId, update)); } @Override
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/exception/IggyErrorCode.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/exception/IggyErrorCode.java index 0abcd7b..aac1115 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/exception/IggyErrorCode.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/exception/IggyErrorCode.java
@@ -57,6 +57,9 @@ PERSONAL_ACCESS_TOKEN_EXPIRED(54), TRANSIENT_NOT_COMMITTED(57), TRANSIENT_NOT_ACCEPTED(58), + INVALID_ENCRYPTION_KEY(70), + CANNOT_ENCRYPT_DATA(71), + CANNOT_DECRYPT_DATA(72), ACCESS_TOKEN_MISSING(77), INVALID_ACCESS_TOKEN(78), @@ -122,6 +125,17 @@ EMPTY_MESSAGE_PAYLOAD(4024), INVALID_MESSAGE_CHECKSUM(4027), + // Resource option errors + UNSUPPORTED_OPTION_KEY(4041), + INVALID_OPTION_VALUE(4042), + OPTIONS_BLOCK_TOO_LARGE(4043), + OPTION_NOT_UPDATABLE(4103), + OPTION_NOT_RESETTABLE(4104), + READ_ONLY_OPTION(4105), + STREAM_ENCRYPTION_UNAVAILABLE(4106), + UNSUPPORTED_CRITICAL_OPTION(4107), + CANNOT_MODIFY_ROOT_USER(4108), + // Consumer group errors CONSUMER_GROUP_ID_NOT_FOUND(5000), INVALID_CONSUMER_GROUP_ID(5002),
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/exception/IggyValidationException.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/exception/IggyValidationException.java index 7ed3814..e46aa52 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/exception/IggyValidationException.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/exception/IggyValidationException.java
@@ -50,7 +50,14 @@ IggyErrorCode.TOO_MANY_MESSAGES, IggyErrorCode.EMPTY_MESSAGE_PAYLOAD, IggyErrorCode.TOO_BIG_MESSAGE_PAYLOAD, - IggyErrorCode.INVALID_MESSAGE_CHECKSUM); + IggyErrorCode.INVALID_MESSAGE_CHECKSUM, + IggyErrorCode.INVALID_ENCRYPTION_KEY, + IggyErrorCode.UNSUPPORTED_OPTION_KEY, + IggyErrorCode.INVALID_OPTION_VALUE, + IggyErrorCode.OPTIONS_BLOCK_TOO_LARGE, + IggyErrorCode.OPTION_NOT_UPDATABLE, + IggyErrorCode.OPTION_NOT_RESETTABLE, + IggyErrorCode.READ_ONLY_OPTION); /** * Constructs a new IggyValidationException.
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/BytesDeserializer.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/BytesDeserializer.java index 957ade6..e8c8828 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/BytesDeserializer.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/BytesDeserializer.java
@@ -55,7 +55,6 @@ import org.apache.iggy.system.ConsumerGroupInfo; import org.apache.iggy.system.OptionSpec; import org.apache.iggy.system.Stats; -import org.apache.iggy.topic.CompressionAlgorithm; import org.apache.iggy.topic.Topic; import org.apache.iggy.topic.TopicDetails; import org.apache.iggy.user.GlobalPermissions; @@ -84,12 +83,12 @@ private static final int CONSUMER_GROUP_ASSIGNMENT_ENTRY_BYTES = Integer.BYTES; private static final int SEND_CONFIRMATION_BYTES = 3 * Integer.BYTES + Long.BYTES; private static final int MIN_CLUSTER_NODE_BYTES = 19; - // A one-character key (length byte plus a byte of name), a kind byte, and - // empty length-prefixed default and description. - private static final int MIN_OPTION_SPEC_BYTES = 2 + 1 + 4 + 4; - // 50-byte fixed part + a one-character name (the server rejects an empty + // A one-character key (length byte plus a byte of name), kind and flags + // bytes, and empty length-prefixed default and description. + private static final int MIN_OPTION_SPEC_BYTES = 2 + 1 + 1 + 4 + 4; + // 33-byte fixed part + a one-character name (the server rejects an empty // one) + two u32 options-block length prefixes. - private static final int MIN_TOPIC_BYTES = 59; + private static final int MIN_TOPIC_BYTES = 42; private static final int OPEN_FILES_FIELDS_BYTES = 2 * Long.BYTES; private BytesDeserializer() {} @@ -102,9 +101,11 @@ var messagesCount = readU64AsBigInteger(response); var nameLength = response.readUnsignedByte(); var name = response.readCharSequence(nameLength, StandardCharsets.UTF_8).toString(); - var options = readOptionsBlock(response, "stream options"); + var options = readOptionsBlock(response, "stream explicit options"); + var derivedOptions = readOptionsBlock(response, "stream derived options"); - return new StreamBase(streamId, createdAt, name, size.toString(), messagesCount, topicsCount, options); + return new StreamBase( + streamId, createdAt, name, size.toString(), messagesCount, topicsCount, options, derivedOptions); } public static StreamDetails readStreamDetails(ByteBuf response) { @@ -159,9 +160,6 @@ var topicId = response.readUnsignedIntLE(); var createdAt = readU64AsBigInteger(response); var partitionsCount = response.readUnsignedIntLE(); - var messageExpiry = readU64AsBigInteger(response); - var compressionAlgorithmCode = response.readByte(); - var maxTopicSize = readU64AsBigInteger(response); var size = readU64AsBigInteger(response); var messagesCount = readU64AsBigInteger(response); var nameLength = response.readUnsignedByte(); @@ -169,17 +167,7 @@ var options = readOptionsBlock(response, "topic explicit options"); var derivedOptions = readOptionsBlock(response, "topic derived options"); return new Topic( - topicId, - createdAt, - name, - size.toString(), - messageExpiry, - CompressionAlgorithm.fromCode(compressionAlgorithmCode), - maxTopicSize, - messagesCount, - partitionsCount, - options, - derivedOptions); + topicId, createdAt, name, size.toString(), messagesCount, partitionsCount, options, derivedOptions); } public static ConsumerGroupDetails readConsumerGroupDetails(ByteBuf response) { @@ -516,7 +504,7 @@ /** * Reads a {@code DescribeOptions} response. * - * <p>Wire format: {@code [count:u32][key_len:u8][key][kind:u8][default_len:u32][default] + * <p>Wire format: {@code [count:u32][key_len:u8][key][kind:u8][flags:u8][default_len:u32][default] * [description_len:u32][description]}*. A scope with no catalog keys answers with a zero count * rather than an error. */ @@ -534,10 +522,12 @@ response.readCharSequence(keyLength, StandardCharsets.UTF_8).toString(); var kindCode = response.readUnsignedByte(); + var flags = response.readUnsignedByte(); var defaultValue = readU32PrefixedBytes(response, "option default value for '" + key + "'"); var description = readU32PrefixedString(response, "option description for '" + key + "'"); - specs.add(new OptionSpec(key, new HeaderValue(HeaderKind.fromCode(kindCode), defaultValue), description)); + specs.add(new OptionSpec( + key, new HeaderValue(HeaderKind.fromCode(kindCode), defaultValue), description, flags)); } return specs; @@ -656,10 +646,9 @@ var usernameLength = response.readUnsignedByte(); var username = response.readCharSequence(usernameLength, StandardCharsets.UTF_8) .toString(); - // Validated and dropped: users have no catalog keys yet, so the server - // refuses every one and the block is always empty. - readOptionsBlock(response, "user options"); - return new UserInfo(userId, createdAt, status, username); + var options = readOptionsBlock(response, "user explicit options"); + var derivedOptions = readOptionsBlock(response, "user derived options"); + return new UserInfo(userId, createdAt, status, username, options, derivedOptions); } public static RawPersonalAccessToken readRawPersonalAccessToken(ByteBuf response) {
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/BytesSerializer.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/BytesSerializer.java index 3adac3a..b0bf015 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/BytesSerializer.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/BytesSerializer.java
@@ -26,24 +26,35 @@ import org.apache.iggy.consumergroup.Consumer; import org.apache.iggy.exception.IggyInvalidArgumentException; import org.apache.iggy.identifier.Identifier; +import org.apache.iggy.identifier.StreamId; +import org.apache.iggy.identifier.TopicId; +import org.apache.iggy.identifier.UserId; import org.apache.iggy.message.HeaderKey; +import org.apache.iggy.message.HeaderKind; import org.apache.iggy.message.HeaderValue; import org.apache.iggy.message.Message; import org.apache.iggy.message.MessageHeader; import org.apache.iggy.message.MessageId; import org.apache.iggy.message.Partitioning; import org.apache.iggy.message.PollingStrategy; +import org.apache.iggy.stream.StreamUpdate; +import org.apache.iggy.system.OptionsScope; +import org.apache.iggy.topic.TopicUpdate; import org.apache.iggy.user.GlobalPermissions; import org.apache.iggy.user.Permissions; import org.apache.iggy.user.StreamPermissions; import org.apache.iggy.user.TopicPermissions; +import org.apache.iggy.user.UserStatus; +import org.apache.iggy.user.UserUpdate; import java.math.BigInteger; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.TreeMap; /** * Unified serializer for both blocking and async clients. @@ -51,6 +62,18 @@ */ public final class BytesSerializer { + /** + * Credential bounds the server enforces, in UTF-8 bytes. Checked before the round trip so a bad value + * fails here instead of as an opaque server error. + */ + public static final int MIN_USERNAME_LENGTH = 3; + + public static final int MAX_USERNAME_LENGTH = 50; + + public static final int MIN_PASSWORD_LENGTH = 3; + + public static final int MAX_PASSWORD_LENGTH = 100; + /** Size of the batch header on the wire; bytes past the stamped fields stay zero. */ static final int BATCH_HEADER_SIZE = 256; @@ -220,6 +243,106 @@ return buffer; } + /** + * An options block: the user-headers TLV with string keys, written in ascending key byte order so one + * set of options has one encoding. A request carries it to the end of its payload. + */ + public static ByteBuf toOptionsBlock(Map<String, HeaderValue> options) { + if (options.isEmpty()) { + return Unpooled.buffer(0); + } + Map<byte[], HeaderValue> sorted = new TreeMap<>(Arrays::compareUnsigned); + options.forEach((key, value) -> sorted.put(key.getBytes(StandardCharsets.UTF_8), value)); + var buffer = Unpooled.buffer(); + for (Map.Entry<byte[], HeaderValue> entry : sorted.entrySet()) { + HeaderKey key = new HeaderKey(HeaderKind.String, entry.getKey()); + checkFieldLength(key.value().length, "key", key); + buffer.writeByte(HeaderKind.String.asCode()); + buffer.writeIntLE(key.value().length); + buffer.writeBytes(key.value()); + + HeaderValue value = entry.getValue(); + checkFieldLength(value.value().length, "value for key", key); + buffer.writeByte(value.kind().asCode()); + buffer.writeIntLE(value.value().length); + buffer.writeBytes(value.value()); + } + return buffer; + } + + /** {@code CreateStream}: {@code [name][options to end]}. */ + public static ByteBuf toCreateStream(String name, Map<String, HeaderValue> options) { + var payload = toBytes(name, "name"); + writeAndRelease(payload, toOptionsBlock(options)); + return payload; + } + + /** + * {@code CreateTopic}: {@code [stream_id][partitions_count:u32][name][options to end]}. + * {@code partitions_count} is an argument of the command, never an option: admission consumes it to + * compute partition assignments. + */ + public static ByteBuf toCreateTopic( + StreamId streamId, String name, long partitionsCount, Map<String, HeaderValue> options) { + var payload = Unpooled.buffer(); + writeAndRelease(payload, toBytes(streamId)); + payload.writeIntLE((int) partitionsCount); + writeAndRelease(payload, toBytes(name, "name")); + writeAndRelease(payload, toOptionsBlock(options)); + return payload; + } + + /** + * {@code CreateUser}: {@code [username][password][status:u8][has_permissions:u8] + * [permissions_len:u32 permissions]?[options to end]}. + */ + public static ByteBuf toCreateUser( + String username, + String password, + UserStatus status, + Optional<Permissions> permissions, + Map<String, HeaderValue> options) { + var payload = Unpooled.buffer(); + writeAndRelease(payload, toBytes(username, "username", MIN_USERNAME_LENGTH, MAX_USERNAME_LENGTH)); + writeAndRelease(payload, toBytes(password, "password", MIN_PASSWORD_LENGTH, MAX_PASSWORD_LENGTH)); + payload.writeByte(status.asCode()); + permissions.ifPresentOrElse( + granted -> { + payload.writeByte(1); + var permissionBytes = toBytes(granted); + payload.writeIntLE(permissionBytes.readableBytes()); + writeAndRelease(payload, permissionBytes); + }, + () -> payload.writeByte(0)); + writeAndRelease(payload, toOptionsBlock(options)); + return payload; + } + + /** {@code UpdateResource} on a stream. */ + public static ByteBuf toUpdateResource(StreamId streamId, StreamUpdate update) { + var target = Unpooled.buffer(); + target.writeByte(OptionsScope.Stream.asCode()); + writeAndRelease(target, toBytes(streamId)); + return updateResource(target, update.resetKeys(), update.entries()); + } + + /** {@code UpdateResource} on a topic. */ + public static ByteBuf toUpdateResource(StreamId streamId, TopicId topicId, TopicUpdate update) { + var target = Unpooled.buffer(); + target.writeByte(OptionsScope.Topic.asCode()); + writeAndRelease(target, toBytes(streamId)); + writeAndRelease(target, toBytes(topicId)); + return updateResource(target, update.resetKeys(), update.entries()); + } + + /** {@code UpdateResource} on a user. */ + public static ByteBuf toUpdateResource(UserId userId, UserUpdate update) { + var target = Unpooled.buffer(); + target.writeByte(OptionsScope.User.asCode()); + writeAndRelease(target, toBytes(userId)); + return updateResource(target, update.resetKeys(), update.entries()); + } + public static ByteBuf toBytesAsU64(BigInteger value) { if (value.signum() == -1) { throw new IggyInvalidArgumentException("Negative value cannot be serialized to unsigned 64: " + value); @@ -441,6 +564,27 @@ return readAllBytes(toBytes(userHeaders)); } + /** + * Completes an {@code UpdateResource} body after its scope and identifiers: + * {@code [reset_count:u8][reset_key]*[options to end]}. + */ + private static ByteBuf updateResource(ByteBuf payload, List<String> reset, Map<String, HeaderValue> set) { + payload.writeByte(reset.size()); + for (String key : reset) { + writeAndRelease(payload, toBytes(key, "reset key")); + } + writeAndRelease(payload, toOptionsBlock(set)); + return payload; + } + + private static void writeAndRelease(ByteBuf payload, ByteBuf part) { + try { + payload.writeBytes(part); + } finally { + part.release(); + } + } + private static byte[] readAllBytes(ByteBuf buffer) { try { var bytes = new byte[buffer.readableBytes()];
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/CommandCode.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/CommandCode.java index 0b86009..b4ebe46 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/CommandCode.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/CommandCode.java
@@ -33,6 +33,7 @@ GET_CLUSTER_METADATA(12), DESCRIBE_OPTIONS(13), BIND_SESSION(15), + UPDATE_RESOURCE(16), GET_ME(20), GET_CLIENT(21), GET_ALL_CLIENTS(22); @@ -54,7 +55,7 @@ GET_ALL(32), CREATE(33), DELETE(34), - UPDATE(35), + // 35 is retired and must never be reused. UPDATE_PERMISSIONS(36), CHANGE_PASSWORD(37), LOGIN(38), @@ -132,8 +133,8 @@ GET(200), GET_ALL(201), CREATE(202), - DELETE(203), - UPDATE(204); + DELETE(203); + // 204 is retired and must never be reused. private final int value; @@ -151,8 +152,8 @@ GET(300), GET_ALL(301), CREATE(302), - DELETE(303), - UPDATE(304); + DELETE(303); + // 304 is retired and must never be reused. private final int value;
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/stream/EncryptionKeySource.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/stream/EncryptionKeySource.java new file mode 100644 index 0000000..adf7c1a --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/stream/EncryptionKeySource.java
@@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.stream; + +import java.util.Optional; + +/** Who chose a stream's data encryption key, as the server reports it. */ +public enum EncryptionKeySource { + /** The client sent the key when it created the stream. */ + STREAM("stream"), + /** The server generated the key because encryption was enabled and the client sent none. */ + SERVER("server"), + /** The stream is stored in plaintext. */ + NONE("none"); + + private final String value; + + EncryptionKeySource(String value) { + this.value = value; + } + + /** The source named {@code value}, or empty for a source this build does not know. */ + public static Optional<EncryptionKeySource> fromValue(String value) { + for (EncryptionKeySource source : values()) { + if (source.value.equals(value)) { + return Optional.of(source); + } + } + return Optional.empty(); + } + + public String value() { + return value; + } +}
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/stream/EncryptionKeys.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/stream/EncryptionKeys.java new file mode 100644 index 0000000..229f131 --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/stream/EncryptionKeys.java
@@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.stream; + +import org.apache.iggy.exception.IggyErrorCode; +import org.apache.iggy.exception.IggyValidationException; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Base64; +import java.util.HexFormat; +import java.util.Optional; + +/** + * Checks and fingerprints a stream's own data encryption key. + * + * <p>A stream key is 32 random bytes. Never derive one from a password: anyone who can read the stream sees + * its fingerprint, which would then become an offline guessing oracle. The server never returns the key, + * only its fingerprint, so comparing {@link #fingerprint(byte[])} with + * {@link StreamDetails#encryptionKeyFingerprint()} tells which key a stream uses. + */ +public final class EncryptionKeys { + + /** Length of a stream key in bytes. */ + public static final int KEY_LENGTH = 32; + + private static final byte[] FINGERPRINT_LABEL = "iggy/stream-key-fingerprint/v1".getBytes(StandardCharsets.UTF_8); + private static final int FINGERPRINT_BYTES = 8; + + private EncryptionKeys() {} + + /** + * The key's fingerprint: the first 8 bytes of {@code SHA-256("iggy/stream-key-fingerprint/v1" || key)} + * as 16 lowercase hex characters. + * + * @throws IggyValidationException when the key is not exactly 32 bytes + */ + public static String fingerprint(byte[] key) { + MessageDigest digest = sha256(); + digest.update(FINGERPRINT_LABEL); + digest.update(requireValid(key)); + byte[] hash = digest.digest(); + return HexFormat.of().formatHex(hash, 0, FINGERPRINT_BYTES); + } + + /** + * The fingerprint of a key given as canonical standard padded base64. + * + * @throws IggyValidationException when the text is not the canonical encoding of 32 bytes + */ + public static String fingerprint(String base64Key) { + return fingerprint(decodeBase64(base64Key)); + } + + /** + * Decodes a key from canonical standard padded base64. + * + * <p>Whitespace, the URL-safe alphabet, missing padding and noncanonical trailing bits are refused, so + * one key has exactly one accepted text form. + * + * @throws IggyValidationException when the text is not the canonical encoding of 32 bytes + */ + public static byte[] decodeBase64(String base64Key) { + byte[] key; + try { + key = Base64.getDecoder().decode(base64Key); + } catch (IllegalArgumentException malformed) { + throw invalidKey("the text is not standard base64"); + } + requireValid(key); + if (!Base64.getEncoder().encodeToString(key).equals(base64Key)) { + throw invalidKey("the text is not the canonical padded base64 of the key"); + } + return key; + } + + /** Returns {@code key} after checking its length, without copying it. */ + static byte[] requireValid(byte[] key) { + if (key.length != KEY_LENGTH) { + throw invalidKey("expected " + KEY_LENGTH + " bytes, got " + key.length); + } + return key; + } + + private static IggyValidationException invalidKey(String reason) { + return new IggyValidationException( + IggyErrorCode.INVALID_ENCRYPTION_KEY, + IggyErrorCode.INVALID_ENCRYPTION_KEY.getCode(), + "Invalid stream encryption key: " + reason, + Optional.of(StreamOptionKey.ENCRYPTION_KEY.value()), + Optional.empty()); + } + + private static MessageDigest sha256() { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException missing) { + // Every Java platform is required to ship SHA-256. + throw new IllegalStateException("SHA-256 is unavailable", missing); + } + } +}
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/stream/EncryptionProjection.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/stream/EncryptionProjection.java new file mode 100644 index 0000000..885a050 --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/stream/EncryptionProjection.java
@@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.stream; + +import org.apache.iggy.message.HeaderKind; +import org.apache.iggy.message.HeaderValue; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** Reads the encryption projections a server reports in a stream's derived options. */ +final class EncryptionProjection { + + private EncryptionProjection() {} + + static Optional<EncryptionKeySource> source(Map<String, HeaderValue> options, Map<String, HeaderValue> derived) { + return text(options, derived, StreamOptionKey.ENCRYPTION_KEY_SOURCE).flatMap(EncryptionKeySource::fromValue); + } + + static Optional<String> keyFingerprint(Map<String, HeaderValue> options, Map<String, HeaderValue> derived) { + return text(options, derived, StreamOptionKey.ENCRYPTION_KEY_FINGERPRINT); + } + + static List<String> kekFingerprints(Map<String, HeaderValue> options, Map<String, HeaderValue> derived) { + return text(options, derived, StreamOptionKey.ENCRYPTION_KEK_FINGERPRINT) + .map(text -> Arrays.asList(text.split(","))) + .map(List::copyOf) + .orElse(List.of()); + } + + private static Optional<String> text( + Map<String, HeaderValue> options, Map<String, HeaderValue> derived, StreamOptionKey key) { + HeaderValue value = derived.get(key.value()); + if (value == null) { + value = options.get(key.value()); + } + return Optional.ofNullable(value) + .filter(found -> found.kind() == HeaderKind.String) + .map(HeaderValue::asString); + } +}
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/stream/StreamBase.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/stream/StreamBase.java index 16aa3c5..2b8cb5e 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/stream/StreamBase.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/stream/StreamBase.java
@@ -22,14 +22,16 @@ import org.apache.iggy.message.HeaderValue; import java.math.BigInteger; +import java.util.List; import java.util.Map; +import java.util.Optional; /** * A stream as the server reports it. * - * <p>{@code options} carries the creation options, keyed by option name. Streams have no - * catalog keys yet, so it is empty until one lands; the field is read rather than skipped - * so the first key needs no client change. + * <p>{@code options} carries the keys the creating client set, {@code derivedOptions} the values the + * server filled or computes, such as the encryption key source and fingerprints. Both are keyed by option + * name and keep the kind the server reports, on every transport. The stream key itself is never returned. */ public record StreamBase( Long id, @@ -38,4 +40,25 @@ String size, BigInteger messagesCount, Long topicsCount, - Map<String, HeaderValue> options) {} + Map<String, HeaderValue> options, + Map<String, HeaderValue> derivedOptions) { + public StreamBase { + options = options == null ? Map.of() : options; + derivedOptions = derivedOptions == null ? Map.of() : derivedOptions; + } + + /** Who chose the stream's data key, empty when the server reported no source. */ + public Optional<EncryptionKeySource> encryptionKeySource() { + return EncryptionProjection.source(options, derivedOptions); + } + + /** Fingerprint of the stream's data key, empty for a plaintext stream. */ + public Optional<String> encryptionKeyFingerprint() { + return EncryptionProjection.keyFingerprint(options, derivedOptions); + } + + /** Fingerprints of the server keys that wrap the stream's key, empty for a plaintext stream. */ + public List<String> encryptionKekFingerprints() { + return EncryptionProjection.kekFingerprints(options, derivedOptions); + } +}
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/stream/StreamDetails.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/stream/StreamDetails.java index 732ec49..7680f62 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/stream/StreamDetails.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/stream/StreamDetails.java
@@ -19,11 +19,18 @@ package org.apache.iggy.stream; +import org.apache.iggy.message.HeaderValue; import org.apache.iggy.topic.Topic; import java.math.BigInteger; import java.util.List; +import java.util.Map; +import java.util.Optional; +/** + * A stream with its topics, as the server reports it. See {@link StreamBase} for what {@code options} and + * {@code derivedOptions} carry. + */ public record StreamDetails( Long id, BigInteger createdAt, @@ -31,8 +38,39 @@ String size, BigInteger messagesCount, Long topicsCount, - List<Topic> topics) { + List<Topic> topics, + Map<String, HeaderValue> options, + Map<String, HeaderValue> derivedOptions) { + public StreamDetails { + options = options == null ? Map.of() : options; + derivedOptions = derivedOptions == null ? Map.of() : derivedOptions; + } + public StreamDetails(StreamBase base, List<Topic> topics) { - this(base.id(), base.createdAt(), base.name(), base.size(), base.messagesCount(), base.topicsCount(), topics); + this( + base.id(), + base.createdAt(), + base.name(), + base.size(), + base.messagesCount(), + base.topicsCount(), + topics, + base.options(), + base.derivedOptions()); + } + + /** Who chose the stream's data key, empty when the server reported no source. */ + public Optional<EncryptionKeySource> encryptionKeySource() { + return EncryptionProjection.source(options, derivedOptions); + } + + /** Fingerprint of the stream's data key, empty for a plaintext stream. */ + public Optional<String> encryptionKeyFingerprint() { + return EncryptionProjection.keyFingerprint(options, derivedOptions); + } + + /** Fingerprints of the server keys that wrap the stream's key, empty for a plaintext stream. */ + public List<String> encryptionKekFingerprints() { + return EncryptionProjection.kekFingerprints(options, derivedOptions); } }
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/stream/StreamOptionKey.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/stream/StreamOptionKey.java new file mode 100644 index 0000000..f11152c --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/stream/StreamOptionKey.java
@@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.stream; + +import org.apache.iggy.system.OptionKey; + +import static org.apache.iggy.system.OptionSpec.FLAG_CRITICAL; +import static org.apache.iggy.system.OptionSpec.FLAG_READ_ONLY; +import static org.apache.iggy.system.OptionSpec.FLAG_SECRET; +import static org.apache.iggy.system.OptionSpec.FLAG_UPDATABLE; + +/** + * The option keys of a stream, in catalog order. + * + * <p>{@code name} is set at create and changed through an update. {@code encryption_key} is accepted only + * at create and never returned. The three projections are computed by the server and only read. + */ +public enum StreamOptionKey implements OptionKey { + NAME("name", FLAG_UPDATABLE), + ENCRYPTION_KEY("encryption_key", FLAG_SECRET | FLAG_CRITICAL), + ENCRYPTION_KEY_SOURCE("encryption_key_source", FLAG_READ_ONLY), + ENCRYPTION_KEY_FINGERPRINT("encryption_key_fingerprint", FLAG_READ_ONLY), + ENCRYPTION_KEK_FINGERPRINT("encryption_kek_fingerprint", FLAG_READ_ONLY); + + private final String value; + private final int flags; + + StreamOptionKey(String value, int flags) { + this.value = value; + this.flags = flags; + } + + @Override + public String value() { + return value; + } + + @Override + public int flags() { + return flags; + } +}
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/stream/StreamOptions.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/stream/StreamOptions.java new file mode 100644 index 0000000..0da64bc --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/stream/StreamOptions.java
@@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.stream; + +import org.apache.iggy.message.HeaderValue; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; + +/** + * The options a stream is created with. + * + * <p>{@code encryption_key} gives the stream its own 32-byte data key. It must be 32 random bytes. It + * travels inside the create request over whatever transport the client uses, so protecting it on the + * network is the job of TLS on that transport. The server stores it wrapped and never returns it, reads + * report only its source and fingerprints. Without a key, a server with encryption enabled generates one + * for the stream. + * + * <pre>{@code + * var options = StreamOptions.builder().encryptionKey(key).build(); + * streamsClient.createStream("payments", options); + * }</pre> + */ +public final class StreamOptions { + + private final byte[] encryptionKey; + private final Map<String, HeaderValue> raw; + + private StreamOptions(byte[] encryptionKey, Map<String, HeaderValue> raw) { + this.encryptionKey = encryptionKey; + this.raw = raw; + } + + public static Builder builder() { + return new Builder(); + } + + /** No options: the server encryption setting decides whether the stream gets a generated key. */ + public static StreamOptions empty() { + return builder().build(); + } + + /** A copy of the stream key, when one is set. */ + public Optional<byte[]> encryptionKey() { + return Optional.ofNullable(encryptionKey).map(byte[]::clone); + } + + /** Keys set for options newer than this build. */ + public Map<String, HeaderValue> raw() { + return raw; + } + + /** + * Every option this value sends, keyed by option name. A typed setting wins over a raw entry of the + * same key. + */ + public Map<String, HeaderValue> entries() { + Map<String, HeaderValue> entries = new LinkedHashMap<>(raw); + if (encryptionKey != null) { + entries.put(StreamOptionKey.ENCRYPTION_KEY.value(), HeaderValue.fromRaw(encryptionKey.clone())); + } + return Collections.unmodifiableMap(entries); + } + + @Override + public String toString() { + return "StreamOptions[encryptionKey=" + (encryptionKey == null ? "none" : "<redacted>") + ", raw=" + + raw.keySet() + "]"; + } + + public static final class Builder { + + private final Map<String, HeaderValue> raw = new LinkedHashMap<>(); + private byte[] encryptionKey; + + private Builder() {} + + /** + * The stream's own data key, exactly 32 random bytes. + * + * @throws org.apache.iggy.exception.IggyValidationException when the key is not 32 bytes + */ + public Builder encryptionKey(byte[] key) { + encryptionKey = EncryptionKeys.requireValid(key).clone(); + return this; + } + + /** + * The stream's own data key as canonical standard padded base64 of 32 bytes. + * + * @throws org.apache.iggy.exception.IggyValidationException when the text is not the + * canonical encoding of 32 bytes + */ + public Builder encryptionKeyBase64(String key) { + encryptionKey = EncryptionKeys.decodeBase64(key); + return this; + } + + /** An option this build has no setter for. A typed setter wins on collision. */ + public Builder raw(String key, HeaderValue value) { + raw.put(key, value); + return this; + } + + public StreamOptions build() { + return new StreamOptions(encryptionKey, Collections.unmodifiableMap(new LinkedHashMap<>(raw))); + } + } +}
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/stream/StreamUpdate.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/stream/StreamUpdate.java new file mode 100644 index 0000000..6d45d52 --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/stream/StreamUpdate.java
@@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.stream; + +import org.apache.iggy.exception.IggyErrorCode; +import org.apache.iggy.message.HeaderValue; +import org.apache.iggy.system.OptionKey; + +import java.util.Collections; +import java.util.EnumSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static org.apache.iggy.system.OptionKey.MAX_NAME_LENGTH; + +/** + * What an update changes on a stream. Every key left out keeps its stored value. + * + * <p>No stream key is resettable in this release, so a non-empty {@code reset} is refused before anything + * is sent, with the error the server would answer with. An update that sets and resets nothing is refused + * the same way. An update with a raw key or a raw reset name gets only the structural checks, the server + * decides the rest. + * + * @param name the new name, or empty to keep it + * @param reset keys to return to their defaults + * @param raw keys this build has no field for, a field wins on collision + * @param rawReset reset names this build has no key constant for + */ +public record StreamUpdate( + Optional<String> name, Set<StreamOptionKey> reset, Map<String, HeaderValue> raw, Set<String> rawReset) { + + public StreamUpdate { + reset = copyOf(reset); + raw = Collections.unmodifiableMap(new LinkedHashMap<>(raw)); + rawReset = Set.copyOf(rawReset); + boolean typedOnly = raw.isEmpty() && rawReset.isEmpty(); + OptionKey.requireValidUpdate( + entries(name, raw).keySet(), reset, rawReset, typedOnly, EnumSet.allOf(StreamOptionKey.class)); + if (typedOnly) { + name.ifPresent(value -> OptionKey.requireTextLength( + StreamOptionKey.NAME.value(), value, 1, MAX_NAME_LENGTH, IggyErrorCode.INVALID_STREAM_NAME)); + } + } + + public static Builder builder() { + return new Builder(); + } + + /** The keys this update sets, keyed by option name. */ + public Map<String, HeaderValue> entries() { + return entries(name, raw); + } + + private static Map<String, HeaderValue> entries(Optional<String> name, Map<String, HeaderValue> raw) { + Map<String, HeaderValue> entries = new LinkedHashMap<>(raw); + name.ifPresent( + value -> entries.put(StreamOptionKey.NAME.value(), OptionKey.textValue(value, 1, MAX_NAME_LENGTH))); + return Collections.unmodifiableMap(entries); + } + + /** The names this update resets, typed and raw together in ascending order. */ + public List<String> resetKeys() { + return OptionKey.resetNames(reset, rawReset); + } + + /** Raw entries print by name only, because this build cannot tell which newer keys are secret. */ + @Override + public String toString() { + return "StreamUpdate[name=" + name + ", reset=" + reset + ", raw=" + raw.keySet() + ", rawReset=" + rawReset + + "]"; + } + + private static Set<StreamOptionKey> copyOf(Set<StreamOptionKey> keys) { + Set<StreamOptionKey> copy = EnumSet.noneOf(StreamOptionKey.class); + copy.addAll(keys); + return Collections.unmodifiableSet(copy); + } + + public static final class Builder { + + private final Set<StreamOptionKey> reset = EnumSet.noneOf(StreamOptionKey.class); + private final Map<String, HeaderValue> raw = new LinkedHashMap<>(); + private final Set<String> rawReset = new LinkedHashSet<>(); + private Optional<String> name = Optional.empty(); + + private Builder() {} + + public Builder name(String name) { + this.name = Optional.of(name); + return this; + } + + public Builder reset(StreamOptionKey key) { + reset.add(key); + return this; + } + + public Builder raw(String key, HeaderValue value) { + raw.put(key, value); + return this; + } + + /** Resets a key this build has no constant for. The server decides whether the reset is allowed. */ + public Builder rawReset(String key) { + rawReset.add(key); + return this; + } + + /** + * @throws org.apache.iggy.exception.IggyValidationException when the server would refuse the update + * for its shape + */ + public StreamUpdate build() { + return new StreamUpdate(name, reset, raw, rawReset); + } + } +}
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/system/OptionKey.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/system/OptionKey.java new file mode 100644 index 0000000..a6f0eae --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/system/OptionKey.java
@@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.system; + +import org.apache.iggy.exception.IggyErrorCode; +import org.apache.iggy.exception.IggyValidationException; +import org.apache.iggy.message.HeaderKind; +import org.apache.iggy.message.HeaderValue; + +import java.nio.charset.StandardCharsets; +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; +import java.util.function.Predicate; + +/** + * A catalog option key of one resource scope, with the facts {@code describeOptions} reports as flags. + * + * <p>The typed update records refuse a patch the server would refuse for its shape before anything is + * sent, with the same error category the server answers with. + */ +public interface OptionKey { + + /** The longest stream or topic name the server accepts, in UTF-8 bytes. */ + int MAX_NAME_LENGTH = 255; + + /** The most reset names one update carries, the reach of its count byte. */ + int MAX_RESET_KEYS = 255; + + /** The longest option key the wire carries, in UTF-8 bytes. */ + int MAX_KEY_LENGTH = 255; + + /** The key as the server spells it. */ + String value(); + + /** The {@link OptionSpec} flag bits of this key. */ + int flags(); + + default boolean updatable() { + return (flags() & OptionSpec.FLAG_UPDATABLE) != 0; + } + + default boolean resettable() { + return (flags() & OptionSpec.FLAG_RESETTABLE) != 0; + } + + default boolean readOnly() { + return (flags() & OptionSpec.FLAG_READ_ONLY) != 0; + } + + /** + * Refuses an update the server would refuse for its shape, in the server's admission order. + * + * <p>A patch with a raw set key or a raw reset name gets only the structural checks, because the server's + * catalog may know keys or flags this build does not. The server decides the rest. + * + * @param setKeys the keys the update sets + * @param reset the typed keys the update returns to their defaults + * @param rawReset reset names this build has no key constant for + * @param typedOnly whether every set key and reset name is a typed field of this build + * @param catalog every key of the update's scope + * @throws IggyValidationException carrying the server's error code for the refusal + */ + static void requireValidUpdate( + Set<String> setKeys, + Collection<? extends OptionKey> reset, + Set<String> rawReset, + boolean typedOnly, + Collection<? extends OptionKey> catalog) { + if (setKeys.isEmpty() && reset.isEmpty() && rawReset.isEmpty()) { + throw refusal(IggyErrorCode.INVALID_COMMAND, "The update sets and resets nothing", Optional.empty()); + } + requireNone( + reset, + key -> setKeys.contains(key.value()), + IggyErrorCode.INVALID_COMMAND, + "A key is both set and reset"); + for (String name : rawReset) { + if (setKeys.contains(name)) { + throw refusal(IggyErrorCode.INVALID_COMMAND, "A key is both set and reset", Optional.of(name)); + } + } + List<String> names = resetNames(reset, rawReset); + for (String name : names) { + if (!textFits(name, 1, MAX_KEY_LENGTH)) { + throw refusal(IggyErrorCode.UNSUPPORTED_OPTION_KEY, "Unsupported option key", Optional.of(name)); + } + } + if (names.size() > MAX_RESET_KEYS) { + // The catalog is far smaller than the limit, so such a list always names a key outside it. + Optional<String> unknown = names.stream() + .filter(name -> + catalog.stream().noneMatch(key -> key.value().equals(name))) + .findFirst(); + throw refusal(IggyErrorCode.UNSUPPORTED_OPTION_KEY, "Unsupported option key", unknown); + } + if (!typedOnly) { + return; + } + requireNone(reset, OptionKey::readOnly, IggyErrorCode.READ_ONLY_OPTION, "Option is read-only"); + requireNone( + reset, + key -> !key.updatable(), + IggyErrorCode.OPTION_NOT_UPDATABLE, + "Option cannot be updated after creation"); + requireNone(reset, key -> !key.resettable(), IggyErrorCode.OPTION_NOT_RESETTABLE, "Option cannot be reset"); + } + + /** + * Refuses a text value outside {@code minBytes} to {@code maxBytes} UTF-8 bytes with {@code code}. + * + * @throws IggyValidationException carrying {@code code} + */ + static void requireTextLength(String key, String text, int minBytes, int maxBytes, IggyErrorCode code) { + if (!textFits(text, minBytes, maxBytes)) { + throw refusal( + code, + "Value must be between " + minBytes + " and " + maxBytes + " bytes when UTF-8 encoded", + Optional.of(key)); + } + } + + /** + * The text as a string value, or a bool the server refuses with the key's own error when the text does not + * fit, so a patch the server decides still reports the server's category. + */ + static HeaderValue textValue(String text, int minBytes, int maxBytes) { + if (!textFits(text, minBytes, maxBytes)) { + return HeaderValue.fromBool(false); + } + return new HeaderValue(HeaderKind.String, text.getBytes(StandardCharsets.UTF_8)); + } + + /** The reset names an update sends, typed and raw together in ascending order without repeats. */ + static List<String> resetNames(Collection<? extends OptionKey> reset, Set<String> rawReset) { + Set<String> names = new TreeSet<>(rawReset); + reset.forEach(key -> names.add(key.value())); + return List.copyOf(names); + } + + private static boolean textFits(String text, int minBytes, int maxBytes) { + int length = text.getBytes(StandardCharsets.UTF_8).length; + return length >= minBytes && length <= maxBytes; + } + + private static void requireNone( + Collection<? extends OptionKey> keys, Predicate<OptionKey> refused, IggyErrorCode code, String reason) { + for (OptionKey key : keys) { + if (refused.test(key)) { + throw refusal(code, reason, Optional.of(key.value())); + } + } + } + + private static IggyValidationException refusal(IggyErrorCode code, String reason, Optional<String> key) { + return new IggyValidationException(code, code.getCode(), reason, key, Optional.empty()); + } +}
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/system/OptionSpec.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/system/OptionSpec.java index 9f69010..d9a0b50 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/system/OptionSpec.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/system/OptionSpec.java
@@ -24,13 +24,50 @@ /** * One entry of a resource's option catalog. * - * <p>This is the discovery surface for the option keys a create command accepts: a key outside the - * catalog is refused at create, and the binary transports carry only the error code back, so there - * is no other way to learn which keys a given server knows. + * <p>This is the discovery surface for the option keys a create or update command accepts: a key outside + * the catalog is refused, and the binary transports carry only the error code back, so there is no other + * way to learn which keys a given server knows. * - * @param key the option key the create command accepts + * @param key the option key * @param defaultValue the key's default, carrying the kind the server encodes it under, or empty for * a key with no default. Reuses {@link HeaderValue} because options ride the user-headers codec. * @param description what the option does, including the bounds its value is checked against + * @param flags what an update may do with the key, as {@code FLAG_*} bits */ -public record OptionSpec(String key, HeaderValue defaultValue, String description) {} +public record OptionSpec(String key, HeaderValue defaultValue, String description, int flags) { + + /** An update may set the key. */ + public static final int FLAG_UPDATABLE = 1; + + /** An update may reset the key to its default. */ + public static final int FLAG_RESETTABLE = 1 << 1; + + /** Write-only: accepted, never returned. */ + public static final int FLAG_SECRET = 1 << 2; + + /** Computed by the server and returned on reads, never accepted. */ + public static final int FLAG_READ_ONLY = 1 << 3; + + /** A node that does not understand the key refuses to serve the resource. */ + public static final int FLAG_CRITICAL = 1 << 4; + + public boolean updatable() { + return (flags & FLAG_UPDATABLE) != 0; + } + + public boolean resettable() { + return (flags & FLAG_RESETTABLE) != 0; + } + + public boolean secret() { + return (flags & FLAG_SECRET) != 0; + } + + public boolean readOnly() { + return (flags & FLAG_READ_ONLY) != 0; + } + + public boolean critical() { + return (flags & FLAG_CRITICAL) != 0; + } +}
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/PartitionResizePolicy.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/PartitionResizePolicy.java new file mode 100644 index 0000000..d36fd8f --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/PartitionResizePolicy.java
@@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.topic; + +/** Whether a topic's partition set may change after create. */ +public enum PartitionResizePolicy { + MUTABLE("mutable"), + FIXED("fixed"); + + private final String value; + + PartitionResizePolicy(String value) { + this.value = value; + } + + public String value() { + return value; + } +}
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/Topic.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/Topic.java index 8a88eb8..93b97c5 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/Topic.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/Topic.java
@@ -23,28 +23,22 @@ import java.math.BigInteger; import java.util.Map; +import java.util.Optional; /** * A topic as the server reports it. * - * <p>{@code options} carries the keys the creating client set explicitly; - * {@code derivedOptions} carries the values admission resolved for the keys it did not. - * Both are keyed by option name, and the split is the provenance: a value in - * {@code derivedOptions} would have resolved differently under another server config. - * The fixed fields above repeat three of those keys and stay for compatibility. - * - * <p>Both transports populate them. The binary protocol carries each value in the kind the server - * stored it under; REST renders values in a readable string form, so over HTTP every value is - * String-kinded. + * <p>{@code options} carries the keys the creating client set explicitly, {@code derivedOptions} the + * values admission resolved for the keys it did not. Both are keyed by option name, and the split is the + * provenance: a value in {@code derivedOptions} would have resolved differently under another server + * config. Values keep the kind the server stored on every transport. Message expiry, compression and the + * size cap live in those blocks only, read them through the accessors below. */ public record Topic( Long id, BigInteger createdAt, String name, String size, - BigInteger messageExpiry, - CompressionAlgorithm compressionAlgorithm, - BigInteger maxTopicSize, BigInteger messagesCount, Long partitionsCount, Map<String, HeaderValue> options, @@ -53,4 +47,19 @@ options = options == null ? Map.of() : options; derivedOptions = derivedOptions == null ? Map.of() : derivedOptions; } + + /** Message expiry in microseconds, {@code 2^64 - 1} meaning never. */ + public Optional<BigInteger> messageExpiry() { + return TopicSettings.uint64(options, derivedOptions, TopicOptionKey.MESSAGE_EXPIRY); + } + + /** The stored compression label. */ + public Optional<CompressionAlgorithm> compressionAlgorithm() { + return TopicSettings.compression(options, derivedOptions); + } + + /** Topic size cap in bytes, {@code 2^64 - 1} meaning unlimited. */ + public Optional<BigInteger> maxTopicSize() { + return TopicSettings.uint64(options, derivedOptions, TopicOptionKey.MAX_TOPIC_SIZE); + } }
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/TopicDetails.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/TopicDetails.java index 78dfd34..8606183 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/TopicDetails.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/TopicDetails.java
@@ -25,6 +25,7 @@ import java.math.BigInteger; import java.util.List; import java.util.Map; +import java.util.Optional; /** * A topic and its partitions, as the server reports them. @@ -36,9 +37,6 @@ BigInteger createdAt, String name, String size, - BigInteger messageExpiry, - CompressionAlgorithm compressionAlgorithm, - BigInteger maxTopicSize, BigInteger messagesCount, Long partitionsCount, List<Partition> partitions, @@ -55,13 +53,25 @@ topic.createdAt(), topic.name(), topic.size(), - topic.messageExpiry(), - topic.compressionAlgorithm(), - topic.maxTopicSize(), topic.messagesCount(), topic.partitionsCount(), partitions, topic.options(), topic.derivedOptions()); } + + /** Message expiry in microseconds, {@code 2^64 - 1} meaning never. */ + public Optional<BigInteger> messageExpiry() { + return TopicSettings.uint64(options, derivedOptions, TopicOptionKey.MESSAGE_EXPIRY); + } + + /** The stored compression label. */ + public Optional<CompressionAlgorithm> compressionAlgorithm() { + return TopicSettings.compression(options, derivedOptions); + } + + /** Topic size cap in bytes, {@code 2^64 - 1} meaning unlimited. */ + public Optional<BigInteger> maxTopicSize() { + return TopicSettings.uint64(options, derivedOptions, TopicOptionKey.MAX_TOPIC_SIZE); + } }
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/TopicOptionKey.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/TopicOptionKey.java new file mode 100644 index 0000000..23e6d6e --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/TopicOptionKey.java
@@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.topic; + +import org.apache.iggy.system.OptionKey; + +import static org.apache.iggy.system.OptionSpec.FLAG_RESETTABLE; +import static org.apache.iggy.system.OptionSpec.FLAG_UPDATABLE; + +/** + * The option keys of a topic, in catalog order. + * + * <p>An update may set {@code name}, {@code message_expiry} and {@code max_topic_size}, and reset the + * last two to their defaults. Every other key is fixed at create. + */ +public enum TopicOptionKey implements OptionKey { + NAME("name", FLAG_UPDATABLE), + COMPRESSION_ALGORITHM("compression_algorithm", 0), + MESSAGE_EXPIRY("message_expiry", FLAG_UPDATABLE | FLAG_RESETTABLE), + MAX_TOPIC_SIZE("max_topic_size", FLAG_UPDATABLE | FLAG_RESETTABLE), + SEGMENT_SIZE("segment_size", 0), + DURABILITY("durability", 0), + CONSUMER_OFFSET_DURABILITY("consumer_offset_durability", 0), + PARTITION_RESIZE_POLICY("partition_resize_policy", 0), + MESSAGES_REQUIRED_TO_SAVE("messages_required_to_save", 0), + SIZE_OF_MESSAGES_REQUIRED_TO_SAVE("size_of_messages_required_to_save", 0), + PREALLOCATE_SEGMENTS("preallocate_segments", 0); + + private final String value; + private final int flags; + + TopicOptionKey(String value, int flags) { + this.value = value; + this.flags = flags; + } + + @Override + public String value() { + return value; + } + + @Override + public int flags() { + return flags; + } +}
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/TopicOptions.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/TopicOptions.java index d134e2b..4e18043 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/TopicOptions.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/TopicOptions.java
@@ -24,31 +24,35 @@ import java.math.BigInteger; import java.util.Collections; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.Map; +import java.util.Set; /** - * Typed builder for the topic option keys that have no parameter of their own on - * {@code createTopic}. + * The options a topic is created with, one typed setter per catalog key. * - * <p>Compression, message expiry and max topic size are named parameters already. The five keys here - * would otherwise have to be spelled as raw strings paired with hand-encoded values, where a typo in - * the key or the wrong kind for the value is only caught by the server refusing the create. - * - * <p>These are create-time keys: an update refuses them by name, because they describe how a - * partition's storage was laid down. {@code describeOptions} enumerates what a given server accepts. + * <p>Every key here is fixed at create except {@code message_expiry} and {@code max_topic_size}, which a + * {@link TopicUpdate} may change later. A key left unset is resolved by the server and reported in the + * topic's derived options. {@code describeOptions} enumerates what a given server accepts. * * <pre>{@code * var options = TopicOptions.builder() + * .messageExpiry(BigInteger.valueOf(604_800_000_000L)) * .segmentSize(BigInteger.valueOf(134_217_728)) * .durability(Durability.PERSISTED) * .build(); - * topicsClient.createTopic(streamId, 1L, CompressionAlgorithm.None, - * BigInteger.ZERO, BigInteger.ZERO, "orders", options); + * topicsClient.createTopic(streamId, "orders", 1, options); * }</pre> */ public final class TopicOptions { - private TopicOptions() {} + private final Map<String, HeaderValue> entries; + private final Set<String> rawKeys; + + private TopicOptions(Map<String, HeaderValue> entries, Set<String> rawKeys) { + this.entries = entries; + this.rawKeys = rawKeys; + } public static Map<String, HeaderValue> withDurabilityDefaults(Map<String, HeaderValue> source) { Map<String, HeaderValue> resolved = new LinkedHashMap<>(); @@ -73,42 +77,82 @@ return new Builder(); } + /** + * Every option this value sends, keyed by option name. A typed setting wins over a raw entry of the + * same key. The encoder writes them in ascending key order, so the order here carries no meaning. + */ + public Map<String, HeaderValue> entries() { + return entries; + } + + /** Raw entries print by name only, because this build cannot tell which newer keys are secret. */ + @Override + public String toString() { + Map<String, HeaderValue> typed = new LinkedHashMap<>(entries); + typed.keySet().removeAll(rawKeys); + return "TopicOptions[options=" + typed + ", raw=" + rawKeys + "]"; + } + public static final class Builder { - private final Map<String, HeaderValue> options = new LinkedHashMap<>(); + private final Map<String, HeaderValue> typed = new LinkedHashMap<>(); + private final Map<String, HeaderValue> raw = new LinkedHashMap<>(); private Builder() { durability(Durability.REPLICATED); consumerOffsetDurability(Durability.REPLICATED); } + /** The stored compression label. The server never applies it to messages. */ + public Builder compressionAlgorithm(CompressionAlgorithm algorithm) { + return set( + TopicOptionKey.COMPRESSION_ALGORITHM, + HeaderValue.fromString(TopicSettings.compressionName(algorithm))); + } + + /** + * Message expiry in microseconds, {@code 2^64 - 1} meaning never. Zero leaves the key to the server + * default. + */ + public Builder messageExpiry(BigInteger micros) { + return setUnlessZero(TopicOptionKey.MESSAGE_EXPIRY, micros); + } + + /** + * Topic size cap in bytes, {@code 2^64 - 1} meaning unlimited. Zero leaves the key to the server + * default. + */ + public Builder maxTopicSize(BigInteger bytes) { + return setUnlessZero(TopicOptionKey.MAX_TOPIC_SIZE, bytes); + } + /** Per-topic segment size in bytes: a 512-byte multiple within the server's bounds. */ public Builder segmentSize(BigInteger bytes) { - options.put("segment_size", HeaderValue.fromUint64(bytes)); - return this; + return set(TopicOptionKey.SEGMENT_SIZE, HeaderValue.fromUint64(bytes)); } /** Message completion policy, independently defaulting to replicated. */ public Builder durability(Durability durability) { - options.put("durability", HeaderValue.fromString(durability.value())); - return this; + return set(TopicOptionKey.DURABILITY, HeaderValue.fromString(durability.value())); } public Builder consumerOffsetDurability(Durability durability) { - options.put("consumer_offset_durability", HeaderValue.fromString(durability.value())); - return this; + return set(TopicOptionKey.CONSUMER_OFFSET_DURABILITY, HeaderValue.fromString(durability.value())); + } + + /** Whether the partition set may change after create. */ + public Builder partitionResizePolicy(PartitionResizePolicy policy) { + return set(TopicOptionKey.PARTITION_RESIZE_POLICY, HeaderValue.fromString(policy.value())); } /** Flush the journal once it holds this many messages. Must be non-zero. */ public Builder messagesRequiredToSave(long messages) { - options.put("messages_required_to_save", HeaderValue.fromUint32(messages)); - return this; + return set(TopicOptionKey.MESSAGES_REQUIRED_TO_SAVE, HeaderValue.fromUint32(messages)); } /** Flush the journal once it holds this many bytes. */ public Builder sizeOfMessagesRequiredToSave(BigInteger bytes) { - options.put("size_of_messages_required_to_save", HeaderValue.fromUint64(bytes)); - return this; + return set(TopicOptionKey.SIZE_OF_MESSAGES_REQUIRED_TO_SAVE, HeaderValue.fromUint64(bytes)); } /** @@ -118,19 +162,34 @@ * {@code segment_size * partitions_count} at admission. */ public Builder preallocateSegments(boolean enabled) { - options.put("preallocate_segments", HeaderValue.fromBool(enabled)); + return set(TopicOptionKey.PREALLOCATE_SEGMENTS, HeaderValue.fromBool(enabled)); + } + + /** An option this build has no setter for. A typed setter wins on collision. */ + public Builder raw(String key, HeaderValue value) { + raw.put(key, value); return this; } - /** - * The keys set on this builder, in the order they were set. - * - * <p>Wrapping a {@link LinkedHashMap} rather than calling {@code Map.copyOf}, whose - * iteration order is salted per JVM run: the encoder walks this map in order, so a - * randomized one would give the same builder a different block on every run. - */ - public Map<String, HeaderValue> build() { - return Collections.unmodifiableMap(new LinkedHashMap<>(options)); + public TopicOptions build() { + Map<String, HeaderValue> entries = new LinkedHashMap<>(raw); + entries.putAll(typed); + Set<String> rawKeys = new LinkedHashSet<>(raw.keySet()); + rawKeys.removeAll(typed.keySet()); + return new TopicOptions(Collections.unmodifiableMap(entries), Collections.unmodifiableSet(rawKeys)); + } + + private Builder set(TopicOptionKey key, HeaderValue value) { + typed.put(key.value(), value); + return this; + } + + private Builder setUnlessZero(TopicOptionKey key, BigInteger value) { + if (value.signum() == 0) { + typed.remove(key.value()); + return this; + } + return set(key, HeaderValue.fromUint64(value)); } } }
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/TopicSettings.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/TopicSettings.java new file mode 100644 index 0000000..224adec --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/TopicSettings.java
@@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.topic; + +import org.apache.iggy.message.HeaderKind; +import org.apache.iggy.message.HeaderValue; + +import java.math.BigInteger; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; + +/** Reads a topic's settings from its option blocks, the explicit value first. */ +final class TopicSettings { + + private TopicSettings() {} + + static Optional<BigInteger> uint64( + Map<String, HeaderValue> options, Map<String, HeaderValue> derived, TopicOptionKey key) { + return value(options, derived, key) + .filter(value -> value.kind() == HeaderKind.Uint64) + .map(HeaderValue::asUint64); + } + + static Optional<CompressionAlgorithm> compression( + Map<String, HeaderValue> options, Map<String, HeaderValue> derived) { + return value(options, derived, TopicOptionKey.COMPRESSION_ALGORITHM) + .filter(value -> value.kind() == HeaderKind.String) + .flatMap(value -> compressionNamed(value.asString())); + } + + static String compressionName(CompressionAlgorithm algorithm) { + return algorithm.name().toLowerCase(Locale.ROOT); + } + + private static Optional<CompressionAlgorithm> compressionNamed(String name) { + for (CompressionAlgorithm algorithm : CompressionAlgorithm.values()) { + if (compressionName(algorithm).equals(name)) { + return Optional.of(algorithm); + } + } + return Optional.empty(); + } + + private static Optional<HeaderValue> value( + Map<String, HeaderValue> options, Map<String, HeaderValue> derived, TopicOptionKey key) { + HeaderValue value = options.get(key.value()); + return Optional.ofNullable(value == null ? derived.get(key.value()) : value); + } +}
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/TopicUpdate.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/TopicUpdate.java new file mode 100644 index 0000000..dad7ba5 --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/TopicUpdate.java
@@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.topic; + +import org.apache.iggy.exception.IggyErrorCode; +import org.apache.iggy.message.HeaderValue; +import org.apache.iggy.system.OptionKey; + +import java.math.BigInteger; +import java.util.Collections; +import java.util.EnumSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static org.apache.iggy.system.OptionKey.MAX_NAME_LENGTH; + +/** + * What an update changes on a topic. Every key left out keeps its stored value. + * + * <p>Only the name, the message expiry and the size cap change after create, and the last two can be + * reset to their defaults. Compression and the other settings are fixed at create, so they have no field + * here. A zero expiry or size cap changes nothing, {@code reset} is what returns a key to its default. + * An update the server would refuse for its shape is refused before anything is sent, with the server's + * error code. An update with a raw key or a raw reset name gets only the structural checks, the server + * decides the rest. + * + * @param name the new name, or empty to keep it + * @param messageExpiry the new expiry in microseconds, {@code 2^64 - 1} meaning never + * @param maxTopicSize the new size cap in bytes, {@code 2^64 - 1} meaning unlimited + * @param reset keys to return to their defaults + * @param raw keys this build has no field for, a field wins on collision + * @param rawReset reset names this build has no key constant for + */ +public record TopicUpdate( + Optional<String> name, + Optional<BigInteger> messageExpiry, + Optional<BigInteger> maxTopicSize, + Set<TopicOptionKey> reset, + Map<String, HeaderValue> raw, + Set<String> rawReset) { + + public TopicUpdate { + reset = copyOf(reset); + raw = Collections.unmodifiableMap(new LinkedHashMap<>(raw)); + rawReset = Set.copyOf(rawReset); + boolean typedOnly = raw.isEmpty() && rawReset.isEmpty(); + OptionKey.requireValidUpdate( + entries(name, messageExpiry, maxTopicSize, raw).keySet(), + reset, + rawReset, + typedOnly, + EnumSet.allOf(TopicOptionKey.class)); + if (typedOnly) { + name.ifPresent(value -> OptionKey.requireTextLength( + TopicOptionKey.NAME.value(), value, 1, MAX_NAME_LENGTH, IggyErrorCode.INVALID_TOPIC_NAME)); + } + } + + public static Builder builder() { + return new Builder(); + } + + /** The keys this update sets, keyed by option name. */ + public Map<String, HeaderValue> entries() { + return entries(name, messageExpiry, maxTopicSize, raw); + } + + private static Map<String, HeaderValue> entries( + Optional<String> name, + Optional<BigInteger> messageExpiry, + Optional<BigInteger> maxTopicSize, + Map<String, HeaderValue> raw) { + Map<String, HeaderValue> entries = new LinkedHashMap<>(raw); + name.ifPresent( + value -> entries.put(TopicOptionKey.NAME.value(), OptionKey.textValue(value, 1, MAX_NAME_LENGTH))); + messageExpiry + .filter(value -> value.signum() != 0) + .ifPresent(value -> entries.put(TopicOptionKey.MESSAGE_EXPIRY.value(), HeaderValue.fromUint64(value))); + maxTopicSize + .filter(value -> value.signum() != 0) + .ifPresent(value -> entries.put(TopicOptionKey.MAX_TOPIC_SIZE.value(), HeaderValue.fromUint64(value))); + return Collections.unmodifiableMap(entries); + } + + /** The names this update resets, typed and raw together in ascending order. */ + public List<String> resetKeys() { + return OptionKey.resetNames(reset, rawReset); + } + + /** Raw entries print by name only, because this build cannot tell which newer keys are secret. */ + @Override + public String toString() { + return "TopicUpdate[name=" + name + ", messageExpiry=" + messageExpiry + ", maxTopicSize=" + maxTopicSize + + ", reset=" + reset + ", raw=" + raw.keySet() + ", rawReset=" + rawReset + "]"; + } + + private static Set<TopicOptionKey> copyOf(Set<TopicOptionKey> keys) { + Set<TopicOptionKey> copy = EnumSet.noneOf(TopicOptionKey.class); + copy.addAll(keys); + return Collections.unmodifiableSet(copy); + } + + public static final class Builder { + + private final Set<TopicOptionKey> reset = EnumSet.noneOf(TopicOptionKey.class); + private final Map<String, HeaderValue> raw = new LinkedHashMap<>(); + private final Set<String> rawReset = new LinkedHashSet<>(); + private Optional<String> name = Optional.empty(); + private Optional<BigInteger> messageExpiry = Optional.empty(); + private Optional<BigInteger> maxTopicSize = Optional.empty(); + + private Builder() {} + + public Builder name(String name) { + this.name = Optional.of(name); + return this; + } + + public Builder messageExpiry(BigInteger micros) { + this.messageExpiry = Optional.of(micros); + return this; + } + + public Builder maxTopicSize(BigInteger bytes) { + this.maxTopicSize = Optional.of(bytes); + return this; + } + + public Builder reset(TopicOptionKey key) { + reset.add(key); + return this; + } + + public Builder raw(String key, HeaderValue value) { + raw.put(key, value); + return this; + } + + /** Resets a key this build has no constant for. The server decides whether the reset is allowed. */ + public Builder rawReset(String key) { + rawReset.add(key); + return this; + } + + /** + * @throws org.apache.iggy.exception.IggyValidationException when the server would refuse the update + * for its shape + */ + public TopicUpdate build() { + return new TopicUpdate(name, messageExpiry, maxTopicSize, reset, raw, rawReset); + } + } +}
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/user/UserInfo.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/user/UserInfo.java index 484fd8a..a3c35af 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/user/UserInfo.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/user/UserInfo.java
@@ -19,6 +19,26 @@ package org.apache.iggy.user; -import java.math.BigInteger; +import org.apache.iggy.message.HeaderValue; -public record UserInfo(Long id, BigInteger createdAt, UserStatus status, String username) {} +import java.math.BigInteger; +import java.util.Map; + +/** + * A user as the server reports it. + * + * <p>{@code options} carries the keys the creating client set, {@code derivedOptions} the values the + * server filled. No user key exists in this release, so both are empty until a server adds one. + */ +public record UserInfo( + Long id, + BigInteger createdAt, + UserStatus status, + String username, + Map<String, HeaderValue> options, + Map<String, HeaderValue> derivedOptions) { + public UserInfo { + options = options == null ? Map.of() : options; + derivedOptions = derivedOptions == null ? Map.of() : derivedOptions; + } +}
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/user/UserInfoDetails.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/user/UserInfoDetails.java index f1f5e19..857bb9a 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/user/UserInfoDetails.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/user/UserInfoDetails.java
@@ -19,12 +19,34 @@ package org.apache.iggy.user; +import org.apache.iggy.message.HeaderValue; + import java.math.BigInteger; +import java.util.Map; import java.util.Optional; +/** A user with its permissions. See {@link UserInfo} for what {@code options} and {@code derivedOptions} carry. */ public record UserInfoDetails( - Long id, BigInteger createdAt, UserStatus status, String username, Optional<Permissions> permissions) { + Long id, + BigInteger createdAt, + UserStatus status, + String username, + Optional<Permissions> permissions, + Map<String, HeaderValue> options, + Map<String, HeaderValue> derivedOptions) { + public UserInfoDetails { + options = options == null ? Map.of() : options; + derivedOptions = derivedOptions == null ? Map.of() : derivedOptions; + } + public UserInfoDetails(UserInfo userInfo, Optional<Permissions> permissions) { - this(userInfo.id(), userInfo.createdAt(), userInfo.status(), userInfo.username(), permissions); + this( + userInfo.id(), + userInfo.createdAt(), + userInfo.status(), + userInfo.username(), + permissions, + userInfo.options(), + userInfo.derivedOptions()); } }
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/user/UserOptionKey.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/user/UserOptionKey.java new file mode 100644 index 0000000..a65fb36 --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/user/UserOptionKey.java
@@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.user; + +import org.apache.iggy.system.OptionKey; + +import static org.apache.iggy.system.OptionSpec.FLAG_UPDATABLE; + +/** + * The option keys of a user, in catalog order. Both are changed through an update and neither resets. + */ +public enum UserOptionKey implements OptionKey { + USERNAME("username", FLAG_UPDATABLE), + STATUS("status", FLAG_UPDATABLE); + + private final String value; + private final int flags; + + UserOptionKey(String value, int flags) { + this.value = value; + this.flags = flags; + } + + @Override + public String value() { + return value; + } + + @Override + public int flags() { + return flags; + } +}
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/user/UserOptions.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/user/UserOptions.java new file mode 100644 index 0000000..5d53d6b --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/user/UserOptions.java
@@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.user; + +import org.apache.iggy.message.HeaderValue; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * The options a user is created with. No user key exists in this release, so the only content is raw + * entries for keys a newer server knows. The server refuses a key outside its catalog by name. + */ +public final class UserOptions { + + private final Map<String, HeaderValue> entries; + + private UserOptions(Map<String, HeaderValue> entries) { + this.entries = entries; + } + + public static Builder builder() { + return new Builder(); + } + + public static UserOptions empty() { + return builder().build(); + } + + /** Every option this value sends, keyed by option name. */ + public Map<String, HeaderValue> entries() { + return entries; + } + + @Override + public String toString() { + return "UserOptions" + entries.keySet(); + } + + public static final class Builder { + + private final Map<String, HeaderValue> raw = new LinkedHashMap<>(); + + private Builder() {} + + /** An option this build has no setter for. */ + public Builder raw(String key, HeaderValue value) { + raw.put(key, value); + return this; + } + + public UserOptions build() { + return new UserOptions(Collections.unmodifiableMap(new LinkedHashMap<>(raw))); + } + } +}
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/user/UserUpdate.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/user/UserUpdate.java new file mode 100644 index 0000000..ed2ce56 --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/user/UserUpdate.java
@@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.user; + +import org.apache.iggy.exception.IggyErrorCode; +import org.apache.iggy.message.HeaderValue; +import org.apache.iggy.system.OptionKey; + +import java.util.Collections; +import java.util.EnumSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static org.apache.iggy.serde.BytesSerializer.MAX_USERNAME_LENGTH; +import static org.apache.iggy.serde.BytesSerializer.MIN_USERNAME_LENGTH; + +/** + * What an update changes on a user. Every key left out keeps its stored value. + * + * <p>No user key is resettable in this release, so a non-empty {@code reset} is refused before anything is + * sent, with the error the server would answer with. An update with a raw key or a raw reset name gets only + * the structural checks, the server decides the rest. Renaming the signed-in user also moves the + * credentials a TCP client signs back in with after a reconnect. + * + * @param username the new username, or empty to keep it + * @param status the new status, or empty to keep it. Deactivating a user ends its sessions. + * @param reset keys to return to their defaults + * @param raw keys this build has no field for, a field wins on collision + * @param rawReset reset names this build has no key constant for + */ +public record UserUpdate( + Optional<String> username, + Optional<UserStatus> status, + Set<UserOptionKey> reset, + Map<String, HeaderValue> raw, + Set<String> rawReset) { + + public UserUpdate { + reset = copyOf(reset); + raw = Collections.unmodifiableMap(new LinkedHashMap<>(raw)); + rawReset = Set.copyOf(rawReset); + boolean typedOnly = raw.isEmpty() && rawReset.isEmpty(); + OptionKey.requireValidUpdate( + entries(username, status, raw).keySet(), + reset, + rawReset, + typedOnly, + EnumSet.allOf(UserOptionKey.class)); + if (typedOnly) { + username.ifPresent(value -> OptionKey.requireTextLength( + UserOptionKey.USERNAME.value(), + value, + MIN_USERNAME_LENGTH, + MAX_USERNAME_LENGTH, + IggyErrorCode.INVALID_USERNAME)); + } + } + + public static Builder builder() { + return new Builder(); + } + + /** The keys this update sets, keyed by option name. */ + public Map<String, HeaderValue> entries() { + return entries(username, status, raw); + } + + private static Map<String, HeaderValue> entries( + Optional<String> username, Optional<UserStatus> status, Map<String, HeaderValue> raw) { + Map<String, HeaderValue> entries = new LinkedHashMap<>(raw); + username.ifPresent(value -> entries.put( + UserOptionKey.USERNAME.value(), OptionKey.textValue(value, MIN_USERNAME_LENGTH, MAX_USERNAME_LENGTH))); + status.ifPresent(value -> entries.put( + UserOptionKey.STATUS.value(), + HeaderValue.fromString(value.name().toLowerCase(Locale.ROOT)))); + return Collections.unmodifiableMap(entries); + } + + /** The names this update resets, typed and raw together in ascending order. */ + public List<String> resetKeys() { + return OptionKey.resetNames(reset, rawReset); + } + + /** Raw entries print by name only, because this build cannot tell which newer keys are secret. */ + @Override + public String toString() { + return "UserUpdate[username=" + username + ", status=" + status + ", reset=" + reset + ", raw=" + raw.keySet() + + ", rawReset=" + rawReset + "]"; + } + + private static Set<UserOptionKey> copyOf(Set<UserOptionKey> keys) { + Set<UserOptionKey> copy = EnumSet.noneOf(UserOptionKey.class); + copy.addAll(keys); + return Collections.unmodifiableSet(copy); + } + + public static final class Builder { + + private final Set<UserOptionKey> reset = EnumSet.noneOf(UserOptionKey.class); + private final Map<String, HeaderValue> raw = new LinkedHashMap<>(); + private final Set<String> rawReset = new LinkedHashSet<>(); + private Optional<String> username = Optional.empty(); + private Optional<UserStatus> status = Optional.empty(); + + private Builder() {} + + public Builder username(String username) { + this.username = Optional.of(username); + return this; + } + + public Builder status(UserStatus status) { + this.status = Optional.of(status); + return this; + } + + public Builder reset(UserOptionKey key) { + reset.add(key); + return this; + } + + public Builder raw(String key, HeaderValue value) { + raw.put(key, value); + return this; + } + + /** Resets a key this build has no constant for. The server decides whether the reset is allowed. */ + public Builder rawReset(String key) { + rawReset.add(key); + return this; + } + + /** + * @throws org.apache.iggy.exception.IggyValidationException when the server would refuse the update + * for its shape + */ + public UserUpdate build() { + return new UserUpdate(username, status, reset, raw, rawReset); + } + } +}
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/ProtocolParityTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/ProtocolParityTest.java index b448fea..7ba82ea 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/ProtocolParityTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/ProtocolParityTest.java
@@ -62,6 +62,7 @@ Map.entry(CommandCode.System.GET_CLUSTER_METADATA, "GET_CLUSTER_METADATA_CODE"), Map.entry(CommandCode.System.DESCRIBE_OPTIONS, "DESCRIBE_OPTIONS_CODE"), Map.entry(CommandCode.System.BIND_SESSION, "BIND_SESSION_CODE"), + Map.entry(CommandCode.System.UPDATE_RESOURCE, "UPDATE_RESOURCE_CODE"), Map.entry(CommandCode.System.GET_ME, "GET_ME_CODE"), Map.entry(CommandCode.System.GET_CLIENT, "GET_CLIENT_CODE"), Map.entry(CommandCode.System.GET_ALL_CLIENTS, "GET_CLIENTS_CODE"), @@ -69,7 +70,6 @@ Map.entry(CommandCode.User.GET_ALL, "GET_USERS_CODE"), Map.entry(CommandCode.User.CREATE, "CREATE_USER_CODE"), Map.entry(CommandCode.User.DELETE, "DELETE_USER_CODE"), - Map.entry(CommandCode.User.UPDATE, "UPDATE_USER_CODE"), Map.entry(CommandCode.User.UPDATE_PERMISSIONS, "UPDATE_PERMISSIONS_CODE"), Map.entry(CommandCode.User.CHANGE_PASSWORD, "CHANGE_PASSWORD_CODE"), Map.entry(CommandCode.User.LOGIN, "LOGIN_USER_CODE"), @@ -90,12 +90,10 @@ Map.entry(CommandCode.Stream.GET_ALL, "GET_STREAMS_CODE"), Map.entry(CommandCode.Stream.CREATE, "CREATE_STREAM_CODE"), Map.entry(CommandCode.Stream.DELETE, "DELETE_STREAM_CODE"), - Map.entry(CommandCode.Stream.UPDATE, "UPDATE_STREAM_CODE"), Map.entry(CommandCode.Topic.GET, "GET_TOPIC_CODE"), Map.entry(CommandCode.Topic.GET_ALL, "GET_TOPICS_CODE"), Map.entry(CommandCode.Topic.CREATE, "CREATE_TOPIC_CODE"), Map.entry(CommandCode.Topic.DELETE, "DELETE_TOPIC_CODE"), - Map.entry(CommandCode.Topic.UPDATE, "UPDATE_TOPIC_CODE"), Map.entry(CommandCode.Partition.CREATE, "CREATE_PARTITIONS_CODE"), Map.entry(CommandCode.Partition.DELETE, "DELETE_PARTITIONS_CODE"), Map.entry(CommandCode.ConsumerGroup.GET, "GET_CONSUMER_GROUP_CODE"),
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/ResourceOptionsFixture.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/ResourceOptionsFixture.java new file mode 100644 index 0000000..81c4ad8 --- /dev/null +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/ResourceOptionsFixture.java
@@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy; + +import io.netty.buffer.ByteBuf; +import org.apache.iggy.identifier.StreamId; +import org.apache.iggy.identifier.TopicId; +import org.apache.iggy.identifier.UserId; +import org.apache.iggy.message.HeaderKind; +import org.apache.iggy.message.HeaderValue; +import org.apache.iggy.stream.StreamOptionKey; +import org.apache.iggy.stream.StreamUpdate; +import org.apache.iggy.system.OptionKey; +import org.apache.iggy.topic.TopicOptionKey; +import org.apache.iggy.topic.TopicUpdate; +import org.apache.iggy.user.UserOptionKey; +import org.apache.iggy.user.UserStatus; +import org.apache.iggy.user.UserUpdate; +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.json.JsonMapper; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.HexFormat; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Consumer; + +/** + * The shared resource options fixture the Rust encoders generate, read from the test classpath. + * + * <p>The JSON copy is byte-identical with every other SDK's, so the vectors here are the server's contract, + * not this SDK's own serializer run twice. + */ +public final class ResourceOptionsFixture { + + private static final JsonNode ROOT = load(); + + private ResourceOptionsFixture() {} + + public static JsonNode section(String name) { + return ROOT.get(name); + } + + public static byte[] bytes(String hex) { + return HexFormat.of().parseHex(hex); + } + + /** The readable bytes of {@code buffer} as lowercase hex, releasing it. */ + public static String hex(ByteBuf buffer) { + try { + byte[] bytes = new byte[buffer.readableBytes()]; + buffer.getBytes(buffer.readerIndex(), bytes); + return HexFormat.of().formatHex(bytes); + } finally { + buffer.release(); + } + } + + public static HeaderKind kind(String name) { + for (HeaderKind kind : HeaderKind.values()) { + if (kind.name().equalsIgnoreCase(name)) { + return kind; + } + } + throw new IllegalArgumentException("unknown kind " + name); + } + + /** A value in the fixture's reversible text form for its kind. */ + public static HeaderValue optionValue(String kind, String text) { + return switch (kind(kind)) { + case String -> HeaderValue.fromString(text); + case Bool -> HeaderValue.fromBool(Boolean.parseBoolean(text)); + case Uint32 -> HeaderValue.fromUint32(Long.parseLong(text)); + case Uint64 -> HeaderValue.fromUint64(new BigInteger(text)); + case Raw -> HeaderValue.fromRaw(Base64.getDecoder().decode(text)); + default -> throw new IllegalArgumentException("fixture kind " + kind); + }; + } + + /** The entries of an option map in the fixture's JSON form whose provenance is {@code explicit}. */ + public static Map<String, HeaderValue> optionMap(JsonNode options, boolean explicit) { + Map<String, HeaderValue> selected = new LinkedHashMap<>(); + for (Map.Entry<String, JsonNode> entry : options.properties()) { + JsonNode option = entry.getValue(); + if (option.get("explicit").asBoolean() == explicit) { + selected.put( + entry.getKey(), + optionValue( + option.get("kind").asString(), + option.get("value").asString())); + } + } + return selected; + } + + public static StreamId streamId(JsonNode target) { + JsonNode id = target.get("stream_id"); + return id.isNumber() ? StreamId.of(id.asLong()) : StreamId.of(id.asString()); + } + + public static TopicId topicId(JsonNode target) { + JsonNode id = target.get("topic_id"); + return id.isNumber() ? TopicId.of(id.asLong()) : TopicId.of(id.asString()); + } + + public static UserId userId(JsonNode target) { + JsonNode id = target.get("user_id"); + return id.isNumber() ? UserId.of(id.asLong()) : UserId.of(id.asString()); + } + + /** The typed update a fixture vector's {@code input} describes, built for its {@code scope}. */ + public static StreamUpdate streamUpdate(JsonNode input) { + var builder = StreamUpdate.builder(); + text(input, "name").ifPresent(builder::name); + resetKeys(input, StreamOptionKey.class, builder::rawReset).forEach(builder::reset); + return builder.build(); + } + + public static TopicUpdate topicUpdate(JsonNode input) { + var builder = TopicUpdate.builder(); + text(input, "name").ifPresent(builder::name); + text(input, "message_expiry").map(BigInteger::new).ifPresent(builder::messageExpiry); + text(input, "max_topic_size").map(BigInteger::new).ifPresent(builder::maxTopicSize); + resetKeys(input, TopicOptionKey.class, builder::rawReset).forEach(builder::reset); + return builder.build(); + } + + public static UserUpdate userUpdate(JsonNode input) { + var builder = UserUpdate.builder(); + text(input, "username").ifPresent(builder::username); + text(input, "status") + .map(status -> status.equals("active") ? UserStatus.Active : UserStatus.Inactive) + .ifPresent(builder::status); + resetKeys(input, UserOptionKey.class, builder::rawReset).forEach(builder::reset); + return builder.build(); + } + + /** The catalog key enum constant spelled {@code value}. */ + public static <K extends Enum<K> & OptionKey> K optionKey(Class<K> keys, String value) { + for (K key : keys.getEnumConstants()) { + if (key.value().equals(value)) { + return key; + } + } + throw new IllegalArgumentException("no " + keys.getSimpleName() + " named " + value); + } + + private static Optional<String> text(JsonNode input, String field) { + return Optional.ofNullable(input.get(field)).map(JsonNode::asString); + } + + /** The typed reset keys of {@code input}, handing every name without a key constant to {@code raw}. */ + private static <K extends Enum<K> & OptionKey> List<K> resetKeys( + JsonNode input, Class<K> keys, Consumer<String> raw) { + List<K> reset = new ArrayList<>(); + JsonNode names = input.get("reset"); + if (names != null) { + names.forEach(name -> Arrays.stream(keys.getEnumConstants()) + .filter(key -> key.value().equals(name.asString())) + .findFirst() + .ifPresentOrElse(reset::add, () -> raw.accept(name.asString()))); + } + return reset; + } + + private static JsonNode load() { + try (InputStream fixture = ResourceOptionsFixture.class.getResourceAsStream("/resource_options.json")) { + if (fixture == null) { + throw new IllegalStateException("resource_options.json is not on the test classpath"); + } + return JsonMapper.builder().build().readTree(fixture); + } catch (IOException unreadable) { + throw new UncheckedIOException(unreadable); + } + } +}
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/TestConstants.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/TestConstants.java index 178d10d..4cb7acb 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/TestConstants.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/TestConstants.java
@@ -21,17 +21,14 @@ import org.apache.iggy.identifier.StreamId; import org.apache.iggy.identifier.TopicId; -import org.apache.iggy.message.HeaderValue; import org.apache.iggy.topic.Durability; import org.apache.iggy.topic.TopicOptions; -import java.util.Map; - public final class TestConstants { public static final StreamId STREAM_NAME = StreamId.of("test-stream"); public static final TopicId TOPIC_NAME = TopicId.of("test-topic"); - public static final Map<String, HeaderValue> PERSISTED_TOPIC_OPTIONS = TopicOptions.builder() + public static final TopicOptions PERSISTED_TOPIC_OPTIONS = TopicOptions.builder() .durability(Durability.PERSISTED) .consumerOffsetDurability(Durability.PERSISTED) .build();
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncClientIntegrationTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncClientIntegrationTest.java index c5e4452..b88c317 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncClientIntegrationTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncClientIntegrationTest.java
@@ -37,7 +37,6 @@ import org.apache.iggy.message.PollingStrategy; import org.apache.iggy.message.SendMessagesResponse; import org.apache.iggy.serde.CommandCode; -import org.apache.iggy.topic.CompressionAlgorithm; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -112,14 +111,7 @@ try { client.streams().createStream(streamName).get(TIMEOUT_SECONDS, TimeUnit.SECONDS); client.topics() - .createTopic( - streamId, - 2L, - CompressionAlgorithm.None, - BigInteger.ZERO, - BigInteger.ZERO, - "test-topic", - PERSISTED_TOPIC_OPTIONS) + .createTopic(streamId, "test-topic", 2L, PERSISTED_TOPIC_OPTIONS) .get(TIMEOUT_SECONDS, TimeUnit.SECONDS); List<Message> messages = new ArrayList<>(); @@ -167,14 +159,7 @@ try { client.streams().createStream(streamName).get(TIMEOUT_SECONDS, TimeUnit.SECONDS); client.topics() - .createTopic( - streamId, - 2L, - CompressionAlgorithm.None, - BigInteger.ZERO, - BigInteger.ZERO, - "test-topic", - PERSISTED_TOPIC_OPTIONS) + .createTopic(streamId, "test-topic", 2L, PERSISTED_TOPIC_OPTIONS) .get(TIMEOUT_SECONDS, TimeUnit.SECONDS); // when — send messages concurrently from multiple threads @@ -230,14 +215,7 @@ try { client.streams().createStream(streamName).get(TIMEOUT_SECONDS, TimeUnit.SECONDS); client.topics() - .createTopic( - streamId, - 2L, - CompressionAlgorithm.None, - BigInteger.ZERO, - BigInteger.ZERO, - "test-topic", - PERSISTED_TOPIC_OPTIONS) + .createTopic(streamId, "test-topic", 2L, PERSISTED_TOPIC_OPTIONS) .get(TIMEOUT_SECONDS, TimeUnit.SECONDS); // when — send messages in concurrent batches @@ -297,14 +275,7 @@ try { client.streams().createStream(streamName).get(TIMEOUT_SECONDS, TimeUnit.SECONDS); client.topics() - .createTopic( - stream, - 1L, - CompressionAlgorithm.None, - BigInteger.ZERO, - BigInteger.ZERO, - "large-replies", - PERSISTED_TOPIC_OPTIONS) + .createTopic(stream, "large-replies", 1L, PERSISTED_TOPIC_OPTIONS) .get(TIMEOUT_SECONDS, TimeUnit.SECONDS); for (int index = 0; index < PAUSED_READER_REQUESTS; index++) { String payload = String.valueOf((char) ('a' + index)).repeat(LARGE_REPLY_BYTES);
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncConnectionPoolAuthTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncConnectionPoolAuthTest.java index 938c7e2..2b255b5 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncConnectionPoolAuthTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncConnectionPoolAuthTest.java
@@ -28,7 +28,6 @@ import org.apache.iggy.message.Message; import org.apache.iggy.message.Partitioning; import org.apache.iggy.message.SendMessagesResponse; -import org.apache.iggy.topic.CompressionAlgorithm; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; @@ -36,7 +35,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.math.BigInteger; import java.util.ArrayList; import java.util.List; import java.util.UUID; @@ -156,14 +154,7 @@ client.streams().createStream(streamName).get(5, TimeUnit.SECONDS); client.topics() - .createTopic( - StreamId.of(streamName), - 1L, - CompressionAlgorithm.None, - BigInteger.ZERO, - BigInteger.ZERO, - "test-topic", - PERSISTED_TOPIC_OPTIONS) + .createTopic(StreamId.of(streamName), "test-topic", 1L, PERSISTED_TOPIC_OPTIONS) .get(5, TimeUnit.SECONDS); // when - fire a burst of concurrent requests to exercise lazy @@ -197,14 +188,7 @@ client.streams().createStream(streamName).get(5, TimeUnit.SECONDS); client.topics() - .createTopic( - StreamId.of(streamName), - 1L, - CompressionAlgorithm.None, - BigInteger.ZERO, - BigInteger.ZERO, - "test-topic", - PERSISTED_TOPIC_OPTIONS) + .createTopic(StreamId.of(streamName), "test-topic", 1L, PERSISTED_TOPIC_OPTIONS) .get(5, TimeUnit.SECONDS); List<CompletableFuture<SendMessagesResponse>> warmupFutures = new ArrayList<>();
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncConsumerGroupsTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncConsumerGroupsTest.java index e7dec17..2c6fa91 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncConsumerGroupsTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncConsumerGroupsTest.java
@@ -27,14 +27,12 @@ import org.apache.iggy.identifier.ConsumerId; import org.apache.iggy.identifier.StreamId; import org.apache.iggy.identifier.TopicId; -import org.apache.iggy.topic.CompressionAlgorithm; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.math.BigInteger; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -81,9 +79,7 @@ .get(TIMEOUT_SECONDS, TimeUnit.SECONDS); client.streams().createStream(TEST_STREAM).get(TIMEOUT_SECONDS, TimeUnit.SECONDS); - client.topics() - .createTopic(STREAM_ID, 1L, CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, TEST_TOPIC) - .get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + client.topics().createTopic(STREAM_ID, TEST_TOPIC, 1L).get(TIMEOUT_SECONDS, TimeUnit.SECONDS); } @AfterAll @@ -260,9 +256,7 @@ try { client.streams().createStream(streamName).get(TIMEOUT_SECONDS, TimeUnit.SECONDS); - client.topics() - .createTopic(streamId, 1L, CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, topicName) - .get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + client.topics().createTopic(streamId, topicName, 1L).get(TIMEOUT_SECONDS, TimeUnit.SECONDS); // when List<CompletableFuture<ConsumerGroupDetails>> futures = new ArrayList<>();
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/ResourceOptionsTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/ResourceOptionsTest.java new file mode 100644 index 0000000..9ace7cb --- /dev/null +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/ResourceOptionsTest.java
@@ -0,0 +1,371 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.client.async; + +import org.apache.iggy.client.BaseIntegrationTest; +import org.apache.iggy.client.async.tcp.AsyncIggyTcpClient; +import org.apache.iggy.consumergroup.Consumer; +import org.apache.iggy.exception.IggyErrorCode; +import org.apache.iggy.exception.IggyServerException; +import org.apache.iggy.identifier.StreamId; +import org.apache.iggy.identifier.TopicId; +import org.apache.iggy.identifier.UserId; +import org.apache.iggy.message.HeaderValue; +import org.apache.iggy.message.Message; +import org.apache.iggy.message.Partitioning; +import org.apache.iggy.message.PollingStrategy; +import org.apache.iggy.stream.EncryptionKeySource; +import org.apache.iggy.stream.EncryptionKeys; +import org.apache.iggy.stream.StreamDetails; +import org.apache.iggy.stream.StreamOptions; +import org.apache.iggy.stream.StreamUpdate; +import org.apache.iggy.topic.CompressionAlgorithm; +import org.apache.iggy.topic.TopicOptionKey; +import org.apache.iggy.topic.TopicOptions; +import org.apache.iggy.topic.TopicUpdate; +import org.apache.iggy.user.GlobalPermissions; +import org.apache.iggy.user.Permissions; +import org.apache.iggy.user.UserOptions; +import org.apache.iggy.user.UserStatus; +import org.apache.iggy.user.UserUpdate; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** The shared resource options cases through the async TCP client. */ +class ResourceOptionsTest extends BaseIntegrationTest { + + private static final int TIMEOUT_SECONDS = 5; + private static final BigInteger HOUR_MICROS = BigInteger.valueOf(3_600_000_000L); + private static final BigInteger GIB = BigInteger.valueOf(1_073_741_824L); + private static final BigInteger U64_MAX = BigInteger.ONE.shiftLeft(64).subtract(BigInteger.ONE); + + private final List<Long> streams = new ArrayList<>(); + private final List<Long> users = new ArrayList<>(); + private AsyncIggyTcpClient client; + + @BeforeEach + void setUp() throws Exception { + client = connect(); + await(client.users().login("iggy", "iggy")); + } + + @AfterEach + void tearDown() throws Exception { + for (Long stream : streams) { + client.streams().deleteStream(stream).exceptionally(ignored -> null).get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + } + for (Long user : users) { + client.users().deleteUser(user).exceptionally(ignored -> null).get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + } + client.close().get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + } + + @Test + void givenStreamKeyWhenCreatingShouldReadMatchingProjection() throws Exception { + // given + byte[] key = randomKey(); + + // when + StreamDetails created = await(client.streams() + .createStream( + unique("async-keyed"), + StreamOptions.builder().encryptionKey(key).build())); + streams.add(created.id()); + StreamDetails fetched = await(client.streams().getStream(created.id())).orElseThrow(); + var listed = await(client.streams().getStreams()).stream() + .filter(stream -> stream.id().equals(created.id())) + .findFirst() + .orElseThrow(); + + // then + String fingerprint = EncryptionKeys.fingerprint(key); + assertThat(created.encryptionKeySource()).contains(EncryptionKeySource.STREAM); + assertThat(fetched.encryptionKeyFingerprint()).contains(fingerprint); + assertThat(listed.encryptionKeyFingerprint()).contains(fingerprint); + assertThat(fetched.encryptionKekFingerprints()).isNotEmpty().isEqualTo(listed.encryptionKekFingerprints()); + for (Map<String, HeaderValue> options : + List.of(fetched.options(), fetched.derivedOptions(), listed.options(), listed.derivedOptions())) { + assertThat(options).doesNotContainKeys("encryption_key", "encryption_key_wrapped", "critical_keys"); + } + } + + @Test + void givenKeylessStreamWhenReadingShouldReportServerGeneratedKey() throws Exception { + // when + StreamDetails created = await(client.streams().createStream(unique("async-keyless"))); + streams.add(created.id()); + StreamDetails fetched = await(client.streams().getStream(created.id())).orElseThrow(); + + // then + assertThat(fetched.encryptionKeySource()).contains(EncryptionKeySource.SERVER); + assertThat(fetched.encryptionKeyFingerprint()) + .hasValueSatisfying(fingerprint -> assertThat(fingerprint).matches("[0-9a-f]{16}")); + assertThat(fetched.encryptionKekFingerprints()).isNotEmpty(); + } + + @Test + void givenStreamRenameWhenUpdatingShouldPreserveEncryptionProjection() throws Exception { + // given + byte[] key = randomKey(); + StreamDetails created = await(client.streams() + .createStream( + unique("async-rename"), + StreamOptions.builder().encryptionKey(key).build())); + streams.add(created.id()); + StreamId streamId = StreamId.of(created.id()); + await(client.topics().createTopic(streamId, "events", 1L)); + await(client.messages() + .sendMessages( + streamId, TopicId.of("events"), Partitioning.partitionId(0L), List.of(Message.of("kept")))); + String renamed = unique("async-renamed"); + + // when + await(client.streams() + .updateStream(streamId, StreamUpdate.builder().name(renamed).build())); + + // then + StreamDetails after = await(client.streams().getStream(created.id())).orElseThrow(); + assertThat(after.name()).isEqualTo(renamed); + assertThat(after.encryptionKeySource()).contains(EncryptionKeySource.STREAM); + assertThat(after.encryptionKeyFingerprint()).contains(EncryptionKeys.fingerprint(key)); + var polled = await(client.messages() + .pollMessages( + streamId, + TopicId.of("events"), + Optional.of(0L), + Consumer.of(0L), + PollingStrategy.offset(BigInteger.ZERO), + 1L, + false)); + assertThat(new String(polled.messages().get(0).payload(), StandardCharsets.UTF_8)) + .isEqualTo("kept"); + } + + @Test + void givenTopicResetWhenUpdatingShouldPreserveUnmentionedValues() throws Exception { + // given + StreamDetails stream = await(client.streams().createStream(unique("async-topic"))); + streams.add(stream.id()); + StreamId streamId = StreamId.of(stream.id()); + var created = await(client.topics() + .createTopic( + streamId, + "patched", + 1L, + TopicOptions.builder() + .compressionAlgorithm(CompressionAlgorithm.Gzip) + .messageExpiry(HOUR_MICROS) + .maxTopicSize(GIB) + .build())); + TopicId topicId = TopicId.of(created.id()); + + // when + await(client.topics() + .updateTopic( + streamId, + topicId, + TopicUpdate.builder() + .messageExpiry(HOUR_MICROS.shiftLeft(1)) + .reset(TopicOptionKey.MAX_TOPIC_SIZE) + .build())); + + // then + var topic = await(client.topics().getTopic(streamId, topicId)).orElseThrow(); + assertThat(topic.name()).isEqualTo("patched"); + assertThat(topic.options().get("message_expiry")).isEqualTo(HeaderValue.fromUint64(HOUR_MICROS.shiftLeft(1))); + assertThat(topic.derivedOptions().get("max_topic_size")).isEqualTo(HeaderValue.fromUint64(U64_MAX)); + assertThat(topic.options().get("compression_algorithm")).isEqualTo(HeaderValue.fromString("gzip")); + } + + @Test + void givenUserOptionsWhenCreatingAndUpdatingShouldPreserveUserBehavior() throws Exception { + // given + String firstName = unique("u1"); + String secondName = unique("u2"); + var first = await(client.users() + .createUser(firstName, "secret", UserStatus.Active, Optional.empty(), UserOptions.empty())); + users.add(first.id()); + var second = await(client.users().createUser(secondName, "secret", UserStatus.Active, Optional.empty())); + users.add(second.id()); + String renamed = unique("u1r"); + + // when + await(client.users() + .updateUser( + first.id(), + UserUpdate.builder() + .username(renamed) + .status(UserStatus.Inactive) + .build())); + + // then + var updated = await(client.users().getUser(first.id())).orElseThrow(); + assertThat(updated.username()).isEqualTo(renamed); + assertThat(updated.status()).isEqualTo(UserStatus.Inactive); + assertThat(updated.options()).isEmpty(); + assertThat(updated.derivedOptions()).isEmpty(); + assertThatThrownBy(() -> await(client.users() + .updateUser( + second.id(), + UserUpdate.builder() + .username(renamed) + .status(UserStatus.Inactive) + .build()))) + .isInstanceOf(IggyServerException.class); + var untouched = await(client.users().getUser(second.id())).orElseThrow(); + assertThat(untouched.username()).isEqualTo(secondName); + assertThat(untouched.status()).isEqualTo(UserStatus.Active); + } + + @Test + void givenLoggedInUserRenameWhenReconnectingShouldKeepNewUsername() throws Exception { + // given + String username = unique("self"); + String renamed = unique("self-renamed"); + var user = + await(client.users().createUser(username, "secret", UserStatus.Active, Optional.of(allPermissions()))); + users.add(user.id()); + AsyncIggyTcpClient self = AsyncIggyTcpClient.builder() + .host(serverHost()) + .port(serverTcpPort()) + .credentials(username, "secret") + .build(); + try { + self.connect().get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + await(self.login()); + StreamDetails stream = await(self.streams().createStream(unique("self-stream"))); + streams.add(stream.id()); + + // when a stream rename and a refused rename leave the credentials alone + await(self.streams() + .updateStream( + stream.id(), + StreamUpdate.builder() + .name(unique("self-stream-renamed")) + .build())); + assertThatThrownBy(() -> await(self.users() + .updateUser( + user.id(), + UserUpdate.builder().username("iggy").build()))) + .isInstanceOf(IggyServerException.class); + await(self.users().logout()); + assertThat(await(self.login()).userId()).isEqualTo(user.id()); + + // and a committed rename of the signed-in user moves them + await(self.users() + .updateUser( + UserId.of(user.id()), + UserUpdate.builder().username(renamed).build())); + await(self.users().logout()); + + // then + assertThat(await(self.login()).userId()).isEqualTo(user.id()); + assertThat(await(client.users().getUser(user.id())).orElseThrow().username()) + .isEqualTo(renamed); + } finally { + self.close().get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + } + } + + @Test + void givenRawResetWhenUpdatingShouldLetTheServerDecide() throws Exception { + // given + StreamDetails stream = await(client.streams().createStream(unique("async-raw-reset"))); + streams.add(stream.id()); + StreamId streamId = StreamId.of(stream.id()); + var created = await(client.topics() + .createTopic( + streamId, + "raw", + 1L, + TopicOptions.builder().messageExpiry(HOUR_MICROS).build())); + TopicId topicId = TopicId.of(created.id()); + + // when + await(client.topics() + .updateTopic( + streamId, + topicId, + TopicUpdate.builder().rawReset("message_expiry").build())); + + // then + var topic = await(client.topics().getTopic(streamId, topicId)).orElseThrow(); + assertThat(topic.derivedOptions().get("message_expiry")).isEqualTo(HeaderValue.fromUint64(U64_MAX)); + assertThatThrownBy(() -> await(client.streams() + .updateStream( + streamId, + StreamUpdate.builder().rawReset("critical_keys").build()))) + .isInstanceOfSatisfying( + IggyServerException.class, + refusal -> assertThat(refusal.getErrorCode()).isEqualTo(IggyErrorCode.READ_ONLY_OPTION)); + } + + private AsyncIggyTcpClient connect() throws Exception { + AsyncIggyTcpClient connected = AsyncIggyTcpClient.builder() + .host(serverHost()) + .port(serverTcpPort()) + .build(); + connected.connect().get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + return connected; + } + + /** Waits for {@code future}, rethrowing an Iggy failure as itself rather than wrapped. */ + private static <T> T await(CompletableFuture<T> future) throws Exception { + try { + return future.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + } catch (ExecutionException failed) { + if (failed.getCause() instanceof RuntimeException cause) { + throw cause; + } + throw failed; + } + } + + private static String unique(String prefix) { + return prefix + "-" + UUID.randomUUID().toString().substring(0, 8); + } + + private static byte[] randomKey() { + byte[] key = new byte[EncryptionKeys.KEY_LENGTH]; + new SecureRandom().nextBytes(key); + return key; + } + + private static Permissions allPermissions() { + return new Permissions( + new GlobalPermissions(true, true, true, true, true, true, true, true, true, true), Map.of()); + } +}
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientCredentialChangeTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientCredentialChangeTest.java new file mode 100644 index 0000000..8ac9908 --- /dev/null +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientCredentialChangeTest.java
@@ -0,0 +1,293 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.client.async.tcp; + +import io.netty.buffer.ByteBufUtil; +import io.netty.buffer.Unpooled; +import org.apache.iggy.client.async.tcp.AsyncIggyTcpClientEndpointFailoverTest.MockNode; +import org.apache.iggy.client.async.tcp.AsyncIggyTcpClientEndpointFailoverTest.RequestHandler; +import org.apache.iggy.client.async.tcp.AsyncIggyTcpClientTransientFailoverTest.Response; +import org.apache.iggy.client.async.tcp.vsr.VsrOperation; +import org.apache.iggy.exception.IggyErrorCode; +import org.apache.iggy.exception.IggyServerException; +import org.apache.iggy.identifier.UserId; +import org.apache.iggy.serde.BytesSerializer; +import org.apache.iggy.serde.CommandCode; +import org.apache.iggy.user.UserUpdate; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.net.InetAddress; +import java.net.ServerSocket; +import java.time.Duration; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.apache.iggy.client.async.tcp.AsyncIggyTcpClientEndpointFailoverTest.FIRST_PASSWORD; +import static org.apache.iggy.client.async.tcp.AsyncIggyTcpClientEndpointFailoverTest.RECOVERY; +import static org.apache.iggy.client.async.tcp.AsyncIggyTcpClientEndpointFailoverTest.SECOND_PASSWORD; +import static org.apache.iggy.client.async.tcp.AsyncIggyTcpClientEndpointFailoverTest.SIGNED_IN_USER_ID; +import static org.apache.iggy.client.async.tcp.AsyncIggyTcpClientEndpointFailoverTest.USERNAME; +import static org.apache.iggy.client.async.tcp.AsyncIggyTcpClientEndpointFailoverTest.limitedClient; +import static org.apache.iggy.client.async.tcp.AsyncIggyTcpClientEndpointFailoverTest.uncertainOperation; +import static org.apache.iggy.client.async.tcp.AsyncIggyTcpClientTransientFailoverTest.CHANGE_PASSWORD_OPERATION; +import static org.apache.iggy.client.async.tcp.AsyncIggyTcpClientTransientFailoverTest.GET_CLUSTER_METADATA_CODE; +import static org.apache.iggy.client.async.tcp.AsyncIggyTcpClientTransientFailoverTest.OPERATION_LOGOUT; +import static org.apache.iggy.client.async.tcp.AsyncIggyTcpClientTransientFailoverTest.OPERATION_NON_REPLICATED; +import static org.apache.iggy.client.async.tcp.AsyncIggyTcpClientTransientFailoverTest.OPERATION_REGISTER; +import static org.apache.iggy.client.async.tcp.AsyncIggyTcpClientTransientFailoverTest.RENAME_USER; +import static org.apache.iggy.client.async.tcp.AsyncIggyTcpClientTransientFailoverTest.UPDATE_RESOURCE_OPERATION; +import static org.apache.iggy.client.async.tcp.AsyncIggyTcpClientTransientFailoverTest.client; +import static org.apache.iggy.client.async.tcp.AsyncIggyTcpClientTransientFailoverTest.committedOperation; +import static org.apache.iggy.client.async.tcp.AsyncIggyTcpClientTransientFailoverTest.registerBody; +import static org.apache.iggy.client.async.tcp.AsyncIggyTcpClientTransientFailoverTest.serve; +import static org.apache.iggy.client.async.tcp.AsyncIggyTcpClientTransientFailoverTest.singleNodeMetadata; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** A committed rename or password change of the signed-in user reaches the credentials of its next sign-in. */ +class AsyncIggyTcpClientCredentialChangeTest { + private static final int CHANGE_PASSWORD_CODE = CommandCode.User.CHANGE_PASSWORD.getValue(); + private static final int UPDATE_RESOURCE_CODE = CommandCode.System.UPDATE_RESOURCE.getValue(); + private static final long OTHER_USER_ID = 2; + private static final String RENAMED = "carol"; + + @Test + void shouldRetainSuccessfulSelfRenameAndPasswordChangeForConfiguredLogin() throws Exception { + InetAddress loopback = InetAddress.getLoopbackAddress(); + try (ServerSocket serverSocket = new ServerSocket(0, 4, loopback)) { + List<String> logins = new CopyOnWriteArrayList<>(); + CompletableFuture<Void> server = serve(serverSocket, request -> { + if (request.operation() == OPERATION_REGISTER) { + logins.add(request.bodyAsText()); + return Response.success(OPERATION_REGISTER, registerBody(logins.size())); + } + if (request.is(GET_CLUSTER_METADATA_CODE, OPERATION_NON_REPLICATED)) { + return Response.success(OPERATION_NON_REPLICATED, singleNodeMetadata(serverSocket.getLocalPort())); + } + if (request.operation() == UPDATE_RESOURCE_OPERATION + || request.operation() == CHANGE_PASSWORD_OPERATION) { + return Response.committed( + committedOperation(request.operation()), + 11, + Unpooled.buffer().writeIntLE(0)); + } + if (request.operation() == OPERATION_LOGOUT) { + return Response.success(OPERATION_LOGOUT, Unpooled.EMPTY_BUFFER); + } + throw new IllegalStateException("Unexpected request: " + request); + }); + AsyncIggyTcpClient client = client(serverSocket); + try { + client.connect().get(5, TimeUnit.SECONDS); + client.login().get(5, TimeUnit.SECONDS); + client.users().updateUser(UserId.of(1L), RENAME_USER).get(5, TimeUnit.SECONDS); + client.users() + .changePassword(UserId.of("renamed-user"), "configured-password", "new-password") + .get(5, TimeUnit.SECONDS); + client.users().logout().get(5, TimeUnit.SECONDS); + client.login().get(5, TimeUnit.SECONDS); + assertThat(logins).hasSize(2); + assertThat(logins.get(0)).contains("configured-user").contains("configured-password"); + assertThat(logins.get(1)) + .contains("renamed-user") + .contains("new-password") + .doesNotContain("configured-user") + .doesNotContain("configured-password"); + } finally { + client.close().get(5, TimeUnit.SECONDS); + } + server.get(5, TimeUnit.SECONDS); + } + } + + /** + * A committed password change or rename of the signed-in user reaches every later sign-in, also when the + * change was sent raw, or ended uncertain until a resolve proved it. Only the request names the new + * credentials. + */ + @ParameterizedTest(name = "raw={0}, rename={1}") + @CsvSource({"false,false", "true,false", "false,true", "true,true"}) + @SuppressWarnings("checkstyle:NPathComplexity") + void shouldSignInWithTheCommittedChangeOfTheSignedInUser(boolean raw, boolean rename) throws Exception { + try (ServerSocket nodeSocket = new ServerSocket(0, 4, InetAddress.getLoopbackAddress())) { + AtomicBoolean committing = new AtomicBoolean(raw); + AtomicBoolean hangUpOnRegister = new AtomicBoolean(); + List<String> signIns = new CopyOnWriteArrayList<>(); + MockNode node = credentialsNode( + nodeSocket, + hangUpOnRegister, + signIns, + request -> committing.get() + ? Response.committed( + committedOperation(request.operation()), + 2, + Unpooled.buffer().writeIntLE(0)) + : Response.error(request.operation(), AsyncTcpConnection.TRANSIENT_NOT_COMMITTED)); + AsyncIggyTcpClient client = limitedClient(nodeSocket.getLocalPort()); + try { + client.connect().get(5, TimeUnit.SECONDS); + client.users().login(USERNAME, FIRST_PASSWORD).get(5, TimeUnit.SECONDS); + // A rename names the user by name and a password change by id, so both ways must match. + UserId self = rename ? UserId.of(USERNAME) : UserId.of(SIGNED_IN_USER_ID); + if (raw) { + client.sendBinaryRequest( + rename ? UPDATE_RESOURCE_CODE : CHANGE_PASSWORD_CODE, + rename + ? renameBody(self, RENAMED) + : changePasswordBody(self, FIRST_PASSWORD, SECOND_PASSWORD)) + .get(5, TimeUnit.SECONDS); + } else { + CompletableFuture<Void> change = rename + ? client.users() + .updateUser( + self, + UserUpdate.builder() + .username(RENAMED) + .build()) + : client.users().changePassword(self, FIRST_PASSWORD, SECOND_PASSWORD); + long operation = uncertainOperation(change); + committing.set(true); + client.resolveOperation(operation, Duration.ofSeconds(1)).get(5, TimeUnit.SECONDS); + } + + assertThat(signInsAfterTheSessionIsLost(client, node, hangUpOnRegister, signIns)) + .allSatisfy(signIn -> assertThat(signIn) + .contains(rename ? RENAMED : USERNAME, rename ? FIRST_PASSWORD : SECOND_PASSWORD) + .doesNotContain(rename ? USERNAME : FIRST_PASSWORD)); + } finally { + client.close().get(5, TimeUnit.SECONDS); + node.close(); + } + } + } + + /** + * A change that the server committed as rejected, such as a wrong current password, and a change of another + * user leave the sign-in as it was. + */ + @ParameterizedTest(name = "rejected={0}") + @ValueSource(booleans = {false, true}) + void shouldKeepTheSignInAfterARejectedChangeOrAChangeOfAnotherUser(boolean rejected) throws Exception { + try (ServerSocket nodeSocket = new ServerSocket(0, 4, InetAddress.getLoopbackAddress())) { + AtomicBoolean hangUpOnRegister = new AtomicBoolean(); + List<String> signIns = new CopyOnWriteArrayList<>(); + MockNode node = credentialsNode( + nodeSocket, + hangUpOnRegister, + signIns, + request -> Response.committed( + request.operation(), + 2, + rejected + ? AsyncIggyTcpClientTransientFailoverTest.transientResult( + IggyErrorCode.INVALID_CREDENTIALS.getCode()) + : Unpooled.buffer().writeIntLE(0))); + AsyncIggyTcpClient client = limitedClient(nodeSocket.getLocalPort()); + try { + client.connect().get(5, TimeUnit.SECONDS); + client.users().login(USERNAME, FIRST_PASSWORD).get(5, TimeUnit.SECONDS); + var change = client.sendBinaryRequest( + CHANGE_PASSWORD_CODE, + changePasswordBody( + UserId.of(rejected ? SIGNED_IN_USER_ID : OTHER_USER_ID), + FIRST_PASSWORD, + SECOND_PASSWORD)); + if (rejected) { + assertThatThrownBy(() -> change.get(5, TimeUnit.SECONDS)) + .cause() + .isInstanceOfSatisfying( + IggyServerException.class, + error -> assertThat(error.getRawErrorCode()) + .isEqualTo(IggyErrorCode.INVALID_CREDENTIALS.getCode())); + } else { + change.get(5, TimeUnit.SECONDS); + } + + assertThat(signInsAfterTheSessionIsLost(client, node, hangUpOnRegister, signIns)) + .allSatisfy(signIn -> assertThat(signIn) + .contains(USERNAME, FIRST_PASSWORD) + .doesNotContain(SECOND_PASSWORD)); + } finally { + client.close().get(5, TimeUnit.SECONDS); + node.close(); + } + } + } + + /** + * Records every Register and hangs up on the first one after {@code hangUpOnRegister} is set. A password + * change or a rename gets the reply of {@code change}. + */ + private static MockNode credentialsNode( + ServerSocket socket, AtomicBoolean hangUpOnRegister, List<String> signIns, RequestHandler change) { + return MockNode.serve(socket, request -> { + if (request.operation() == OPERATION_REGISTER) { + signIns.add(request.bodyAsText()); + return hangUpOnRegister.getAndSet(false) + ? Response.disconnect() + : Response.success(OPERATION_REGISTER, registerBody(signIns.size())); + } + if (request.operation() == VsrOperation.CHANGE_PASSWORD + || request.operation() == VsrOperation.UPDATE_RESOURCE) { + return change.handle(request); + } + return Response.success(request.operation(), Unpooled.EMPTY_BUFFER); + }); + } + + /** + * Ends the session and returns the sign-ins that restore it: the one on the next link of the connection, + * which hangs up, and the one that the redial replays. A roster read never waits for a session, so it ends + * when that sign-in hangs up. + */ + private static List<String> signInsAfterTheSessionIsLost( + AsyncIggyTcpClient client, MockNode node, AtomicBoolean hangUpOnRegister, List<String> signIns) + throws Exception { + int before = signIns.size(); + node.rejectBindings = true; + hangUpOnRegister.set(true); + node.dropConnections(); + assertThatThrownBy(() -> client.system().getClusterMetadata().get(5, TimeUnit.SECONDS)) + .isInstanceOf(ExecutionException.class); + AsyncIggyTcpClientEventLoopGroupTest.await(() -> signIns.size() >= before + 2, "the redial sign-in", RECOVERY); + List<String> all = List.copyOf(signIns); + return all.subList(before, all.size()); + } + + /** The body that {@code UsersClient.changePassword} writes. */ + private static byte[] changePasswordBody(UserId user, String currentPassword, String newPassword) { + return ByteBufUtil.getBytes(BytesSerializer.toBytes(user) + .writeBytes(BytesSerializer.toBytes(currentPassword, "current password")) + .writeBytes(BytesSerializer.toBytes(newPassword, "new password"))); + } + + /** The body that {@code UsersClient.updateUser} writes for a rename: a username and no status. */ + private static byte[] renameBody(UserId user, String username) { + return ByteBufUtil.getBytes(BytesSerializer.toUpdateResource( + user, UserUpdate.builder().username(username).build())); + } +}
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEndpointFailoverTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEndpointFailoverTest.java index 1361b1b..105fd89 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEndpointFailoverTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEndpointFailoverTest.java
@@ -76,6 +76,13 @@ * kept separate so a death mid-connection cannot disturb that suite's server. */ class AsyncIggyTcpClientEndpointFailoverTest { + static final Duration RECOVERY = Duration.ofSeconds(5); + // The user that registerBody signs in. + static final long SIGNED_IN_USER_ID = 1; + static final String USERNAME = "alice"; + static final String FIRST_PASSWORD = "first-secret"; + static final String SECOND_PASSWORD = "second-secret"; + private static final int OPERATION_REGISTER = 1; private static final int OPERATION_NON_REPLICATED = 2; private static final int GET_CLUSTER_METADATA_CODE = 12; @@ -88,18 +95,8 @@ // One retry per burst, so an outage of a second outlasts many bursts. private static final RetryPolicy ONE_RETRY = RetryPolicy.fixedDelay(1, Duration.ofMillis(50)); private static final Duration OUTAGE = Duration.ofSeconds(1); - private static final Duration RECOVERY = Duration.ofSeconds(5); private static final int LOGIN_CODE = CommandCode.User.LOGIN.getValue(); - private static final int CHANGE_PASSWORD_CODE = CommandCode.User.CHANGE_PASSWORD.getValue(); - private static final int UPDATE_USER_CODE = CommandCode.User.UPDATE.getValue(); - // The user that registerBody signs in. - private static final long SIGNED_IN_USER_ID = 1; - private static final long OTHER_USER_ID = 2; - private static final String USERNAME = "alice"; - private static final String RENAMED = "carol"; private static final String OTHER_USERNAME = "bob"; - private static final String FIRST_PASSWORD = "first-secret"; - private static final String SECOND_PASSWORD = "second-secret"; @Test void shouldCloseAnUnmanagedBootstrapAfterRetargeting() throws Exception { @@ -1184,113 +1181,6 @@ } /** - * A committed password change or rename of the signed-in user reaches every later sign-in, also when the - * change was sent raw, or ended uncertain until a resolve proved it. Only the request names the new - * credentials. - */ - @ParameterizedTest(name = "raw={0}, rename={1}") - @CsvSource({"false,false", "true,false", "false,true", "true,true"}) - @SuppressWarnings("checkstyle:NPathComplexity") - void shouldSignInWithTheCommittedChangeOfTheSignedInUser(boolean raw, boolean rename) throws Exception { - try (ServerSocket nodeSocket = new ServerSocket(0, 4, InetAddress.getLoopbackAddress())) { - AtomicBoolean committing = new AtomicBoolean(raw); - AtomicBoolean hangUpOnRegister = new AtomicBoolean(); - List<String> signIns = new CopyOnWriteArrayList<>(); - MockNode node = credentialsNode( - nodeSocket, - hangUpOnRegister, - signIns, - request -> committing.get() - ? Response.committed( - request.operation(), 2, Unpooled.buffer().writeIntLE(0)) - : Response.error(request.operation(), AsyncTcpConnection.TRANSIENT_NOT_COMMITTED)); - AsyncIggyTcpClient client = limitedClient(nodeSocket.getLocalPort()); - try { - client.connect().get(5, TimeUnit.SECONDS); - client.users().login(USERNAME, FIRST_PASSWORD).get(5, TimeUnit.SECONDS); - // A rename names the user by name and a password change by id, so both ways must match. - UserId self = rename ? UserId.of(USERNAME) : UserId.of(SIGNED_IN_USER_ID); - if (raw) { - client.sendBinaryRequest( - rename ? UPDATE_USER_CODE : CHANGE_PASSWORD_CODE, - rename - ? renameBody(self, RENAMED) - : changePasswordBody(self, FIRST_PASSWORD, SECOND_PASSWORD)) - .get(5, TimeUnit.SECONDS); - } else { - CompletableFuture<Void> change = rename - ? client.users().updateUser(self, Optional.of(RENAMED), Optional.empty()) - : client.users().changePassword(self, FIRST_PASSWORD, SECOND_PASSWORD); - long operation = uncertainOperation(change); - committing.set(true); - client.resolveOperation(operation, Duration.ofSeconds(1)).get(5, TimeUnit.SECONDS); - } - - assertThat(signInsAfterTheSessionIsLost(client, node, hangUpOnRegister, signIns)) - .allSatisfy(signIn -> assertThat(signIn) - .contains(rename ? RENAMED : USERNAME, rename ? FIRST_PASSWORD : SECOND_PASSWORD) - .doesNotContain(rename ? USERNAME : FIRST_PASSWORD)); - } finally { - client.close().get(5, TimeUnit.SECONDS); - node.close(); - } - } - } - - /** - * A change that the server committed as rejected, such as a wrong current password, and a change of another - * user leave the sign-in as it was. - */ - @ParameterizedTest(name = "rejected={0}") - @ValueSource(booleans = {false, true}) - void shouldKeepTheSignInAfterARejectedChangeOrAChangeOfAnotherUser(boolean rejected) throws Exception { - try (ServerSocket nodeSocket = new ServerSocket(0, 4, InetAddress.getLoopbackAddress())) { - AtomicBoolean hangUpOnRegister = new AtomicBoolean(); - List<String> signIns = new CopyOnWriteArrayList<>(); - MockNode node = credentialsNode( - nodeSocket, - hangUpOnRegister, - signIns, - request -> Response.committed( - request.operation(), - 2, - rejected - ? AsyncIggyTcpClientTransientFailoverTest.transientResult( - IggyErrorCode.INVALID_CREDENTIALS.getCode()) - : Unpooled.buffer().writeIntLE(0))); - AsyncIggyTcpClient client = limitedClient(nodeSocket.getLocalPort()); - try { - client.connect().get(5, TimeUnit.SECONDS); - client.users().login(USERNAME, FIRST_PASSWORD).get(5, TimeUnit.SECONDS); - var change = client.sendBinaryRequest( - CHANGE_PASSWORD_CODE, - changePasswordBody( - UserId.of(rejected ? SIGNED_IN_USER_ID : OTHER_USER_ID), - FIRST_PASSWORD, - SECOND_PASSWORD)); - if (rejected) { - assertThatThrownBy(() -> change.get(5, TimeUnit.SECONDS)) - .cause() - .isInstanceOfSatisfying( - IggyServerException.class, - error -> assertThat(error.getRawErrorCode()) - .isEqualTo(IggyErrorCode.INVALID_CREDENTIALS.getCode())); - } else { - change.get(5, TimeUnit.SECONDS); - } - - assertThat(signInsAfterTheSessionIsLost(client, node, hangUpOnRegister, signIns)) - .allSatisfy(signIn -> assertThat(signIn) - .contains(USERNAME, FIRST_PASSWORD) - .doesNotContain(SECOND_PASSWORD)); - } finally { - client.close().get(5, TimeUnit.SECONDS); - node.close(); - } - } - } - - /** * Every link of the session signs in with the committed password of the signed-in user. Here that is the poll * link on the survivor, which the coordinator moves to and back from before the change commits, and which * becomes the coordinator when the leader moves there. An uncertain change commits only when a resolve proves @@ -1575,47 +1465,7 @@ } } - /** - * Records every Register and hangs up on the first one after {@code hangUpOnRegister} is set. A password - * change or a rename gets the reply of {@code change}. - */ - private static MockNode credentialsNode( - ServerSocket socket, AtomicBoolean hangUpOnRegister, List<String> signIns, RequestHandler change) { - return MockNode.serve(socket, request -> { - if (request.operation() == OPERATION_REGISTER) { - signIns.add(request.bodyAsText()); - return hangUpOnRegister.getAndSet(false) - ? Response.disconnect() - : Response.success(OPERATION_REGISTER, registerBody(signIns.size())); - } - if (request.operation() == VsrOperation.CHANGE_PASSWORD - || request.operation() == VsrOperation.UPDATE_USER) { - return change.handle(request); - } - return Response.success(request.operation(), Unpooled.EMPTY_BUFFER); - }); - } - - /** - * Ends the session and returns the sign-ins that restore it: the one on the next link of the connection, - * which hangs up, and the one that the redial replays. A roster read never waits for a session, so it ends - * when that sign-in hangs up. - */ - private static List<String> signInsAfterTheSessionIsLost( - AsyncIggyTcpClient client, MockNode node, AtomicBoolean hangUpOnRegister, List<String> signIns) - throws Exception { - int before = signIns.size(); - node.rejectBindings = true; - hangUpOnRegister.set(true); - node.dropConnections(); - assertThatThrownBy(() -> client.system().getClusterMetadata().get(5, TimeUnit.SECONDS)) - .isInstanceOf(ExecutionException.class); - AsyncIggyTcpClientEventLoopGroupTest.await(() -> signIns.size() >= before + 2, "the redial sign-in", RECOVERY); - List<String> all = List.copyOf(signIns); - return all.subList(before, all.size()); - } - - private static long uncertainOperation(CompletableFuture<?> call) { + static long uncertainOperation(CompletableFuture<?> call) { AtomicLong operation = new AtomicLong(); assertThatThrownBy(() -> call.get(5, TimeUnit.SECONDS)) .cause() @@ -1632,22 +1482,7 @@ .writeBytes(BytesSerializer.toBytes(password, "password"))); } - /** The body that {@code UsersClient.changePassword} writes. */ - private static byte[] changePasswordBody(UserId user, String currentPassword, String newPassword) { - return ByteBufUtil.getBytes(BytesSerializer.toBytes(user) - .writeBytes(BytesSerializer.toBytes(currentPassword, "current password")) - .writeBytes(BytesSerializer.toBytes(newPassword, "new password"))); - } - - /** The body that {@code UsersClient.updateUser} writes for a rename: a username and no status. */ - private static byte[] renameBody(UserId user, String username) { - return ByteBufUtil.getBytes(BytesSerializer.toBytes(user) - .writeByte(1) - .writeBytes(BytesSerializer.toBytes(username, "username")) - .writeByte(0)); - } - - private static AsyncIggyTcpClient limitedClient(int port) { + static AsyncIggyTcpClient limitedClient(int port) { return AsyncIggyTcpClient.builder() .host(InetAddress.getLoopbackAddress().getHostAddress()) .port(port) @@ -1676,7 +1511,7 @@ body.writeIntLE(0); body.writeIntLE(1); body.writeLongLE(session); - body.writeIntLE((11 << 10) | 1); + body.writeIntLE(12 << 10); body.writeByte(0); return body; }
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEventLoopGroupTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEventLoopGroupTest.java index aa22c3c..8951463 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEventLoopGroupTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEventLoopGroupTest.java
@@ -455,7 +455,7 @@ .putInt(0) .putInt(1) .putLong(42) - .putInt((11 << 10) | 1) + .putInt(12 << 10) .put((byte) 0) .array(); }
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java index 31f7671..036cc71 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java
@@ -33,6 +33,7 @@ import org.apache.iggy.message.PolledMessages; import org.apache.iggy.message.PollingStrategy; import org.apache.iggy.serde.CommandCode; +import org.apache.iggy.user.UserUpdate; import org.junit.jupiter.api.Test; import java.io.EOFException; @@ -73,6 +74,11 @@ static final int EVICTION_STALE_CLIENT = 13; static final int OPERATION_CREATE_STREAM = 128; static final int CREATE_STREAM_CODE = 202; + static final int OPERATION_LOGOUT = 3; + static final int UPDATE_RESOURCE_OPERATION = 150; + static final int CHANGE_PASSWORD_OPERATION = 144; + static final UserUpdate RENAME_USER = + UserUpdate.builder().username("renamed-user").build(); private static final int HEADER_SIZE = 256; private static final int SIZE_OFFSET = 48; @@ -92,10 +98,8 @@ private static final int COMMAND_REPLY = 8; private static final int COMMAND_EVICTION = 13; - private static final int OPERATION_LOGOUT = 3; private static final int GROUP_JOIN_OPERATION = 148; - private static final int UPDATE_USER_OPERATION = 142; - private static final int CHANGE_PASSWORD_OPERATION = 144; + private static final int UPDATE_USER_RESOLVED_OPERATION = 74; private static final int TRANSIENT_NOT_COMMITTED = 57; private static final int POLL_PARAMETERS_BYTES = 14; private static final Duration TEST_POLL_TIMEOUT = Duration.ofSeconds(2); @@ -425,7 +429,7 @@ AtomicInteger denials = new AtomicInteger(); AtomicInteger nextOperations = new AtomicInteger(); List<String> logins = new CopyOnWriteArrayList<>(); - int mutation = rename ? UPDATE_USER_OPERATION : CHANGE_PASSWORD_OPERATION; + int mutation = rename ? UPDATE_RESOURCE_OPERATION : CHANGE_PASSWORD_OPERATION; CompletableFuture<Void> oldLeader = serve(oldSocket, request -> { if (request.operation() == OPERATION_REGISTER) { return Response.success(OPERATION_REGISTER, registerBody(1)); @@ -467,7 +471,9 @@ } if (request.operation() == mutation) { return Response.committed( - mutation, 11, Unpooled.buffer().writeIntLE(0)); + committedOperation(mutation), + 11, + Unpooled.buffer().writeIntLE(0)); } if (request.operation() == OPERATION_CREATE_STREAM) { return nextOperations.incrementAndGet() == 1 @@ -488,7 +494,7 @@ client.connect().get(5, TimeUnit.SECONDS); client.users().login("manual-user", "manual-password").get(5, TimeUnit.SECONDS); CompletableFuture<Void> mutationResult = rename - ? client.users().updateUser(UserId.of(1L), Optional.of("renamed-user"), Optional.empty()) + ? client.users().updateUser(UserId.of(1L), RENAME_USER) : client.users().changePassword(UserId.of(1L), "manual-password", "new-password"); mutationResult.get(10, TimeUnit.SECONDS); assertThat(client.getConnectionInfo().port()).isEqualTo(newSocket.getLocalPort()); @@ -786,54 +792,6 @@ } @Test - void shouldRetainSuccessfulSelfRenameAndPasswordChangeForConfiguredLogin() throws Exception { - InetAddress loopback = InetAddress.getLoopbackAddress(); - try (ServerSocket serverSocket = new ServerSocket(0, 4, loopback)) { - List<String> logins = new CopyOnWriteArrayList<>(); - CompletableFuture<Void> server = serve(serverSocket, request -> { - if (request.operation() == OPERATION_REGISTER) { - logins.add(request.bodyAsText()); - return Response.success(OPERATION_REGISTER, registerBody(logins.size())); - } - if (request.is(GET_CLUSTER_METADATA_CODE, OPERATION_NON_REPLICATED)) { - return Response.success(OPERATION_NON_REPLICATED, singleNodeMetadata(serverSocket.getLocalPort())); - } - if (request.operation() == UPDATE_USER_OPERATION || request.operation() == CHANGE_PASSWORD_OPERATION) { - return Response.committed( - request.operation(), 11, Unpooled.buffer().writeIntLE(0)); - } - if (request.operation() == OPERATION_LOGOUT) { - return Response.success(OPERATION_LOGOUT, Unpooled.EMPTY_BUFFER); - } - throw new IllegalStateException("Unexpected request: " + request); - }); - AsyncIggyTcpClient client = client(serverSocket); - try { - client.connect().get(5, TimeUnit.SECONDS); - client.login().get(5, TimeUnit.SECONDS); - client.users() - .updateUser(UserId.of(1L), Optional.of("renamed-user"), Optional.empty()) - .get(5, TimeUnit.SECONDS); - client.users() - .changePassword(UserId.of("renamed-user"), "configured-password", "new-password") - .get(5, TimeUnit.SECONDS); - client.users().logout().get(5, TimeUnit.SECONDS); - client.login().get(5, TimeUnit.SECONDS); - assertThat(logins).hasSize(2); - assertThat(logins.get(0)).contains("configured-user").contains("configured-password"); - assertThat(logins.get(1)) - .contains("renamed-user") - .contains("new-password") - .doesNotContain("configured-user") - .doesNotContain("configured-password"); - } finally { - client.close().get(5, TimeUnit.SECONDS); - } - server.get(5, TimeUnit.SECONDS); - } - } - - @Test @SuppressWarnings({"checkstyle:CyclomaticComplexity", "checkstyle:NPathComplexity"}) void shouldPreserveCapturedContextWhileWaitingAndRefreshItForTheNextCall() throws Exception { for (boolean holdAttachment : new boolean[] {false, true}) { @@ -1163,10 +1121,12 @@ OPERATION_NON_REPLICATED, pollRoute(request, 1, Math.max(1, request.metadataOp()), primarySocket.getLocalPort())); } - if (request.operation() == UPDATE_USER_OPERATION + if (request.operation() == UPDATE_RESOURCE_OPERATION || request.operation() == CHANGE_PASSWORD_OPERATION) { return Response.committed( - request.operation(), 11, Unpooled.buffer().writeIntLE(0)); + committedOperation(request.operation()), + 11, + Unpooled.buffer().writeIntLE(0)); } throw new IllegalStateException("Unexpected coordinator request: " + request); }); @@ -1203,9 +1163,7 @@ .satisfies(error -> assertThat(((IggyServerException) error.getCause()).getRawErrorCode()) .isEqualTo(failure == -1 ? TRANSIENT_NOT_COMMITTED : failure)); assertThat(polls).hasValue(1); - client.users() - .updateUser(UserId.of(1L), Optional.of("renamed-user"), Optional.empty()) - .get(5, TimeUnit.SECONDS); + client.users().updateUser(UserId.of(1L), RENAME_USER).get(5, TimeUnit.SECONDS); client.users() .changePassword(UserId.of("renamed-user"), "manual-password", "new-password") .get(5, TimeUnit.SECONDS); @@ -1812,12 +1770,17 @@ output.flush(); } + /** The server commits a generic user patch under its resolved operation. */ + static int committedOperation(int requestOperation) { + return requestOperation == UPDATE_RESOURCE_OPERATION ? UPDATE_USER_RESOLVED_OPERATION : requestOperation; + } + static ByteBuf registerBody(long session) { ByteBuf body = Unpooled.buffer(); body.writeIntLE(0); body.writeIntLE(1); body.writeLongLE(session); - body.writeIntLE((11 << 10) | 1); + body.writeIntLE(12 << 10); body.writeByte(0); return body; }
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionConcurrencyTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionConcurrencyTest.java index e27e187..5c8778a 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionConcurrencyTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionConcurrencyTest.java
@@ -829,7 +829,7 @@ .putInt(0) .putInt(1) .putLong(42) - .putInt((11 << 10) | 1) + .putInt(12 << 10) .put((byte) 0) .array(); }
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/TopicsTcpClientPayloadTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/TopicsTcpClientPayloadTest.java index a58b72a..e5c70b4 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/TopicsTcpClientPayloadTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/TopicsTcpClientPayloadTest.java
@@ -22,7 +22,9 @@ import io.netty.buffer.ByteBuf; import org.apache.iggy.identifier.StreamId; import org.apache.iggy.message.HeaderKind; +import org.apache.iggy.serde.BytesSerializer; import org.apache.iggy.topic.CompressionAlgorithm; +import org.apache.iggy.topic.TopicOptions; import org.junit.jupiter.api.Test; import java.math.BigInteger; @@ -40,8 +42,7 @@ @Test void shouldWritePartitionsCountAsFixedFieldBeforeName() { - ByteBuf payload = TopicsTcpClient.createTopicPayload( - StreamId.of(1L), 3L, CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, "orders", Map.of()); + ByteBuf payload = BytesSerializer.toCreateTopic(StreamId.of(1L), "orders", 3L, Map.of()); assertThat(payload.readUnsignedByte()).isEqualTo((short) 1); // numeric identifier kind assertThat(payload.readUnsignedByte()).isEqualTo((short) 4); // identifier length @@ -53,38 +54,39 @@ @Test void shouldOmitServerDefaultOptions() { - ByteBuf payload = TopicsTcpClient.createTopicPayload( - StreamId.of("my-stream"), - 1L, - CompressionAlgorithm.None, - BigInteger.ZERO, - BigInteger.ZERO, - "events", - Map.of()); + var options = TopicOptions.builder() + .messageExpiry(BigInteger.ZERO) + .maxTopicSize(BigInteger.ZERO) + .build(); + ByteBuf payload = BytesSerializer.toCreateTopic(StreamId.of("my-stream"), "events", 1L, options.entries()); payload.skipBytes(2 + "my-stream".length()); assertThat(payload.readUnsignedIntLE()).isEqualTo(1L); assertThat(readString(payload)).isEqualTo("events"); - assertThat(payload.isReadable()).isFalse(); + assertThat(readOptions(payload)).containsOnlyKeys("consumer_offset_durability", "durability"); } @Test void shouldEncodeExplicitOptionsWithoutPartitionsCount() { - ByteBuf payload = TopicsTcpClient.createTopicPayload( - StreamId.of(1L), - 2L, - CompressionAlgorithm.Gzip, - BigInteger.valueOf(60_000_000L), - BigInteger.valueOf(1024L), - "orders", - Map.of()); + var topicOptions = TopicOptions.builder() + .compressionAlgorithm(CompressionAlgorithm.Gzip) + .messageExpiry(BigInteger.valueOf(60_000_000L)) + .maxTopicSize(BigInteger.valueOf(1024L)) + .build(); + ByteBuf payload = BytesSerializer.toCreateTopic(StreamId.of(1L), "orders", 2L, topicOptions.entries()); payload.skipBytes(6); assertThat(payload.readUnsignedIntLE()).isEqualTo(2L); readString(payload); Map<String, TypedValue> options = readOptions(payload); - assertThat(options).containsOnlyKeys("compression_algorithm", "message_expiry", "max_topic_size"); + assertThat(options) + .containsOnlyKeys( + "compression_algorithm", + "consumer_offset_durability", + "durability", + "max_topic_size", + "message_expiry"); assertThat(options.get("compression_algorithm").kind()).isEqualTo(HeaderKind.String); assertThat(new String(options.get("compression_algorithm").value(), StandardCharsets.UTF_8)) .isEqualTo("gzip");
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrOperationTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrOperationTest.java new file mode 100644 index 0000000..08514fe --- /dev/null +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrOperationTest.java
@@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.client.async.tcp.vsr; + +import org.junit.jupiter.api.Test; +import tools.jackson.databind.JsonNode; + +import java.util.Map; + +import static org.apache.iggy.ResourceOptionsFixture.section; +import static org.assertj.core.api.Assertions.assertThat; + +class VsrOperationTest { + + @Test + void givenFixtureOperationsWhenMirroringShouldMatchCodesAndCorrelation() { + JsonNode constants = section("constants"); + + for (Map.Entry<String, JsonNode> command : + constants.get("command_codes").properties()) { + JsonNode operation = command.getValue().get("operation"); + int expected = operation.isNull() ? VsrOperation.NON_REPLICATED : operation.asInt(); + assertThat(VsrOperation.operationForCode( + command.getValue().get("code").asInt())) + .as(command.getKey()) + .isEqualTo(expected); + } + for (Map.Entry<String, JsonNode> retired : + constants.get("retired_command_codes").properties()) { + assertThat(VsrOperation.operationForCode(retired.getValue().asInt())) + .as(retired.getKey()) + .isEqualTo(VsrOperation.NON_REPLICATED); + } + for (Map.Entry<String, JsonNode> operation : constants.get("operations").properties()) { + int value = operation.getValue().get("value").asInt(); + assertThat(VsrOperation.isKnown(value)).as(operation.getKey()).isTrue(); + assertThat(VsrOperation.isMetadata(value)).as(operation.getKey()).isTrue(); + assertThat(VsrOperation.correlationOperation(value)) + .as(operation.getKey()) + .isEqualTo(operation.getValue().get("request_operation").asInt()); + } + for (Map.Entry<String, JsonNode> retired : + constants.get("retired_operations").properties()) { + assertThat(VsrOperation.isKnown(retired.getValue().asInt())) + .as(retired.getKey()) + .isFalse(); + } + assertThat(VsrLoginCodec.PROTOCOL_VERSION) + .isEqualTo(constants.get("protocol_version").get("packed").asInt()) + .isEqualTo(constants.get("protocol_version_min").get("packed").asInt()); + } +}
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrRequestEncoderTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrRequestEncoderTest.java index a1c1ee6..d514d58 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrRequestEncoderTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrRequestEncoderTest.java
@@ -122,6 +122,20 @@ ByteBuf first = encoder.encode(alloc, CREATE_STREAM_CODE, Unpooled.EMPTY_BUFFER); ByteBuf second = encoder.encode(alloc, CREATE_STREAM_CODE, Unpooled.EMPTY_BUFFER); try { + byte[] firstIdentifier = new byte[16]; + byte[] secondIdentifier = new byte[16]; + first.getBytes(VsrHeaders.REQUEST_CHECKSUM_OFFSET, firstIdentifier); + second.getBytes(VsrHeaders.REQUEST_CHECKSUM_OFFSET, secondIdentifier); + assertThat(firstIdentifier).isNotEqualTo(new byte[16]); + assertThat(firstIdentifier).isNotEqualTo(secondIdentifier); + ByteBuf retry = first.retainedDuplicate(); + try { + byte[] retryIdentifier = new byte[16]; + retry.getBytes(VsrHeaders.REQUEST_CHECKSUM_OFFSET, retryIdentifier); + assertThat(retryIdentifier).isEqualTo(firstIdentifier); + } finally { + retry.release(); + } assertThat(first.getLongLE(VsrHeaders.REQUEST_ID_OFFSET)).isEqualTo(1); assertThat(second.getLongLE(VsrHeaders.REQUEST_ID_OFFSET)).isEqualTo(2); assertThat(second.getLongLE(VsrHeaders.REQUEST_SESSION_OFFSET)).isEqualTo(42);
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandlerTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandlerTest.java index 372081a..de6374f 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandlerTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandlerTest.java
@@ -37,6 +37,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -485,6 +486,32 @@ } @Test + void givenResolvedUpdateReplyWhenCorrelatingShouldAcceptOnlyMatchingRequestIdentity() { + int updateResource = 150; + for (int resolved : new int[] {72, 73, 74}) { + assertThat(correlates(updateResource, resolved, reply -> {})) + .as("resolved %s", resolved) + .isTrue(); + assertThat(correlates( + updateResource, + resolved, + reply -> reply.setByte( + VsrHeaders.REPLY_CLIENT_OFFSET, reply.getByte(VsrHeaders.REPLY_CLIENT_OFFSET) + 1))) + .as("client mismatch for %s", resolved) + .isFalse(); + assertThat(correlates( + updateResource, resolved, reply -> reply.setLongLE(VsrHeaders.REPLY_CHECKSUM_OFFSET, 14))) + .as("request checksum mismatch for %s", resolved) + .isFalse(); + assertThat(correlates(VsrOperation.CREATE_STREAM, resolved, reply -> {})) + .as("unrelated request for %s", resolved) + .isFalse(); + } + assertThat(correlates(updateResource, VsrOperation.CREATE_TOPIC_WITH_ASSIGNMENTS, reply -> {})) + .isFalse(); + } + + @Test void shouldRejectUnrelatedReplyOperation() { CompletableFuture<ByteBuf> pending = enqueue(VsrOperation.CREATE_TOPIC, 7); CompletableFuture<ByteBuf> other = enqueue(VsrOperation.CREATE_STREAM, 8); @@ -565,6 +592,32 @@ return future; } + /** Registers one committed request on a fresh link and reports whether the reply completes it. */ + private boolean correlates(int requestOperation, int replyOperation, Consumer<ByteBuf> alteration) { + VsrResponseHandler link = new VsrResponseHandler(session, errorCode -> {}); + EmbeddedChannel linkChannel = new EmbeddedChannel(link); + try { + CompletableFuture<ByteBuf> result = new CompletableFuture<>(); + ByteBuf request = requestFrame(requestOperation, 1); + request.setLongLE(VsrHeaders.REQUEST_CHECKSUM_OFFSET, 13); + link.registerRequest(linkChannel, request, result, System.nanoTime() + TimeUnit.MINUTES.toNanos(1), 16); + request.release(); + ByteBuf reply = replyFrame(replyOperation, 1, Unpooled.buffer().writeIntLE(0)); + reply.setLongLE(VsrHeaders.REPLY_CHECKSUM_OFFSET, 13); + alteration.accept(reply); + linkChannel.writeInbound(reply); + assertThat(result).isDone(); + if (result.isCompletedExceptionally()) { + assertThatThrownBy(result::join).hasCauseInstanceOf(IggyConnectionException.class); + return false; + } + result.join().release(); + return true; + } finally { + linkChannel.finishAndReleaseAll(); + } + } + /** * Encodes a partition send off the live session and registers it the way * the connection does, returning the request id the encoder minted.
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/IntegrationTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/IntegrationTest.java index 8d33e90..b4a90b9 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/IntegrationTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/IntegrationTest.java
@@ -21,11 +21,9 @@ import org.apache.iggy.client.BaseIntegrationTest; import org.apache.iggy.stream.StreamDetails; -import org.apache.iggy.topic.CompressionAlgorithm; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; -import java.math.BigInteger; import java.util.ArrayList; import java.util.List; @@ -91,15 +89,7 @@ protected void setUpStreamAndTopic() { setUpStream(); - client.topics() - .createTopic( - STREAM_NAME, - 1L, - CompressionAlgorithm.None, - BigInteger.ZERO, - BigInteger.ZERO, - TOPIC_NAME.getName(), - PERSISTED_TOPIC_OPTIONS); + client.topics().createTopic(STREAM_NAME, TOPIC_NAME.getName(), 1L, PERSISTED_TOPIC_OPTIONS); } protected void login() {
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/StreamClientBaseTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/StreamClientBaseTest.java index b2be6d3..bef20b9 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/StreamClientBaseTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/StreamClientBaseTest.java
@@ -19,13 +19,41 @@ package org.apache.iggy.client.blocking; +import org.apache.iggy.consumergroup.Consumer; +import org.apache.iggy.exception.IggyErrorCode; +import org.apache.iggy.exception.IggyValidationException; +import org.apache.iggy.identifier.StreamId; +import org.apache.iggy.identifier.TopicId; +import org.apache.iggy.message.HeaderValue; +import org.apache.iggy.message.Message; +import org.apache.iggy.message.Partitioning; +import org.apache.iggy.message.PollingStrategy; +import org.apache.iggy.stream.EncryptionKeySource; +import org.apache.iggy.stream.EncryptionKeys; +import org.apache.iggy.stream.StreamDetails; +import org.apache.iggy.stream.StreamOptionKey; +import org.apache.iggy.stream.StreamOptions; +import org.apache.iggy.stream.StreamUpdate; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.security.SecureRandom; +import java.util.Arrays; +import java.util.Base64; +import java.util.List; +import java.util.Map; +import java.util.Optional; + import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; public abstract class StreamClientBaseTest extends IntegrationTest { + private static final List<String> SECRET_KEYS = + List.of("encryption_key", "encryption_key_wrapped", "critical_keys"); + StreamsClient streamsClient; @BeforeEach @@ -69,7 +97,9 @@ trackStream(streamDetails.id()); // when - streamsClient.updateStream(streamDetails.id(), "test-stream-new"); + streamsClient.updateStream( + streamDetails.id(), + StreamUpdate.builder().name("test-stream-new").build()); // then var streamOptional = streamsClient.getStream(streamDetails.id()); @@ -87,4 +117,127 @@ // then assertThat(stream).isEmpty(); } + + @Test + void givenStreamKeyWhenCreatingShouldReadMatchingProjection() { + // given + byte[] key = randomKey(); + var options = StreamOptions.builder().encryptionKey(key).build(); + + // when + var created = streamsClient.createStream("keyed-stream", options); + trackStream(created.id()); + var fetched = streamsClient.getStream(created.id()).orElseThrow(); + var listed = streamsClient.getStreams().stream() + .filter(stream -> stream.id().equals(created.id())) + .findFirst() + .orElseThrow(); + + // then + String fingerprint = EncryptionKeys.fingerprint(key); + for (StreamDetails stream : List.of(created, fetched)) { + assertThat(stream.encryptionKeySource()).contains(EncryptionKeySource.STREAM); + assertThat(stream.encryptionKeyFingerprint()).contains(fingerprint); + assertThat(stream.encryptionKekFingerprints()).isNotEmpty(); + assertNoKeyMaterial(key, stream.options(), stream.derivedOptions()); + } + assertThat(listed.encryptionKeySource()).contains(EncryptionKeySource.STREAM); + assertThat(listed.encryptionKeyFingerprint()).contains(fingerprint); + assertThat(listed.encryptionKekFingerprints()).isEqualTo(fetched.encryptionKekFingerprints()); + assertNoKeyMaterial(key, listed.options(), listed.derivedOptions()); + } + + @Test + void givenKeylessStreamWhenReadingShouldReportServerGeneratedKey() { + // when + var created = streamsClient.createStream("keyless-stream"); + trackStream(created.id()); + var fetched = streamsClient.getStream(created.id()).orElseThrow(); + + // then + assertThat(fetched.encryptionKeySource()).contains(EncryptionKeySource.SERVER); + assertThat(fetched.encryptionKeyFingerprint()) + .hasValueSatisfying(fingerprint -> assertThat(fingerprint).matches("[0-9a-f]{16}")); + assertThat(fetched.encryptionKekFingerprints()).isNotEmpty(); + assertThat(created.encryptionKeyFingerprint()).isEqualTo(fetched.encryptionKeyFingerprint()); + assertThat(fetched.derivedOptions()).doesNotContainKeys(SECRET_KEYS.toArray(String[]::new)); + } + + @Test + void givenStreamRenameWhenUpdatingShouldPreserveEncryptionProjection() { + // given + byte[] key = randomKey(); + var created = streamsClient.createStream( + "rename-keyed-stream", + StreamOptions.builder().encryptionKey(key).build()); + trackStream(created.id()); + var streamId = StreamId.of(created.id()); + client.topics().createTopic(streamId, "events", 1L); + client.messages() + .sendMessages( + streamId, TopicId.of("events"), Partitioning.partitionId(0L), List.of(Message.of("before"))); + var before = streamsClient.getStream(streamId).orElseThrow(); + + // when + streamsClient.updateStream( + streamId, StreamUpdate.builder().name("renamed-keyed-stream").build()); + + // then + var after = streamsClient.getStream(created.id()).orElseThrow(); + assertThat(after.id()).isEqualTo(created.id()); + assertThat(after.name()).isEqualTo("renamed-keyed-stream"); + assertThat(after.encryptionKeySource()).isEqualTo(before.encryptionKeySource()); + assertThat(after.encryptionKeyFingerprint()).contains(EncryptionKeys.fingerprint(key)); + assertThat(after.encryptionKekFingerprints()).isEqualTo(before.encryptionKekFingerprints()); + var polled = client.messages() + .pollMessages( + streamId, + TopicId.of("events"), + Optional.of(0L), + Consumer.of(0L), + PollingStrategy.offset(BigInteger.ZERO), + 1L, + false); + assertThat(polled.messages()).hasSize(1); + assertThat(new String(polled.messages().get(0).payload(), StandardCharsets.UTF_8)) + .isEqualTo("before"); + } + + @Test + void givenStreamKeyResetWhenUpdatingShouldRejectBeforeSendingAndKeepState() { + // given + byte[] key = randomKey(); + var created = streamsClient.createStream( + "reset-keyed-stream", StreamOptions.builder().encryptionKey(key).build()); + trackStream(created.id()); + + // when + var update = StreamUpdate.builder().reset(StreamOptionKey.ENCRYPTION_KEY); + + // then + assertThatThrownBy(update::build) + .isInstanceOfSatisfying( + IggyValidationException.class, + refusal -> assertThat(refusal.getErrorCode()).isEqualTo(IggyErrorCode.OPTION_NOT_UPDATABLE)); + var after = streamsClient.getStream(created.id()).orElseThrow(); + assertThat(after.name()).isEqualTo("reset-keyed-stream"); + assertThat(after.encryptionKeyFingerprint()).contains(EncryptionKeys.fingerprint(key)); + } + + private static byte[] randomKey() { + byte[] key = new byte[EncryptionKeys.KEY_LENGTH]; + new SecureRandom().nextBytes(key); + return key; + } + + private static void assertNoKeyMaterial( + byte[] key, Map<String, HeaderValue> explicit, Map<String, HeaderValue> derived) { + String base64 = Base64.getEncoder().encodeToString(key); + for (Map<String, HeaderValue> options : List.of(explicit, derived)) { + assertThat(options).doesNotContainKeys(SECRET_KEYS.toArray(String[]::new)); + assertThat(options.values()) + .noneMatch(value -> Arrays.equals(value.value(), key) + || value.toStringValue().contains(base64)); + } + } }
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/SystemClientBaseTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/SystemClientBaseTest.java index ccdee76..93ee820 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/SystemClientBaseTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/SystemClientBaseTest.java
@@ -19,16 +19,24 @@ package org.apache.iggy.client.blocking; +import org.apache.commons.lang3.StringUtils; import org.apache.iggy.cluster.ClusterNodeRole; import org.apache.iggy.cluster.ClusterNodeStatus; import org.apache.iggy.message.HeaderKind; +import org.apache.iggy.stream.StreamOptionKey; import org.apache.iggy.system.OptionSpec; import org.apache.iggy.system.OptionsScope; +import org.apache.iggy.user.UserOptionKey; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import tools.jackson.databind.JsonNode; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; import java.util.stream.Collectors; +import static org.apache.iggy.ResourceOptionsFixture.section; import static org.assertj.core.api.Assertions.assertThat; public abstract class SystemClientBaseTest extends IntegrationTest { @@ -57,11 +65,36 @@ } @Test - void shouldDescribeEmptyCatalogsForScopesWithoutKeys() { - // Streams and users have no catalog keys yet, so the scope answers with - // an empty list rather than an error. - assertThat(systemClient.describeOptions(OptionsScope.Stream)).isEmpty(); - assertThat(systemClient.describeOptions(OptionsScope.User)).isEmpty(); + void shouldDescribeStreamAndUserOptionCatalogs() { + var stream = systemClient.describeOptions(OptionsScope.Stream).stream() + .collect(Collectors.toMap(OptionSpec::key, spec -> spec)); + assertThat(stream).containsKeys(StreamOptionKey.NAME.value(), StreamOptionKey.ENCRYPTION_KEY.value()); + assertThat(stream.get(StreamOptionKey.ENCRYPTION_KEY.value()).secret()).isTrue(); + assertThat(stream.get(StreamOptionKey.NAME.value()).updatable()).isTrue(); + + var user = systemClient.describeOptions(OptionsScope.User).stream() + .collect(Collectors.toMap(OptionSpec::key, spec -> spec)); + assertThat(user).containsKeys(UserOptionKey.USERNAME.value(), UserOptionKey.STATUS.value()); + } + + @Test + void shouldDescribeEveryScopeWithTheFixtureKindsAndFlags() { + for (JsonNode vector : section("describe_options")) { + var scope = OptionsScope.valueOf( + StringUtils.capitalize(vector.get("scope").asString())); + List<String> expected = new ArrayList<>(); + vector.get("entries") + .forEach(entry -> expected.add(entry.get("key").asString() + " " + + entry.get("kind").asString() + " " + + entry.get("flags").asInt())); + + var described = systemClient.describeOptions(scope).stream() + .map(spec -> spec.key() + " " + + spec.defaultValue().kind().name().toLowerCase(Locale.ROOT) + " " + spec.flags()) + .toList(); + + assertThat(described).as(scope.name()).isEqualTo(expected); + } } @Test
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/TopicsClientBaseTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/TopicsClientBaseTest.java index 48e64d2..74bd1c6 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/TopicsClientBaseTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/TopicsClientBaseTest.java
@@ -19,9 +19,14 @@ package org.apache.iggy.client.blocking; +import org.apache.iggy.exception.IggyErrorCode; +import org.apache.iggy.exception.IggyValidationException; import org.apache.iggy.identifier.TopicId; import org.apache.iggy.message.HeaderValue; import org.apache.iggy.topic.CompressionAlgorithm; +import org.apache.iggy.topic.TopicOptionKey; +import org.apache.iggy.topic.TopicOptions; +import org.apache.iggy.topic.TopicUpdate; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -34,6 +39,10 @@ public abstract class TopicsClientBaseTest extends IntegrationTest { + private static final BigInteger HOUR_MICROS = BigInteger.valueOf(3_600_000_000L); + private static final BigInteger GIB = BigInteger.valueOf(1_073_741_824L); + private static final BigInteger U64_MAX = BigInteger.ONE.shiftLeft(64).subtract(BigInteger.ONE); + protected TopicsClient topicsClient; @BeforeEach @@ -47,8 +56,7 @@ @Test void shouldCreateAndDeleteTopic() { // when - var topicDetails = topicsClient.createTopic( - STREAM_NAME, 1L, CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, "test-topic"); + var topicDetails = topicsClient.createTopic(STREAM_NAME, "test-topic", 1L); var topicId = TopicId.of("test-topic"); var topicOptional = topicsClient.getTopic(STREAM_NAME, topicId); @@ -67,36 +75,29 @@ @Test void shouldUpdateTopic() { // given - var topic = topicsClient.createTopic( - STREAM_NAME, 1L, CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, "test-topic"); + var topic = topicsClient.createTopic(STREAM_NAME, "test-topic", 1L); // when topicsClient.updateTopic( STREAM_NAME, TopicId.of(topic.id()), - CompressionAlgorithm.None, - BigInteger.valueOf(5000), - BigInteger.ZERO, - "new-name"); + TopicUpdate.builder() + .name("new-name") + .messageExpiry(BigInteger.valueOf(5000)) + .build()); // then var updatedTopic = topicsClient.getTopic(STREAM_NAME, TopicId.of(topic.id())).get(); assertThat(updatedTopic.name()).isEqualTo("new-name"); - assertThat(updatedTopic.messageExpiry()).isEqualTo(BigInteger.valueOf(5000)); + assertThat(updatedTopic.messageExpiry()).contains(BigInteger.valueOf(5000)); } @Test void shouldCarryOptionKeysWithNoParameterOfTheirOwn() { // when var created = topicsClient.createTopic( - STREAM_NAME, - 1L, - CompressionAlgorithm.None, - BigInteger.ZERO, - BigInteger.ZERO, - "options-topic", - Map.of("durability", HeaderValue.fromString("persisted"))); + STREAM_NAME, "options-topic", 1L, Map.of("durability", HeaderValue.fromString("persisted"))); // then var topic = topicsClient.getTopic(STREAM_NAME, TopicId.of(created.id())).orElseThrow(); @@ -111,17 +112,96 @@ @Test void shouldRejectAnOptionKeyOutsideTheCatalog() { assertThatThrownBy(() -> topicsClient.createTopic( - STREAM_NAME, - 1L, - CompressionAlgorithm.None, - BigInteger.ZERO, - BigInteger.ZERO, - "bad-options-topic", - Map.of("not_a_real_option", HeaderValue.fromString("1")))) + STREAM_NAME, "bad-options-topic", 1L, Map.of("not_a_real_option", HeaderValue.fromString("1")))) .isInstanceOf(RuntimeException.class); } @Test + void givenTopicResetWhenUpdatingShouldPreserveUnmentionedValues() { + // given + var created = topicsClient.createTopic( + STREAM_NAME, + "patched-topic", + 1L, + TopicOptions.builder() + .compressionAlgorithm(CompressionAlgorithm.Gzip) + .messageExpiry(HOUR_MICROS) + .maxTopicSize(GIB) + .build()); + var topicId = TopicId.of(created.id()); + + // when + topicsClient.updateTopic( + STREAM_NAME, + topicId, + TopicUpdate.builder() + .messageExpiry(HOUR_MICROS.shiftLeft(1)) + .reset(TopicOptionKey.MAX_TOPIC_SIZE) + .build()); + + // then + var topic = topicsClient.getTopic(STREAM_NAME, topicId).orElseThrow(); + assertThat(topic.name()).isEqualTo("patched-topic"); + assertThat(topic.options().get("message_expiry")).isEqualTo(HeaderValue.fromUint64(HOUR_MICROS.shiftLeft(1))); + assertThat(topic.options()).doesNotContainKey("max_topic_size"); + assertThat(topic.derivedOptions().get("max_topic_size")).isEqualTo(HeaderValue.fromUint64(U64_MAX)); + assertThat(topic.options().get("compression_algorithm")).isEqualTo(HeaderValue.fromString("gzip")); + assertThat(topic.compressionAlgorithm()).contains(CompressionAlgorithm.Gzip); + assertThat(topic.maxTopicSize()).contains(U64_MAX); + } + + @Test + void givenTopicRenameWhenUpdatingShouldKeepSettings() { + // given + var created = topicsClient.createTopic( + STREAM_NAME, + "rename-topic", + 1L, + TopicOptions.builder() + .messageExpiry(HOUR_MICROS) + .maxTopicSize(GIB) + .build()); + var topicId = TopicId.of(created.id()); + + // when + topicsClient.updateTopic( + STREAM_NAME, + topicId, + TopicUpdate.builder().name("renamed-topic").build()); + + // then + var topic = topicsClient.getTopic(STREAM_NAME, topicId).orElseThrow(); + assertThat(topic.name()).isEqualTo("renamed-topic"); + assertThat(topic.options().get("message_expiry")).isEqualTo(HeaderValue.fromUint64(HOUR_MICROS)); + assertThat(topic.options().get("max_topic_size")).isEqualTo(HeaderValue.fromUint64(GIB)); + } + + @Test + void givenCreateOnlyTopicKeyWhenUpdatingShouldRefuseOnClientAndServer() { + // given + var created = topicsClient.createTopic(STREAM_NAME, "fixed-topic", 1L); + var topicId = TopicId.of(created.id()); + + // when + var reset = TopicUpdate.builder().reset(TopicOptionKey.COMPRESSION_ALGORITHM); + var rawSet = TopicUpdate.builder() + .raw("compression_algorithm", HeaderValue.fromString("gzip")) + .build(); + + // then + assertThatThrownBy(reset::build) + .isInstanceOfSatisfying( + IggyValidationException.class, + refusal -> assertThat(refusal.getErrorCode()).isEqualTo(IggyErrorCode.OPTION_NOT_UPDATABLE)); + assertThatThrownBy(() -> topicsClient.updateTopic(STREAM_NAME, topicId, rawSet)) + .isInstanceOfSatisfying( + IggyValidationException.class, + refusal -> assertThat(refusal.getErrorCode()).isEqualTo(IggyErrorCode.OPTION_NOT_UPDATABLE)); + var topic = topicsClient.getTopic(STREAM_NAME, topicId).orElseThrow(); + assertThat(topic.compressionAlgorithm()).contains(CompressionAlgorithm.None); + } + + @Test void shouldReturnEmptyForNonExistingTopic() { // when var topic = topicsClient.getTopic(STREAM_NAME, TopicId.of(404L));
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/UsersClientBaseTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/UsersClientBaseTest.java index a5724ba..f88b36c 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/UsersClientBaseTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/UsersClientBaseTest.java
@@ -19,12 +19,18 @@ package org.apache.iggy.client.blocking; +import org.apache.iggy.exception.IggyErrorCode; +import org.apache.iggy.exception.IggyServerException; +import org.apache.iggy.exception.IggyValidationException; import org.apache.iggy.user.GlobalPermissions; import org.apache.iggy.user.IdentityInfo; import org.apache.iggy.user.Permissions; import org.apache.iggy.user.UserInfo; import org.apache.iggy.user.UserInfoDetails; +import org.apache.iggy.user.UserOptionKey; +import org.apache.iggy.user.UserOptions; import org.apache.iggy.user.UserStatus; +import org.apache.iggy.user.UserUpdate; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -33,6 +39,7 @@ import java.util.Optional; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; public abstract class UsersClientBaseTest extends IntegrationTest { @@ -112,7 +119,8 @@ trackUser(user.id()); // when - usersClient.updateUser(user.id(), Optional.of("new-name"), Optional.empty()); + usersClient.updateUser( + user.id(), UserUpdate.builder().username("new-name").build()); // then var updatedUser = usersClient.getUser(user.id()); @@ -129,7 +137,12 @@ trackUser(user.id()); // when - usersClient.updateUser(user.id(), Optional.of("renamed"), Optional.of(UserStatus.Inactive)); + usersClient.updateUser( + user.id(), + UserUpdate.builder() + .username("renamed") + .status(UserStatus.Inactive) + .build()); // then var updatedUser = usersClient.getUser(user.id()); @@ -146,7 +159,8 @@ trackUser(user.id()); // when - usersClient.updateUser(user.id(), Optional.empty(), Optional.of(UserStatus.Inactive)); + usersClient.updateUser( + user.id(), UserUpdate.builder().status(UserStatus.Inactive).build()); // then List<UserInfo> users = usersClient.getUsers(); @@ -209,6 +223,52 @@ } @Test + void givenUserOptionsWhenCreatingAndUpdatingShouldPreserveUserBehavior() { + // given + login(); + UserInfoDetails first = usersClient.createUser( + "options-user", "secret", UserStatus.Active, Optional.empty(), UserOptions.empty()); + trackUser(first.id()); + UserInfoDetails second = usersClient.createUser("other-user", "secret", UserStatus.Active, Optional.empty()); + trackUser(second.id()); + + // when + usersClient.updateUser( + first.id(), + UserUpdate.builder() + .username("options-user-renamed") + .status(UserStatus.Inactive) + .build()); + + // then + var updated = usersClient.getUser(first.id()).orElseThrow(); + assertThat(updated.username()).isEqualTo("options-user-renamed"); + assertThat(updated.status()).isEqualTo(UserStatus.Inactive); + assertThat(updated.options()).isEmpty(); + assertThat(updated.derivedOptions()).isEmpty(); + assertThat(first.options()).isEmpty(); + + // when a duplicate username rides with a status change + assertThatThrownBy(() -> usersClient.updateUser( + second.id(), + UserUpdate.builder() + .username("options-user-renamed") + .status(UserStatus.Inactive) + .build())) + .isInstanceOf(IggyServerException.class); + + // then neither change applied + var untouched = usersClient.getUser(second.id()).orElseThrow(); + assertThat(untouched.username()).isEqualTo("other-user"); + assertThat(untouched.status()).isEqualTo(UserStatus.Active); + assertThatThrownBy( + () -> UserUpdate.builder().reset(UserOptionKey.STATUS).build()) + .isInstanceOfSatisfying( + IggyValidationException.class, + refusal -> assertThat(refusal.getErrorCode()).isEqualTo(IggyErrorCode.OPTION_NOT_RESETTABLE)); + } + + @Test void shouldReturnEmptyForNonExistingUser() { // given login();
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/http/HttpOptionValueTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/http/HttpOptionValueTest.java new file mode 100644 index 0000000..a2e8710 --- /dev/null +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/http/HttpOptionValueTest.java
@@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.client.blocking.http; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import org.apache.iggy.exception.IggyMalformedResponseException; +import org.apache.iggy.message.HeaderKind; +import org.apache.iggy.message.HeaderValue; +import org.apache.iggy.serde.BytesDeserializer; +import org.apache.iggy.stream.StreamBase; +import org.apache.iggy.topic.Topic; +import org.apache.iggy.user.UserInfo; +import org.junit.jupiter.api.Test; +import tools.jackson.core.type.TypeReference; +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.ObjectMapper; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HexFormat; +import java.util.List; +import java.util.Map; + +import static org.apache.iggy.ResourceOptionsFixture.bytes; +import static org.apache.iggy.ResourceOptionsFixture.optionMap; +import static org.apache.iggy.ResourceOptionsFixture.section; +import static org.apache.iggy.ResourceOptionsFixture.streamUpdate; +import static org.apache.iggy.ResourceOptionsFixture.topicUpdate; +import static org.apache.iggy.ResourceOptionsFixture.userUpdate; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class HttpOptionValueTest { + + private static final ObjectMapper MAPPER = ObjectMapperFactory.getInstance(); + private static final TypeReference<Map<String, HttpOptionValue>> OPTIONS = new TypeReference<>() {}; + + @Test + void givenServerDescribeJsonWhenMappingShouldRebuildFixtureFlags() { + JsonNode flagBits = section("constants").get("option_flags"); + for (JsonNode vector : section("describe_options")) { + var json = MAPPER.createArrayNode(); + for (JsonNode entry : vector.get("entries")) { + int flags = entry.get("flags").asInt(); + var spec = json.addObject() + .put("key", entry.get("key").asString()) + .put("kind", entry.get("kind").asString()) + .put("description", ""); + spec.putArray("default_value"); + for (String flag : List.of("updatable", "resettable", "secret", "read_only", "critical")) { + spec.put(flag, (flags & flagBits.get(flag).asInt()) != 0); + } + } + List<SystemHttpClient.HttpOptionSpec> specs = + MAPPER.readValue(MAPPER.writeValueAsString(json), new TypeReference<>() {}); + List<String> expected = new ArrayList<>(); + vector.get("entries") + .forEach(entry -> expected.add(entry.get("key").asString() + " " + + entry.get("flags").asInt())); + + assertThat(specs.stream() + .map(SystemHttpClient.HttpOptionSpec::toOptionSpec) + .map(spec -> spec.key() + " " + spec.flags()) + .toList()) + .as(vector.get("scope").asString()) + .isEqualTo(expected); + } + } + + @Test + void givenSharedResponsesWhenDecodingHttpOptionsShouldMatchBinaryBlocks() { + for (JsonNode vector : section("stream_responses")) { + StreamBase stream = BytesDeserializer.readStreamBase(response(vector)); + assertSameBlocks(vector, stream.options(), stream.derivedOptions()); + } + for (JsonNode vector : section("topic_responses")) { + Topic topic = BytesDeserializer.readTopic(response(vector)); + assertSameBlocks(vector, topic.options(), topic.derivedOptions()); + } + for (JsonNode vector : section("user_responses")) { + UserInfo user = BytesDeserializer.readUserInfo(response(vector)); + assertSameBlocks(vector, user.options(), user.derivedOptions()); + } + } + + @Test + void givenTypedUpdatesWhenRenderingPutBodyShouldMatchWireSetAndReset() { + Map<String, JsonNode> bodiesByWire = new HashMap<>(); + for (JsonNode vector : section("update_resource")) { + JsonNode body = render(updateBody(vector)); + String name = vector.get("case").asString(); + var expectedSet = optionMap(vector.get("wire_set"), true); + List<String> expectedReset = new ArrayList<>(); + vector.get("wire_reset").forEach(key -> expectedReset.add(key.asString())); + List<String> keys = new ArrayList<>(); + body.properties().forEach(entry -> keys.add(entry.getKey())); + List<String> expectedKeys = new ArrayList<>(expectedSet.keySet()); + if (!expectedReset.isEmpty()) { + expectedKeys.add("reset"); + } + assertThat(keys).as(name).containsExactlyInAnyOrderElementsOf(expectedKeys); + expectedSet.forEach((key, value) -> + assertThat(body.get(key).asString()).as(name + " " + key).isEqualTo(value.toStringValue())); + List<String> reset = new ArrayList<>(); + if (body.has("reset")) { + body.get("reset").forEach(key -> reset.add(key.asString())); + } + assertThat(reset).as(name).isEqualTo(expectedReset); + bodiesByWire.put(vector.get("body_hex").asString(), body); + } + int matched = 0; + for (JsonNode vector : section("http_update")) { + JsonNode wire = vector.get("update_resource_hex"); + if (vector.get("outcome").asString().equals("accepted") && bodiesByWire.containsKey(wire.asString())) { + assertThat(bodiesByWire.get(wire.asString())) + .as(vector.get("case").asString()) + .isEqualTo(MAPPER.readTree(vector.get("body").asString())); + matched++; + } + } + assertThat(matched).isEqualTo(3); + } + + @Test + void givenEveryHeaderKindWhenDecodingHttpTextShouldRebuildExactBytes() { + Map<String, HttpOptionValue> options = Map.of( + "int8", new HttpOptionValue("-5", HeaderKind.Int8, true), + "int128", new HttpOptionValue("-170141183460469231731687303715884105728", HeaderKind.Int128, true), + "uint128", new HttpOptionValue("340282366920938463463374607431768211455", HeaderKind.Uint128, true), + "uint16", new HttpOptionValue("65535", HeaderKind.Uint16, true), + "nan", new HttpOptionValue("0x7fc00001", HeaderKind.Float32, true), + "negative_zero", new HttpOptionValue("0x8000000000000000", HeaderKind.Float64, true), + "raw", new HttpOptionValue("AAEC", HeaderKind.Raw, true), + "bool", new HttpOptionValue("false", HeaderKind.Bool, true)); + + Map<String, HeaderValue> decoded = HttpOptionValue.select(options, true); + + assertThat(hex(decoded.get("int8"))).isEqualTo("fb"); + assertThat(hex(decoded.get("int128"))).isEqualTo("00000000000000000000000000000080"); + assertThat(hex(decoded.get("uint128"))).isEqualTo("ffffffffffffffffffffffffffffffff"); + assertThat(hex(decoded.get("uint16"))).isEqualTo("ffff"); + assertThat(hex(decoded.get("nan"))).isEqualTo("0100c07f"); + assertThat(hex(decoded.get("negative_zero"))).isEqualTo("0000000000000080"); + assertThat(hex(decoded.get("raw"))).isEqualTo("000102"); + assertThat(decoded.get("bool")).isEqualTo(HeaderValue.fromBool(false)); + for (HttpOptionValue noncanonical : List.of( + new HttpOptionValue("+5", HeaderKind.Int8, true), + new HttpOptionValue("05", HeaderKind.Uint32, true), + new HttpOptionValue("256", HeaderKind.Uint8, true), + new HttpOptionValue("AAE", HeaderKind.Raw, true), + new HttpOptionValue("0x7FC00001", HeaderKind.Float32, true), + new HttpOptionValue("TRUE", HeaderKind.Bool, true))) { + assertThatThrownBy(() -> HttpOptionValue.select(Map.of("key", noncanonical), true)) + .as(noncanonical.toString()) + .isInstanceOf(IggyMalformedResponseException.class); + } + } + + private static void assertSameBlocks( + JsonNode vector, Map<String, HeaderValue> explicit, Map<String, HeaderValue> derived) { + Map<String, HttpOptionValue> options = + MAPPER.convertValue(vector.get("expected").get("options"), OPTIONS); + String name = vector.get("case").asString(); + assertThat(HttpOptionValue.select(options, true)).as(name).isEqualTo(explicit); + assertThat(HttpOptionValue.select(options, false)).as(name).isEqualTo(derived); + } + + private static Map<String, Object> updateBody(JsonNode vector) { + JsonNode input = vector.get("input"); + return switch (vector.get("scope").asString()) { + case "topic" -> { + var update = topicUpdate(input); + yield HttpOptionValue.updateBody(update.entries(), update.resetKeys()); + } + case "stream" -> { + var update = streamUpdate(input); + yield HttpOptionValue.updateBody(update.entries(), update.resetKeys()); + } + default -> { + var update = userUpdate(input); + yield HttpOptionValue.updateBody(update.entries(), update.resetKeys()); + } + }; + } + + private static JsonNode render(Map<String, Object> body) { + return MAPPER.readTree(MAPPER.writeValueAsString(body)); + } + + private static ByteBuf response(JsonNode vector) { + return Unpooled.wrappedBuffer(bytes(vector.get("response_hex").asString())); + } + + private static String hex(HeaderValue value) { + return HexFormat.of().formatHex(value.value()); + } +}
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/http/InternalHttpClientLoggingTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/http/InternalHttpClientLoggingTest.java new file mode 100644 index 0000000..93d4136 --- /dev/null +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/http/InternalHttpClientLoggingTest.java
@@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.client.blocking.http; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; + +class InternalHttpClientLoggingTest { + + private static final String ENCRYPTION_KEY = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8="; + private static final String PASSWORD = "secret-password"; + + @Test + void givenDebugLoggingWhenPreparingBodiesShouldNotLogSecrets() throws Exception { + var logger = (Logger) LoggerFactory.getLogger(InternalHttpClient.class); + var previousLevel = logger.getLevel(); + var appender = new ListAppender<ILoggingEvent>(); + appender.start(); + logger.addAppender(appender); + logger.setLevel(Level.DEBUG); + try (var client = + new InternalHttpClient("http://127.0.0.1:1", Optional.empty(), Optional.empty(), Optional.empty())) { + client.preparePostRequest( + "/streams", Map.of("name", "keyed", "options", Map.of("encryption_key", ENCRYPTION_KEY))); + client.preparePutRequest("/users/1/password", Map.of("new_password", PASSWORD)); + } finally { + logger.detachAppender(appender); + logger.setLevel(previousLevel); + } + + assertThat(appender.list) + .extracting(ILoggingEvent::getFormattedMessage) + .noneMatch(message -> message.contains(ENCRYPTION_KEY) || message.contains(PASSWORD)); + } +}
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/http/MessagesHttpClientSendContextTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/http/MessagesHttpClientSendContextTest.java index 1a449c5..3229b1d 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/http/MessagesHttpClientSendContextTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/http/MessagesHttpClientSendContextTest.java
@@ -30,7 +30,6 @@ import org.apache.iggy.message.Partitioning; import org.apache.iggy.message.PollingStrategy; import org.apache.iggy.partition.PartitionContext; -import org.apache.iggy.topic.CompressionAlgorithm; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; @@ -208,8 +207,7 @@ send(client, Partitioning.partitionId(3L)); client.streams().createStream("stream"); - client.topics() - .createTopic(STREAM, 1L, CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, "topic"); + client.topics().createTopic(STREAM, "topic", 1L); send(client, Partitioning.partitionId(3L)); assertThat(server.requests).filteredOn(GET_TOPIC::equals).hasSize(1);
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/http/StreamHttpClientTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/http/StreamHttpClientTest.java index 30ae81b..4d40ffe 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/http/StreamHttpClientTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/http/StreamHttpClientTest.java
@@ -21,6 +21,15 @@ import org.apache.iggy.client.blocking.IggyBaseClient; import org.apache.iggy.client.blocking.StreamClientBaseTest; +import org.apache.iggy.message.HeaderKind; +import org.apache.iggy.stream.EncryptionKeys; +import org.apache.iggy.stream.StreamBase; +import org.apache.iggy.stream.StreamOptions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; + +import static org.assertj.core.api.Assertions.assertThat; class StreamHttpClientTest extends StreamClientBaseTest { @@ -28,4 +37,32 @@ protected IggyBaseClient getClient() { return HttpClientFactory.create(serverHost(), serverHttpPort()); } + + @Test + void givenHttpStreamOptionsWhenListingAndCreatingShouldPreserveBothBlocks() { + // given + byte[] key = new byte[EncryptionKeys.KEY_LENGTH]; + Arrays.fill(key, (byte) 3); + + // when + var created = client.streams() + .createStream( + "http-options-stream", + StreamOptions.builder().encryptionKey(key).build()); + trackStream(created.id()); + StreamBase listed = client.streams().getStreams().stream() + .filter(stream -> stream.id().equals(created.id())) + .findFirst() + .orElseThrow(); + + // then + assertThat(created.options()).isEmpty(); + assertThat(listed.options()).isEmpty(); + assertThat(created.derivedOptions()).isEqualTo(listed.derivedOptions()); + assertThat(listed.derivedOptions()) + .containsKeys("encryption_key_source", "encryption_key_fingerprint", "encryption_kek_fingerprint"); + assertThat(listed.derivedOptions().values()) + .allSatisfy(value -> assertThat(value.kind()).isEqualTo(HeaderKind.String)); + assertThat(listed.encryptionKeyFingerprint()).contains(EncryptionKeys.fingerprint(key)); + } }
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/MessagesTcpClientTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/MessagesTcpClientTest.java index 3bdf86f..dbb96d2 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/MessagesTcpClientTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/MessagesTcpClientTest.java
@@ -27,10 +27,8 @@ import org.apache.iggy.message.Message; import org.apache.iggy.message.Partitioning; import org.apache.iggy.message.PollingStrategy; -import org.apache.iggy.topic.CompressionAlgorithm; import org.junit.jupiter.api.Test; -import java.math.BigInteger; import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Optional; @@ -80,15 +78,7 @@ var topicId = TopicId.of("тема-日本語"); var stream = client.streams().createStream(streamId.getName()); trackStream(stream.id()); - client.topics() - .createTopic( - streamId, - 1L, - CompressionAlgorithm.None, - BigInteger.ZERO, - BigInteger.ZERO, - topicId.getName(), - PERSISTED_TOPIC_OPTIONS); + client.topics().createTopic(streamId, topicId.getName(), 1L, PERSISTED_TOPIC_OPTIONS); String text = "wiadomość 世界 😀"; // when
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/TlsConnectionTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/TlsConnectionTest.java index f284ad5..941e509 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/TlsConnectionTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/TlsConnectionTest.java
@@ -27,7 +27,6 @@ import org.apache.iggy.message.PolledMessages; import org.apache.iggy.message.PollingStrategy; import org.apache.iggy.stream.StreamDetails; -import org.apache.iggy.topic.CompressionAlgorithm; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; import org.slf4j.Logger; @@ -129,8 +128,7 @@ StreamDetails stream = client.streams().createStream(streamName); assertThat(stream).isNotNull(); - client.topics() - .createTopic(streamId, 1L, CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, topicName); + client.topics().createTopic(streamId, topicName, 1L); List<Message> messages = List.of(Message.of("tls-message-1"), Message.of("tls-message-2"), Message.of("tls-message-3"));
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/TopicsTcpClientTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/TopicsTcpClientTest.java index d76e35c..cf75940 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/TopicsTcpClientTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/TopicsTcpClientTest.java
@@ -24,10 +24,8 @@ import org.apache.iggy.identifier.TopicId; import org.apache.iggy.message.HeaderKind; import org.apache.iggy.message.HeaderValue; -import org.apache.iggy.topic.CompressionAlgorithm; import org.junit.jupiter.api.Test; -import java.math.BigInteger; import java.util.Map; import static org.apache.iggy.TestConstants.STREAM_NAME; @@ -47,13 +45,7 @@ // arrives back in the key's catalog kind whatever kind it was sent as. // Only the binary transport carries kinds: REST renders values as strings. var created = topicsClient.createTopic( - STREAM_NAME, - 1L, - CompressionAlgorithm.None, - BigInteger.ZERO, - BigInteger.ZERO, - "canonical-kind-topic", - Map.of("durability", HeaderValue.fromString("persisted"))); + STREAM_NAME, "canonical-kind-topic", 1L, Map.of("durability", HeaderValue.fromString("persisted"))); var topic = topicsClient.getTopic(STREAM_NAME, TopicId.of(created.id())).orElseThrow(); assertThat(topic.options().get("durability").kind()).isEqualTo(HeaderKind.String); @@ -62,13 +54,7 @@ @Test void shouldRejectAnOptionKeyOutsideTheCatalog() { assertThatThrownBy(() -> topicsClient.createTopic( - STREAM_NAME, - 1L, - CompressionAlgorithm.None, - BigInteger.ZERO, - BigInteger.ZERO, - "bad-options-topic", - Map.of("not_a_real_option", HeaderValue.fromString("1")))) + STREAM_NAME, "bad-options-topic", 1L, Map.of("not_a_real_option", HeaderValue.fromString("1")))) .isInstanceOf(RuntimeException.class); } }
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/exception/IggyValidationExceptionTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/exception/IggyValidationExceptionTest.java index 51f5f7a..676fc03 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/exception/IggyValidationExceptionTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/exception/IggyValidationExceptionTest.java
@@ -64,7 +64,14 @@ "TOO_MANY_MESSAGES", "EMPTY_MESSAGE_PAYLOAD", "TOO_BIG_MESSAGE_PAYLOAD", - "INVALID_MESSAGE_CHECKSUM" + "INVALID_MESSAGE_CHECKSUM", + "INVALID_ENCRYPTION_KEY", + "UNSUPPORTED_OPTION_KEY", + "INVALID_OPTION_VALUE", + "OPTIONS_BLOCK_TOO_LARGE", + "OPTION_NOT_UPDATABLE", + "OPTION_NOT_RESETTABLE", + "READ_ONLY_OPTION" }) void matchesReturnsTrueForValidationRelatedCodes(IggyErrorCode code) { assertThat(IggyValidationException.matches(code)).isTrue(); @@ -92,7 +99,14 @@ "TOO_MANY_MESSAGES", "EMPTY_MESSAGE_PAYLOAD", "TOO_BIG_MESSAGE_PAYLOAD", - "INVALID_MESSAGE_CHECKSUM" + "INVALID_MESSAGE_CHECKSUM", + "INVALID_ENCRYPTION_KEY", + "UNSUPPORTED_OPTION_KEY", + "INVALID_OPTION_VALUE", + "OPTIONS_BLOCK_TOO_LARGE", + "OPTION_NOT_UPDATABLE", + "OPTION_NOT_RESETTABLE", + "READ_ONLY_OPTION" }, mode = Mode.EXCLUDE) void matchesReturnsFalseForNonValidationRelatedCodes(IggyErrorCode code) {
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/BytesDeserializerTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/BytesDeserializerTest.java index f77a48b..9f10f17 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/BytesDeserializerTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/BytesDeserializerTest.java
@@ -29,8 +29,8 @@ import org.apache.iggy.message.HeaderKey; import org.apache.iggy.message.HeaderKind; import org.apache.iggy.message.HeaderValue; +import org.apache.iggy.stream.EncryptionKeySource; import org.apache.iggy.system.CacheMetricsKey; -import org.apache.iggy.topic.CompressionAlgorithm; import org.apache.iggy.user.UserStatus; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -84,9 +84,6 @@ buffer.writeIntLE(10); // topic ID writeU64(buffer, BigInteger.valueOf(1000)); // created at buffer.writeIntLE(4); // partitions count - writeU64(buffer, BigInteger.ZERO); // message expiry - buffer.writeByte(CompressionAlgorithm.None.asCode()); // compression - writeU64(buffer, BigInteger.valueOf(10000)); // max topic size writeU64(buffer, BigInteger.valueOf(500)); // size writeU64(buffer, BigInteger.valueOf(50)); // messages count buffer.writeByte(4); // name length @@ -100,9 +97,6 @@ buffer.writeIntLE(10); writeU64(buffer, BigInteger.valueOf(1000)); buffer.writeIntLE(4); - writeU64(buffer, BigInteger.ZERO); - buffer.writeByte(CompressionAlgorithm.None.asCode()); - writeU64(buffer, BigInteger.valueOf(10000)); writeU64(buffer, BigInteger.valueOf(500)); writeU64(buffer, BigInteger.valueOf(50)); buffer.writeByte(4); @@ -202,6 +196,8 @@ buffer.writeByte(11); // name length buffer.writeBytes("test-stream".getBytes(StandardCharsets.UTF_8)); writeOptionsBlock(buffer, Map.of()); + writeOptionsBlock( + buffer, Map.of(HeaderKey.fromString("encryption_key_source"), HeaderValue.fromString("server"))); // when var stream = readStreamBase(buffer); @@ -211,6 +207,8 @@ assertThat(stream.createdAt()).isEqualTo(BigInteger.valueOf(1000)); assertThat(stream.topicsCount()).isEqualTo(2L); assertThat(stream.name()).isEqualTo("test-stream"); + assertThat(stream.options()).isEmpty(); + assertThat(stream.encryptionKeySource()).contains(EncryptionKeySource.SERVER); } @Test @@ -226,6 +224,7 @@ buffer.writeByte(6); buffer.writeBytes("stream".getBytes()); writeOptionsBlock(buffer, Map.of()); + writeOptionsBlock(buffer, Map.of()); // Write one topic writeTopicData(buffer); @@ -812,6 +811,7 @@ buffer.writeByte(4); // username length buffer.writeBytes("user".getBytes()); writeOptionsBlock(buffer, Map.of()); + writeOptionsBlock(buffer, Map.of()); // when var userInfo = readUserInfo(buffer); @@ -833,6 +833,7 @@ buffer.writeByte(5); buffer.writeBytes("admin".getBytes()); writeOptionsBlock(buffer, Map.of()); + writeOptionsBlock(buffer, Map.of()); buffer.writeIntLE(0); // no-permissions marker: u32_le(0) // when @@ -853,6 +854,7 @@ buffer.writeByte(5); buffer.writeBytes("admin".getBytes()); writeOptionsBlock(buffer, Map.of()); + writeOptionsBlock(buffer, Map.of()); buffer.writeBoolean(true); // has permissions buffer.writeIntLE(10); // permissions length (ignored but required) // Write global permissions (10 booleans)
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/ResourceOptionsTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/ResourceOptionsTest.java new file mode 100644 index 0000000..cbb032b --- /dev/null +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/ResourceOptionsTest.java
@@ -0,0 +1,670 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.serde; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import org.apache.iggy.exception.IggyErrorCode; +import org.apache.iggy.exception.IggyValidationException; +import org.apache.iggy.identifier.StreamId; +import org.apache.iggy.message.HeaderKind; +import org.apache.iggy.message.HeaderValue; +import org.apache.iggy.stream.EncryptionKeySource; +import org.apache.iggy.stream.EncryptionKeys; +import org.apache.iggy.stream.StreamBase; +import org.apache.iggy.stream.StreamDetails; +import org.apache.iggy.stream.StreamOptionKey; +import org.apache.iggy.stream.StreamOptions; +import org.apache.iggy.stream.StreamUpdate; +import org.apache.iggy.system.OptionKey; +import org.apache.iggy.system.OptionSpec; +import org.apache.iggy.system.OptionsScope; +import org.apache.iggy.topic.CompressionAlgorithm; +import org.apache.iggy.topic.Durability; +import org.apache.iggy.topic.Topic; +import org.apache.iggy.topic.TopicDetails; +import org.apache.iggy.topic.TopicOptionKey; +import org.apache.iggy.topic.TopicOptions; +import org.apache.iggy.topic.TopicUpdate; +import org.apache.iggy.user.UserInfo; +import org.apache.iggy.user.UserInfoDetails; +import org.apache.iggy.user.UserOptionKey; +import org.apache.iggy.user.UserOptions; +import org.apache.iggy.user.UserStatus; +import org.apache.iggy.user.UserUpdate; +import org.assertj.core.api.ThrowableAssert.ThrowingCallable; +import org.junit.jupiter.api.Test; +import tools.jackson.databind.JsonNode; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HexFormat; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.IntStream; + +import static org.apache.iggy.ResourceOptionsFixture.bytes; +import static org.apache.iggy.ResourceOptionsFixture.hex; +import static org.apache.iggy.ResourceOptionsFixture.kind; +import static org.apache.iggy.ResourceOptionsFixture.optionMap; +import static org.apache.iggy.ResourceOptionsFixture.section; +import static org.apache.iggy.ResourceOptionsFixture.streamId; +import static org.apache.iggy.ResourceOptionsFixture.streamUpdate; +import static org.apache.iggy.ResourceOptionsFixture.topicId; +import static org.apache.iggy.ResourceOptionsFixture.topicUpdate; +import static org.apache.iggy.ResourceOptionsFixture.userId; +import static org.apache.iggy.ResourceOptionsFixture.userUpdate; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.SoftAssertions.assertSoftly; + +/** + * The shared resource options vectors against this SDK's own encoder and decoder. + * + * <p>The Java SDK encodes the wire itself, so every request here must match the Rust encoder byte for byte + * and every response must decode to the projection the Rust reader returns. + */ +class ResourceOptionsTest { + + @Test + void givenSharedVectorsWhenEncodingShouldMatchServerContract() { + for (JsonNode vector : section("update_resource")) { + String name = vector.get("case").asString(); + assertThat(hex(encodeUpdate(vector))) + .as(name) + .isEqualTo(vector.get("body_hex").asString()); + } + for (JsonNode vector : section("create_stream")) { + JsonNode input = vector.get("input"); + var options = StreamOptions.builder(); + Optional.ofNullable(input.get("encryption_key")) + .ifPresent(key -> + options.encryptionKey(bytes(key.get("raw_hex").asString()))); + Optional.ofNullable(input.get("raw")) + .ifPresent(raw -> raw.properties() + .forEach(entry -> options.raw( + entry.getKey(), + HeaderValue.fromString(entry.getValue().asString())))); + ByteBuf body = BytesSerializer.toCreateStream( + input.get("name").asString(), options.build().entries()); + assertThat(hex(body)) + .as(vector.get("case").asString()) + .isEqualTo(vector.get("body_hex").asString()); + } + for (JsonNode vector : section("create_topic")) { + JsonNode input = vector.get("input"); + var options = TopicOptions.builder() + .compressionAlgorithm(CompressionAlgorithm.Gzip) + .messageExpiry(new BigInteger(input.get("message_expiry").asString())) + .maxTopicSize(new BigInteger(input.get("max_topic_size").asString())) + .segmentSize(new BigInteger(input.get("segment_size").asString())) + .durability(Durability.PERSISTED) + .consumerOffsetDurability(Durability.REPLICATED) + .build(); + assertThat(input.get("compression_algorithm").asString()).isEqualTo("gzip"); + ByteBuf body = BytesSerializer.toCreateTopic( + StreamId.of(input.get("stream_id").asLong()), + input.get("name").asString(), + input.get("partitions_count").asLong(), + options.entries()); + assertThat(hex(body)) + .as(vector.get("case").asString()) + .isEqualTo(vector.get("body_hex").asString()); + } + for (JsonNode vector : section("create_user")) { + JsonNode input = vector.get("input"); + ByteBuf body = BytesSerializer.toCreateUser( + input.get("username").asString(), + input.get("password").asString(), + UserStatus.Active, + Optional.empty(), + UserOptions.empty().entries()); + assertThat(hex(body)) + .as(vector.get("case").asString()) + .isEqualTo(vector.get("body_hex").asString()); + } + } + + @Test + void givenSharedVectorsWhenSplittingUpdatesShouldMatchWireSetAndReset() { + for (JsonNode vector : section("update_resource")) { + JsonNode input = vector.get("input"); + var expectedSet = optionMap(vector.get("wire_set"), true); + List<String> expectedReset = new ArrayList<>(); + vector.get("wire_reset").forEach(key -> expectedReset.add(key.asString())); + String name = vector.get("case").asString(); + switch (vector.get("scope").asString()) { + case "topic" -> { + assertThat(topicUpdate(input).entries()).as(name).isEqualTo(expectedSet); + assertThat(topicUpdate(input).resetKeys()).as(name).isEqualTo(expectedReset); + } + case "stream" -> { + assertThat(streamUpdate(input).entries()).as(name).isEqualTo(expectedSet); + assertThat(streamUpdate(input).resetKeys()).as(name).isEqualTo(expectedReset); + } + default -> { + assertThat(userUpdate(input).entries()).as(name).isEqualTo(expectedSet); + assertThat(userUpdate(input).resetKeys()).as(name).isEqualTo(expectedReset); + } + } + } + } + + @Test + void givenSharedUpdateErrorsWhenBuildingShouldRefuseWithServerCategoryBeforeSending() { + for (JsonNode vector : section("update_resource_errors")) { + JsonNode error = vector.get("error"); + if (namesKeyWithoutConstant(vector)) { + // A reset this build has no constant for travels raw, and the server decides it. + assertThat(hex(encodeUpdate(vector))) + .as(vector.get("case").asString()) + .isNotEmpty(); + continue; + } + assertThatThrownBy(() -> encodeUpdate(vector)) + .as(vector.get("case").asString()) + .isInstanceOfSatisfying(IggyValidationException.class, refusal -> { + assertThat(refusal.getRawErrorCode()) + .isEqualTo(error.get("code").asInt()); + assertThat(refusal.getErrorCode().name().toLowerCase(Locale.ROOT)) + .isEqualTo(error.get("name").asString()); + }); + } + } + + @Test + void givenValuesTheServerRefusesWhenBuildingShouldReportTheServerCode() { + assertRefusal(() -> TopicUpdate.builder().name("").build(), IggyErrorCode.INVALID_TOPIC_NAME); + assertRefusal(() -> StreamUpdate.builder().name("n".repeat(256)).build(), IggyErrorCode.INVALID_STREAM_NAME); + assertRefusal(() -> UserUpdate.builder().username("ab").build(), IggyErrorCode.INVALID_USERNAME); + assertRefusal( + () -> TopicUpdate.builder() + .name("") + .reset(TopicOptionKey.SEGMENT_SIZE) + .build(), + IggyErrorCode.OPTION_NOT_UPDATABLE); + assertRefusal(() -> StreamOptions.builder().encryptionKey(new byte[31]), IggyErrorCode.INVALID_ENCRYPTION_KEY); + assertRefusal(() -> StreamOptions.builder().encryptionKeyBase64("AAAA"), IggyErrorCode.INVALID_ENCRYPTION_KEY); + assertThat(StreamUpdate.builder().name(" ").build().entries().get(StreamOptionKey.NAME.value())) + .isEqualTo(new HeaderValue(HeaderKind.String, new byte[] {' '})); + } + + @Test + void givenRawResetWhenBuildingUpdateShouldSendItForTheServerToDecide() { + assertThat(UserUpdate.builder().rawReset("critical_keys").build().resetKeys()) + .containsExactly("critical_keys"); + TopicUpdate topic = TopicUpdate.builder() + .name("") + .reset(TopicOptionKey.SEGMENT_SIZE) + .reset(TopicOptionKey.MESSAGE_EXPIRY) + .rawReset("message_expiry") + .rawReset("future_key") + .build(); + assertThat(topic.resetKeys()).containsExactly("future_key", "message_expiry", "segment_size"); + assertThat(topic.entries().get(TopicOptionKey.NAME.value()).kind()).isEqualTo(HeaderKind.Bool); + assertRefusal( + () -> TopicUpdate.builder() + .messageExpiry(BigInteger.ONE) + .rawReset("message_expiry") + .build(), + IggyErrorCode.INVALID_COMMAND); + assertRefusal(() -> StreamUpdate.builder().build(), IggyErrorCode.INVALID_COMMAND); + } + + @Test + void givenMoreResetNamesThanTheCountByteCarriesWhenBuildingShouldRefuseAnUnsupportedKey() { + StreamUpdate.Builder atLimit = StreamUpdate.builder(); + futureKeys(255).forEach(atLimit::rawReset); + ByteBuf payload = BytesSerializer.toUpdateResource(StreamId.of(1L), atLimit.build()); + try { + int resetCountIndex = 1 + 6; + assertThat(payload.getUnsignedByte(resetCountIndex)).isEqualTo((short) 255); + } finally { + payload.release(); + } + + TopicUpdate.Builder overLimit = + TopicUpdate.builder().reset(TopicOptionKey.MESSAGE_EXPIRY).rawReset("compression_algorithm"); + futureKeys(255).forEach(overLimit::rawReset); + assertThatThrownBy(overLimit::build).isInstanceOfSatisfying(IggyValidationException.class, refusal -> { + assertThat(refusal.getErrorCode()).isEqualTo(IggyErrorCode.UNSUPPORTED_OPTION_KEY); + assertThat(refusal.getField()).contains("future_000"); + }); + + UserUpdate.Builder rawOnly = UserUpdate.builder(); + futureKeys(256).forEach(rawOnly::rawReset); + assertRefusal(rawOnly::build, IggyErrorCode.UNSUPPORTED_OPTION_KEY); + } + + @Test + void givenResetNameTheWireCannotCarryWhenBuildingShouldRefuseAnUnsupportedKey() { + String longest = "é".repeat(127) + "k"; + ByteBuf payload = BytesSerializer.toUpdateResource( + StreamId.of(1L), StreamUpdate.builder().rawReset(longest).build()); + payload.release(); + for (String name : List.of("", "é".repeat(128))) { + assertThatThrownBy(() -> UserUpdate.builder().rawReset(name).build()) + .isInstanceOfSatisfying(IggyValidationException.class, refusal -> { + assertThat(refusal.getErrorCode()).isEqualTo(IggyErrorCode.UNSUPPORTED_OPTION_KEY); + assertThat(refusal.getField()).contains(name); + }); + } + } + + @Test + void givenRawSecretEntriesWhenPrintingOptionsShouldShowNamesWithoutValues() { + JsonNode key = section("stream_key"); + String keyHex = key.get("key_hex").asString(); + String keyBase64 = key.get("key_base64").asString(); + HeaderValue rawKey = HeaderValue.fromRaw(bytes(keyHex)); + HeaderValue keyText = HeaderValue.fromString(keyBase64); + List<Object> printed = List.of( + StreamOptions.builder() + .raw("encryption_key", rawKey) + .raw("future_secret", keyText) + .build(), + StreamUpdate.builder() + .raw("encryption_key", rawKey) + .raw("future_secret", keyText) + .build(), + TopicOptions.builder() + .raw("encryption_key", rawKey) + .raw("future_secret", keyText) + .build(), + TopicUpdate.builder() + .raw("encryption_key", rawKey) + .raw("future_secret", keyText) + .build(), + UserOptions.builder() + .raw("encryption_key", rawKey) + .raw("future_secret", keyText) + .build(), + UserUpdate.builder() + .raw("encryption_key", rawKey) + .raw("future_secret", keyText) + .build()); + assertSoftly(softly -> printed.forEach(options -> softly.assertThat(options.toString()) + .as(options.getClass().getSimpleName()) + .contains("encryption_key", "future_secret") + .doesNotContain(keyHex) + .doesNotContain(keyBase64))); + } + + @Test + void givenSharedResponsesWhenDecodingShouldMatchProjection() { + for (JsonNode vector : section("stream_responses")) { + JsonNode expected = vector.get("expected"); + StreamDetails stream = BytesDeserializer.readStreamDetails(responseOf(vector)); + String name = vector.get("case").asString(); + assertThat(stream.id()).as(name).isEqualTo(expected.get("id").asLong()); + assertThat(stream.createdAt()) + .as(name) + .isEqualTo(new BigInteger(expected.get("created_at").asString())); + assertThat(stream.name()).as(name).isEqualTo(expected.get("name").asString()); + assertThat(stream.topicsCount()) + .as(name) + .isEqualTo(expected.get("topics_count").asLong()); + assertThat(stream.size()) + .as(name) + .isEqualTo(expected.get("size_bytes").asString()); + assertThat(stream.messagesCount()) + .as(name) + .isEqualTo(BigInteger.valueOf(expected.get("messages_count").asLong())); + assertThat(stream.options()).as(name).isEqualTo(optionMap(expected.get("options"), true)); + assertThat(stream.derivedOptions()).as(name).isEqualTo(optionMap(expected.get("options"), false)); + assertThat(stream.encryptionKeySource().map(EncryptionKeySource::value)) + .as(name) + .contains(expected.get("encryption_key_source").asString()); + JsonNode fingerprint = expected.get("encryption_key_fingerprint"); + assertThat(stream.encryptionKeyFingerprint()) + .as(name) + .isEqualTo(fingerprint.isNull() ? Optional.empty() : Optional.of(fingerprint.asString())); + List<String> keks = new ArrayList<>(); + expected.get("encryption_kek_fingerprints").forEach(kek -> keks.add(kek.asString())); + assertThat(stream.encryptionKekFingerprints()).as(name).isEqualTo(keks); + assertNoSecretKeys( + name, stream.options().keySet(), stream.derivedOptions().keySet()); + } + for (JsonNode vector : section("topic_responses")) { + JsonNode expected = vector.get("expected"); + Topic topic = BytesDeserializer.readTopic(responseOf(vector)); + String name = vector.get("case").asString(); + assertThat(topic.id()).as(name).isEqualTo(expected.get("id").asLong()); + assertThat(topic.name()).as(name).isEqualTo(expected.get("name").asString()); + assertThat(topic.partitionsCount()) + .as(name) + .isEqualTo(expected.get("partitions_count").asLong()); + assertThat(topic.size()) + .as(name) + .isEqualTo(expected.get("size_bytes").asString()); + assertThat(topic.options()).as(name).isEqualTo(optionMap(expected.get("options"), true)); + assertThat(topic.derivedOptions()).as(name).isEqualTo(optionMap(expected.get("options"), false)); + assertThat(topic.compressionAlgorithm()).as(name).contains(CompressionAlgorithm.Gzip); + assertThat(expected.get("compression_algorithm").asString()).isEqualTo("gzip"); + assertThat(topic.messageExpiry()) + .as(name) + .contains(new BigInteger(expected.get("message_expiry").asString())); + assertThat(topic.maxTopicSize()) + .as(name) + .contains(new BigInteger(expected.get("max_topic_size").asString())); + } + for (JsonNode vector : section("user_responses")) { + JsonNode expected = vector.get("expected"); + UserInfo user = BytesDeserializer.readUserInfo(responseOf(vector)); + String name = vector.get("case").asString(); + assertThat(user.id()).as(name).isEqualTo(expected.get("id").asLong()); + assertThat(user.username()) + .as(name) + .isEqualTo(expected.get("username").asString()); + assertThat(user.status().name().toLowerCase(Locale.ROOT)) + .as(name) + .isEqualTo(expected.get("status").asString()); + assertThat(user.options()).as(name).isEqualTo(optionMap(expected.get("options"), true)); + assertThat(user.derivedOptions()).as(name).isEqualTo(optionMap(expected.get("options"), false)); + } + } + + @Test + void givenDescribeVectorsWhenDecodingShouldReportKindsDefaultsAndFlags() { + for (JsonNode vector : section("describe_options")) { + String scope = vector.get("scope").asString(); + OptionsScope optionsScope = + OptionsScope.valueOf(scope.substring(0, 1).toUpperCase(Locale.ROOT) + scope.substring(1)); + assertThat(HexFormat.of().formatHex(new byte[] {(byte) optionsScope.asCode()})) + .isEqualTo(vector.get("request_hex").asString()); + List<OptionSpec> specs = BytesDeserializer.readOptionSpecs(responseOf(vector)); + List<String> expected = new ArrayList<>(); + vector.get("entries") + .forEach(entry -> expected.add(describe( + entry.get("key").asString(), + entry.get("kind").asString(), + entry.get("flags").asInt(), + entry.get("default").isNull() + ? "" + : entry.get("default").asString()))); + assertThat(specs.stream() + .map(spec -> describe( + spec.key(), + spec.defaultValue().kind().name().toLowerCase(Locale.ROOT), + spec.flags(), + spec.defaultValue().value().length == 0 + ? "" + : spec.defaultValue().toStringValue())) + .toList()) + .as(scope) + .isEqualTo(expected); + } + } + + @Test + void givenLiveCatalogWhenComparingKeyEnumsShouldMatchKeysAndFlags() { + assertThat(catalogKeys("stream")).isEqualTo(enumKeys(StreamOptionKey.values())); + assertThat(catalogKeys("topic")).isEqualTo(enumKeys(TopicOptionKey.values())); + assertThat(catalogKeys("user")).isEqualTo(enumKeys(UserOptionKey.values())); + } + + @Test + void givenFixtureCatalogWhenComparingTypedSurfaceShouldExposeEveryKey() { + Map<String, List<Class<?>>> create = Map.of( + "stream", List.of(StreamOptions.Builder.class), + "topic", List.of(TopicOptions.Builder.class), + "user", List.of(UserOptions.Builder.class)); + Map<String, List<Class<?>>> update = Map.of( + "stream", List.of(StreamUpdate.Builder.class), + "topic", List.of(TopicUpdate.Builder.class), + "user", List.of(UserUpdate.Builder.class)); + Map<String, List<Class<?>>> reads = Map.of( + "stream", List.of(StreamBase.class, StreamDetails.class), + "topic", List.of(Topic.class, TopicDetails.class), + "user", List.of(UserInfo.class, UserInfoDetails.class)); + for (JsonNode row : section("catalog")) { + if (!row.get("described").asBoolean()) { + continue; + } + String key = row.get("key").asString(); + String mutability = row.get("mutability").asString(); + boolean mapKey = "map".equals(row.get("storage").asString()); + for (JsonNode scopeNode : row.get("scopes")) { + String scope = scopeNode.asString(); + if (mapKey && !"read_only".equals(mutability)) { + assertTypedMember(create.get(scope), key, scope + " create " + key); + } + if ("updatable".equals(mutability)) { + assertTypedMember(update.get(scope), key, scope + " update " + key); + } + if ("read_only".equals(mutability)) { + assertTypedMember(reads.get(scope), key, scope + " read " + key); + } + if ((row.get("flags").asInt() & OptionSpec.FLAG_RESETTABLE) != 0) { + assertThat(scopeKeys(scope)).as(scope + " reset " + key).anySatisfy(constant -> { + assertThat(constant.value()).isEqualTo(key); + assertThat(constant.resettable()).isTrue(); + }); + } + } + } + } + + @Test + void givenFixtureConstantsWhenMirroringShouldMatchCodesKindsAndFlags() { + JsonNode constants = section("constants"); + assertThat(IggyErrorCode.CANNOT_MODIFY_ROOT_USER.getCode()) + .isEqualTo(constants + .get("error_codes") + .get("cannot_modify_root_user") + .asInt()); + JsonNode codes = constants.get("command_codes"); + assertThat(CommandCode.System.UPDATE_RESOURCE.getValue()) + .isEqualTo(codes.get("update_resource").get("code").asInt()); + assertThat(CommandCode.System.DESCRIBE_OPTIONS.getValue()) + .isEqualTo(codes.get("describe_options").get("code").asInt()); + assertThat(CommandCode.Stream.CREATE.getValue()) + .isEqualTo(codes.get("create_stream").get("code").asInt()); + assertThat(CommandCode.Topic.CREATE.getValue()) + .isEqualTo(codes.get("create_topic").get("code").asInt()); + assertThat(CommandCode.User.CREATE.getValue()) + .isEqualTo(codes.get("create_user").get("code").asInt()); + List<Integer> liveCodes = new ArrayList<>(); + for (Class<?> group : CommandCode.class.getDeclaredClasses()) { + for (Object code : group.getEnumConstants()) { + liveCodes.add(((CommandCode) code).getValue()); + } + } + constants + .get("retired_command_codes") + .forEach(code -> assertThat(liveCodes).doesNotContain(code.asInt())); + JsonNode scopes = constants.get("scope_codes"); + assertThat(OptionsScope.Topic.asCode()).isEqualTo(scopes.get("topic").asInt()); + assertThat(OptionsScope.Stream.asCode()).isEqualTo(scopes.get("stream").asInt()); + assertThat(OptionsScope.User.asCode()).isEqualTo(scopes.get("user").asInt()); + constants + .get("header_kind_codes") + .properties() + .forEach(kind -> assertThat(kind(kind.getKey()).asCode()) + .isEqualTo(kind.getValue().asInt())); + JsonNode flags = constants.get("option_flags"); + assertThat(OptionSpec.FLAG_UPDATABLE).isEqualTo(flags.get("updatable").asInt()); + assertThat(OptionSpec.FLAG_RESETTABLE).isEqualTo(flags.get("resettable").asInt()); + assertThat(OptionSpec.FLAG_SECRET).isEqualTo(flags.get("secret").asInt()); + assertThat(OptionSpec.FLAG_READ_ONLY).isEqualTo(flags.get("read_only").asInt()); + assertThat(OptionSpec.FLAG_CRITICAL).isEqualTo(flags.get("critical").asInt()); + assertThat(UserStatus.Active.asCode()) + .isEqualTo(constants.get("user_status_codes").get("active").asInt()); + assertThat(UserStatus.Inactive.asCode()) + .isEqualTo(constants.get("user_status_codes").get("inactive").asInt()); + assertThat(EncryptionKeys.KEY_LENGTH) + .isEqualTo(constants.get("stream_key_length").asInt()); + } + + @Test + void givenFixedKeyWhenFingerprintingShouldMatchFixture() { + JsonNode key = section("stream_key"); + String expected = key.get("fingerprint").asString(); + + assertThat(EncryptionKeys.fingerprint(bytes(key.get("key_hex").asString()))) + .isEqualTo(expected); + assertThat(EncryptionKeys.fingerprint(key.get("key_base64").asString())).isEqualTo(expected); + assertThat(EncryptionKeys.decodeBase64(key.get("key_base64").asString())) + .isEqualTo(bytes(key.get("key_hex").asString())); + } + + @Test + void givenInvalidKeyWhenEncodingShouldRejectBeforeNetworkIo() { + for (JsonNode vector : section("stream_key").get("invalid_keys")) { + JsonNode input = vector.get("input"); + assertThat(vector.get("error").get("code").asInt()) + .isEqualTo(IggyErrorCode.INVALID_ENCRYPTION_KEY.getCode()); + if (input.has("raw_hex")) { + byte[] key = bytes(input.get("raw_hex").asString()); + assertThatThrownBy(() -> StreamOptions.builder().encryptionKey(key)) + .as(vector.get("case").asString()) + .isInstanceOfSatisfying(IggyValidationException.class, ResourceOptionsTest::assertInvalidKey); + assertThatThrownBy(() -> EncryptionKeys.fingerprint(key)) + .isInstanceOfSatisfying(IggyValidationException.class, ResourceOptionsTest::assertInvalidKey); + } else { + String key = input.get("base64").asString(); + assertThatThrownBy(() -> StreamOptions.builder().encryptionKeyBase64(key)) + .as(vector.get("case").asString()) + .isInstanceOfSatisfying(IggyValidationException.class, ResourceOptionsTest::assertInvalidKey) + .satisfies(error -> assertThat(error.getMessage()).doesNotContain(key.strip())); + assertThatThrownBy(() -> EncryptionKeys.fingerprint(key)) + .isInstanceOfSatisfying(IggyValidationException.class, ResourceOptionsTest::assertInvalidKey); + } + } + JsonNode valid = section("stream_key"); + byte[] key = bytes(valid.get("key_hex").asString()); + StreamOptions options = StreamOptions.builder().encryptionKey(key).build(); + assertThat(options.encryptionKey()).hasValue(key); + assertThat(options.toString()) + .doesNotContain(valid.get("key_base64").asString()) + .doesNotContain(valid.get("key_hex").asString()) + .contains("<redacted>"); + assertThat(StreamOptions.builder() + .encryptionKeyBase64(valid.get("key_base64").asString()) + .build() + .encryptionKey()) + .hasValue(key); + assertThat(options.entries().get(StreamOptionKey.ENCRYPTION_KEY.value())) + .isEqualTo(new HeaderValue(HeaderKind.Raw, key)); + } + + private static void assertInvalidKey(IggyValidationException refusal) { + assertThat(refusal.getErrorCode()).isEqualTo(IggyErrorCode.INVALID_ENCRYPTION_KEY); + } + + private static void assertTypedMember(List<Class<?>> owners, String key, String description) { + String[] words = key.split("_"); + StringBuilder camel = new StringBuilder(words[0]); + for (int index = 1; index < words.length; index++) { + camel.append(Character.toUpperCase(words[index].charAt(0))).append(words[index].substring(1)); + } + String member = camel.toString(); + assertThat(owners.stream().flatMap(owner -> Arrays.stream(owner.getMethods()))) + .as(description) + .anyMatch(method -> + method.getName().equals(member) || method.getName().equals(member + "s")); + } + + private static OptionKey[] scopeKeys(String scope) { + return switch (scope) { + case "topic" -> TopicOptionKey.values(); + case "stream" -> StreamOptionKey.values(); + default -> UserOptionKey.values(); + }; + } + + private static boolean namesKeyWithoutConstant(JsonNode vector) { + OptionKey[] keys = scopeKeys(vector.get("scope").asString()); + JsonNode reset = vector.get("input").get("reset"); + if (reset == null) { + return false; + } + for (JsonNode name : reset) { + if (Arrays.stream(keys).noneMatch(key -> key.value().equals(name.asString()))) { + return true; + } + } + return false; + } + + private static List<String> futureKeys(int count) { + return IntStream.range(0, count) + .mapToObj(index -> String.format(Locale.ROOT, "future_%03d", index)) + .toList(); + } + + private static void assertRefusal(ThrowingCallable build, IggyErrorCode expected) { + assertThatThrownBy(build) + .isInstanceOfSatisfying( + IggyValidationException.class, + refusal -> assertThat(refusal.getErrorCode()).isEqualTo(expected)); + } + + private static ByteBuf encodeUpdate(JsonNode vector) { + JsonNode target = vector.get("target"); + JsonNode input = vector.get("input"); + return switch (vector.get("scope").asString()) { + case "topic" -> BytesSerializer.toUpdateResource(streamId(target), topicId(target), topicUpdate(input)); + case "stream" -> BytesSerializer.toUpdateResource(streamId(target), streamUpdate(input)); + default -> BytesSerializer.toUpdateResource(userId(target), userUpdate(input)); + }; + } + + private static ByteBuf responseOf(JsonNode vector) { + return Unpooled.wrappedBuffer(bytes(vector.get("response_hex").asString())); + } + + private static String describe(String key, String kind, int flags, String defaultValue) { + return key + " " + kind + " " + flags + " " + defaultValue; + } + + private static List<String> catalogKeys(String scope) { + List<String> keys = new ArrayList<>(); + for (JsonNode row : section("catalog")) { + boolean inScope = false; + for (JsonNode rowScope : row.get("scopes")) { + inScope |= rowScope.asString().equals(scope); + } + if (inScope && row.get("described").asBoolean()) { + keys.add(row.get("key").asString() + " " + row.get("flags").asInt()); + } + } + return keys; + } + + private static List<String> enumKeys(OptionKey[] keys) { + List<String> described = new ArrayList<>(); + for (OptionKey key : keys) { + described.add(key.value() + " " + key.flags()); + } + return described; + } + + private static void assertNoSecretKeys(String name, Set<String> explicit, Set<String> derived) { + section("constants").get("never_returned_keys").forEach(secret -> { + assertThat(explicit).as(name).doesNotContain(secret.asString()); + assertThat(derived).as(name).doesNotContain(secret.asString()); + }); + } +}
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/stream/StreamDetailsTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/stream/StreamDetailsTest.java index e8bc7fb..b870d06 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/stream/StreamDetailsTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/stream/StreamDetailsTest.java
@@ -19,7 +19,7 @@ package org.apache.iggy.stream; -import org.apache.iggy.topic.CompressionAlgorithm; +import org.apache.iggy.message.HeaderValue; import org.apache.iggy.topic.Topic; import org.junit.jupiter.api.Test; @@ -32,19 +32,10 @@ class StreamDetailsTest { @Test void constructorWithStreamBaseCreatesExpectedStreamDetails() { - var base = new StreamBase(10L, BigInteger.valueOf(500L), "name", "size", BigInteger.ZERO, 1L, Map.of()); - var topics = List.of(new Topic( - 1L, - BigInteger.ZERO, - "name", - "size", - BigInteger.TEN, - CompressionAlgorithm.None, - BigInteger.ONE, - BigInteger.ZERO, - 2L, - Map.of(), - Map.of())); + var derived = Map.of("encryption_key_source", HeaderValue.fromString("server")); + var base = + new StreamBase(10L, BigInteger.valueOf(500L), "name", "size", BigInteger.ZERO, 1L, Map.of(), derived); + var topics = List.of(new Topic(1L, BigInteger.ZERO, "name", "size", BigInteger.ZERO, 2L, Map.of(), Map.of())); var streamDetails = new StreamDetails(base, topics); @@ -55,5 +46,7 @@ assertThat(streamDetails.messagesCount()).isEqualTo(BigInteger.ZERO); assertThat(streamDetails.topicsCount()).isEqualTo(1L); assertThat(streamDetails.topics()).isEqualTo(topics); + assertThat(streamDetails.derivedOptions()).isEqualTo(derived); + assertThat(streamDetails.encryptionKeySource()).contains(EncryptionKeySource.SERVER); } }
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/topic/TopicDetailsTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/topic/TopicDetailsTest.java index dba379e..778ff0a 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/topic/TopicDetailsTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/topic/TopicDetailsTest.java
@@ -19,6 +19,7 @@ package org.apache.iggy.topic; +import org.apache.iggy.message.HeaderValue; import org.apache.iggy.partition.Partition; import org.apache.iggy.partition.PartitionContext; import org.junit.jupiter.api.Test; @@ -37,13 +38,14 @@ BigInteger.TEN, "foo", "size", - BigInteger.valueOf(10000L), - CompressionAlgorithm.Gzip, - BigInteger.TWO, BigInteger.ZERO, 1L, - Map.of(), - Map.of()); + Map.of( + "message_expiry", + HeaderValue.fromUint64(BigInteger.valueOf(10000L)), + "compression_algorithm", + HeaderValue.fromString("gzip")), + Map.of("max_topic_size", HeaderValue.fromUint64(BigInteger.TWO))); var partitions = List.of( new Partition(1L, BigInteger.TEN, 2L, BigInteger.ZERO, "size", BigInteger.ONE, PartitionContext.EMPTY)); @@ -53,9 +55,9 @@ assertThat(topicDetails.createdAt()).isEqualTo(BigInteger.TEN); assertThat(topicDetails.name()).isEqualTo("foo"); assertThat(topicDetails.size()).isEqualTo("size"); - assertThat(topicDetails.messageExpiry()).isEqualTo(BigInteger.valueOf(10000L)); - assertThat(topicDetails.compressionAlgorithm()).isEqualTo(CompressionAlgorithm.Gzip); - assertThat(topicDetails.maxTopicSize()).isEqualTo(BigInteger.TWO); + assertThat(topicDetails.messageExpiry()).contains(BigInteger.valueOf(10000L)); + assertThat(topicDetails.compressionAlgorithm()).contains(CompressionAlgorithm.Gzip); + assertThat(topicDetails.maxTopicSize()).contains(BigInteger.TWO); assertThat(topicDetails.messagesCount()).isEqualTo(BigInteger.ZERO); assertThat(topicDetails.partitionsCount()).isEqualTo(1L); assertThat(topicDetails.partitions()).isEqualTo(partitions);
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/topic/TopicOptionsTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/topic/TopicOptionsTest.java index a958627..646b9b7 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/topic/TopicOptionsTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/topic/TopicOptionsTest.java
@@ -38,9 +38,8 @@ @Test void shouldEmitOnlyTheKeysThatWereSet() { - var options = TopicOptions.builder() - .durability(org.apache.iggy.topic.Durability.PERSISTED) - .build(); + var options = + TopicOptions.builder().durability(Durability.PERSISTED).build().entries(); assertThat(options).containsOnlyKeys("durability", "consumer_offset_durability"); assertThat(options.get("durability").kind()).isEqualTo(HeaderKind.String); @@ -51,11 +50,12 @@ void shouldEncodeEveryKeyInItsCatalogKind() { var options = TopicOptions.builder() .segmentSize(BigInteger.valueOf(134_217_728)) - .durability(org.apache.iggy.topic.Durability.REPLICATED) + .durability(Durability.REPLICATED) .messagesRequiredToSave(1024) .sizeOfMessagesRequiredToSave(BigInteger.valueOf(1_048_576)) .preallocateSegments(true) - .build(); + .build() + .entries(); assertThat(options) .containsOnlyKeys( @@ -78,8 +78,9 @@ var options = TopicOptions.builder() .preallocateSegments(true) .segmentSize(BigInteger.valueOf(134_217_728)) - .durability(org.apache.iggy.topic.Durability.REPLICATED) - .build(); + .durability(Durability.REPLICATED) + .build() + .entries(); assertThat(options.keySet()) .containsExactly("durability", "consumer_offset_durability", "preallocate_segments", "segment_size");
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/user/UserInfoDetailsTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/user/UserInfoDetailsTest.java index 3d4350d..2702d33 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/user/UserInfoDetailsTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/user/UserInfoDetailsTest.java
@@ -30,7 +30,7 @@ class UserInfoDetailsTest { @Test void constructorWithUserInfoCreatesUserInfoDetailsWithExpectedValues() { - var userInfo = new UserInfo(123L, BigInteger.TEN, UserStatus.Active, "foo"); + var userInfo = new UserInfo(123L, BigInteger.TEN, UserStatus.Active, "foo", Map.of(), Map.of()); var globalPermissions = new GlobalPermissions(true, false, false, false, false, false, false, false, false, false); var permissions = Optional.of(new Permissions(globalPermissions, Map.of()));
diff --git a/foreign/java/java-sdk/src/test/resources/resource_options.json b/foreign/java/java-sdk/src/test/resources/resource_options.json new file mode 100644 index 0000000..c5cc2ac --- /dev/null +++ b/foreign/java/java-sdk/src/test/resources/resource_options.json
@@ -0,0 +1,1619 @@ +{ + "schema_version": 1, + "generated_by": "core/common/examples/resource_options_fixtures.rs", + "conventions": { + "byte_strings": "Every *_hex field is lowercase hex without separators.", + "errors": "name is the snake_case IggyError name and code its numeric code.", + "max_topic_size": "Bytes. 18446744073709551615 means unlimited.", + "message_expiry": "Microseconds. 18446744073709551615 means never expire.", + "never_returned_keys": "No response, decoded projection or accessor may contain these keys or their values.", + "option_maps": "Option maps use the HTTP JSON form: value as reversible text, kind, and explicit provenance. Request blocks mark every entry explicit.", + "options_block_order": "Options block entries are encoded in ascending key byte order.", + "request_bodies": "body_hex and update_resource_hex are command payloads without framing. constants.command_codes maps each command to its code and VSR operation.", + "reset_order": "The typed encoders and the HTTP handler write reset keys in ascending key byte order.", + "stream_state": "state is the stored server state a stream response was projected from. SDKs need only response_hex and expected.", + "u64_values": "u64 option values and typed u64 inputs are decimal strings, so values above 2^53 keep their precision." + }, + "constants": { + "protocol_version": { + "packed": 12288, + "text": "0.12.0", + "major": 0, + "minor": 12, + "patch": 0 + }, + "protocol_version_min": { + "packed": 12288, + "text": "0.12.0", + "major": 0, + "minor": 12, + "patch": 0 + }, + "command_codes": { + "create_stream": { + "code": 202, + "operation": 128 + }, + "create_topic": { + "code": 302, + "operation": 132 + }, + "create_user": { + "code": 33, + "operation": 141 + }, + "describe_options": { + "code": 13, + "operation": null + }, + "get_stream": { + "code": 200, + "operation": null + }, + "get_streams": { + "code": 201, + "operation": null + }, + "get_topic": { + "code": 300, + "operation": null + }, + "get_topics": { + "code": 301, + "operation": null + }, + "get_user": { + "code": 31, + "operation": null + }, + "get_users": { + "code": 32, + "operation": null + }, + "update_resource": { + "code": 16, + "operation": 150 + } + }, + "error_codes": { + "cannot_modify_root_user": 4108 + }, + "retired_command_codes": { + "update_stream": 204, + "update_topic": 304, + "update_user": 35 + }, + "operations": { + "rewrap_stream_keys": { + "value": 75, + "internal": true, + "request_operation": 75 + }, + "update_resource": { + "value": 150, + "internal": false, + "request_operation": 150 + }, + "update_stream_resolved": { + "value": 72, + "internal": true, + "request_operation": 150 + }, + "update_topic_resolved": { + "value": 73, + "internal": true, + "request_operation": 150 + }, + "update_user_resolved": { + "value": 74, + "internal": true, + "request_operation": 150 + } + }, + "retired_operations": { + "update_stream": 129, + "update_topic": 133, + "update_user": 142 + }, + "scope_codes": { + "stream": 2, + "topic": 1, + "user": 3 + }, + "header_kind_codes": { + "raw": 1, + "string": 2, + "bool": 3, + "uint32": 11, + "uint64": 12 + }, + "option_flags": { + "critical": 16, + "read_only": 8, + "resettable": 2, + "secret": 4, + "updatable": 1 + }, + "user_status_codes": { + "active": 1, + "inactive": 2 + }, + "stream_key_length": 32, + "never_returned_keys": [ + "encryption_key", + "encryption_key_wrapped", + "critical_keys" + ] + }, + "catalog": [ + { + "key": "name", + "scopes": [ + "stream", + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "column", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 1, + "described": true, + "default": null + }, + { + "key": "username", + "scopes": [ + "user" + ], + "kind": "string", + "kind_code": 2, + "storage": "column", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 1, + "described": true, + "default": null + }, + { + "key": "status", + "scopes": [ + "user" + ], + "kind": "string", + "kind_code": 2, + "storage": "column", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 1, + "described": true, + "default": null + }, + { + "key": "compression_algorithm", + "scopes": [ + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "none" + }, + { + "key": "message_expiry", + "scopes": [ + "topic" + ], + "kind": "uint64", + "kind_code": 12, + "storage": "map", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 3, + "described": true, + "default": "18446744073709551615" + }, + { + "key": "max_topic_size", + "scopes": [ + "topic" + ], + "kind": "uint64", + "kind_code": 12, + "storage": "map", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 3, + "described": true, + "default": "18446744073709551615" + }, + { + "key": "segment_size", + "scopes": [ + "topic" + ], + "kind": "uint64", + "kind_code": 12, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "1073741824" + }, + { + "key": "durability", + "scopes": [ + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "replicated" + }, + { + "key": "consumer_offset_durability", + "scopes": [ + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "replicated" + }, + { + "key": "partition_resize_policy", + "scopes": [ + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "mutable" + }, + { + "key": "messages_required_to_save", + "scopes": [ + "topic" + ], + "kind": "uint32", + "kind_code": 11, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "1024" + }, + { + "key": "size_of_messages_required_to_save", + "scopes": [ + "topic" + ], + "kind": "uint64", + "kind_code": 12, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "1048576" + }, + { + "key": "preallocate_segments", + "scopes": [ + "topic" + ], + "kind": "bool", + "kind_code": 3, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "false" + }, + { + "key": "encryption_key", + "scopes": [ + "stream" + ], + "kind": "raw", + "kind_code": 1, + "storage": "map", + "mutability": "create_only", + "visibility": "secret", + "critical": true, + "flags": 20, + "described": true, + "default": null + }, + { + "key": "encryption_key_wrapped", + "scopes": [ + "stream" + ], + "kind": "raw", + "kind_code": 1, + "storage": "map", + "mutability": "create_only", + "visibility": "internal", + "critical": true, + "flags": 16, + "described": false, + "default": null + }, + { + "key": "encryption_key_source", + "scopes": [ + "stream" + ], + "kind": "string", + "kind_code": 2, + "storage": "computed", + "mutability": "read_only", + "visibility": "read_only", + "critical": false, + "flags": 8, + "described": true, + "default": null + }, + { + "key": "encryption_key_fingerprint", + "scopes": [ + "stream" + ], + "kind": "string", + "kind_code": 2, + "storage": "computed", + "mutability": "read_only", + "visibility": "read_only", + "critical": false, + "flags": 8, + "described": true, + "default": null + }, + { + "key": "encryption_kek_fingerprint", + "scopes": [ + "stream" + ], + "kind": "string", + "kind_code": 2, + "storage": "computed", + "mutability": "read_only", + "visibility": "read_only", + "critical": false, + "flags": 8, + "described": true, + "default": null + }, + { + "key": "critical_keys", + "scopes": [ + "stream", + "topic", + "user" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "internal", + "critical": false, + "flags": 0, + "described": false, + "default": null + } + ], + "describe_options": [ + { + "scope": "topic", + "request_hex": "01", + "response_hex": "0b000000046e616d650201000000007e0000005265736f75726365206e616d652c203120746f203235352062797465732c20756e697175652077697468696e2069747320706172656e742e20536574206279207468652063726561746520636f6d6d616e642773206e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e15636f6d7072657373696f6e5f616c676f726974686d0200040000006e6f6e6550000000436f6d7072657373696f6e20616c676f726974686d20286e6f6e652c20677a6970292e2053746f72656420616e64207265706f727465642c206e6f74206170706c69656420746f206d657373616765730e6d6573736167655f6578706972790c0308000000ffffffffffffffff430000004d6573736167652065787069727920696e206d6963726f7365636f6e64732c206f7220612068756d616e74696d6520737472696e672028652e672e20372064617973290e6d61785f746f7069635f73697a650c0308000000ffffffffffffffff9d000000546f7069632d77696465207365616c65642d7365676d656e742073697a65206361702c2073706c6974206163726f737320616c6c20706172746974696f6e732c20696e206279746573206f72206120627974652d73697a6520737472696e672028652e672e203120476942293b2066696e6974652076616c756573206d757374206265206174206c6561737420746865207365676d656e742073697a650c7365676d656e745f73697a650c000800000000000040000000006c0000005365676d656e742073697a6520696e2062797465732c206f72206120627974652d73697a6520737472696e672028652e672e20313238204d6942293b2061203531322d62797465206d756c7469706c652077697468696e20313034383537362e2e3d313037333734313832340a6475726162696c69747902000a0000007265706c6963617465643e0100004d65737361676520636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20412073696e676c65746f6e2071756f72756d20686173206f6e6520636f70792e20496e207265706c6963617465642067726f7570732c20706572736973746564206d65737361676573207573652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c2057414c207265636c616d6174696f6e2e205468652066756c6c20626f64792073697a65207374696c6c20636f756e747320616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e1a636f6e73756d65725f6f66667365745f6475726162696c69747902000a0000007265706c6963617465646e0100004578706c69636974206f666673657420636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20506f6c6c206175746f2d636f6d6d69742072656d61696e73206173796e6368726f6e6f75732e20496e207265706c6963617465642067726f7570732c20706572736973746564206f66667365747320616c736f20656e61626c652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c207265636c616d6174696f6e2c206576656e2077697468207265706c696361746564206d657373616765206475726162696c6974792e2046756c6c20626f64792073697a657320636f756e7420616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e17706172746974696f6e5f726573697a655f706f6c6963790200070000006d757461626c65c8000000496d6d757461626c65206166746572206372656174696f6e3a206d757461626c65207065726d69747320706172746974696f6e206164646974696f6e2f64656c6574696f6e20776974686f7574206120626f756e646564206375746f766572206f72207065722d6b6579206f72646572696e67206163726f7373206c61796f7574733b2066697865642072656a6563747320626f74682e20526574656e74696f6e20616e6420746f7069632064656c6574696f6e2072656d61696e2064657374727563746976652e196d657373616765735f72657175697265645f746f5f736176650b000400000000040000870000004f7264696e617279206d6573736167652d636f756e7420666c75736820747269676765723b20312e2e3d31363737373231362e2052657175697265642070657273697374656e63652c2063617061636974792070726573737572652c20616e64206c6966656379636c65206f7065726174696f6e732063616e20666c757368206561726c6965722173697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c0008000000000010000000000083000000466c75736820746865206a6f75726e616c206f6e636520697420686f6c64732074686973206d616e792062797465732c206f72206120627974652d73697a6520737472696e673b20776869636865766572207468726573686f6c6420747269707320666972737420666c75736865732e204174206d6f7374203130373337343138323414707265616c6c6f636174655f7365676d656e7473030001000000001201000052657175657374207365676d656e745f73697a65206279746573206f66206469736b207265736572766174696f6e207768656e2065616368207365676d656e74206f70656e732e20556e737570706f72746564206f72206661696c6564207265736572766174696f6e732066616c6c206261636b20746f206f7264696e61727920616c6c6f636174696f6e20776974682061207761726e696e672e205265736572766174696f6e2072756e7320696e6c696e65206f6e20746865206f776e696e672073686172642e207365676d656e745f73697a65202a20706172746974696f6e735f636f756e74206973206361707065642061742036383731393437363733362062797465732070657220637265617465", + "entries": [ + { + "key": "name", + "kind": "string", + "flags": 1, + "default": null + }, + { + "key": "compression_algorithm", + "kind": "string", + "flags": 0, + "default": "none" + }, + { + "key": "message_expiry", + "kind": "uint64", + "flags": 3, + "default": "18446744073709551615" + }, + { + "key": "max_topic_size", + "kind": "uint64", + "flags": 3, + "default": "18446744073709551615" + }, + { + "key": "segment_size", + "kind": "uint64", + "flags": 0, + "default": "1073741824" + }, + { + "key": "durability", + "kind": "string", + "flags": 0, + "default": "replicated" + }, + { + "key": "consumer_offset_durability", + "kind": "string", + "flags": 0, + "default": "replicated" + }, + { + "key": "partition_resize_policy", + "kind": "string", + "flags": 0, + "default": "mutable" + }, + { + "key": "messages_required_to_save", + "kind": "uint32", + "flags": 0, + "default": "1024" + }, + { + "key": "size_of_messages_required_to_save", + "kind": "uint64", + "flags": 0, + "default": "1048576" + }, + { + "key": "preallocate_segments", + "kind": "bool", + "flags": 0, + "default": "false" + } + ] + }, + { + "scope": "stream", + "request_hex": "02", + "response_hex": "05000000046e616d650201000000007e0000005265736f75726365206e616d652c203120746f203235352062797465732c20756e697175652077697468696e2069747320706172656e742e20536574206279207468652063726561746520636f6d6d616e642773206e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e0e656e6372797074696f6e5f6b6579011400000000a30100005468652073747265616d2773206f776e206461746120656e6372797074696f6e206b65793a2033322072616e646f6d2062797465732c20726177206f72207374616e6461726420706164646564206261736536342e204372656174652d6f6e6c7920616e642077726974652d6f6e6c793a20746865207365727665722073746f726573206974207772617070656420756e6465722061206b657920646572697665642066726f6d207468652073657276657220656e6372797074696f6e206b657920616e64206e657665722072657475726e732069742e205265717569726573207468652073657276657220656e6372797074696f6e206b657920746f20626520636f6e666967757265642e20576974686f75742069742c206120736572766572207769746820656e6372797074696f6e20656e61626c65642067656e6572617465732061206b657920666f72207468652073747265616d2e20546865206b65792074726176656c7320696e20746865206372656174652072657175657374206f76657220746865207472616e73706f72742074686520636c69656e7420757365732e15656e6372797074696f6e5f6b65795f736f757263650208000000009f00000057686f2063686f7365207468652073747265616d27732061742d72657374206b65793a2073747265616d207768656e2074686520636c69656e742073656e742069742c20736572766572207768656e20746865207365727665722067656e6572617465642069742c206e6f6e65207768656e207468652073747265616d2773206d65737361676573206172652073746f72656420756e656e637279707465641a656e6372797074696f6e5f6b65795f66696e6765727072696e740208000000009f0000005468652066697273742038206279746573206f66205348412d323536206f76657220746865206c6162656c20696767792f73747265616d2d6b65792d66696e6765727072696e742f763120616e64207468652073747265616d2773206b65792c206173203136206c6f776572636173652068657820636861726163746572732e20416273656e74207768656e2074686520736f75726365206973206e6f6e651a656e6372797074696f6e5f6b656b5f66696e6765727072696e74020800000000b800000046696e6765727072696e7473206f6620746865206b65792d656e6372797074696f6e206b65797320746861742077726170207468652073747265616d2773206b65792c20636f6d6d612d7365706172617465643a206f6e65206f7574736964652061206b657920726f746174696f6e2c206f6e652070657220636f6e6669677572656420736572766572206b657920647572696e672069742e20416273656e74207768656e2074686520736f75726365206973206e6f6e65", + "entries": [ + { + "key": "name", + "kind": "string", + "flags": 1, + "default": null + }, + { + "key": "encryption_key", + "kind": "raw", + "flags": 20, + "default": null + }, + { + "key": "encryption_key_source", + "kind": "string", + "flags": 8, + "default": null + }, + { + "key": "encryption_key_fingerprint", + "kind": "string", + "flags": 8, + "default": null + }, + { + "key": "encryption_kek_fingerprint", + "kind": "string", + "flags": 8, + "default": null + } + ] + }, + { + "scope": "user", + "request_hex": "03", + "response_hex": "0200000008757365726e616d650201000000006b00000055736572206e616d652c203320746f2035302062797465732c20756e697175652e20536574206279207468652063726561746520636f6d6d616e64277320757365726e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e067374617475730201000000008f00000055736572207374617475732c20616374697665206f7220696e6163746976652e20536574206279207468652063726561746520636f6d6d616e64277320737461747573206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e20446561637469766174696e672061207573657220656e6473206974732073657373696f6e732e", + "entries": [ + { + "key": "username", + "kind": "string", + "flags": 1, + "default": null + }, + { + "key": "status", + "kind": "string", + "flags": 1, + "default": null + } + ] + } + ], + "stream_key": { + "key_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "key_base64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", + "fingerprint_label": "iggy/stream-key-fingerprint/v1", + "fingerprint": "8441e38538cbfa57", + "invalid_keys": [ + { + "case": "raw_31_bytes", + "input": { + "raw_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + }, + { + "case": "raw_33_bytes", + "input": { + "raw_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + }, + { + "case": "base64_with_whitespace", + "input": { + "base64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=\n" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + }, + { + "case": "base64_missing_padding", + "input": { + "base64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + }, + { + "case": "base64_noncanonical_trailing_bits", + "input": { + "base64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh9=" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + } + ] + }, + "update_resource": [ + { + "case": "topic_set_expiry_without_name_reset_max_topic_size", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "message_expiry": "604800000000", + "reset": [ + "max_topic_size" + ] + }, + "wire_set": { + "message_expiry": { + "value": "604800000000", + "kind": "uint64", + "explicit": true + } + }, + "wire_reset": [ + "max_topic_size" + ], + "body_hex": "01010401000000010402000000010e6d61785f746f7069635f73697a65020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000" + }, + { + "case": "topic_rename_only_named_identifiers", + "scope": "topic", + "target": { + "stream_id": "orders", + "topic_id": "events" + }, + "input": { + "name": "events-v2" + }, + "wire_set": { + "name": { + "value": "events-v2", + "kind": "string", + "explicit": true + } + }, + "wire_reset": [], + "body_hex": "0102066f726465727302066576656e74730002040000006e616d6502090000006576656e74732d7632" + }, + { + "case": "topic_rename_set_expiry_reset_max_topic_size", + "scope": "topic", + "target": { + "stream_id": 7, + "topic_id": "events" + }, + "input": { + "name": "events-v3", + "message_expiry": "86400000000", + "reset": [ + "max_topic_size" + ] + }, + "wire_set": { + "message_expiry": { + "value": "86400000000", + "kind": "uint64", + "explicit": true + }, + "name": { + "value": "events-v3", + "kind": "string", + "explicit": true + } + }, + "wire_reset": [ + "max_topic_size" + ], + "body_hex": "0101040700000002066576656e7473010e6d61785f746f7069635f73697a65020e0000006d6573736167655f6578706972790c080000000060d71d1400000002040000006e616d6502090000006576656e74732d7633" + }, + { + "case": "topic_reset_both_defaults", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "reset": [ + "max_topic_size", + "message_expiry" + ] + }, + "wire_set": {}, + "wire_reset": [ + "max_topic_size", + "message_expiry" + ], + "body_hex": "01010401000000010402000000020e6d61785f746f7069635f73697a650e6d6573736167655f657870697279" + }, + { + "case": "topic_max_topic_size_above_2_pow_53", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "max_topic_size": "9007199254740993" + }, + "wire_set": { + "max_topic_size": { + "value": "9007199254740993", + "kind": "uint64", + "explicit": true + } + }, + "wire_reset": [], + "body_hex": "0101040100000001040200000000020e0000006d61785f746f7069635f73697a650c080000000100000000002000" + }, + { + "case": "stream_rename", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "input": { + "name": "payments-v2" + }, + "wire_set": { + "name": { + "value": "payments-v2", + "kind": "string", + "explicit": true + } + }, + "wire_reset": [], + "body_hex": "020104030000000002040000006e616d65020b0000007061796d656e74732d7632" + }, + { + "case": "user_username_and_status", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "username": "alice2", + "status": "inactive" + }, + "wire_set": { + "status": { + "value": "inactive", + "kind": "string", + "explicit": true + }, + "username": { + "value": "alice2", + "kind": "string", + "explicit": true + } + }, + "wire_reset": [], + "body_hex": "030104050000000002060000007374617475730208000000696e6163746976650208000000757365726e616d650206000000616c69636532" + } + ], + "update_resource_errors": [ + { + "case": "topic_empty_update", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": {}, + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_set_and_reset_same_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "message_expiry": "604800000000", + "reset": [ + "message_expiry" + ] + }, + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_reset_create_only_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "reset": [ + "compression_algorithm" + ] + }, + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "stream_reset_name", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "input": { + "reset": [ + "name" + ] + }, + "error": { + "name": "option_not_resettable", + "code": 4104 + } + }, + { + "case": "user_reset_status", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "reset": [ + "status" + ] + }, + "error": { + "name": "option_not_resettable", + "code": 4104 + } + }, + { + "case": "stream_reset_read_only_key", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "input": { + "reset": [ + "encryption_key_source" + ] + }, + "error": { + "name": "read_only_option", + "code": 4105 + } + }, + { + "case": "user_reset_internal_key", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "reset": [ + "critical_keys" + ] + }, + "error": { + "name": "read_only_option", + "code": 4105 + } + }, + { + "case": "topic_empty_name", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "name": "" + }, + "error": { + "name": "invalid_topic_name", + "code": 2014 + } + }, + { + "case": "topic_empty_name_and_create_only_reset", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "name": "", + "reset": [ + "segment_size" + ] + }, + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "stream_empty_name", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "input": { + "name": "" + }, + "error": { + "name": "invalid_stream_name", + "code": 1013 + } + }, + { + "case": "user_short_username", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "username": "a" + }, + "error": { + "name": "invalid_username", + "code": 43 + } + }, + { + "case": "user_short_username_and_status_reset", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "username": "a", + "reset": [ + "status" + ] + }, + "error": { + "name": "option_not_resettable", + "code": 4104 + } + } + ], + "create_stream": [ + { + "case": "stream_without_key", + "input": { + "name": "orders" + }, + "wire_options": {}, + "body_hex": "066f7264657273", + "http_body": "{\"name\":\"orders\"}", + "admission_error": null + }, + { + "case": "stream_with_key", + "input": { + "name": "orders", + "encryption_key": { + "raw_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f" + } + }, + "wire_options": { + "encryption_key": { + "value": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", + "kind": "raw", + "explicit": true + } + }, + "body_hex": "066f7264657273020e000000656e6372797074696f6e5f6b65790120000000000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "http_body": "{\"name\":\"orders\",\"options\":{\"encryption_key\":\"AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=\"}}", + "admission_error": null + }, + { + "case": "stream_with_raw_extra_key", + "input": { + "name": "orders", + "raw": { + "future_option": "enabled" + } + }, + "wire_options": { + "future_option": { + "value": "enabled", + "kind": "string", + "explicit": true + } + }, + "body_hex": "066f7264657273020d0000006675747572655f6f7074696f6e0207000000656e61626c6564", + "http_body": "{\"name\":\"orders\",\"options\":{\"future_option\":\"enabled\"}}", + "admission_error": { + "name": "unsupported_option_key", + "code": 4041 + } + } + ], + "create_topic": [ + { + "case": "topic_with_options", + "input": { + "stream_id": 1, + "name": "events", + "partitions_count": 3, + "compression_algorithm": "gzip", + "message_expiry": "604800000000", + "max_topic_size": "9007199254740993", + "segment_size": "134217728", + "durability": "persisted", + "consumer_offset_durability": "replicated" + }, + "wire_options": { + "compression_algorithm": { + "value": "gzip", + "kind": "string", + "explicit": true + }, + "consumer_offset_durability": { + "value": "replicated", + "kind": "string", + "explicit": true + }, + "durability": { + "value": "persisted", + "kind": "string", + "explicit": true + }, + "max_topic_size": { + "value": "9007199254740993", + "kind": "uint64", + "explicit": true + }, + "message_expiry": { + "value": "604800000000", + "kind": "uint64", + "explicit": true + }, + "segment_size": { + "value": "134217728", + "kind": "uint64", + "explicit": true + } + }, + "body_hex": "01040100000003000000066576656e74730215000000636f6d7072657373696f6e5f616c676f726974686d0204000000677a6970021a000000636f6e73756d65725f6f66667365745f6475726162696c697479020a0000007265706c696361746564020a0000006475726162696c6974790209000000706572736973746564020e0000006d61785f746f7069635f73697a650c080000000100000000002000020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000020c0000007365676d656e745f73697a650c080000000000000800000000", + "http_body": "{\"partitions_count\":3,\"name\":\"events\",\"options\":{\"compression_algorithm\":\"gzip\",\"consumer_offset_durability\":\"replicated\",\"durability\":\"persisted\",\"max_topic_size\":\"9007199254740993\",\"message_expiry\":604800000000,\"segment_size\":134217728}}", + "admission_error": null + } + ], + "create_user": [ + { + "case": "user_with_empty_options", + "input": { + "username": "alice", + "password": "s3cret-pass", + "status": "active" + }, + "wire_options": {}, + "body_hex": "05616c6963650b7333637265742d706173730100", + "http_body": "{\"username\":\"alice\",\"password\":\"s3cret-pass\",\"status\":\"active\",\"permissions\":null}", + "admission_error": null + } + ], + "http_update": [ + { + "case": "topic_flat_keys", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"name\":\"events-v2\",\"message_expiry\":\"7 days\"}", + "update_resource_hex": "0101040100000001040200000000020e0000006d6573736167655f657870697279020600000037206461797302040000006e616d6502090000006576656e74732d7632", + "outcome": "accepted", + "error": null + }, + { + "case": "topic_reset_array", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":604800000000,\"reset\":[\"max_topic_size\"]}", + "update_resource_hex": "01010401000000010402000000010e6d61785f746f7069635f73697a65020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000", + "outcome": "accepted", + "error": null + }, + { + "case": "topic_null_ignored", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"name\":null,\"max_topic_size\":null,\"message_expiry\":\"1 day\"}", + "update_resource_hex": "0101040100000001040200000000020e0000006d6573736167655f65787069727902050000003120646179", + "outcome": "accepted", + "error": null + }, + { + "case": "topic_u64_above_2_pow_53_as_string", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"max_topic_size\":\"9007199254740993\"}", + "update_resource_hex": "0101040100000001040200000000020e0000006d61785f746f7069635f73697a65021000000039303037313939323534373430393933", + "outcome": "accepted", + "error": null + }, + { + "case": "stream_rename", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "body": "{\"name\":\"payments-v2\"}", + "update_resource_hex": "020104030000000002040000006e616d65020b0000007061796d656e74732d7632", + "outcome": "accepted", + "error": null + }, + { + "case": "user_username_and_status", + "scope": "user", + "target": { + "user_id": 5 + }, + "body": "{\"username\":\"alice2\",\"status\":\"inactive\"}", + "update_resource_hex": "030104050000000002060000007374617475730208000000696e6163746976650208000000757365726e616d650206000000616c69636532", + "outcome": "accepted", + "error": null + }, + { + "case": "topic_duplicate_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":\"1 day\",\"message_expiry\":\"2 days\"}", + "update_resource_hex": null, + "outcome": "json_refused", + "error": null + }, + { + "case": "topic_nested_object", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":{\"value\":\"1 day\"}}", + "update_resource_hex": null, + "outcome": "json_refused", + "error": null + }, + { + "case": "topic_null_only", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"max_topic_size\":null}", + "update_resource_hex": "0101040100000001040200000000", + "outcome": "decode_refused", + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_zero_sentinel_only", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":0}", + "update_resource_hex": "0101040100000001040200000000020e0000006d6573736167655f6578706972790c080000000000000000000000", + "outcome": "admission_refused", + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_reset_names_set_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":\"1 day\",\"reset\":[\"message_expiry\"]}", + "update_resource_hex": "01010401000000010402000000010e6d6573736167655f657870697279020e0000006d6573736167655f65787069727902050000003120646179", + "outcome": "decode_refused", + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_unknown_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"retention\":\"1 day\"}", + "update_resource_hex": "01010401000000010402000000000209000000726574656e74696f6e02050000003120646179", + "outcome": "admission_refused", + "error": { + "name": "unsupported_option_key", + "code": 4041 + } + }, + { + "case": "topic_set_create_only_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"compression_algorithm\":\"gzip\"}", + "update_resource_hex": "01010401000000010402000000000215000000636f6d7072657373696f6e5f616c676f726974686d0204000000677a6970", + "outcome": "admission_refused", + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "topic_reset_create_only_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"reset\":[\"compression_algorithm\"]}", + "update_resource_hex": "010104010000000104020000000115636f6d7072657373696f6e5f616c676f726974686d", + "outcome": "admission_refused", + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "stream_reset_name", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "body": "{\"reset\":[\"name\"]}", + "update_resource_hex": "0201040300000001046e616d65", + "outcome": "admission_refused", + "error": { + "name": "option_not_resettable", + "code": 4104 + } + }, + { + "case": "topic_name_as_number", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"name\":5}", + "update_resource_hex": "010104010000000104020000000002040000006e616d650c080000000500000000000000", + "outcome": "admission_refused", + "error": { + "name": "invalid_topic_name", + "code": 2014 + } + }, + { + "case": "stream_empty_name", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "body": "{\"name\":\"\"}", + "update_resource_hex": "020104030000000002040000006e616d65030100000000", + "outcome": "admission_refused", + "error": { + "name": "invalid_stream_name", + "code": 1013 + } + }, + { + "case": "user_status_as_bool", + "scope": "user", + "target": { + "user_id": 5 + }, + "body": "{\"status\":true}", + "update_resource_hex": "03010405000000000206000000737461747573030100000001", + "outcome": "admission_refused", + "error": { + "name": "invalid_user_status", + "code": 45 + } + }, + { + "case": "topic_create_only_key_as_bool", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"segment_size\":true}", + "update_resource_hex": "0101040100000001040200000000020c0000007365676d656e745f73697a65030100000001", + "outcome": "admission_refused", + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "stream_read_only_key_as_number", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "body": "{\"encryption_key_source\":1}", + "update_resource_hex": "02010403000000000215000000656e6372797074696f6e5f6b65795f736f757263650c080000000100000000000000", + "outcome": "admission_refused", + "error": { + "name": "read_only_option", + "code": 4105 + } + }, + { + "case": "topic_invalid_value", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":\"soon\"}", + "update_resource_hex": "0101040100000001040200000000020e0000006d6573736167655f6578706972790204000000736f6f6e", + "outcome": "admission_refused", + "error": { + "name": "invalid_option_value", + "code": 4042 + } + } + ], + "stream_responses": [ + { + "case": "stream_source_none", + "state": { + "encryption_key_source": "none", + "stored_extra": {} + }, + "response_hex": "0100000000e090663c1306000000000000100000000000000c000000000000000c706c61696e2d73747265616d00000000230000000215000000656e6372797074696f6e5f6b65795f736f7572636502040000006e6f6e65", + "expected": { + "id": 1, + "created_at": 1710000000000000, + "name": "plain-stream", + "topics_count": 0, + "size_bytes": 4096, + "messages_count": 12, + "options": { + "encryption_key_source": { + "value": "none", + "kind": "string", + "explicit": false + } + }, + "encryption_key_source": "none", + "encryption_key_fingerprint": null, + "encryption_kek_fingerprints": [] + } + }, + { + "case": "stream_source_stream_with_unknown_explicit_key", + "state": { + "encryption_key_source": "stream", + "data_key": { + "raw_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f" + }, + "kek_config_keys": [ + "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=" + ], + "stored_extra": { + "future_option": { + "value": "enabled", + "kind": "string", + "explicit": true + } + } + }, + "response_hex": "0200000000e090663c1306000000000000100000000000000c0000000000000011636c69656e742d6b65792d73747265616d1e000000020d0000006675747572655f6f7074696f6e0207000000656e61626c65648d000000021a000000656e6372797074696f6e5f6b656b5f66696e6765727072696e74021000000033396534663066393563333639336337021a000000656e6372797074696f6e5f6b65795f66696e6765727072696e740210000000383434316533383533386362666135370215000000656e6372797074696f6e5f6b65795f736f75726365020600000073747265616d", + "expected": { + "id": 2, + "created_at": 1710000000000000, + "name": "client-key-stream", + "topics_count": 0, + "size_bytes": 4096, + "messages_count": 12, + "options": { + "encryption_kek_fingerprint": { + "value": "39e4f0f95c3693c7", + "kind": "string", + "explicit": false + }, + "encryption_key_fingerprint": { + "value": "8441e38538cbfa57", + "kind": "string", + "explicit": false + }, + "encryption_key_source": { + "value": "stream", + "kind": "string", + "explicit": false + }, + "future_option": { + "value": "enabled", + "kind": "string", + "explicit": true + } + }, + "encryption_key_source": "stream", + "encryption_key_fingerprint": "8441e38538cbfa57", + "encryption_kek_fingerprints": [ + "39e4f0f95c3693c7" + ] + } + }, + { + "case": "stream_source_server_two_keks", + "state": { + "encryption_key_source": "server", + "data_key": { + "raw_hex": "202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f" + }, + "kek_config_keys": [ + "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=", + "AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA=" + ], + "stored_extra": {} + }, + "response_hex": "0300000000e090663c1306000000000000100000000000000c00000000000000117365727665722d6b65792d73747265616d000000009e000000021a000000656e6372797074696f6e5f6b656b5f66696e6765727072696e740221000000333965346630663935633336393363372c64353564343739383239626332393336021a000000656e6372797074696f6e5f6b65795f66696e6765727072696e740210000000306563616666316261323931666530640215000000656e6372797074696f6e5f6b65795f736f757263650206000000736572766572", + "expected": { + "id": 3, + "created_at": 1710000000000000, + "name": "server-key-stream", + "topics_count": 0, + "size_bytes": 4096, + "messages_count": 12, + "options": { + "encryption_kek_fingerprint": { + "value": "39e4f0f95c3693c7,d55d479829bc2936", + "kind": "string", + "explicit": false + }, + "encryption_key_fingerprint": { + "value": "0ecaff1ba291fe0d", + "kind": "string", + "explicit": false + }, + "encryption_key_source": { + "value": "server", + "kind": "string", + "explicit": false + } + }, + "encryption_key_source": "server", + "encryption_key_fingerprint": "0ecaff1ba291fe0d", + "encryption_kek_fingerprints": [ + "39e4f0f95c3693c7", + "d55d479829bc2936" + ] + } + } + ], + "topic_responses": [ + { + "case": "topic_explicit_and_derived", + "response_hex": "0400000000e090663c1306000000000000080000000000000700000000000000066576656e7473430000000215000000636f6d7072657373696f6e5f616c676f726974686d0204000000677a6970020e0000006d6573736167655f6578706972790c0800000000a0e3d08c0000002b010000021a000000636f6e73756d65725f6f66667365745f6475726162696c697479020a0000007265706c696361746564020a0000006475726162696c697479020a0000007265706c696361746564020e0000006d61785f746f7069635f73697a650c08000000ffffffffffffffff02190000006d657373616765735f72657175697265645f746f5f736176650b04000000000400000217000000706172746974696f6e5f726573697a655f706f6c69637902070000006d757461626c650214000000707265616c6c6f636174655f7365676d656e7473030100000000020c0000007365676d656e745f73697a650c080000000000004000000000022100000073697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c080000000000100000000000", + "expected": { + "id": 4, + "created_at": 1710000000000000, + "name": "events", + "partitions_count": 0, + "size_bytes": 2048, + "messages_count": 7, + "options": { + "compression_algorithm": { + "value": "gzip", + "kind": "string", + "explicit": true + }, + "consumer_offset_durability": { + "value": "replicated", + "kind": "string", + "explicit": false + }, + "durability": { + "value": "replicated", + "kind": "string", + "explicit": false + }, + "max_topic_size": { + "value": "18446744073709551615", + "kind": "uint64", + "explicit": false + }, + "message_expiry": { + "value": "604800000000", + "kind": "uint64", + "explicit": true + }, + "messages_required_to_save": { + "value": "1024", + "kind": "uint32", + "explicit": false + }, + "partition_resize_policy": { + "value": "mutable", + "kind": "string", + "explicit": false + }, + "preallocate_segments": { + "value": "false", + "kind": "bool", + "explicit": false + }, + "segment_size": { + "value": "1073741824", + "kind": "uint64", + "explicit": false + }, + "size_of_messages_required_to_save": { + "value": "1048576", + "kind": "uint64", + "explicit": false + } + }, + "compression_algorithm": "gzip", + "message_expiry": "604800000000", + "max_topic_size": "18446744073709551615" + } + } + ], + "user_responses": [ + { + "case": "user_empty_blocks", + "response_hex": "0500000000e090663c1306000105616c696365000000000000000000000000", + "expected": { + "id": 5, + "created_at": 1710000000000000, + "status": "active", + "username": "alice", + "options": {} + } + }, + { + "case": "user_populated_blocks", + "response_hex": "0600000000e090663c1306000203626f621e000000020d0000006675747572655f6f7074696f6e0207000000656e61626c65641e000000020c0000006675747572655f71756f74610c08000000010000000000200000000000", + "expected": { + "id": 6, + "created_at": 1710000000000000, + "status": "inactive", + "username": "bob", + "options": { + "future_option": { + "value": "enabled", + "kind": "string", + "explicit": true + }, + "future_quota": { + "value": "9007199254740993", + "kind": "uint64", + "explicit": false + } + } + } + } + ] +}
diff --git a/foreign/node/README.md b/foreign/node/README.md index f3dda93..b6f5b4c 100644 --- a/foreign/node/README.md +++ b/foreign/node/README.md
@@ -22,7 +22,7 @@ npm i --save apache-iggy@edge ``` -Use matching SDK and server builds from the coordinated VSR release. Older builds are incompatible. This release requires fresh server storage (`IGGY-BOUND-GRANTS-5`). Existing incompatible data is refused unchanged. For a local server with the example credentials, follow the [example setup](../../examples/node/README.md#running-examples). +Use matching SDK and server builds from the coordinated VSR release. Older builds are incompatible. This release requires fresh server storage (`IGGY-RESOURCE-OPTIONS-1`). Existing incompatible data is refused unchanged. For a local server with the example credentials, follow the [example setup](../../examples/node/README.md#running-examples). ## basic usage @@ -319,6 +319,49 @@ `0`, `unlimited`, `disabled` and `none` map to zero. Unit-less numbers such as `5` are rejected. +### resource options + +A stream can carry its own data encryption key. Build it with `StreamEncryptionKey.from` from exactly 32 random bytes, or from their canonical padded base64, and pass it to `stream.create` as `encryptionKey`. Never derive the key from a password, because its fingerprint is readable by anyone who can read the stream. A key that is not 32 bytes or canonical base64 is refused with error 70 before anything is sent. + +A `StreamEncryptionKey` keeps its own copy of the bytes and prints only its fingerprint through `util.inspect`, `JSON.stringify` and string conversion, so create arguments carrying it can be logged. Entries passed through `options` print as given, so an `encryption_key` entry there is refused with error 70 before anything is sent. + +The key travels inside the create request over whatever transport the client uses, so connect over TLS when the key must stay confidential. The server stores the key wrapped and never returns it: reads report only `encryptionKeySource`, `encryptionKeyFingerprint` and `encryptionKekFingerprints`. Without a key, a server with encryption enabled generates one for the stream and reports the source `server`. + +```ts +import { randomBytes } from "node:crypto"; +import { Client, StreamEncryptionKey } from "apache-iggy"; + +const client = new Client({ + transport: "TLS", + options: { host: "localhost", port: 8090 }, + credentials: { username: "iggy", password: "iggy" }, +}); + +const key = StreamEncryptionKey.from(randomBytes(32)); +const stream = await client.stream.create({ name: "orders", encryptionKey: key }); +console.log(stream.encryptionKeySource); // "stream" +console.log(stream.encryptionKeyFingerprint === key.fingerprint()); // true +``` + +`stream.update`, `topic.update` and `user.update` change only the keys they name and leave every other setting as it is, so an update without `name` keeps the name. `reset` returns a key to the server default. Use the `StreamOptionKey`, `TopicOptionKey` and `UserOptionKey` constants to name keys. + +Only the topic keys `message_expiry` and `max_topic_size` can be reset. Settings fixed at create, such as the topic compression algorithm, the partition resize policy or the stream key, have no update field. + +The SDK checks the typed fields and `reset` before sending, in the server's order and with its error codes (4041 for an unknown key, 4105 for a read-only key, 4103 for a create-only key, 4104 for a key without a default, then the name, username or status error). For a key this build has no field or constant for, pass `options` entries and `rawReset` names. The SDK sends those as they are and the server decides, so a refusal comes back from the server. + +The server returns 4108 when a non-root caller tries to rename root or change root's status, or when root tries to deactivate itself. + +```ts +await client.topic.update({ + streamId: "orders", + topicId: "events", + messageExpiry: 7_200_000_000n, + reset: ["max_topic_size"], +}); +``` + +Reads return every setting in `options` (keys the client set) and `derivedOptions` (keys the server filled). `u64` values stay `bigint`, so values above 2^53 keep their precision. `system.describeOptions({ scope: OptionsScope.Stream })` lists the keys a server accepts per scope, with `updatable`, `resettable`, `secret`, `readOnly` and `critical`. + ## use sources Cluster polling uses command 15 (`BindSession`), route discovery 105 and poll @@ -326,6 +369,8 @@ server/SDK upgrade and restart them to rejoin their groups. There is no fallback to retired polling commands. +Stream, topic and user updates use the generic update command 16. The former update commands 35, 204 and 304 are retired. + ### Install ```bash @@ -353,6 +398,8 @@ e2e test expect an iggy-server at tcp://127.0.0.1:8090 +The resource options tests also expect the server to run with encryption enabled and a config key (`IGGY_ENCRYPTION_ENABLED=true` and `IGGY_ENCRYPTION_KEY` set to the base64 of 32 bytes), so a stream created without a key reports a server-generated one. + ```bash npm run test:e2e ```
diff --git a/foreign/node/package-lock.json b/foreign/node/package-lock.json index a0263af..131e6ba 100644 --- a/foreign/node/package-lock.json +++ b/foreign/node/package-lock.json
@@ -1,12 +1,12 @@ { "name": "apache-iggy", - "version": "0.10.1-edge.1", + "version": "0.10.1-edge.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "apache-iggy", - "version": "0.10.1-edge.1", + "version": "0.10.1-edge.2", "license": "Apache-2.0", "dependencies": { "@node-rs/xxhash": "1.7.8",
diff --git a/foreign/node/package.json b/foreign/node/package.json index c2b441c..f845707 100644 --- a/foreign/node/package.json +++ b/foreign/node/package.json
@@ -1,7 +1,7 @@ { "name": "apache-iggy", "type": "module", - "version": "0.10.1-edge.1", + "version": "0.10.1-edge.2", "description": "Official Apache Iggy NodeJS SDK", "keywords": [ "iggy",
diff --git a/foreign/node/scripts/check-vsr-protocol.mjs b/foreign/node/scripts/check-vsr-protocol.mjs index dbf4758..d094b85 100644 --- a/foreign/node/scripts/check-vsr-protocol.mjs +++ b/foreign/node/scripts/check-vsr-protocol.mjs
@@ -294,6 +294,28 @@ ); } +// Retry identity: a committed reply can carry the operation the primary +// resolved the request into, and reply correlation maps it back. +const rustRequestOperationBody = rustOperation.match( + /fn request_operation\(&self\) -> Self \{\s*match self \{([\s\S]*?)\n\s*\}\n\s*\}/ +)?.[1] ?? ''; +const rustRequestOperations = new Map(); +for (const [, sources, target] of rustRequestOperationBody.matchAll( + /((?:Self::[A-Za-z0-9]+\s*\|?\s*)+)=>\s*\{?\s*Self::([A-Za-z0-9]+)/g +)) + for (const [, source] of sources.matchAll(/Self::([A-Za-z0-9]+)/g)) + rustRequestOperations.set(source, target); +assert.ok( + rustRequestOperations.size > 0, + 'Rust request_operation mapping not found' +); +for (const [name, value] of rustOperations) + assert.equal( + operationModule.requestOperation?.(value), + rustOperations.get(rustRequestOperations.get(name) ?? name), + `Node requestOperation(${name}) differs from Rust request_operation` + ); + const protocolVersion = rustProtocolVersion.match(/pub const IGGY_PROTOCOL_VERSION: u32 = pack_protocol_version\(([0-9]+), ([0-9]+), ([0-9]+)\);/); assert.ok(protocolVersion, 'binary protocol wire version is missing');
diff --git a/foreign/node/src/bdd/README.md b/foreign/node/src/bdd/README.md index c584e97..7f723ec 100644 --- a/foreign/node/src/bdd/README.md +++ b/foreign/node/src/bdd/README.md
@@ -29,6 +29,8 @@ for iterating against a server you started yourself. note: bdd test expect an iggy-server started with the same root credentials +The stream options feature also needs the server to run with encryption enabled and a config key (`IGGY_ENCRYPTION_ENABLED=true`, `IGGY_ENCRYPTION_KEY`), as `scripts/run-bdd-tests.sh` starts it. + from [/foreign/node](../../) run ```bash
diff --git a/foreign/node/src/bdd/resource-options.ts b/foreign/node/src/bdd/resource-options.ts new file mode 100644 index 0000000..6ce17e3 --- /dev/null +++ b/foreign/node/src/bdd/resource-options.ts
@@ -0,0 +1,333 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import assert from 'node:assert/strict'; +import { Given, When, Then } from '@cucumber/cucumber'; +import { + Consumer, HeaderKeyFactory, HeaderValue, OptionsScope, Partitioning, PollingStrategy, + StreamEncryptionKey, UserStatus, encryptionKeyFingerprint, + type OptionSpec, type StreamOptionKey, type TopicOptionKey +} from '../wire/index.js'; +import type { Headers } from '../wire/message/header.utils.js'; +import type { TestWorld } from './world.js'; + +const NEVER_RETURNED = ['encryption_key', 'encryption_key_wrapped', 'critical_keys']; + +const createStream = async (world: TestWorld, name: string, key?: Buffer) => { + world.stream = await world.client.stream.create({ + name, encryptionKey: key === undefined ? undefined : StreamEncryptionKey.from(key) + }); + world.streamKey = key; +}; + +const fetchStream = async (world: TestWorld) => { + const stream = await world.client.stream.get({ streamId: world.stream.id }); + assert.ok(stream, 'the stream exists'); + return stream; +}; + +const fetchTopic = async (world: TestWorld) => { + const topic = await world.client.topic.get({ + streamId: world.stream.id, topicId: world.topic.id + }); + assert.ok(topic, 'the topic exists'); + return topic; +}; + +const fetchUser = async (world: TestWorld) => { + const user = await world.client.user.get({ userId: world.userId! }); + assert.ok(user, 'the user exists'); + return user; +}; + +const optionSpec = (world: TestWorld, key: string): OptionSpec => { + const spec = world.optionSpecs?.find((candidate) => candidate.key === key); + assert.ok(spec, `the server describes ${key}`); + return spec; +}; + +const sendMessage = async (world: TestWorld, payload: string, headers: Headers = []) => { + assert.ok(await world.client.message.send({ + streamId: world.stream.id, + topicId: world.topic.id, + partition: Partitioning.PartitionId(0), + messages: [{ payload, headers }] + })); +}; + +const pollFirst = async (world: TestWorld) => { + const { messages } = await world.client.message.poll({ + streamId: world.stream.id, + topicId: world.topic.id, + consumer: Consumer.Single, + partitionId: 0, + pollingStrategy: PollingStrategy.First, + count: 1, + autocommit: false + }); + assert.equal(messages.length, 1); + return messages[0]; +}; + +const updateTopic = async ( + world: TestWorld, + update: { name?: string, messageExpiry?: bigint, reset?: TopicOptionKey[] } +) => { + assert.ok(await world.client.topic.update({ + streamId: world.stream.id, topicId: world.topic.id, ...update + })); +}; + +When( + 'I create a stream named {string} with an encryption key of 32 bytes valued {int}', + async function (this: TestWorld, name: string, byte: number) { + await createStream(this, name, Buffer.alloc(32, byte)); + } +); + +Given( + 'a stream named {string} with an encryption key of 32 bytes valued {int} exists', + async function (this: TestWorld, name: string, byte: number) { + await createStream(this, name, Buffer.alloc(32, byte)); + } +); + +When( + 'I create a stream named {string} without an encryption key', + async function (this: TestWorld, name: string) { + await createStream(this, name); + } +); + +Given('a stream named {string} exists', async function (this: TestWorld, name: string) { + await createStream(this, name); +}); + +Then( + 'the stream should report encryption key source {string}', + async function (this: TestWorld, source: string) { + assert.equal((await fetchStream(this)).encryptionKeySource, source); + } +); + +Then( + 'the stream encryption key fingerprint should match the key', + async function (this: TestWorld) { + assert.ok(this.streamKey, 'the stream was created with a key'); + assert.equal( + (await fetchStream(this)).encryptionKeyFingerprint, + encryptionKeyFingerprint(this.streamKey) + ); + } +); + +Then('the stream should report an encryption key fingerprint', async function (this: TestWorld) { + assert.match((await fetchStream(this)).encryptionKeyFingerprint ?? '', /^[0-9a-f]{16}$/); +}); + +Then('the stream should report at least one KEK fingerprint', async function (this: TestWorld) { + assert.ok((await fetchStream(this)).encryptionKekFingerprints.length >= 1); +}); + +Then('the stream should expose no encryption key material', async function (this: TestWorld) { + const stream = await fetchStream(this); + for (const block of [stream.options, stream.derivedOptions]) { + for (const key of NEVER_RETURNED) + assert.equal(key in block, false, `${key} must never be returned`); + for (const value of Object.values(block)) + assert.ok(!(Buffer.isBuffer(value) && this.streamKey?.equals(value)), 'a value holds the key'); + } +}); + +When( + 'I create a topic named {string} with {int} partition(s) in the stream', + async function (this: TestWorld, name: string, partitionCount: number) { + this.topic = await this.client.topic.create({ + streamId: this.stream.id, name, partitionCount + }); + } +); + +When( + 'I send a message with payload {string} to the topic', + async function (this: TestWorld, payload: string) { + await sendMessage(this, payload); + } +); + +When( + 'I send a message with payload {string} and header {string} set to {string} to the topic', + async function (this: TestWorld, payload: string, header: string, value: string) { + await sendMessage(this, payload, [ + { key: HeaderKeyFactory.String(header), value: HeaderValue.String(value) } + ]); + } +); + +Then( + 'polling the topic should return the payload {string}', + async function (this: TestWorld, payload: string) { + assert.equal((await pollFirst(this)).payload.toString(), payload); + } +); + +Then( + 'polling the topic should return the payload {string} and header {string} set to {string}', + async function (this: TestWorld, payload: string, header: string, value: string) { + const message = await pollFirst(this); + assert.equal(message.payload.toString(), payload); + const entry = message.userHeaders.find((candidate) => candidate.key.value === header); + assert.ok(entry, `header ${header} is present`); + assert.equal(entry.value.value, value); + } +); + +When('I rename the stream to {string}', async function (this: TestWorld, name: string) { + assert.ok(await this.client.stream.update({ streamId: this.stream.id, name })); +}); + +Then('the stream name should be {string}', async function (this: TestWorld, name: string) { + assert.equal((await fetchStream(this)).name, name); +}); + +When( + 'I request a reset of the stream option {string}', + async function (this: TestWorld, key: string) { + try { + await this.client.stream.update({ + streamId: this.stream.id, reset: [key as StreamOptionKey] + }); + this.updateError = undefined; + } catch (error) { + this.updateError = error; + } + } +); + +Then('the stream update should be rejected', function (this: TestWorld) { + assert.ok(this.updateError, 'the update must fail'); +}); + +When('I describe the stream options', async function (this: TestWorld) { + this.optionSpecs = await this.client.system.describeOptions({ scope: OptionsScope.Stream }); +}); + +Then( + 'the stream option {string} should be updatable and not secret', + function (this: TestWorld, key: string) { + const spec = optionSpec(this, key); + assert.ok(spec.updatable && !spec.secret && !spec.readOnly); + } +); + +Then( + 'the stream option {string} should be create only and secret', + function (this: TestWorld, key: string) { + const spec = optionSpec(this, key); + assert.ok(!spec.updatable && !spec.resettable && spec.secret && !spec.readOnly); + } +); + +Then('the stream option {string} should be read only', function (this: TestWorld, key: string) { + const spec = optionSpec(this, key); + assert.ok(spec.readOnly && !spec.updatable && !spec.secret); +}); + +Given( + /^a topic named "([^"]+)" exists in the stream with message expiry (\d+) microseconds and max topic size (\d+) bytes$/, + async function (this: TestWorld, name: string, expiry: string, maxSize: string) { + this.topic = await this.client.topic.create({ + streamId: this.stream.id, name, partitionCount: 1, + messageExpiry: BigInt(expiry), maxTopicSize: BigInt(maxSize) + }); + } +); + +When( + /^I update the topic message expiry to (\d+) microseconds without a name$/, + async function (this: TestWorld, expiry: string) { + await updateTopic(this, { messageExpiry: BigInt(expiry) }); + } +); + +When('I reset the topic option {string}', async function (this: TestWorld, key: string) { + await updateTopic(this, { reset: [key as TopicOptionKey] }); +}); + +When('I rename the topic to {string}', async function (this: TestWorld, name: string) { + await updateTopic(this, { name }); +}); + +Then('the topic name should be {string}', async function (this: TestWorld, name: string) { + assert.equal((await fetchTopic(this)).name, name); +}); + +Then( + /^the topic message expiry should be (\d+) microseconds set explicitly$/, + async function (this: TestWorld, expiry: string) { + const topic = await fetchTopic(this); + assert.equal(topic.messageExpiry, BigInt(expiry)); + assert.equal(topic.options.message_expiry, BigInt(expiry)); + } +); + +Then( + /^the topic max topic size should be (\d+) bytes set explicitly$/, + async function (this: TestWorld, maxSize: string) { + const topic = await fetchTopic(this); + assert.equal(topic.maxTopicSize, BigInt(maxSize)); + assert.equal(topic.options.max_topic_size, BigInt(maxSize)); + } +); + +Then( + 'the topic max topic size should be derived from the server default', + async function (this: TestWorld) { + const topic = await fetchTopic(this); + assert.equal('max_topic_size' in topic.options, false); + assert.equal(topic.derivedOptions.max_topic_size, topic.maxTopicSize); + } +); + +Given( + 'a user named {string} with password {string} exists', + async function (this: TestWorld, username: string, password: string) { + const user = await this.client.user.create({ + username, password, status: UserStatus.Active, options: [] + }); + this.userId = user.id; + } +); + +When( + 'I update the user username to {string} and status to {string}', + async function (this: TestWorld, username: string, status: string) { + assert.ok(await this.client.user.update({ + userId: this.userId!, + username, + status: status === 'active' ? UserStatus.Active : UserStatus.Inactive + })); + } +); + +Then('the user should have username {string}', async function (this: TestWorld, username: string) { + assert.equal((await fetchUser(this)).userName, username); +}); + +Then('the user should have status {string}', async function (this: TestWorld, status: string) { + assert.equal((await fetchUser(this)).status.toLowerCase(), status); +});
diff --git a/foreign/node/src/bdd/world.ts b/foreign/node/src/bdd/world.ts index cd689b4..4d26e7d 100644 --- a/foreign/node/src/bdd/world.ts +++ b/foreign/node/src/bdd/world.ts
@@ -17,7 +17,7 @@ import type { Stream } from '../wire/stream/stream.utils.js'; import type { Topic } from '../wire/topic/topic.utils.js'; -import type { CreateMessage, Message } from 'src/wire/index.js'; +import type { CreateMessage, Message, OptionSpec } from 'src/wire/index.js'; import type { Client } from '../client/index.js'; export interface TestWorld { @@ -28,5 +28,10 @@ polledMessages: Message[], rawResponse?: Buffer, rawError?: Error, + /** Key the last stream was created with, absent for a keyless stream */ + streamKey?: Buffer, + updateError?: unknown, + optionSpecs?: OptionSpec[], + userId?: number, callError?: Error };
diff --git a/foreign/node/src/client/client.operation.test.ts b/foreign/node/src/client/client.operation.test.ts index d8db701..e6a0756 100644 --- a/foreign/node/src/client/client.operation.test.ts +++ b/foreign/node/src/client/client.operation.test.ts
@@ -31,6 +31,21 @@ lanes.enter(lane, COMMAND_CODE.CreateStream, body, EMPTY_PARTITION_CONTEXT, true, state, deadline, signal); describe('mutation admission and retained ownership', () => { + it('retains the request identifier for a replay and mints one for a restart', async () => { + const lanes = new MutationLanes(1, frameSize); + const state = session(); + const operation = await enter(lanes, state, 'metadata'); + const span = [REQUEST_OFFSET.requestChecksum, REQUEST_OFFSET.requestChecksum + 16] as const; + const first = Buffer.from(operation.frame.buffer.subarray(...span)); + const replay = operation.frame.buffer; + assert.deepEqual(replay.subarray(...span), first); + + await lanes.restart(operation, body, EMPTY_PARTITION_CONTEXT, state, Date.now() + 1_000, + new AbortController().signal); + assert.notDeepEqual(operation.frame.buffer.subarray(...span), first); + lanes.complete(operation); + }); + it('retains one lane while independent groups keep their own order', async () => { const lanes = new MutationLanes(3, 3 * frameSize); const state = session();
diff --git a/foreign/node/src/client/client.socket.test.ts b/foreign/node/src/client/client.socket.test.ts index 6503e07..c4f16c6 100644 --- a/foreign/node/src/client/client.socket.test.ts +++ b/foreign/node/src/client/client.socket.test.ts
@@ -29,6 +29,9 @@ import { LEAVE_GROUP } from '../wire/consumer-group/leave-group.command.js'; import { LOGIN, LOGIN_WITH_TOKEN, LOGOUT } from '../wire/index.js'; import { UPDATE_USER } from '../wire/user/update-user.command.js'; +import { UPDATE_TOPIC } from '../wire/topic/update-topic.command.js'; +import { UPDATE_STREAM } from '../wire/stream/update-stream.command.js'; +import { OptionsScope } from '../wire/system/describe-options.command.js'; import { CHANGE_PASSWORD } from '../wire/user/change-password.command.js'; import { POLL_MESSAGES, pollMessages } from '../wire/message/poll-messages.command.js'; import { SEND_MESSAGES, sendMessages } from '../wire/message/send-messages.command.js'; @@ -374,6 +377,19 @@ return body; }; +const RESOLVED_UPDATES = new Map<number, number>([ + [OptionsScope.Topic, Operation.UpdateTopicResolved], + [OptionsScope.Stream, Operation.UpdateStreamResolved], + [OptionsScope.User, Operation.UpdateUserResolved] +]); + +/** The operation a committed reply carries: the primary resolves an update by its scope. */ +const committedOperation = (frame: Buffer): number => { + const operation = frame[REQUEST_OFFSET.operation]; + return operation === Operation.UpdateResource + ? RESOLVED_UPDATES.get(frame[HEADER_SIZE]) ?? operation : operation; +}; + /** A single node that also commits offset writes. */ const offsetCommittingHandler = (port: number): FrameHandler => (frame, socket) => { const operation = frame.readUInt8(REQUEST_OFFSET.operation); @@ -513,10 +529,10 @@ socket.write(replyFrame(frame, Operation.NonReplicated, Buffer.alloc(0))); return true; }, (frame, socket) => { - if (frame[REQUEST_OFFSET.operation] === Operation.UpdateUser) { + if (frame[REQUEST_OFFSET.operation] === Operation.UpdateResource) { watermark = 19n; incarnation = 8n; - const response = replyFrame(frame, Operation.UpdateUser, metadataResult(0)); + const response = replyFrame(frame, Operation.UpdateUserResolved, metadataResult(0)); response.writeBigUInt64LE(watermark, REPLY_OFFSET.commit); response.writeBigUInt64LE(watermark, REPLY_OFFSET.op); response[REPLY_OFFSET.reserved] = 1; @@ -533,7 +549,7 @@ ? groupPollPayload() : GET_OFFSET.serialize({ streamId: 1, topicId: 2, consumer: { kind: ConsumerKind.Single, id: 1 }, partitionId: 3 }); await cluster.client.sendCommand(command, payload); - await cluster.client.sendCommand(COMMAND_CODE.UpdateUser, Buffer.alloc(0)); + await cluster.client.sendCommand(COMMAND_CODE.UpdateResource, Buffer.alloc(0)); await cluster.client.sendCommand(command, payload); assert.deepEqual(histories, [7n, 8n]); assert.equal(countCommand(cluster.coordinator, COMMAND_CODE.GetPartitionRoute), 2); @@ -672,9 +688,9 @@ socket.write(response); return true; }, (frame, socket) => { - if (frame[REQUEST_OFFSET.operation] !== Operation.UpdateTopic) + if (frame[REQUEST_OFFSET.operation] !== Operation.UpdateResource) return false; - const response = replyFrame(frame, Operation.UpdateTopic, Buffer.alloc(4)); + const response = replyFrame(frame, Operation.UpdateTopicResolved, Buffer.alloc(4)); response.writeBigUInt64LE(15n, REPLY_OFFSET.commit); response.writeBigUInt64LE(15n, REPLY_OFFSET.op); response[REPLY_OFFSET.reserved] = 1; @@ -686,7 +702,7 @@ await cluster.client.sendCommand(COMMAND_CODE.SendMessages, explicitSend); await cluster.client.sendCommand(POLL_MESSAGES.code, groupPollPayload()); assert.equal(countCommand(cluster.coordinator, COMMAND_CODE.GetPartitionRoute), 2); - await cluster.client.sendCommand(COMMAND_CODE.UpdateTopic, Buffer.alloc(0), { handleResponse: false }); + await cluster.client.sendCommand(COMMAND_CODE.UpdateResource, Buffer.alloc(0), { handleResponse: false }); await cluster.client.sendCommand(POLL_MESSAGES.code, groupPollPayload()); const attachments = cluster.primary.frames.filter((frame) => frame.readUInt32LE(REQUEST_OFFSET.reserved) === COMMAND_CODE.BindSession); @@ -939,9 +955,9 @@ firstPoll({ socket, frame }); return true; }, (frame, socket) => { - if (frame[REQUEST_OFFSET.operation] !== Operation.UpdateTopic) + if (frame[REQUEST_OFFSET.operation] !== Operation.UpdateResource) return false; - const response = replyFrame(frame, Operation.UpdateTopic, Buffer.alloc(4)); + const response = replyFrame(frame, Operation.UpdateTopicResolved, Buffer.alloc(4)); response.writeBigUInt64LE(21n, REPLY_OFFSET.commit); response.writeBigUInt64LE(21n, REPLY_OFFSET.op); response[REPLY_OFFSET.reserved] = 1; @@ -952,7 +968,7 @@ const first = cluster.client.sendCommand(POLL_MESSAGES.code, groupPollPayload()); const { socket, frame } = await admitted; const second = cluster.client.sendCommand(POLL_MESSAGES.code, groupPollPayload()); - await cluster.client.sendCommand(COMMAND_CODE.UpdateTopic, Buffer.alloc(0)); + await cluster.client.sendCommand(COMMAND_CODE.UpdateResource, Buffer.alloc(0)); assert.equal(polls, 1, 'progress polls on one partition share its mutation lane'); socket.write(replyFrame(frame, Operation.NonReplicated, Buffer.from('primary'))); await Promise.all([first, second]); @@ -1093,9 +1109,9 @@ it('binds primary connections after credential changes without another login', async () => { const cluster = await startPollCluster(undefined, (frame, socket) => { const operation = frame[REQUEST_OFFSET.operation]; - if (operation !== Operation.UpdateUser && operation !== Operation.ChangePassword) + if (operation !== Operation.UpdateResource && operation !== Operation.ChangePassword) return false; - socket.write(replyFrame(frame, operation, Buffer.alloc(4))); + socket.write(replyFrame(frame, committedOperation(frame), Buffer.alloc(4))); return true; }); try { @@ -1177,9 +1193,9 @@ const cluster = await startPollCluster(undefined, (frame, socket) => { coordinatorSocket = socket; const operation = frame[REQUEST_OFFSET.operation]; - if (operation !== Operation.UpdateUser && operation !== Operation.ChangePassword) + if (operation !== Operation.UpdateResource && operation !== Operation.ChangePassword) return false; - socket.write(replyFrame(frame, operation, Buffer.alloc(4))); + socket.write(replyFrame(frame, committedOperation(frame), Buffer.alloc(4))); return true; }); try { @@ -1206,6 +1222,66 @@ } }); + it('keeps the signed-in identity when a stream or topic rename names it', async () => { + let coordinatorSocket!: Socket; + const cluster = await startPollCluster(undefined, (frame, socket) => { + coordinatorSocket = socket; + if (frame[REQUEST_OFFSET.operation] !== Operation.UpdateResource) + return false; + socket.write(replyFrame(frame, committedOperation(frame), Buffer.alloc(4))); + return true; + }); + try { + await cluster.client.sendCommand(POLL_MESSAGES.code, groupPollPayload()); + for (const streamId of [7, 'iggy']) + await cluster.client.sendCommand(UPDATE_TOPIC.code, + UPDATE_TOPIC.serialize({ streamId, topicId: 'iggy', name: 'renamed' })); + for (const streamId of [7, 'iggy']) + await cluster.client.sendCommand(UPDATE_STREAM.code, + UPDATE_STREAM.serialize({ streamId, name: 'renamed' })); + const reset = once(cluster.client, 'sessionReset'); + coordinatorSocket.write(evictionFrame(EvictionReason.SessionError)); + await reset; + await cluster.client.sendCommand(POLL_MESSAGES.code, groupPollPayload()); + const login = cluster.coordinator.frames.filter((frame) => + frame[REQUEST_OFFSET.operation] === Operation.Register).at(-1)!; + assert.equal(readWireName(registerCredentialsBody(login), 0).value, 'iggy'); + } finally { + await cluster.close(); + } + }); + + it('keeps the old identity when its rename is refused', async () => { + let coordinatorSocket!: Socket; + const cluster = await startPollCluster(undefined, (frame, socket) => { + coordinatorSocket = socket; + if (frame[REQUEST_OFFSET.operation] !== Operation.UpdateResource) + return false; + const response = replyFrame(frame, committedOperation(frame), metadataResult(UNAUTHORIZED)); + response.writeBigUInt64LE(3n, REPLY_OFFSET.commit); + response.writeBigUInt64LE(3n, REPLY_OFFSET.op); + response[REPLY_OFFSET.reserved] = 1; + socket.write(response); + return true; + }); + try { + await cluster.client.sendCommand(POLL_MESSAGES.code, groupPollPayload()); + await assert.rejects( + cluster.client.sendCommand(UPDATE_USER.code, UPDATE_USER.serialize({ userId: 7, username: 'renamed' })), + (error: unknown) => error instanceof ResponseError && error.errorCode === UNAUTHORIZED + ); + const reset = once(cluster.client, 'sessionReset'); + coordinatorSocket.write(evictionFrame(EvictionReason.SessionError)); + await reset; + await cluster.client.sendCommand(POLL_MESSAGES.code, groupPollPayload()); + const login = cluster.coordinator.frames.filter((frame) => + frame[REQUEST_OFFSET.operation] === Operation.Register).at(-1)!; + assert.equal(readWireName(registerCredentialsBody(login), 0).value, 'iggy'); + } finally { + await cluster.close(); + } + }); + it('closes a waiting primary exchange on coordinator logout without replay', async () => { let admitted!: () => void; const pendingAdmission = new Promise<void>((resolve) => { admitted = resolve; });
diff --git a/foreign/node/src/client/client.socket.ts b/foreign/node/src/client/client.socket.ts index ee40fc7..6f338d8 100644 --- a/foreign/node/src/client/client.socket.ts +++ b/foreign/node/src/client/client.socket.ts
@@ -33,6 +33,8 @@ import { ConsumerKind } from '../wire/offset/offset.utils.js'; import { COMMAND_CODE } from '../wire/command.code.js'; import { serializeIdentifier } from '../wire/identifier.utils.js'; +import { deserializeOptions } from '../wire/options.utils.js'; +import { OptionsScope } from '../wire/system/describe-options.command.js'; import { PartitionKind } from '../wire/message/partitioning.utils.js'; import { PollingStrategyKind } from '../wire/message/poll.utils.js'; import { @@ -1098,22 +1100,32 @@ }; } else if (command === LOGIN_WITH_TOKEN.code) { this.signInCredentials = { token: readWireName(payload, 0).value }; - } else if ((command === COMMAND_CODE.ChangePassword || command === COMMAND_CODE.UpdateUser) && + } else if ((command === COMMAND_CODE.ChangePassword || + (command === COMMAND_CODE.UpdateResource && payload[0] === OptionsScope.User)) && this.userId !== undefined && this.signInCredentials && 'username' in this.signInCredentials) { const credentials = this.signInCredentials; + // An update leads with its scope byte, a password change with the user id. + const start = command === COMMAND_CODE.ChangePassword ? 0 : 1; const identifier = [serializeIdentifier(this.userId), serializeIdentifier(credentials.username)] - .find((encoded) => payload.subarray(0, encoded.length).equals(encoded)); + .find((encoded) => payload.subarray(start, start + encoded.length).equals(encoded)); if (!identifier) return; if (command === COMMAND_CODE.ChangePassword) { const currentPassword = readWireName(payload, identifier.length); const password = readWireName(payload, currentPassword.next).value; this.signInCredentials = { ...credentials, password }; - } else if (payload[identifier.length] === 1) { - const username = readWireName(payload, identifier.length + 1).value; - this.signInCredentials = { ...credentials, username }; + return; } + let position = start + identifier.length; + const resetCount = payload.readUInt8(position); + position += 1; + for (let index = 0; index < resetCount; index++) + position = readWireName(payload, position).next; + const username = deserializeOptions(payload, position).username; + if (typeof username !== 'string') + return; + this.signInCredentials = { ...credentials, username }; } }
diff --git a/foreign/node/src/e2e/resource-options.e2e.ts b/foreign/node/src/e2e/resource-options.e2e.ts new file mode 100644 index 0000000..4be2df2 --- /dev/null +++ b/foreign/node/src/e2e/resource-options.e2e.ts
@@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// These tests need a server started with encryption enabled and a config key +// (IGGY_ENCRYPTION_ENABLED=true, IGGY_ENCRYPTION_KEY), so a stream created +// without a key reports a server-generated one. + +import { Client } from '../client/client.js'; +import { getIggyAddress } from '../tcp.sm.utils.js'; +import { EncryptionKeySource } from '../wire/index.js'; +import { describeResourceOptions } from './resource-options.suite.js'; + +const [host, port] = getIggyAddress(); + +describeResourceOptions('e2e -> resource options', 'e2e-ro', (credentials) => new Client({ + transport: 'TCP', options: { host, port }, credentials +}), EncryptionKeySource.Server);
diff --git a/foreign/node/src/e2e/resource-options.suite.ts b/foreign/node/src/e2e/resource-options.suite.ts new file mode 100644 index 0000000..51f8f5b --- /dev/null +++ b/foreign/node/src/e2e/resource-options.suite.ts
@@ -0,0 +1,323 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { randomBytes } from 'node:crypto'; +import { after, describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import type { Client } from '../client/client.js'; +import { HeaderValue } from '../wire/message/header.utils.js'; +import { ResponseError } from '../wire/error.utils.js'; +import { + Consumer, EncryptionKeySource, OptionsScope, PartitionResizePolicy, Partitioning, + PollingStrategy, StreamEncryptionKey, UserStatus, encryptionKeyFingerprint, type ParsedOptions +} from '../wire/index.js'; +import type { Stream } from '../wire/stream/stream.utils.js'; + +const NEVER_RETURNED = ['encryption_key', 'encryption_key_wrapped', 'critical_keys']; +const FINGERPRINT = /^[0-9a-f]{16}$/; +const UNLIMITED = 18446744073709551615n; +const UNSUPPORTED_OPTION_KEY = 4041; +const OPTION_NOT_UPDATABLE = 4103; + +type Credentials = { username: string, password: string }; + +const assertProjection = ( + stream: Stream, source: EncryptionKeySource, fingerprint?: string +) => { + assert.equal(stream.encryptionKeySource, source); + if (source === EncryptionKeySource.None) { + assert.equal(stream.encryptionKeyFingerprint, null); + assert.deepEqual(stream.encryptionKekFingerprints, []); + return; + } + assert.match(stream.encryptionKeyFingerprint ?? '', FINGERPRINT); + if (fingerprint !== undefined) + assert.equal(stream.encryptionKeyFingerprint, fingerprint); + assert.ok(stream.encryptionKekFingerprints.length >= 1); + for (const kek of stream.encryptionKekFingerprints) + assert.match(kek, FINGERPRINT); +}; + +const assertNoKeyMaterial = (stream: Stream, key?: Buffer) => { + for (const block of [stream.options, stream.derivedOptions] as ParsedOptions[]) { + for (const name of NEVER_RETURNED) + assert.equal(name in block, false, `${name} was returned`); + if (key) + for (const value of Object.values(block)) + assert.ok( + !(Buffer.isBuffer(value) && value.equals(key)) && value !== key.toString('base64'), + 'a returned option holds the key' + ); + } +}; + +/** + * Registers the resource options cases for one transport. `connect` builds a + * client of that transport signed in with the given credentials. + * `keylessSource` is what a stream created without a key reports: `server` + * when the server runs with encryption enabled, `none` otherwise. + */ +export const describeResourceOptions = ( + label: string, prefix: string, connect: (credentials: Credentials) => Client, + keylessSource: EncryptionKeySource, skip: string | false = false +) => describe(label, { skip }, async () => { + + const c = connect({ username: 'iggy', password: 'iggy' }); + + it('given stream key when creating should read matching projection', async () => { + const key = randomBytes(32); + const name = `${prefix}-keyed`; + const created = await c.stream.create({ name, encryptionKey: StreamEncryptionKey.from(key) }); + try { + assertProjection(created, EncryptionKeySource.Stream, encryptionKeyFingerprint(key)); + assertNoKeyMaterial(created, key); + + const fetched = await c.stream.get({ streamId: name }); + assert.ok(fetched); + assertProjection(fetched, EncryptionKeySource.Stream, encryptionKeyFingerprint(key)); + assert.deepEqual(fetched.encryptionKekFingerprints, created.encryptionKekFingerprints); + assertNoKeyMaterial(fetched, key); + + const listed = (await c.stream.list()).find((stream) => stream.id === created.id); + assert.ok(listed); + assertProjection(listed, EncryptionKeySource.Stream, encryptionKeyFingerprint(key)); + assertNoKeyMaterial(listed, key); + + const base64 = `${prefix}-keyed-base64`; + const fromText = await c.stream.create({ + name: base64, encryptionKey: StreamEncryptionKey.from(key.toString('base64')) + }); + assert.equal(fromText.encryptionKeyFingerprint, encryptionKeyFingerprint(key)); + await c.stream.delete({ streamId: fromText.id }); + } finally { + await c.stream.delete({ streamId: created.id }); + } + }); + + it(`given keyless stream when reading should report key source ${keylessSource}`, async () => { + const created = await c.stream.create({ name: `${prefix}-keyless` }); + try { + assertProjection(created, keylessSource); + const fetched = await c.stream.get({ streamId: created.id }); + assert.ok(fetched); + assertProjection(fetched, keylessSource, created.encryptionKeyFingerprint ?? undefined); + assertNoKeyMaterial(fetched); + } finally { + await c.stream.delete({ streamId: created.id }); + } + }); + + it('given stream rename when updating should preserve encryption projection', async () => { + const key = randomBytes(32); + const created = await c.stream.create({ + name: `${prefix}-rename`, encryptionKey: StreamEncryptionKey.from(key) + }); + try { + await c.topic.create({ streamId: created.id, name: 'events', partitionCount: 1 }); + await c.message.send({ + streamId: created.id, topicId: 'events', partition: Partitioning.PartitionId(0), + messages: [{ payload: 'before-rename', headers: [] }] + }); + + assert.ok(await c.stream.update({ streamId: created.id, name: `${prefix}-renamed` })); + await assert.rejects( + c.stream.update({ streamId: created.id, reset: ['encryption_key'] }), + (error: unknown) => error instanceof ResponseError && + error.errorCode === OPTION_NOT_UPDATABLE && !error.fromServer + ); + + const renamed = await c.stream.get({ streamId: created.id }); + assert.ok(renamed); + assert.equal(renamed.id, created.id); + assert.equal(renamed.name, `${prefix}-renamed`); + assertProjection(renamed, EncryptionKeySource.Stream, encryptionKeyFingerprint(key)); + assert.deepEqual(renamed.encryptionKekFingerprints, created.encryptionKekFingerprints); + + const polled = await c.message.poll({ + streamId: created.id, topicId: 'events', consumer: Consumer.Single, partitionId: 0, + pollingStrategy: PollingStrategy.First, count: 10, autocommit: false + }); + assert.deepEqual(polled.messages.map((message) => message.payload.toString()), ['before-rename']); + } finally { + await c.stream.delete({ streamId: created.id }); + } + }); + + it('given topic reset when updating should preserve unmentioned values', async () => { + const stream = await c.stream.create({ name: `${prefix}-topic-reset` }); + try { + const topic = await c.topic.create({ + streamId: stream.id, name: 'settings', partitionCount: 1, + compressionAlgorithm: 2, messageExpiry: 3_600_000_000n, maxTopicSize: 1_073_741_824n + }); + assert.ok(await c.topic.update({ + streamId: stream.id, topicId: topic.id, + messageExpiry: 7_200_000_000n, reset: ['max_topic_size'] + })); + + const updated = await c.topic.get({ streamId: stream.id, topicId: topic.id }); + assert.ok(updated); + assert.equal(updated.name, 'settings'); + assert.equal(updated.options.compression_algorithm, 'gzip'); + assert.equal(updated.compressionAlgorithm, 2); + assert.equal(updated.options.message_expiry, 7_200_000_000n); + assert.equal(updated.messageExpiry, 7_200_000_000n); + assert.equal('max_topic_size' in updated.options, false); + assert.equal(updated.derivedOptions.max_topic_size, UNLIMITED); + assert.equal(updated.maxTopicSize, UNLIMITED); + assert.deepEqual(updated.derivedOptions, { + ...topic.derivedOptions, max_topic_size: UNLIMITED + }); + } finally { + await c.stream.delete({ streamId: stream.id }); + } + }); + + it('given user options when creating and updating should preserve user behavior', async () => { + const password = 'e2e-ro-password'; + const first = await c.user.create({ + username: `${prefix}-user`, password, status: UserStatus.Active, options: [] + }); + const second = await c.user.create({ + username: `${prefix}-user-taken`, password, status: UserStatus.Active + }); + try { + assert.deepEqual(first.options, {}); + assert.deepEqual(first.derivedOptions, {}); + + assert.ok(await c.user.update({ + userId: first.id, username: `${prefix}-user-renamed`, status: UserStatus.Inactive + })); + const updated = await c.user.get({ userId: first.id }); + assert.ok(updated); + assert.equal(updated.userName, `${prefix}-user-renamed`); + assert.equal(updated.status, 'Inactive'); + assert.deepEqual(updated.options, {}); + + await assert.rejects( + c.user.update({ userId: first.id, username: `${prefix}-user-taken`, status: UserStatus.Active }), + (error: unknown) => error instanceof ResponseError && error.fromServer + ); + const unchanged = await c.user.get({ userId: first.id }); + assert.ok(unchanged); + assert.equal(unchanged.userName, `${prefix}-user-renamed`); + assert.equal(unchanged.status, 'Inactive'); + } finally { + await c.user.delete({ userId: first.id }); + await c.user.delete({ userId: second.id }); + } + }); + + it('given logged in user rename when reconnecting should keep new username', async () => { + const password = 'e2e-ro-self-password'; + const user = await c.user.create({ + username: `${prefix}-self`, password, status: UserStatus.Active, + permissions: { + global: { + ManageServers: false, ReadServers: false, ManageUsers: true, ReadUsers: true, + ManageStreams: true, ReadStreams: true, ManageTopics: true, ReadTopics: true, + PollMessages: false, SendMessages: false + }, + streams: [] + } + }); + const taken = await c.user.create({ + username: `${prefix}-self-taken`, password, status: UserStatus.Active + }); + const self = connect({ username: `${prefix}-self`, password }); + const signedInAs = async (username: string) => { + await self.session.logout(); + await self.session.login({ username, password }); + const me = await self.client.getMe(); + return (await c.user.get({ userId: me.userId }))?.userName; + }; + try { + assert.equal((await self.client.getMe()).userId, user.id); + + await assert.rejects( + self.user.update({ userId: user.id, username: `${prefix}-self-taken` }), + (error: unknown) => error instanceof ResponseError && error.fromServer + ); + assert.equal(await signedInAs(`${prefix}-self`), `${prefix}-self`); + + const stream = await self.stream.create({ name: `${prefix}-self-stream` }); + await self.stream.update({ streamId: stream.id, name: `${prefix}-self` }); + const topic = await self.topic.create({ streamId: stream.id, name: 'events', partitionCount: 1 }); + await self.topic.update({ streamId: stream.id, topicId: topic.id, name: `${prefix}-self-topic` }); + await self.stream.delete({ streamId: stream.id }); + assert.equal(await signedInAs(`${prefix}-self`), `${prefix}-self`); + + assert.ok(await self.user.update({ userId: user.id, username: `${prefix}-self-renamed` })); + assert.equal(await signedInAs(`${prefix}-self-renamed`), `${prefix}-self-renamed`); + } finally { + await self.destroy(); + await c.user.delete({ userId: user.id }); + await c.user.delete({ userId: taken.id }); + } + }); + + it('given raw create-only key or unknown raw reset when updating should let the server refuse', async () => { + const stream = await c.stream.create({ name: `${prefix}-raw` }); + try { + const topic = await c.topic.create({ + streamId: stream.id, name: 'fixed', partitionCount: 1, + partitionResizePolicy: PartitionResizePolicy.Fixed + }); + assert.equal(topic.options.partition_resize_policy, PartitionResizePolicy.Fixed); + const fromServer = (code: number) => (error: unknown) => + error instanceof ResponseError && error.errorCode === code && error.fromServer; + await assert.rejects( + c.topic.update({ + streamId: stream.id, topicId: topic.id, + options: [{ key: 'compression_algorithm', value: HeaderValue.String('gzip') }] + }), + fromServer(OPTION_NOT_UPDATABLE) + ); + await assert.rejects( + c.topic.update({ streamId: stream.id, topicId: topic.id, rawReset: ['future_option'] }), + fromServer(UNSUPPORTED_OPTION_KEY) + ); + const unchanged = await c.topic.get({ streamId: stream.id, topicId: topic.id }); + assert.equal(unchanged?.compressionAlgorithm, topic.compressionAlgorithm); + } finally { + await c.stream.delete({ streamId: stream.id }); + } + }); + + it('given stream scope when describing options should report sensitivity and mutability', async () => { + const specs = await c.system.describeOptions({ scope: OptionsScope.Stream }); + const byKey = new Map(specs.map((spec) => [spec.key, spec])); + assert.deepEqual([...byKey.keys()], [ + 'name', 'encryption_key', 'encryption_key_source', + 'encryption_key_fingerprint', 'encryption_kek_fingerprint' + ]); + assert.equal(byKey.get('name')?.updatable, true); + assert.equal(byKey.get('name')?.secret, false); + assert.equal(byKey.get('encryption_key')?.updatable, false); + assert.equal(byKey.get('encryption_key')?.secret, true); + assert.equal(byKey.get('encryption_key')?.critical, true); + assert.equal(byKey.get('encryption_key_fingerprint')?.readOnly, true); + }); + + it(`${label}::cleanup`, async () => { + assert.ok(await c.session.logout()); + }); + + after(() => { + c.destroy(); + }); +});
diff --git a/foreign/node/src/e2e/tcp.token.e2e.ts b/foreign/node/src/e2e/tcp.token.e2e.ts index 88f5ba4..90a25b8 100644 --- a/foreign/node/src/e2e/tcp.token.e2e.ts +++ b/foreign/node/src/e2e/tcp.token.e2e.ts
@@ -24,15 +24,21 @@ const c = getTestClient(); const tokenName = 'yolo-token-test'; + const lifetimeMs = 30 * 60 * 1000; + let createdAt = 0; it('e2e -> token::create', async () => { - const tk = await c.token.create({ name: tokenName, expiry: 1800n }); + createdAt = Date.now(); + const tk = await c.token.create({ name: tokenName, expiry: BigInt(lifetimeMs) * 1000n }); assert.ok(tk.token.length > 1); }); it('e2e -> token::list', async () => { const tks = await c.token.list(); - assert.ok(tks.length > 0); + const listed = tks.find((tk) => tk.name === tokenName); + assert.ok(listed?.expiry, 'the token lists its expiry'); + const remainingMs = listed.expiry.getTime() - createdAt; + assert.ok(Math.abs(remainingMs - lifetimeMs) < 5000, `expiry ${remainingMs} ms after create, want ${lifetimeMs}`); }); it('e2e -> token::delete', async () => {
diff --git a/foreign/node/src/e2e/tls.system.e2e.ts b/foreign/node/src/e2e/tls.system.e2e.ts index aafce9f..71b21b2 100644 --- a/foreign/node/src/e2e/tls.system.e2e.ts +++ b/foreign/node/src/e2e/tls.system.e2e.ts
@@ -17,11 +17,13 @@ // TLS integration tests for the Node.js SDK. // -// These tests require a TLS-enabled Iggy server and are skipped by default. -// To run them locally: +// These tests require a TLS-enabled Iggy server with an encryption key and are +// skipped by default. Encryption stays disabled, the shipped default, so a +// stream created without a key is plaintext. To run them locally: // // 1. Start the server with TLS: // IGGY_ROOT_USERNAME=iggy IGGY_ROOT_PASSWORD=iggy \ +// IGGY_ENCRYPTION_KEY=<base64 of 32 bytes> \ // IGGY_TCP_TLS_ENABLED=true \ // IGGY_TCP_TLS_CERT_FILE=core/certs/iggy_cert.pem \ // IGGY_TCP_TLS_KEY_FILE=core/certs/iggy_key.pem \ @@ -38,8 +40,9 @@ import assert from 'node:assert/strict'; import { Client } from '../client/client.js'; import type { TlsOption } from '../client/client.type.js'; -import { Partitioning, Consumer, PollingStrategy } from '../wire/index.js'; +import { Partitioning, Consumer, EncryptionKeySource, PollingStrategy } from '../wire/index.js'; import { getIggyAddress } from '../tcp.sm.utils.js'; +import { describeResourceOptions } from './resource-options.suite.js'; const tlsEnabled = process.env.IGGY_TCP_TLS_ENABLED === 'true'; @@ -48,7 +51,7 @@ const caCertPath = process.env.E2E_ROOT_CA_CERT || resolve(process.cwd(), '../../core/certs/iggy_ca_cert.pem'); -const getTlsClient = () => { +const getTlsClient = (credentials = { username: 'iggy', password: 'iggy' }) => { const [, port] = getIggyAddress(); const caCert = readFileSync(caCertPath); @@ -59,12 +62,19 @@ options: { port, host: 'localhost', + // Reconnects after a session reset dial the advertised node IP. + servername: 'localhost', ca: caCert, }, - credentials: { username: 'iggy', password: 'iggy' }, + credentials, }); }; +describeResourceOptions( + 'e2e -> tls resource options', 'tls-e2e-ro', getTlsClient, EncryptionKeySource.None, + !tlsEnabled && 'IGGY_TCP_TLS_ENABLED is not set' +); + describe('e2e -> tls', { skip: !tlsEnabled && 'IGGY_TCP_TLS_ENABLED is not set' }, async () => { const c = getTlsClient();
diff --git a/foreign/node/src/index.ts b/foreign/node/src/index.ts index 4942bdd..200972f 100644 --- a/foreign/node/src/index.ts +++ b/foreign/node/src/index.ts
@@ -24,6 +24,23 @@ HeaderKeyFactory, type SendMessagesConfirmation, type SendMessagesResponse, + CompressionAlgorithm, + Durability, + PartitionResizePolicy, + EncryptionKeySource, + OptionsScope, + StreamOptionKey, + TopicOptionKey, + UserOptionKey, + UserStatus, + StreamEncryptionKey, + encryptionKeyFingerprint, + type OptionEntry, + type OptionSpec, + type ParsedOptions, + type StreamOptions, + type TopicOptions, + type UserOptions, } from "./wire/index.js"; export * from "./client/index.js";
diff --git a/foreign/node/src/wire/command.code.ts b/foreign/node/src/wire/command.code.ts index 38eb83f..86924c7 100644 --- a/foreign/node/src/wire/command.code.ts +++ b/foreign/node/src/wire/command.code.ts
@@ -24,6 +24,7 @@ GetClusterMetadata: 12, DescribeOptions: 13, BindSession: 15, + UpdateResource: 16, GetMe: 20, GetClient: 21, GetClients: 22, @@ -31,7 +32,7 @@ GetUsers: 32, CreateUser: 33, DeleteUser: 34, - UpdateUser: 35, + // 35 is retired and must never be reused. UpdatePermissions: 36, ChangePassword: 37, LoginUser: 38, @@ -54,12 +55,12 @@ GetStreams: 201, CreateStream: 202, DeleteStream: 203, - UpdateStream: 204, + // 204 is retired and must never be reused. GetTopic: 300, GetTopics: 301, CreateTopic: 302, DeleteTopic: 303, - UpdateTopic: 304, + // 304 is retired and must never be reused. CreatePartitions: 402, DeletePartitions: 403, DeleteSegments: 503,
diff --git a/foreign/node/src/wire/error.code.test.ts b/foreign/node/src/wire/error.code.test.ts index 63c497b..a2646cb 100644 --- a/foreign/node/src/wire/error.code.test.ts +++ b/foreign/node/src/wire/error.code.test.ts
@@ -85,3 +85,12 @@ assert.equal(translateErrorCode(5010), 'Too many consumer groups'); assert.equal(translateErrorCode(5011), 'error'); }); + +it('translates the resource option refusals', () => { + assert.equal(translateErrorCode(4103), 'Option cannot be updated after creation: {0}'); + assert.equal(translateErrorCode(4104), 'Option cannot be reset: {0}'); + assert.equal(translateErrorCode(4105), 'Option is read-only: {0}'); + assert.equal(translateErrorCode(4106), 'Stream encryption key requires a server encryption key'); + assert.equal(translateErrorCode(4107), 'Unsupported critical option: {0}'); + assert.equal(translateErrorCode(4108), 'Cannot modify root user'); +});
diff --git a/foreign/node/src/wire/error.code.ts b/foreign/node/src/wire/error.code.ts index 6c33297..1740d82 100644 --- a/foreign/node/src/wire/error.code.ts +++ b/foreign/node/src/wire/error.code.ts
@@ -231,6 +231,12 @@ case '4100': return "Invalid offset: {0}"; case '4101': return "Invalid reserved field value: {0}, expected: 0"; case '4102': return "Segment file size on disk: {0} does not match expected size: {1}"; + case '4103': return "Option cannot be updated after creation: {0}"; + case '4104': return "Option cannot be reset: {0}"; + case '4105': return "Option is read-only: {0}"; + case '4106': return "Stream encryption key requires a server encryption key"; + case '4107': return "Unsupported critical option: {0}"; + case '4108': return "Cannot modify root user"; // CONSUMER GROUP case '5000': return "Consumer group with ID: {0} for topic with ID: {1} was not found.";
diff --git a/foreign/node/src/wire/fixtures/resource_options.json b/foreign/node/src/wire/fixtures/resource_options.json new file mode 100644 index 0000000..c5cc2ac --- /dev/null +++ b/foreign/node/src/wire/fixtures/resource_options.json
@@ -0,0 +1,1619 @@ +{ + "schema_version": 1, + "generated_by": "core/common/examples/resource_options_fixtures.rs", + "conventions": { + "byte_strings": "Every *_hex field is lowercase hex without separators.", + "errors": "name is the snake_case IggyError name and code its numeric code.", + "max_topic_size": "Bytes. 18446744073709551615 means unlimited.", + "message_expiry": "Microseconds. 18446744073709551615 means never expire.", + "never_returned_keys": "No response, decoded projection or accessor may contain these keys or their values.", + "option_maps": "Option maps use the HTTP JSON form: value as reversible text, kind, and explicit provenance. Request blocks mark every entry explicit.", + "options_block_order": "Options block entries are encoded in ascending key byte order.", + "request_bodies": "body_hex and update_resource_hex are command payloads without framing. constants.command_codes maps each command to its code and VSR operation.", + "reset_order": "The typed encoders and the HTTP handler write reset keys in ascending key byte order.", + "stream_state": "state is the stored server state a stream response was projected from. SDKs need only response_hex and expected.", + "u64_values": "u64 option values and typed u64 inputs are decimal strings, so values above 2^53 keep their precision." + }, + "constants": { + "protocol_version": { + "packed": 12288, + "text": "0.12.0", + "major": 0, + "minor": 12, + "patch": 0 + }, + "protocol_version_min": { + "packed": 12288, + "text": "0.12.0", + "major": 0, + "minor": 12, + "patch": 0 + }, + "command_codes": { + "create_stream": { + "code": 202, + "operation": 128 + }, + "create_topic": { + "code": 302, + "operation": 132 + }, + "create_user": { + "code": 33, + "operation": 141 + }, + "describe_options": { + "code": 13, + "operation": null + }, + "get_stream": { + "code": 200, + "operation": null + }, + "get_streams": { + "code": 201, + "operation": null + }, + "get_topic": { + "code": 300, + "operation": null + }, + "get_topics": { + "code": 301, + "operation": null + }, + "get_user": { + "code": 31, + "operation": null + }, + "get_users": { + "code": 32, + "operation": null + }, + "update_resource": { + "code": 16, + "operation": 150 + } + }, + "error_codes": { + "cannot_modify_root_user": 4108 + }, + "retired_command_codes": { + "update_stream": 204, + "update_topic": 304, + "update_user": 35 + }, + "operations": { + "rewrap_stream_keys": { + "value": 75, + "internal": true, + "request_operation": 75 + }, + "update_resource": { + "value": 150, + "internal": false, + "request_operation": 150 + }, + "update_stream_resolved": { + "value": 72, + "internal": true, + "request_operation": 150 + }, + "update_topic_resolved": { + "value": 73, + "internal": true, + "request_operation": 150 + }, + "update_user_resolved": { + "value": 74, + "internal": true, + "request_operation": 150 + } + }, + "retired_operations": { + "update_stream": 129, + "update_topic": 133, + "update_user": 142 + }, + "scope_codes": { + "stream": 2, + "topic": 1, + "user": 3 + }, + "header_kind_codes": { + "raw": 1, + "string": 2, + "bool": 3, + "uint32": 11, + "uint64": 12 + }, + "option_flags": { + "critical": 16, + "read_only": 8, + "resettable": 2, + "secret": 4, + "updatable": 1 + }, + "user_status_codes": { + "active": 1, + "inactive": 2 + }, + "stream_key_length": 32, + "never_returned_keys": [ + "encryption_key", + "encryption_key_wrapped", + "critical_keys" + ] + }, + "catalog": [ + { + "key": "name", + "scopes": [ + "stream", + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "column", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 1, + "described": true, + "default": null + }, + { + "key": "username", + "scopes": [ + "user" + ], + "kind": "string", + "kind_code": 2, + "storage": "column", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 1, + "described": true, + "default": null + }, + { + "key": "status", + "scopes": [ + "user" + ], + "kind": "string", + "kind_code": 2, + "storage": "column", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 1, + "described": true, + "default": null + }, + { + "key": "compression_algorithm", + "scopes": [ + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "none" + }, + { + "key": "message_expiry", + "scopes": [ + "topic" + ], + "kind": "uint64", + "kind_code": 12, + "storage": "map", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 3, + "described": true, + "default": "18446744073709551615" + }, + { + "key": "max_topic_size", + "scopes": [ + "topic" + ], + "kind": "uint64", + "kind_code": 12, + "storage": "map", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 3, + "described": true, + "default": "18446744073709551615" + }, + { + "key": "segment_size", + "scopes": [ + "topic" + ], + "kind": "uint64", + "kind_code": 12, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "1073741824" + }, + { + "key": "durability", + "scopes": [ + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "replicated" + }, + { + "key": "consumer_offset_durability", + "scopes": [ + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "replicated" + }, + { + "key": "partition_resize_policy", + "scopes": [ + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "mutable" + }, + { + "key": "messages_required_to_save", + "scopes": [ + "topic" + ], + "kind": "uint32", + "kind_code": 11, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "1024" + }, + { + "key": "size_of_messages_required_to_save", + "scopes": [ + "topic" + ], + "kind": "uint64", + "kind_code": 12, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "1048576" + }, + { + "key": "preallocate_segments", + "scopes": [ + "topic" + ], + "kind": "bool", + "kind_code": 3, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "false" + }, + { + "key": "encryption_key", + "scopes": [ + "stream" + ], + "kind": "raw", + "kind_code": 1, + "storage": "map", + "mutability": "create_only", + "visibility": "secret", + "critical": true, + "flags": 20, + "described": true, + "default": null + }, + { + "key": "encryption_key_wrapped", + "scopes": [ + "stream" + ], + "kind": "raw", + "kind_code": 1, + "storage": "map", + "mutability": "create_only", + "visibility": "internal", + "critical": true, + "flags": 16, + "described": false, + "default": null + }, + { + "key": "encryption_key_source", + "scopes": [ + "stream" + ], + "kind": "string", + "kind_code": 2, + "storage": "computed", + "mutability": "read_only", + "visibility": "read_only", + "critical": false, + "flags": 8, + "described": true, + "default": null + }, + { + "key": "encryption_key_fingerprint", + "scopes": [ + "stream" + ], + "kind": "string", + "kind_code": 2, + "storage": "computed", + "mutability": "read_only", + "visibility": "read_only", + "critical": false, + "flags": 8, + "described": true, + "default": null + }, + { + "key": "encryption_kek_fingerprint", + "scopes": [ + "stream" + ], + "kind": "string", + "kind_code": 2, + "storage": "computed", + "mutability": "read_only", + "visibility": "read_only", + "critical": false, + "flags": 8, + "described": true, + "default": null + }, + { + "key": "critical_keys", + "scopes": [ + "stream", + "topic", + "user" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "internal", + "critical": false, + "flags": 0, + "described": false, + "default": null + } + ], + "describe_options": [ + { + "scope": "topic", + "request_hex": "01", + "response_hex": "0b000000046e616d650201000000007e0000005265736f75726365206e616d652c203120746f203235352062797465732c20756e697175652077697468696e2069747320706172656e742e20536574206279207468652063726561746520636f6d6d616e642773206e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e15636f6d7072657373696f6e5f616c676f726974686d0200040000006e6f6e6550000000436f6d7072657373696f6e20616c676f726974686d20286e6f6e652c20677a6970292e2053746f72656420616e64207265706f727465642c206e6f74206170706c69656420746f206d657373616765730e6d6573736167655f6578706972790c0308000000ffffffffffffffff430000004d6573736167652065787069727920696e206d6963726f7365636f6e64732c206f7220612068756d616e74696d6520737472696e672028652e672e20372064617973290e6d61785f746f7069635f73697a650c0308000000ffffffffffffffff9d000000546f7069632d77696465207365616c65642d7365676d656e742073697a65206361702c2073706c6974206163726f737320616c6c20706172746974696f6e732c20696e206279746573206f72206120627974652d73697a6520737472696e672028652e672e203120476942293b2066696e6974652076616c756573206d757374206265206174206c6561737420746865207365676d656e742073697a650c7365676d656e745f73697a650c000800000000000040000000006c0000005365676d656e742073697a6520696e2062797465732c206f72206120627974652d73697a6520737472696e672028652e672e20313238204d6942293b2061203531322d62797465206d756c7469706c652077697468696e20313034383537362e2e3d313037333734313832340a6475726162696c69747902000a0000007265706c6963617465643e0100004d65737361676520636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20412073696e676c65746f6e2071756f72756d20686173206f6e6520636f70792e20496e207265706c6963617465642067726f7570732c20706572736973746564206d65737361676573207573652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c2057414c207265636c616d6174696f6e2e205468652066756c6c20626f64792073697a65207374696c6c20636f756e747320616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e1a636f6e73756d65725f6f66667365745f6475726162696c69747902000a0000007265706c6963617465646e0100004578706c69636974206f666673657420636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20506f6c6c206175746f2d636f6d6d69742072656d61696e73206173796e6368726f6e6f75732e20496e207265706c6963617465642067726f7570732c20706572736973746564206f66667365747320616c736f20656e61626c652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c207265636c616d6174696f6e2c206576656e2077697468207265706c696361746564206d657373616765206475726162696c6974792e2046756c6c20626f64792073697a657320636f756e7420616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e17706172746974696f6e5f726573697a655f706f6c6963790200070000006d757461626c65c8000000496d6d757461626c65206166746572206372656174696f6e3a206d757461626c65207065726d69747320706172746974696f6e206164646974696f6e2f64656c6574696f6e20776974686f7574206120626f756e646564206375746f766572206f72207065722d6b6579206f72646572696e67206163726f7373206c61796f7574733b2066697865642072656a6563747320626f74682e20526574656e74696f6e20616e6420746f7069632064656c6574696f6e2072656d61696e2064657374727563746976652e196d657373616765735f72657175697265645f746f5f736176650b000400000000040000870000004f7264696e617279206d6573736167652d636f756e7420666c75736820747269676765723b20312e2e3d31363737373231362e2052657175697265642070657273697374656e63652c2063617061636974792070726573737572652c20616e64206c6966656379636c65206f7065726174696f6e732063616e20666c757368206561726c6965722173697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c0008000000000010000000000083000000466c75736820746865206a6f75726e616c206f6e636520697420686f6c64732074686973206d616e792062797465732c206f72206120627974652d73697a6520737472696e673b20776869636865766572207468726573686f6c6420747269707320666972737420666c75736865732e204174206d6f7374203130373337343138323414707265616c6c6f636174655f7365676d656e7473030001000000001201000052657175657374207365676d656e745f73697a65206279746573206f66206469736b207265736572766174696f6e207768656e2065616368207365676d656e74206f70656e732e20556e737570706f72746564206f72206661696c6564207265736572766174696f6e732066616c6c206261636b20746f206f7264696e61727920616c6c6f636174696f6e20776974682061207761726e696e672e205265736572766174696f6e2072756e7320696e6c696e65206f6e20746865206f776e696e672073686172642e207365676d656e745f73697a65202a20706172746974696f6e735f636f756e74206973206361707065642061742036383731393437363733362062797465732070657220637265617465", + "entries": [ + { + "key": "name", + "kind": "string", + "flags": 1, + "default": null + }, + { + "key": "compression_algorithm", + "kind": "string", + "flags": 0, + "default": "none" + }, + { + "key": "message_expiry", + "kind": "uint64", + "flags": 3, + "default": "18446744073709551615" + }, + { + "key": "max_topic_size", + "kind": "uint64", + "flags": 3, + "default": "18446744073709551615" + }, + { + "key": "segment_size", + "kind": "uint64", + "flags": 0, + "default": "1073741824" + }, + { + "key": "durability", + "kind": "string", + "flags": 0, + "default": "replicated" + }, + { + "key": "consumer_offset_durability", + "kind": "string", + "flags": 0, + "default": "replicated" + }, + { + "key": "partition_resize_policy", + "kind": "string", + "flags": 0, + "default": "mutable" + }, + { + "key": "messages_required_to_save", + "kind": "uint32", + "flags": 0, + "default": "1024" + }, + { + "key": "size_of_messages_required_to_save", + "kind": "uint64", + "flags": 0, + "default": "1048576" + }, + { + "key": "preallocate_segments", + "kind": "bool", + "flags": 0, + "default": "false" + } + ] + }, + { + "scope": "stream", + "request_hex": "02", + "response_hex": "05000000046e616d650201000000007e0000005265736f75726365206e616d652c203120746f203235352062797465732c20756e697175652077697468696e2069747320706172656e742e20536574206279207468652063726561746520636f6d6d616e642773206e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e0e656e6372797074696f6e5f6b6579011400000000a30100005468652073747265616d2773206f776e206461746120656e6372797074696f6e206b65793a2033322072616e646f6d2062797465732c20726177206f72207374616e6461726420706164646564206261736536342e204372656174652d6f6e6c7920616e642077726974652d6f6e6c793a20746865207365727665722073746f726573206974207772617070656420756e6465722061206b657920646572697665642066726f6d207468652073657276657220656e6372797074696f6e206b657920616e64206e657665722072657475726e732069742e205265717569726573207468652073657276657220656e6372797074696f6e206b657920746f20626520636f6e666967757265642e20576974686f75742069742c206120736572766572207769746820656e6372797074696f6e20656e61626c65642067656e6572617465732061206b657920666f72207468652073747265616d2e20546865206b65792074726176656c7320696e20746865206372656174652072657175657374206f76657220746865207472616e73706f72742074686520636c69656e7420757365732e15656e6372797074696f6e5f6b65795f736f757263650208000000009f00000057686f2063686f7365207468652073747265616d27732061742d72657374206b65793a2073747265616d207768656e2074686520636c69656e742073656e742069742c20736572766572207768656e20746865207365727665722067656e6572617465642069742c206e6f6e65207768656e207468652073747265616d2773206d65737361676573206172652073746f72656420756e656e637279707465641a656e6372797074696f6e5f6b65795f66696e6765727072696e740208000000009f0000005468652066697273742038206279746573206f66205348412d323536206f76657220746865206c6162656c20696767792f73747265616d2d6b65792d66696e6765727072696e742f763120616e64207468652073747265616d2773206b65792c206173203136206c6f776572636173652068657820636861726163746572732e20416273656e74207768656e2074686520736f75726365206973206e6f6e651a656e6372797074696f6e5f6b656b5f66696e6765727072696e74020800000000b800000046696e6765727072696e7473206f6620746865206b65792d656e6372797074696f6e206b65797320746861742077726170207468652073747265616d2773206b65792c20636f6d6d612d7365706172617465643a206f6e65206f7574736964652061206b657920726f746174696f6e2c206f6e652070657220636f6e6669677572656420736572766572206b657920647572696e672069742e20416273656e74207768656e2074686520736f75726365206973206e6f6e65", + "entries": [ + { + "key": "name", + "kind": "string", + "flags": 1, + "default": null + }, + { + "key": "encryption_key", + "kind": "raw", + "flags": 20, + "default": null + }, + { + "key": "encryption_key_source", + "kind": "string", + "flags": 8, + "default": null + }, + { + "key": "encryption_key_fingerprint", + "kind": "string", + "flags": 8, + "default": null + }, + { + "key": "encryption_kek_fingerprint", + "kind": "string", + "flags": 8, + "default": null + } + ] + }, + { + "scope": "user", + "request_hex": "03", + "response_hex": "0200000008757365726e616d650201000000006b00000055736572206e616d652c203320746f2035302062797465732c20756e697175652e20536574206279207468652063726561746520636f6d6d616e64277320757365726e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e067374617475730201000000008f00000055736572207374617475732c20616374697665206f7220696e6163746976652e20536574206279207468652063726561746520636f6d6d616e64277320737461747573206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e20446561637469766174696e672061207573657220656e6473206974732073657373696f6e732e", + "entries": [ + { + "key": "username", + "kind": "string", + "flags": 1, + "default": null + }, + { + "key": "status", + "kind": "string", + "flags": 1, + "default": null + } + ] + } + ], + "stream_key": { + "key_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "key_base64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", + "fingerprint_label": "iggy/stream-key-fingerprint/v1", + "fingerprint": "8441e38538cbfa57", + "invalid_keys": [ + { + "case": "raw_31_bytes", + "input": { + "raw_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + }, + { + "case": "raw_33_bytes", + "input": { + "raw_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + }, + { + "case": "base64_with_whitespace", + "input": { + "base64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=\n" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + }, + { + "case": "base64_missing_padding", + "input": { + "base64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + }, + { + "case": "base64_noncanonical_trailing_bits", + "input": { + "base64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh9=" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + } + ] + }, + "update_resource": [ + { + "case": "topic_set_expiry_without_name_reset_max_topic_size", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "message_expiry": "604800000000", + "reset": [ + "max_topic_size" + ] + }, + "wire_set": { + "message_expiry": { + "value": "604800000000", + "kind": "uint64", + "explicit": true + } + }, + "wire_reset": [ + "max_topic_size" + ], + "body_hex": "01010401000000010402000000010e6d61785f746f7069635f73697a65020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000" + }, + { + "case": "topic_rename_only_named_identifiers", + "scope": "topic", + "target": { + "stream_id": "orders", + "topic_id": "events" + }, + "input": { + "name": "events-v2" + }, + "wire_set": { + "name": { + "value": "events-v2", + "kind": "string", + "explicit": true + } + }, + "wire_reset": [], + "body_hex": "0102066f726465727302066576656e74730002040000006e616d6502090000006576656e74732d7632" + }, + { + "case": "topic_rename_set_expiry_reset_max_topic_size", + "scope": "topic", + "target": { + "stream_id": 7, + "topic_id": "events" + }, + "input": { + "name": "events-v3", + "message_expiry": "86400000000", + "reset": [ + "max_topic_size" + ] + }, + "wire_set": { + "message_expiry": { + "value": "86400000000", + "kind": "uint64", + "explicit": true + }, + "name": { + "value": "events-v3", + "kind": "string", + "explicit": true + } + }, + "wire_reset": [ + "max_topic_size" + ], + "body_hex": "0101040700000002066576656e7473010e6d61785f746f7069635f73697a65020e0000006d6573736167655f6578706972790c080000000060d71d1400000002040000006e616d6502090000006576656e74732d7633" + }, + { + "case": "topic_reset_both_defaults", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "reset": [ + "max_topic_size", + "message_expiry" + ] + }, + "wire_set": {}, + "wire_reset": [ + "max_topic_size", + "message_expiry" + ], + "body_hex": "01010401000000010402000000020e6d61785f746f7069635f73697a650e6d6573736167655f657870697279" + }, + { + "case": "topic_max_topic_size_above_2_pow_53", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "max_topic_size": "9007199254740993" + }, + "wire_set": { + "max_topic_size": { + "value": "9007199254740993", + "kind": "uint64", + "explicit": true + } + }, + "wire_reset": [], + "body_hex": "0101040100000001040200000000020e0000006d61785f746f7069635f73697a650c080000000100000000002000" + }, + { + "case": "stream_rename", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "input": { + "name": "payments-v2" + }, + "wire_set": { + "name": { + "value": "payments-v2", + "kind": "string", + "explicit": true + } + }, + "wire_reset": [], + "body_hex": "020104030000000002040000006e616d65020b0000007061796d656e74732d7632" + }, + { + "case": "user_username_and_status", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "username": "alice2", + "status": "inactive" + }, + "wire_set": { + "status": { + "value": "inactive", + "kind": "string", + "explicit": true + }, + "username": { + "value": "alice2", + "kind": "string", + "explicit": true + } + }, + "wire_reset": [], + "body_hex": "030104050000000002060000007374617475730208000000696e6163746976650208000000757365726e616d650206000000616c69636532" + } + ], + "update_resource_errors": [ + { + "case": "topic_empty_update", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": {}, + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_set_and_reset_same_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "message_expiry": "604800000000", + "reset": [ + "message_expiry" + ] + }, + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_reset_create_only_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "reset": [ + "compression_algorithm" + ] + }, + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "stream_reset_name", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "input": { + "reset": [ + "name" + ] + }, + "error": { + "name": "option_not_resettable", + "code": 4104 + } + }, + { + "case": "user_reset_status", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "reset": [ + "status" + ] + }, + "error": { + "name": "option_not_resettable", + "code": 4104 + } + }, + { + "case": "stream_reset_read_only_key", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "input": { + "reset": [ + "encryption_key_source" + ] + }, + "error": { + "name": "read_only_option", + "code": 4105 + } + }, + { + "case": "user_reset_internal_key", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "reset": [ + "critical_keys" + ] + }, + "error": { + "name": "read_only_option", + "code": 4105 + } + }, + { + "case": "topic_empty_name", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "name": "" + }, + "error": { + "name": "invalid_topic_name", + "code": 2014 + } + }, + { + "case": "topic_empty_name_and_create_only_reset", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "name": "", + "reset": [ + "segment_size" + ] + }, + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "stream_empty_name", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "input": { + "name": "" + }, + "error": { + "name": "invalid_stream_name", + "code": 1013 + } + }, + { + "case": "user_short_username", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "username": "a" + }, + "error": { + "name": "invalid_username", + "code": 43 + } + }, + { + "case": "user_short_username_and_status_reset", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "username": "a", + "reset": [ + "status" + ] + }, + "error": { + "name": "option_not_resettable", + "code": 4104 + } + } + ], + "create_stream": [ + { + "case": "stream_without_key", + "input": { + "name": "orders" + }, + "wire_options": {}, + "body_hex": "066f7264657273", + "http_body": "{\"name\":\"orders\"}", + "admission_error": null + }, + { + "case": "stream_with_key", + "input": { + "name": "orders", + "encryption_key": { + "raw_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f" + } + }, + "wire_options": { + "encryption_key": { + "value": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", + "kind": "raw", + "explicit": true + } + }, + "body_hex": "066f7264657273020e000000656e6372797074696f6e5f6b65790120000000000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "http_body": "{\"name\":\"orders\",\"options\":{\"encryption_key\":\"AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=\"}}", + "admission_error": null + }, + { + "case": "stream_with_raw_extra_key", + "input": { + "name": "orders", + "raw": { + "future_option": "enabled" + } + }, + "wire_options": { + "future_option": { + "value": "enabled", + "kind": "string", + "explicit": true + } + }, + "body_hex": "066f7264657273020d0000006675747572655f6f7074696f6e0207000000656e61626c6564", + "http_body": "{\"name\":\"orders\",\"options\":{\"future_option\":\"enabled\"}}", + "admission_error": { + "name": "unsupported_option_key", + "code": 4041 + } + } + ], + "create_topic": [ + { + "case": "topic_with_options", + "input": { + "stream_id": 1, + "name": "events", + "partitions_count": 3, + "compression_algorithm": "gzip", + "message_expiry": "604800000000", + "max_topic_size": "9007199254740993", + "segment_size": "134217728", + "durability": "persisted", + "consumer_offset_durability": "replicated" + }, + "wire_options": { + "compression_algorithm": { + "value": "gzip", + "kind": "string", + "explicit": true + }, + "consumer_offset_durability": { + "value": "replicated", + "kind": "string", + "explicit": true + }, + "durability": { + "value": "persisted", + "kind": "string", + "explicit": true + }, + "max_topic_size": { + "value": "9007199254740993", + "kind": "uint64", + "explicit": true + }, + "message_expiry": { + "value": "604800000000", + "kind": "uint64", + "explicit": true + }, + "segment_size": { + "value": "134217728", + "kind": "uint64", + "explicit": true + } + }, + "body_hex": "01040100000003000000066576656e74730215000000636f6d7072657373696f6e5f616c676f726974686d0204000000677a6970021a000000636f6e73756d65725f6f66667365745f6475726162696c697479020a0000007265706c696361746564020a0000006475726162696c6974790209000000706572736973746564020e0000006d61785f746f7069635f73697a650c080000000100000000002000020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000020c0000007365676d656e745f73697a650c080000000000000800000000", + "http_body": "{\"partitions_count\":3,\"name\":\"events\",\"options\":{\"compression_algorithm\":\"gzip\",\"consumer_offset_durability\":\"replicated\",\"durability\":\"persisted\",\"max_topic_size\":\"9007199254740993\",\"message_expiry\":604800000000,\"segment_size\":134217728}}", + "admission_error": null + } + ], + "create_user": [ + { + "case": "user_with_empty_options", + "input": { + "username": "alice", + "password": "s3cret-pass", + "status": "active" + }, + "wire_options": {}, + "body_hex": "05616c6963650b7333637265742d706173730100", + "http_body": "{\"username\":\"alice\",\"password\":\"s3cret-pass\",\"status\":\"active\",\"permissions\":null}", + "admission_error": null + } + ], + "http_update": [ + { + "case": "topic_flat_keys", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"name\":\"events-v2\",\"message_expiry\":\"7 days\"}", + "update_resource_hex": "0101040100000001040200000000020e0000006d6573736167655f657870697279020600000037206461797302040000006e616d6502090000006576656e74732d7632", + "outcome": "accepted", + "error": null + }, + { + "case": "topic_reset_array", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":604800000000,\"reset\":[\"max_topic_size\"]}", + "update_resource_hex": "01010401000000010402000000010e6d61785f746f7069635f73697a65020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000", + "outcome": "accepted", + "error": null + }, + { + "case": "topic_null_ignored", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"name\":null,\"max_topic_size\":null,\"message_expiry\":\"1 day\"}", + "update_resource_hex": "0101040100000001040200000000020e0000006d6573736167655f65787069727902050000003120646179", + "outcome": "accepted", + "error": null + }, + { + "case": "topic_u64_above_2_pow_53_as_string", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"max_topic_size\":\"9007199254740993\"}", + "update_resource_hex": "0101040100000001040200000000020e0000006d61785f746f7069635f73697a65021000000039303037313939323534373430393933", + "outcome": "accepted", + "error": null + }, + { + "case": "stream_rename", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "body": "{\"name\":\"payments-v2\"}", + "update_resource_hex": "020104030000000002040000006e616d65020b0000007061796d656e74732d7632", + "outcome": "accepted", + "error": null + }, + { + "case": "user_username_and_status", + "scope": "user", + "target": { + "user_id": 5 + }, + "body": "{\"username\":\"alice2\",\"status\":\"inactive\"}", + "update_resource_hex": "030104050000000002060000007374617475730208000000696e6163746976650208000000757365726e616d650206000000616c69636532", + "outcome": "accepted", + "error": null + }, + { + "case": "topic_duplicate_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":\"1 day\",\"message_expiry\":\"2 days\"}", + "update_resource_hex": null, + "outcome": "json_refused", + "error": null + }, + { + "case": "topic_nested_object", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":{\"value\":\"1 day\"}}", + "update_resource_hex": null, + "outcome": "json_refused", + "error": null + }, + { + "case": "topic_null_only", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"max_topic_size\":null}", + "update_resource_hex": "0101040100000001040200000000", + "outcome": "decode_refused", + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_zero_sentinel_only", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":0}", + "update_resource_hex": "0101040100000001040200000000020e0000006d6573736167655f6578706972790c080000000000000000000000", + "outcome": "admission_refused", + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_reset_names_set_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":\"1 day\",\"reset\":[\"message_expiry\"]}", + "update_resource_hex": "01010401000000010402000000010e6d6573736167655f657870697279020e0000006d6573736167655f65787069727902050000003120646179", + "outcome": "decode_refused", + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_unknown_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"retention\":\"1 day\"}", + "update_resource_hex": "01010401000000010402000000000209000000726574656e74696f6e02050000003120646179", + "outcome": "admission_refused", + "error": { + "name": "unsupported_option_key", + "code": 4041 + } + }, + { + "case": "topic_set_create_only_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"compression_algorithm\":\"gzip\"}", + "update_resource_hex": "01010401000000010402000000000215000000636f6d7072657373696f6e5f616c676f726974686d0204000000677a6970", + "outcome": "admission_refused", + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "topic_reset_create_only_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"reset\":[\"compression_algorithm\"]}", + "update_resource_hex": "010104010000000104020000000115636f6d7072657373696f6e5f616c676f726974686d", + "outcome": "admission_refused", + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "stream_reset_name", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "body": "{\"reset\":[\"name\"]}", + "update_resource_hex": "0201040300000001046e616d65", + "outcome": "admission_refused", + "error": { + "name": "option_not_resettable", + "code": 4104 + } + }, + { + "case": "topic_name_as_number", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"name\":5}", + "update_resource_hex": "010104010000000104020000000002040000006e616d650c080000000500000000000000", + "outcome": "admission_refused", + "error": { + "name": "invalid_topic_name", + "code": 2014 + } + }, + { + "case": "stream_empty_name", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "body": "{\"name\":\"\"}", + "update_resource_hex": "020104030000000002040000006e616d65030100000000", + "outcome": "admission_refused", + "error": { + "name": "invalid_stream_name", + "code": 1013 + } + }, + { + "case": "user_status_as_bool", + "scope": "user", + "target": { + "user_id": 5 + }, + "body": "{\"status\":true}", + "update_resource_hex": "03010405000000000206000000737461747573030100000001", + "outcome": "admission_refused", + "error": { + "name": "invalid_user_status", + "code": 45 + } + }, + { + "case": "topic_create_only_key_as_bool", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"segment_size\":true}", + "update_resource_hex": "0101040100000001040200000000020c0000007365676d656e745f73697a65030100000001", + "outcome": "admission_refused", + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "stream_read_only_key_as_number", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "body": "{\"encryption_key_source\":1}", + "update_resource_hex": "02010403000000000215000000656e6372797074696f6e5f6b65795f736f757263650c080000000100000000000000", + "outcome": "admission_refused", + "error": { + "name": "read_only_option", + "code": 4105 + } + }, + { + "case": "topic_invalid_value", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":\"soon\"}", + "update_resource_hex": "0101040100000001040200000000020e0000006d6573736167655f6578706972790204000000736f6f6e", + "outcome": "admission_refused", + "error": { + "name": "invalid_option_value", + "code": 4042 + } + } + ], + "stream_responses": [ + { + "case": "stream_source_none", + "state": { + "encryption_key_source": "none", + "stored_extra": {} + }, + "response_hex": "0100000000e090663c1306000000000000100000000000000c000000000000000c706c61696e2d73747265616d00000000230000000215000000656e6372797074696f6e5f6b65795f736f7572636502040000006e6f6e65", + "expected": { + "id": 1, + "created_at": 1710000000000000, + "name": "plain-stream", + "topics_count": 0, + "size_bytes": 4096, + "messages_count": 12, + "options": { + "encryption_key_source": { + "value": "none", + "kind": "string", + "explicit": false + } + }, + "encryption_key_source": "none", + "encryption_key_fingerprint": null, + "encryption_kek_fingerprints": [] + } + }, + { + "case": "stream_source_stream_with_unknown_explicit_key", + "state": { + "encryption_key_source": "stream", + "data_key": { + "raw_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f" + }, + "kek_config_keys": [ + "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=" + ], + "stored_extra": { + "future_option": { + "value": "enabled", + "kind": "string", + "explicit": true + } + } + }, + "response_hex": "0200000000e090663c1306000000000000100000000000000c0000000000000011636c69656e742d6b65792d73747265616d1e000000020d0000006675747572655f6f7074696f6e0207000000656e61626c65648d000000021a000000656e6372797074696f6e5f6b656b5f66696e6765727072696e74021000000033396534663066393563333639336337021a000000656e6372797074696f6e5f6b65795f66696e6765727072696e740210000000383434316533383533386362666135370215000000656e6372797074696f6e5f6b65795f736f75726365020600000073747265616d", + "expected": { + "id": 2, + "created_at": 1710000000000000, + "name": "client-key-stream", + "topics_count": 0, + "size_bytes": 4096, + "messages_count": 12, + "options": { + "encryption_kek_fingerprint": { + "value": "39e4f0f95c3693c7", + "kind": "string", + "explicit": false + }, + "encryption_key_fingerprint": { + "value": "8441e38538cbfa57", + "kind": "string", + "explicit": false + }, + "encryption_key_source": { + "value": "stream", + "kind": "string", + "explicit": false + }, + "future_option": { + "value": "enabled", + "kind": "string", + "explicit": true + } + }, + "encryption_key_source": "stream", + "encryption_key_fingerprint": "8441e38538cbfa57", + "encryption_kek_fingerprints": [ + "39e4f0f95c3693c7" + ] + } + }, + { + "case": "stream_source_server_two_keks", + "state": { + "encryption_key_source": "server", + "data_key": { + "raw_hex": "202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f" + }, + "kek_config_keys": [ + "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=", + "AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA=" + ], + "stored_extra": {} + }, + "response_hex": "0300000000e090663c1306000000000000100000000000000c00000000000000117365727665722d6b65792d73747265616d000000009e000000021a000000656e6372797074696f6e5f6b656b5f66696e6765727072696e740221000000333965346630663935633336393363372c64353564343739383239626332393336021a000000656e6372797074696f6e5f6b65795f66696e6765727072696e740210000000306563616666316261323931666530640215000000656e6372797074696f6e5f6b65795f736f757263650206000000736572766572", + "expected": { + "id": 3, + "created_at": 1710000000000000, + "name": "server-key-stream", + "topics_count": 0, + "size_bytes": 4096, + "messages_count": 12, + "options": { + "encryption_kek_fingerprint": { + "value": "39e4f0f95c3693c7,d55d479829bc2936", + "kind": "string", + "explicit": false + }, + "encryption_key_fingerprint": { + "value": "0ecaff1ba291fe0d", + "kind": "string", + "explicit": false + }, + "encryption_key_source": { + "value": "server", + "kind": "string", + "explicit": false + } + }, + "encryption_key_source": "server", + "encryption_key_fingerprint": "0ecaff1ba291fe0d", + "encryption_kek_fingerprints": [ + "39e4f0f95c3693c7", + "d55d479829bc2936" + ] + } + } + ], + "topic_responses": [ + { + "case": "topic_explicit_and_derived", + "response_hex": "0400000000e090663c1306000000000000080000000000000700000000000000066576656e7473430000000215000000636f6d7072657373696f6e5f616c676f726974686d0204000000677a6970020e0000006d6573736167655f6578706972790c0800000000a0e3d08c0000002b010000021a000000636f6e73756d65725f6f66667365745f6475726162696c697479020a0000007265706c696361746564020a0000006475726162696c697479020a0000007265706c696361746564020e0000006d61785f746f7069635f73697a650c08000000ffffffffffffffff02190000006d657373616765735f72657175697265645f746f5f736176650b04000000000400000217000000706172746974696f6e5f726573697a655f706f6c69637902070000006d757461626c650214000000707265616c6c6f636174655f7365676d656e7473030100000000020c0000007365676d656e745f73697a650c080000000000004000000000022100000073697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c080000000000100000000000", + "expected": { + "id": 4, + "created_at": 1710000000000000, + "name": "events", + "partitions_count": 0, + "size_bytes": 2048, + "messages_count": 7, + "options": { + "compression_algorithm": { + "value": "gzip", + "kind": "string", + "explicit": true + }, + "consumer_offset_durability": { + "value": "replicated", + "kind": "string", + "explicit": false + }, + "durability": { + "value": "replicated", + "kind": "string", + "explicit": false + }, + "max_topic_size": { + "value": "18446744073709551615", + "kind": "uint64", + "explicit": false + }, + "message_expiry": { + "value": "604800000000", + "kind": "uint64", + "explicit": true + }, + "messages_required_to_save": { + "value": "1024", + "kind": "uint32", + "explicit": false + }, + "partition_resize_policy": { + "value": "mutable", + "kind": "string", + "explicit": false + }, + "preallocate_segments": { + "value": "false", + "kind": "bool", + "explicit": false + }, + "segment_size": { + "value": "1073741824", + "kind": "uint64", + "explicit": false + }, + "size_of_messages_required_to_save": { + "value": "1048576", + "kind": "uint64", + "explicit": false + } + }, + "compression_algorithm": "gzip", + "message_expiry": "604800000000", + "max_topic_size": "18446744073709551615" + } + } + ], + "user_responses": [ + { + "case": "user_empty_blocks", + "response_hex": "0500000000e090663c1306000105616c696365000000000000000000000000", + "expected": { + "id": 5, + "created_at": 1710000000000000, + "status": "active", + "username": "alice", + "options": {} + } + }, + { + "case": "user_populated_blocks", + "response_hex": "0600000000e090663c1306000203626f621e000000020d0000006675747572655f6f7074696f6e0207000000656e61626c65641e000000020c0000006675747572655f71756f74610c08000000010000000000200000000000", + "expected": { + "id": 6, + "created_at": 1710000000000000, + "status": "inactive", + "username": "bob", + "options": { + "future_option": { + "value": "enabled", + "kind": "string", + "explicit": true + }, + "future_quota": { + "value": "9007199254740993", + "kind": "uint64", + "explicit": false + } + } + } + } + ] +}
diff --git a/foreign/node/src/wire/index.ts b/foreign/node/src/wire/index.ts index e82ba84..1f256a3 100644 --- a/foreign/node/src/wire/index.ts +++ b/foreign/node/src/wire/index.ts
@@ -28,4 +28,8 @@ export * from './user/index.js'; export * from './command-set.js'; export { type Id } from './identifier.utils.js'; +export { + StreamOptionKey, TopicOptionKey, UserOptionKey, + type OptionEntry, type ParsedOptions +} from './options.utils.js';
diff --git a/foreign/node/src/wire/options.utils.ts b/foreign/node/src/wire/options.utils.ts index c3eb16a..0572770 100644 --- a/foreign/node/src/wire/options.utils.ts +++ b/foreign/node/src/wire/options.utils.ts
@@ -24,6 +24,7 @@ type HeaderValue, type ParsedHeaderValue, } from './message/header.utils.js'; +import { OptionFlag, OptionsScope } from './system/describe-options.command.js'; /** Maximum number of key-value entries in one options block. */ export const MAX_OPTIONS = 1024; @@ -43,6 +44,77 @@ */ const MAX_HEADER_FIELD_LENGTH = 255; +/** Stream option keys a client can name: create, update, reset or read. */ +export const StreamOptionKey = { + Name: 'name', + EncryptionKey: 'encryption_key', + EncryptionKeySource: 'encryption_key_source', + EncryptionKeyFingerprint: 'encryption_key_fingerprint', + EncryptionKekFingerprint: 'encryption_kek_fingerprint' +} as const; + +export type StreamOptionKey = typeof StreamOptionKey[keyof typeof StreamOptionKey]; + +/** Topic option keys a client can name: create, update, reset or read. */ +export const TopicOptionKey = { + Name: 'name', + CompressionAlgorithm: 'compression_algorithm', + MessageExpiry: 'message_expiry', + MaxTopicSize: 'max_topic_size', + SegmentSize: 'segment_size', + Durability: 'durability', + ConsumerOffsetDurability: 'consumer_offset_durability', + PartitionResizePolicy: 'partition_resize_policy', + MessagesRequiredToSave: 'messages_required_to_save', + SizeOfMessagesRequiredToSave: 'size_of_messages_required_to_save', + PreallocateSegments: 'preallocate_segments' +} as const; + +export type TopicOptionKey = typeof TopicOptionKey[keyof typeof TopicOptionKey]; + +/** User option keys a client can name: create, update, reset or read. */ +export const UserOptionKey = { + Username: 'username', + Status: 'status' +} as const; + +export type UserOptionKey = typeof UserOptionKey[keyof typeof UserOptionKey]; + +/** What the server catalog lets a client do with one key. */ +export type OptionDescriptor = { + scopes: readonly OptionsScope[], + /** `OptionFlag` bits, as `describeOptions` reports them */ + flags: number, + /** Written only by the server, never accepted and never returned */ + internal: boolean +}; + +const { Topic, Stream, User } = OptionsScope; +const { Updatable, Resettable, Secret, ReadOnly, Critical } = OptionFlag; + +/** The server option catalog in catalog order. */ +export const OPTION_CATALOG: ReadonlyMap<string, OptionDescriptor> = new Map([ + ['name', { scopes: [Stream, Topic], flags: Updatable, internal: false }], + ['username', { scopes: [User], flags: Updatable, internal: false }], + ['status', { scopes: [User], flags: Updatable, internal: false }], + ['compression_algorithm', { scopes: [Topic], flags: 0, internal: false }], + ['message_expiry', { scopes: [Topic], flags: Updatable | Resettable, internal: false }], + ['max_topic_size', { scopes: [Topic], flags: Updatable | Resettable, internal: false }], + ['segment_size', { scopes: [Topic], flags: 0, internal: false }], + ['durability', { scopes: [Topic], flags: 0, internal: false }], + ['consumer_offset_durability', { scopes: [Topic], flags: 0, internal: false }], + ['partition_resize_policy', { scopes: [Topic], flags: 0, internal: false }], + ['messages_required_to_save', { scopes: [Topic], flags: 0, internal: false }], + ['size_of_messages_required_to_save', { scopes: [Topic], flags: 0, internal: false }], + ['preallocate_segments', { scopes: [Topic], flags: 0, internal: false }], + ['encryption_key', { scopes: [Stream], flags: Secret | Critical, internal: false }], + ['encryption_key_wrapped', { scopes: [Stream], flags: Critical, internal: true }], + ['encryption_key_source', { scopes: [Stream], flags: ReadOnly, internal: false }], + ['encryption_key_fingerprint', { scopes: [Stream], flags: ReadOnly, internal: false }], + ['encryption_kek_fingerprint', { scopes: [Stream], flags: ReadOnly, internal: false }], + ['critical_keys', { scopes: [Stream, Topic, User], flags: 0, internal: true }] +]); + /** A resource option entry: UTF-8 string key with a typed value. */ export type OptionEntry = { key: string, @@ -75,6 +147,14 @@ }; /** + * Orders entries by key bytes, the order the server's own encoders write, so a + * command block is byte-identical to the one any other SDK sends. + */ +export const sortOptions = (options: OptionEntry[]): OptionEntry[] => + [...options].sort((left, right) => + Buffer.compare(Buffer.from(left.key), Buffer.from(right.key))); + +/** * Rejects a key or value the codec cannot express. * * `serializeHeaders` writes whatever field length it is handed, so without
diff --git a/foreign/node/src/wire/stream/create-stream.command.ts b/foreign/node/src/wire/stream/create-stream.command.ts index 44ba2f4..72879d3 100644 --- a/foreign/node/src/wire/stream/create-stream.command.ts +++ b/foreign/node/src/wire/stream/create-stream.command.ts
@@ -18,7 +18,14 @@ import type { CommandResponse } from '../../client/client.type.js'; import { wrapCommand } from '../command.utils.js'; import { COMMAND_CODE } from '../command.code.js'; +import { responseError } from '../error.utils.js'; +import { HeaderValue } from '../message/header.utils.js'; +import { StreamOptionKey, dedupeOptions, serializeOptions, sortOptions } from '../options.utils.js'; import { deserializeToStream, type Stream } from './stream.utils.js'; +import { StreamEncryptionKey, type StreamOptions } from './stream-options.js'; + +const INVALID_ENCRYPTION_KEY = 70; +const INVALID_COMMAND = 3; /** * Parameters for the create stream command. @@ -28,28 +35,48 @@ streamId?: number, /** Stream name (1-255 bytes) */ name: string -}; +} & StreamOptions; /** * Create stream command definition. * Creates a new stream with the specified name. + * Layout: `[name_len:u8][name]` followed by the options block running to the + * end of the payload. */ export const CREATE_STREAM = { code: COMMAND_CODE.CreateStream, - serialize: ({name}: CreateStream) => { + serialize: ({ name, encryptionKey, options: extraOptions = [] }: CreateStream) => { // Stream ID is now auto-assigned by the server, not sent in the protocol const bName = Buffer.from(name); if (bName.length < 1 || bName.length > 255) throw new Error('Stream name should be between 1 and 255 bytes'); + // A plain entry prints its value wherever the caller logs the arguments. + if (extraOptions.some(({ key }) => key === StreamOptionKey.EncryptionKey)) + throw responseError( + COMMAND_CODE.CreateStream, INVALID_COMMAND, + 'stream key must be passed as encryptionKey, not as an options entry' + ); + + const options = [...extraOptions]; + if (encryptionKey !== undefined) { + if (!(encryptionKey instanceof StreamEncryptionKey)) + throw responseError( + COMMAND_CODE.CreateStream, INVALID_ENCRYPTION_KEY, + 'stream key must be a StreamEncryptionKey' + ); + options.push({ key: StreamOptionKey.EncryptionKey, value: HeaderValue.Raw(encryptionKey.asBytes()) }); + } + const b = Buffer.allocUnsafe(1); b.writeUInt8(bName.length, 0); return Buffer.concat([ b, - bName + bName, + serializeOptions(sortOptions(dedupeOptions(options))) ]); },
diff --git a/foreign/node/src/wire/stream/index.ts b/foreign/node/src/wire/stream/index.ts index 4a9421a..7690f13 100644 --- a/foreign/node/src/wire/stream/index.ts +++ b/foreign/node/src/wire/stream/index.ts
@@ -21,3 +21,4 @@ export * from './get-streams.command.js'; export * from './update-stream.command.js'; export * from './ensure-stream.virtual.command.js'; +export * from './stream-options.js';
diff --git a/foreign/node/src/wire/stream/stream-options.test.ts b/foreign/node/src/wire/stream/stream-options.test.ts new file mode 100644 index 0000000..5d52cba --- /dev/null +++ b/foreign/node/src/wire/stream/stream-options.test.ts
@@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { randomBytes } from 'node:crypto'; +import { format, inspect } from 'node:util'; +import type { ClientProvider } from '../../client/client.type.js'; +import { ResponseError } from '../error.utils.js'; +import { HeaderValue } from '../message/header.utils.js'; +import { StreamOptionKey } from '../options.utils.js'; +import { CREATE_STREAM, createStream } from './create-stream.command.js'; +import { StreamEncryptionKey, encryptionKeyFingerprint } from './stream-options.js'; + +const INVALID_ENCRYPTION_KEY = 70; +const INVALID_COMMAND = 3; + +const keySpellings = (bytes: Buffer): string[] => [ + bytes.toString('hex'), + bytes.toString('base64'), + [...bytes.subarray(0, 4)].map((byte) => byte.toString(16).padStart(2, '0')).join(' '), + [...bytes.subarray(0, 4)].join(', ') +]; + +describe('given stream key when printing create arguments should show only its fingerprint', () => { + it('prints through inspect, format, JSON and string conversion without key bytes', () => { + const bytes = randomBytes(32); + const key = StreamEncryptionKey.from(bytes); + const fingerprint = encryptionKeyFingerprint(bytes); + const args = { name: 'orders', encryptionKey: key }; + const printed = [ + inspect(key), + inspect(args, { depth: null, showHidden: true }), + inspect(args, { depth: null, showHidden: true, customInspect: false }), + format('%o %O %j %s', args, args, args, args), + JSON.stringify(args), + String(key), + `${key}` + ]; + for (const text of printed) { + for (const spelling of keySpellings(bytes)) + assert.ok(!text.includes(spelling), `key bytes in ${text}`); + } + for (const text of printed.filter((_, index) => index !== 2)) + assert.ok(text.includes(fingerprint), `no fingerprint in ${text}`); + }); + + it('keeps its own copy of the key bytes', () => { + const bytes = randomBytes(32); + const fingerprint = encryptionKeyFingerprint(bytes); + const key = StreamEncryptionKey.from(bytes); + bytes.fill(0); + assert.equal(key.fingerprint(), fingerprint); + assert.equal(StreamEncryptionKey.from(key.asBytes()).fingerprint(), fingerprint); + }); + + it('refuses a plain buffer or string passed as the key, naming no key bytes', () => { + const bytes = randomBytes(32); + for (const input of [bytes, bytes.toString('base64')]) + assert.throws( + () => CREATE_STREAM.serialize({ + name: 'orders', encryptionKey: input as unknown as StreamEncryptionKey + }), + (error: unknown) => { + assert.ok(error instanceof ResponseError); + assert.equal(error.errorCode, INVALID_ENCRYPTION_KEY); + for (const spelling of keySpellings(bytes)) + assert.ok(!`${error.message} ${error.stack}`.includes(spelling)); + return true; + } + ); + }); + + it('refuses a raw encryption_key options entry before network io, naming no key bytes', async () => { + const bytes = randomBytes(32); + const unexpectedNetwork: ClientProvider = () => { + throw new Error('a raw stream key reached the network'); + }; + const redacted = (error: unknown) => { + assert.ok(error instanceof ResponseError); + assert.equal(error.errorCode, INVALID_COMMAND); + assert.equal(error.fromServer, false); + for (const spelling of keySpellings(bytes)) + assert.ok(!`${error.message} ${error.stack}`.includes(spelling)); + return true; + }; + const rawEntries = [ + HeaderValue.Raw(bytes), + HeaderValue.String(bytes.toString('base64')) + ].map((value) => [{ key: StreamOptionKey.EncryptionKey, value }]); + for (const options of rawEntries) { + for (const encryptionKey of [undefined, StreamEncryptionKey.from(randomBytes(32))]) { + const args = { name: 'orders', encryptionKey, options }; + assert.throws(() => CREATE_STREAM.serialize(args), redacted); + await assert.rejects(createStream(unexpectedNetwork)(args), redacted); + } + } + }); +});
diff --git a/foreign/node/src/wire/stream/stream-options.ts b/foreign/node/src/wire/stream/stream-options.ts new file mode 100644 index 0000000..227c7ca --- /dev/null +++ b/foreign/node/src/wire/stream/stream-options.ts
@@ -0,0 +1,140 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { createHash } from 'node:crypto'; +import { inspect } from 'node:util'; +import { COMMAND_CODE } from '../command.code.js'; +import { responseError } from '../error.utils.js'; +import type { OptionEntry } from '../options.utils.js'; + +const STREAM_KEY_BYTES = 32; +const INVALID_ENCRYPTION_KEY = 70; +const FINGERPRINT_LABEL = 'iggy/stream-key-fingerprint/v1'; + +/** + * Options a stream takes at create. + * + * `encryptionKey` is the stream's own data key. It travels inside the create + * request, so protect the connection with TLS. The server stores it wrapped + * and never returns it, reads report only its source and fingerprints. + * Without a key, a server with encryption enabled generates one for the + * stream. + */ +export type StreamOptions = { + /** The stream's data key, set once at create */ + encryptionKey?: StreamEncryptionKey, + /** + * Option keys with no field of their own, for a key the server catalog + * gained after this build shipped. These entries print as given, so an + * `encryption_key` entry is refused with error 70 before anything is sent. + */ + options?: OptionEntry[] +}; + +/** + * A stream data key: 32 random bytes. It prints only its fingerprint through + * `util.inspect`, `JSON.stringify` and string conversion, so arguments + * carrying it are safe to log. Never derive it from a password, because the + * fingerprint is then an offline guessing oracle. + */ +export class StreamEncryptionKey { + readonly #bytes: Buffer; + + private constructor(bytes: Buffer) { + this.#bytes = bytes; + } + + /** + * @param key - 32 raw bytes, or their canonical padded base64. The bytes are copied + * @throws ResponseError with code 70 (invalid encryption key) for any other input + */ + static from(key: Buffer | string): StreamEncryptionKey { + return new StreamEncryptionKey(Buffer.from(streamEncryptionKey(key, COMMAND_CODE.CreateStream))); + } + + /** The value `getStream` reports as `encryptionKeyFingerprint` for this key */ + fingerprint(): string { + return encryptionKeyFingerprint(this.#bytes); + } + + /** A copy of the raw key, for the request encoder */ + asBytes(): Buffer { + return Buffer.from(this.#bytes); + } + + toString(): string { + return `StreamEncryptionKey { fingerprint: ${this.fingerprint()} }`; + } + + toJSON(): { fingerprint: string } { + return { fingerprint: this.fingerprint() }; + } + + [inspect.custom](): string { + return this.toString(); + } +} + +/** Who chose a stream's at-rest key. */ +export const EncryptionKeySource = { + /** The client sent the key at create */ + Stream: 'stream', + /** The server generated the key at create */ + Server: 'server', + /** The stream is plaintext */ + None: 'none' +} as const; + +export type EncryptionKeySource = typeof EncryptionKeySource[keyof typeof EncryptionKeySource]; + +/** + * Fingerprint of a stream data key, the value `getStream` reports as + * `encryptionKeyFingerprint`: the first 8 bytes of SHA-256 over a fixed label + * and the key, as lowercase hex. + * + * @param key - 32 raw bytes, or their canonical padded base64 + * @returns 16 lowercase hex characters + * @throws ResponseError with code 70 (invalid encryption key) for any other input + */ +export const encryptionKeyFingerprint = (key: Buffer | string): string => + createHash('sha256') + .update(FINGERPRINT_LABEL) + .update(streamEncryptionKey(key, 0)) + .digest() + .subarray(0, 8) + .toString('hex'); + +/** + * The 32 key bytes of a stream key input. Text is accepted only as the + * canonical padded standard base64 of 32 bytes, so whitespace, the URL-safe + * alphabet, missing padding and noncanonical trailing bits are refused. + * + * @param key - 32 raw bytes, or their canonical padded base64 + * @param commandCode - Command the refusal is reported against + * @returns The key bytes + * @throws ResponseError with code 70 (invalid encryption key), naming no key bytes + */ +export const streamEncryptionKey = (key: Buffer | string, commandCode: number): Buffer => { + const bytes = typeof key === 'string' ? Buffer.from(key, 'base64') : key; + const canonical = typeof key !== 'string' || bytes.toString('base64') === key; + if (!Buffer.isBuffer(bytes) || bytes.length !== STREAM_KEY_BYTES || !canonical) + throw responseError( + commandCode, INVALID_ENCRYPTION_KEY, + `stream key must be ${STREAM_KEY_BYTES} bytes or their canonical padded base64` + ); + return bytes; +};
diff --git a/foreign/node/src/wire/stream/stream.utils.ts b/foreign/node/src/wire/stream/stream.utils.ts index 4c453b6..a82e1e0 100644 --- a/foreign/node/src/wire/stream/stream.utils.ts +++ b/foreign/node/src/wire/stream/stream.utils.ts
@@ -17,6 +17,7 @@ import { toDate } from '../serialize.utils.js'; import { deserializePrefixedOptions, type ParsedOptions } from '../options.utils.js'; +import { EncryptionKeySource } from './stream-options.js'; /** * Stream information returned from the server. @@ -35,7 +36,15 @@ /** Stream creation timestamp */ createdAt: Date, /** Options the client explicitly sent at create */ - options: ParsedOptions + options: ParsedOptions, + /** Options the server filled or computes, such as the encryption key source */ + derivedOptions: ParsedOptions, + /** Who chose the stream's at-rest key, `null` when the server reported none */ + encryptionKeySource: EncryptionKeySource | null, + /** Fingerprint of the stream's data key, `null` for a plaintext stream */ + encryptionKeyFingerprint: string | null, + /** Fingerprints of the server keys wrapping the data key, empty for a plaintext stream */ + encryptionKekFingerprints: string[] } /** @@ -67,13 +76,25 @@ const messagesCount = r.readBigUint64LE(pos + 24); const nameLength = r.readUInt8(pos + 32); const name = r.subarray(pos + 33, pos + 33 + nameLength).toString(); - const { bytesRead: optionsBytes, options } = - deserializePrefixedOptions(r, pos + 33 + nameLength); + const explicit = deserializePrefixedOptions(r, pos + 33 + nameLength); + const derived = deserializePrefixedOptions(r, pos + 33 + nameLength + explicit.bytesRead); + const text = (key: string) => { + const value = derived.options[key] ?? explicit.options[key]; + return typeof value === 'string' ? value : null; + }; + const source = text('encryption_key_source'); + const keks = text('encryption_kek_fingerprint'); return { - bytesRead: 33 + nameLength + optionsBytes, + bytesRead: 33 + nameLength + explicit.bytesRead + derived.bytesRead, data: { - id, name, topicsCount, messagesCount, sizeBytes, createdAt, options + id, name, topicsCount, messagesCount, sizeBytes, createdAt, + options: explicit.options, + derivedOptions: derived.options, + encryptionKeySource: Object.values<string>(EncryptionKeySource).includes(source ?? '') + ? source as EncryptionKeySource : null, + encryptionKeyFingerprint: text('encryption_key_fingerprint'), + encryptionKekFingerprints: keks === null ? [] : keks.split(',') } }; };
diff --git a/foreign/node/src/wire/stream/update-stream.command.ts b/foreign/node/src/wire/stream/update-stream.command.ts index 6aa61ab..c711831 100644 --- a/foreign/node/src/wire/stream/update-stream.command.ts +++ b/foreign/node/src/wire/stream/update-stream.command.ts
@@ -15,44 +15,54 @@ // specific language governing permissions and limitations // under the License. -import { deserializeVoidResponse } from '../../client/client.utils.js'; import { wrapCommand } from '../command.utils.js'; -import { COMMAND_CODE } from '../command.code.js'; -import { serializeIdentifier, type Id } from '../identifier.utils.js'; -import { uint8ToBuf } from '../number.utils.js'; +import type { Id } from '../identifier.utils.js'; +import type { OptionEntry, StreamOptionKey } from '../options.utils.js'; +import { OptionsScope } from '../system/describe-options.command.js'; +import { UPDATE_RESOURCE, columnOption } from '../system/update-resource.command.js'; /** - * Parameters for the update stream command. + * Parameters for the update stream command. Keys left out keep their values. */ export type UpdateStream = { /** Stream identifier (ID or name) */ streamId: Id, /** New stream name (1-255 bytes) */ - name: string + name?: string, + /** + * Keys to reset to their defaults. No stream key is resettable yet, so any + * key is refused before the request is sent. + */ + reset?: StreamOptionKey[], + /** + * Reset names with no typed member, for a resettable key the server catalog + * gained after this build shipped. The server decides on them. + */ + rawReset?: readonly string[], + /** + * Option keys with no field of their own, for an updatable key the server + * catalog gained after this build shipped. A field above wins on collision. + * The server decides on them, so a create-only or unknown key comes back as + * a server error. + */ + options?: OptionEntry[] } /** * Update stream command definition. - * Updates a stream's name. + * Changes only the keys it names, through the generic update command. */ export const UPDATE_STREAM = { - code: COMMAND_CODE.UpdateStream, + code: UPDATE_RESOURCE.code, - serialize: ({streamId, name}: UpdateStream) => { - const bId = serializeIdentifier(streamId); - const bName = Buffer.from(name); - - if (bName.length < 1 || bName.length > 255) - throw new Error('Stream name should be between 1 and 255 bytes'); - - return Buffer.concat([ - bId, - uint8ToBuf(bName.length), - bName - ]); + serialize: ({ streamId, name, reset, rawReset, options: rawOptions }: UpdateStream) => { + const options = name === undefined ? [] : [columnOption('name', name)]; + return UPDATE_RESOURCE.serialize({ + target: { scope: OptionsScope.Stream, streamId }, options, reset, rawOptions, rawReset + }); }, - deserialize: deserializeVoidResponse + deserialize: UPDATE_RESOURCE.deserialize };
diff --git a/foreign/node/src/wire/system/describe-options.command.ts b/foreign/node/src/wire/system/describe-options.command.ts index 930af01..d146dbd 100644 --- a/foreign/node/src/wire/system/describe-options.command.ts +++ b/foreign/node/src/wire/system/describe-options.command.ts
@@ -28,18 +28,36 @@ export type OptionsScope = typeof OptionsScope[keyof typeof OptionsScope]; +/** Bits of the `flags` byte a `DescribeOptions` entry carries. */ +export const OptionFlag = { + Updatable: 1, + Resettable: 2, + Secret: 4, + ReadOnly: 8, + Critical: 16 +} as const; + /** - * One catalog entry: the key a create command accepts, the kind the server - * encodes its default under, that default, and what the option does. + * One catalog entry: the key, the kind the server encodes its default under, + * what a client may do with it, that default, and what the option does. * * `kind` is this key's canonical kind: what the server encodes its default - * under, and what a value set at create is stored as whatever kind it was sent - * in, since create admission re-encodes the block from its own parse. An update - * stores the client's bytes verbatim and is the exception. + * under, and what a value set at create or update is stored as whatever kind it + * was sent in, since admission re-encodes every block from its own parse. */ export type OptionSpec = { key: string, kind: number, + /** An update may set the key */ + updatable: boolean, + /** An update may reset the key to its default */ + resettable: boolean, + /** Write-only: accepted at create, never returned */ + secret: boolean, + /** Computed by the server and returned on reads, never accepted */ + readOnly: boolean, + /** A node that does not understand the key must not serve the resource */ + critical: boolean, defaultValue: Buffer, description: string }; @@ -61,7 +79,8 @@ position += keyLength; const kind = b.readUInt8(position); - position += 1; + const flags = b.readUInt8(position + 1); + position += 2; const defaultLength = b.readUInt32LE(position); position += 4; @@ -81,7 +100,17 @@ ).toString(); position += descriptionLength; - specs.push({ key, kind, defaultValue, description }); + specs.push({ + key, + kind, + updatable: (flags & OptionFlag.Updatable) !== 0, + resettable: (flags & OptionFlag.Resettable) !== 0, + secret: (flags & OptionFlag.Secret) !== 0, + readOnly: (flags & OptionFlag.ReadOnly) !== 0, + critical: (flags & OptionFlag.Critical) !== 0, + defaultValue, + description + }); } return specs; };
diff --git a/foreign/node/src/wire/system/index.ts b/foreign/node/src/wire/system/index.ts index 1e181a9..949aa97 100644 --- a/foreign/node/src/wire/system/index.ts +++ b/foreign/node/src/wire/system/index.ts
@@ -19,3 +19,4 @@ export * from './get-stats.command.js'; export * from './ping.command.js'; export * from './snapshot.command.js'; +export * from './update-resource.command.js';
diff --git a/foreign/node/src/wire/system/update-resource.command.test.ts b/foreign/node/src/wire/system/update-resource.command.test.ts new file mode 100644 index 0000000..b5ab031 --- /dev/null +++ b/foreign/node/src/wire/system/update-resource.command.test.ts
@@ -0,0 +1,680 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import type { ClientProvider } from '../../client/client.type.js'; +import { COMMAND_CODE } from '../command.code.js'; +import { ResponseError } from '../error.utils.js'; +import { serializeIdentifier, type Id } from '../identifier.utils.js'; +import { HeaderValue } from '../message/header.utils.js'; +import { + OPTION_CATALOG, StreamOptionKey, TopicOptionKey, UserOptionKey, deserializeOptions, + type OptionEntry, type ParsedOptions +} from '../options.utils.js'; +import { CREATE_STREAM, createStream } from '../stream/create-stream.command.js'; +import { UPDATE_STREAM } from '../stream/update-stream.command.js'; +import { StreamEncryptionKey, encryptionKeyFingerprint } from '../stream/stream-options.js'; +import { deserializeToStream, type Stream } from '../stream/stream.utils.js'; +import { CREATE_TOPIC } from '../topic/create-topic.command.js'; +import { UPDATE_TOPIC } from '../topic/update-topic.command.js'; +import { + CompressionAlgorithm, compressionAlgorithmName, deserializeTopic, Durability, PartitionResizePolicy +} from '../topic/topic.utils.js'; +import { CREATE_USER } from '../user/create-user.command.js'; +import { translateErrorCode } from '../error.code.js'; +import { UPDATE_USER } from '../user/update-user.command.js'; +import { deserializeUser, UserStatus } from '../user/user.utils.js'; +import { Command, COMMITTED_RESULT_MARKER, HEADER_SIZE, REPLY_OFFSET, REQUEST_OFFSET } from '../vsr/header.js'; +import { VsrSession } from '../vsr/index.js'; +import { + Operation, isInternal, isKnownOperation, operationForCode, requestOperation +} from '../vsr/operation.js'; +import { IGGY_PROTOCOL_VERSION } from '../vsr/register.js'; +import { validateReplyForRequest } from '../vsr/reply.js'; +import { DESCRIBE_OPTIONS, OptionsScope, type OptionSpec } from './describe-options.command.js'; +import { UPDATE_RESOURCE } from './update-resource.command.js'; + +type FixtureError = { name: string, code: number }; +type FixtureOption = { value: string, kind: string, explicit: boolean }; +type FixtureOptions = Record<string, FixtureOption>; +type FixtureKey = { raw_hex?: string, base64?: string }; +type Scope = 'topic' | 'stream' | 'user'; +type Target = { stream_id?: Id, topic_id?: Id, user_id?: Id }; +type UpdateInput = { + name?: string, username?: string, status?: string, + message_expiry?: string, max_topic_size?: string, reset?: string[] +}; + +type Fixture = { + constants: { + protocol_version: { packed: number }, + command_codes: Record<string, { code: number, operation: number | null }>, + error_codes: Record<string, number>, + retired_command_codes: Record<string, number>, + operations: Record<string, { value: number, internal: boolean, request_operation: number }>, + retired_operations: Record<string, number>, + scope_codes: Record<Scope, number>, + header_kind_codes: Record<string, number>, + option_flags: Record<string, number>, + user_status_codes: Record<string, number>, + stream_key_length: number, + never_returned_keys: string[] + }, + catalog: { + key: string, scopes: Scope[], storage: string, mutability: string, visibility: string, + flags: number, described: boolean + }[], + describe_options: { + scope: Scope, request_hex: string, response_hex: string, + entries: { key: string, kind: string, flags: number, default: string | null }[] + }[], + stream_key: { + key_hex: string, key_base64: string, fingerprint: string, + invalid_keys: { case: string, input: FixtureKey, error: FixtureError }[] + }, + update_resource: { + case: string, scope: Scope, target: Target, input: UpdateInput, body_hex: string + }[], + update_resource_errors: { + case: string, scope: Scope, target: Target, input: UpdateInput, error: FixtureError + }[], + create_stream: { + case: string, + input: { name: string, encryption_key?: FixtureKey, raw?: Record<string, string> }, + body_hex: string + }[], + create_topic: { + case: string, + input: { + stream_id: Id, name: string, partitions_count: number, compression_algorithm: string, + message_expiry: string, max_topic_size: string, segment_size: string, + durability: Durability, consumer_offset_durability: Durability + }, + body_hex: string + }[], + create_user: { + case: string, input: { username: string, password: string, status: string }, body_hex: string + }[], + stream_responses: { + case: string, response_hex: string, + expected: { + id: number, created_at: number, name: string, topics_count: number, + size_bytes: number, messages_count: number, options: FixtureOptions, + encryption_key_source: string, encryption_key_fingerprint: string | null, + encryption_kek_fingerprints: string[] + } + }[], + topic_responses: { + case: string, response_hex: string, + expected: { + id: number, created_at: number, name: string, partitions_count: number, + size_bytes: number, messages_count: number, options: FixtureOptions, + compression_algorithm: string, message_expiry: string, max_topic_size: string + } + }[], + user_responses: { + case: string, response_hex: string, + expected: { + id: number, created_at: number, status: string, username: string, options: FixtureOptions + } + }[] +}; + +const fixture: Fixture = JSON.parse( + readFileSync(new URL('../fixtures/resource_options.json', import.meta.url), 'utf8') +); +const { constants } = fixture; +const INVALID_ENCRYPTION_KEY = 70; +const UNSUPPORTED_OPTION_KEY = 4041; +const UPDATE_RESOURCE_CODE = constants.command_codes.update_resource.code; + +const fromHex = (hex: string) => Buffer.from(hex, 'hex'); +const keyInput = (key: FixtureKey): Buffer | string => + key.raw_hex !== undefined ? fromHex(key.raw_hex) : key.base64!; +const userStatus = (status: string) => + status === 'active' ? UserStatus.Active : UserStatus.Inactive; +const optionalBigInt = (value?: string) => value === undefined ? undefined : BigInt(value); + +const kindName = (value: unknown): string => { + if (typeof value === 'bigint') return 'uint64'; + if (typeof value === 'number') return 'uint32'; + if (typeof value === 'boolean') return 'bool'; + if (Buffer.isBuffer(value)) return 'raw'; + return 'string'; +}; + +const optionText = (value: unknown): string => + Buffer.isBuffer(value) ? value.toString('base64') : String(value); + +/** Both blocks in the fixture's HTTP form, so provenance is compared too. */ +const withProvenance = (explicit: ParsedOptions, derived: ParsedOptions): FixtureOptions => { + const merged: FixtureOptions = {}; + for (const [options, isExplicit] of [[derived, false], [explicit, true]] as const) + for (const [key, value] of Object.entries(options)) + merged[key] = { value: optionText(value), kind: kindName(value), explicit: isExplicit }; + return merged; +}; + +const assertNoSecrets = (explicit: ParsedOptions, derived: ParsedOptions) => { + for (const key of constants.never_returned_keys) { + assert.equal(key in explicit, false, `explicit block returned ${key}`); + assert.equal(key in derived, false, `derived block returned ${key}`); + } +}; + +const encodeUpdate = (scope: Scope, target: Target, input: UpdateInput): Buffer => { + const reset = input.reset as never[] | undefined; + switch (scope) { + case 'topic': + return UPDATE_TOPIC.serialize({ + streamId: target.stream_id!, topicId: target.topic_id!, name: input.name, + messageExpiry: optionalBigInt(input.message_expiry), + maxTopicSize: optionalBigInt(input.max_topic_size), reset + }); + case 'stream': + return UPDATE_STREAM.serialize({ streamId: target.stream_id!, name: input.name, reset }); + case 'user': + return UPDATE_USER.serialize({ + userId: target.user_id!, username: input.username, + status: input.status === undefined ? undefined : userStatus(input.status), reset + }); + } +}; + +const isClientError = (code: number, commandCode: number) => (error: unknown) => + error instanceof ResponseError && error.errorCode === code && + error.commandCode === commandCode && !error.fromServer; + +const defaultText = (spec: OptionSpec): string | null => { + if (spec.defaultValue.length === 0) return null; + switch (spec.kind) { + case constants.header_kind_codes.uint64: return spec.defaultValue.readBigUInt64LE(0).toString(); + case constants.header_kind_codes.uint32: return spec.defaultValue.readUInt32LE(0).toString(); + case constants.header_kind_codes.bool: return String(spec.defaultValue.readUInt8(0) === 1); + default: return spec.defaultValue.toString(); + } +}; + +const specFlags = (spec: OptionSpec): number => + (spec.updatable ? constants.option_flags.updatable : 0) | + (spec.resettable ? constants.option_flags.resettable : 0) | + (spec.secret ? constants.option_flags.secret : 0) | + (spec.readOnly ? constants.option_flags.read_only : 0) | + (spec.critical ? constants.option_flags.critical : 0); + +const kindNames = new Map( + Object.entries(constants.header_kind_codes).map(([name, code]) => [code, name]) +); + +describe('given shared vectors when encoding should match server contract', () => { + + it('encodes every update vector byte for byte through the typed scope commands', () => { + for (const vector of fixture.update_resource) { + const encoded = encodeUpdate(vector.scope, vector.target, vector.input); + assert.equal(encoded.toString('hex'), vector.body_hex, vector.case); + } + for (const command of [UPDATE_STREAM, UPDATE_TOPIC, UPDATE_USER, UPDATE_RESOURCE]) + assert.equal(command.code, UPDATE_RESOURCE_CODE); + }); + + it('refuses every invalid update before sending, with the server category', () => { + for (const vector of fixture.update_resource_errors) + assert.throws( + () => encodeUpdate(vector.scope, vector.target, vector.input), + isClientError(vector.error.code, UPDATE_RESOURCE_CODE), + vector.case + ); + }); + + it('refuses each reset category in the server order', () => { + const cases: [Scope, string[], number][] = [ + ['topic', ['encryption_key'], 4041], + ['stream', ['future_option'], 4041], + ['stream', ['encryption_key_fingerprint'], 4105], + ['stream', ['critical_keys'], 4105], + ['topic', ['segment_size', 'name'], 4103], + ['topic', ['name'], 4104], + ['user', ['username'], 4104], + ['topic', ['name', 'encryption_key'], 4041] + ]; + for (const [scope, reset, code] of cases) + assert.throws( + () => encodeUpdate(scope, { stream_id: 1, topic_id: 2, user_id: 3 }, { reset }), + isClientError(code, UPDATE_RESOURCE_CODE), + `${scope} reset ${reset.join(',')}` + ); + }); + + it('lets a typed field win over a raw entry of the same key', () => { + const typed = UPDATE_TOPIC.serialize({ + streamId: 1, topicId: 2, maxTopicSize: 9007199254740993n, + options: [{ key: 'max_topic_size', value: HeaderValue.String('1 GiB') }] + }); + const vector = fixture.update_resource + .find((entry) => entry.case === 'topic_max_topic_size_above_2_pow_53')!; + assert.equal(typed.toString('hex'), vector.body_hex); + }); + + it('sends raw set keys and raw reset names for the server to decide', () => { + const body = UPDATE_TOPIC.serialize({ + streamId: 1, topicId: 2, reset: ['name'], rawReset: ['future_option'], + options: [{ key: 'compression_algorithm', value: HeaderValue.String('gzip') }] + }); + for (const name of ['name', 'future_option', 'compression_algorithm']) + assert.ok(body.includes(Buffer.from(name)), name); + assert.doesNotThrow(() => UPDATE_STREAM.serialize({ streamId: 3, rawReset: ['name'] })); + assert.doesNotThrow(() => UPDATE_USER.serialize({ + userId: 5, options: [{ key: 'username', value: HeaderValue.String('a') }] + })); + assert.throws( + () => UPDATE_USER.serialize({ userId: 5, status: UserStatus.Active, rawReset: ['status'] }), + isClientError(3, UPDATE_RESOURCE_CODE) + ); + assert.throws( + () => UPDATE_TOPIC.serialize({ streamId: 1, topicId: 2, rawReset: [] }), + isClientError(3, UPDATE_RESOURCE_CODE) + ); + }); + + it('refuses a raw reset list its count byte or a name length byte cannot carry', () => { + const futureNames = (count: number) => + Array.from({ length: count }, (_, index) => `future_${index}`); + for (const rawReset of [futureNames(256), [''], ['x'.repeat(256)]]) + assert.throws( + () => UPDATE_TOPIC.serialize({ streamId: 1, topicId: 2, rawReset }), + isClientError(UNSUPPORTED_OPTION_KEY, UPDATE_RESOURCE_CODE), + `${rawReset.length} names, first ${rawReset[0].length} bytes` + ); + const body = UPDATE_TOPIC.serialize({ streamId: 1, topicId: 2, rawReset: futureNames(255) }); + assert.equal(body[1 + 2 * serializeIdentifier(1).length], 255); + }); + + it('writes typed and raw reset names together in ascending name order', () => { + const body = UPDATE_TOPIC.serialize({ + streamId: 1, topicId: 2, reset: ['message_expiry', 'max_topic_size'], rawReset: ['a_future'] + }); + const resets = body.subarray(1 + 2 * serializeIdentifier(1).length); + const names: string[] = []; + for (let pos = 1; names.length < resets[0]; pos += 1 + resets[pos]) + names.push(resets.subarray(pos + 1, pos + 1 + resets[pos]).toString()); + assert.deepEqual(names, ['a_future', 'max_topic_size', 'message_expiry']); + }); + + it('exposes the typed surface for every public catalog row', () => { + const rows = (scope: Scope, accept: (row: Fixture['catalog'][number]) => boolean) => + fixture.catalog.filter((row) => row.scopes.includes(scope) && accept(row)) + .map((row) => row.key).sort(); + const optionsAt = (body: Buffer, offset: number) => deserializeOptions(body, offset); + const idLength = serializeIdentifier(1).length; + const topic = { streamId: 1, name: 'events', partitionCount: 1 }; + const topicBlock = (fields: Partial<Parameters<typeof CREATE_TOPIC.serialize>[0]>) => + optionsAt(CREATE_TOPIC.serialize({ ...topic, ...fields }), idLength + 5 + topic.name.length); + const typedCreate: Record<Scope, Record<string, () => ParsedOptions>> = { + stream: { + encryption_key: () => optionsAt(CREATE_STREAM.serialize({ + name: 'orders', encryptionKey: StreamEncryptionKey.from(fromHex(fixture.stream_key.key_hex)) + }), 1 + 'orders'.length) + }, + topic: { + compression_algorithm: () => topicBlock({ compressionAlgorithm: CompressionAlgorithm.Gzip }), + message_expiry: () => topicBlock({ messageExpiry: 1n }), + max_topic_size: () => topicBlock({ maxTopicSize: 1n }), + segment_size: () => topicBlock({ segmentSize: 1048576n }), + durability: () => topicBlock({ durability: Durability.Persisted }), + consumer_offset_durability: () => topicBlock({ consumerOffsetDurability: Durability.Persisted }), + partition_resize_policy: () => + topicBlock({ partitionResizePolicy: PartitionResizePolicy.Fixed }), + messages_required_to_save: () => topicBlock({ messagesRequiredToSave: 1 }), + size_of_messages_required_to_save: () => topicBlock({ sizeOfMessagesRequiredToSave: 1n }), + preallocate_segments: () => topicBlock({ preallocateSegments: true }) + }, + user: {} + }; + const baseCreate: Record<Scope, ParsedOptions> = { + stream: optionsAt(CREATE_STREAM.serialize({ name: 'orders' }), 1 + 'orders'.length), + topic: topicBlock({}), + user: {} + }; + const updateBlock = (body: Buffer, ids: number) => optionsAt(body, 1 + ids * idLength + 1); + const typedUpdate: Record<Scope, Record<string, () => ParsedOptions>> = { + stream: { name: () => updateBlock(UPDATE_STREAM.serialize({ streamId: 1, name: 'renamed' }), 1) }, + topic: { + name: () => updateBlock(UPDATE_TOPIC.serialize({ streamId: 1, topicId: 1, name: 'renamed' }), 2), + message_expiry: () => + updateBlock(UPDATE_TOPIC.serialize({ streamId: 1, topicId: 1, messageExpiry: 1n }), 2), + max_topic_size: () => + updateBlock(UPDATE_TOPIC.serialize({ streamId: 1, topicId: 1, maxTopicSize: 1n }), 2) + }, + user: { + username: () => updateBlock(UPDATE_USER.serialize({ userId: 1, username: 'alice2' }), 1), + status: () => updateBlock(UPDATE_USER.serialize({ userId: 1, status: UserStatus.Inactive }), 1) + } + }; + const typedReset: Record<Scope, (key: string) => Buffer> = { + stream: (key) => UPDATE_STREAM.serialize({ streamId: 1, reset: [key as StreamOptionKey] }), + topic: (key) => UPDATE_TOPIC.serialize({ streamId: 1, topicId: 1, reset: [key as TopicOptionKey] }), + user: (key) => UPDATE_USER.serialize({ userId: 1, reset: [key as UserOptionKey] }) + }; + const resetMembers: Record<Scope, readonly string[]> = { + stream: Object.values(StreamOptionKey), + topic: Object.values(TopicOptionKey), + user: Object.values(UserOptionKey) + }; + const readAccessors: Record<Scope, Record<string, keyof Stream>> = { + stream: { + encryption_key_source: 'encryptionKeySource', + encryption_key_fingerprint: 'encryptionKeyFingerprint', + encryption_kek_fingerprint: 'encryptionKekFingerprints' + }, + topic: {}, + user: {} + }; + const stream = deserializeToStream(fromHex(fixture.stream_responses[0].response_hex)).data; + + for (const scope of ['stream', 'topic', 'user'] as const) { + const createKeys = rows(scope, (row) => row.storage === 'map' && + ['public', 'secret'].includes(row.visibility) && + ['create_only', 'updatable'].includes(row.mutability)); + assert.deepEqual(Object.keys(typedCreate[scope]).sort(), createKeys, `${scope} create`); + for (const [key, block] of Object.entries(typedCreate[scope])) { + const options = block(); + assert.ok(key in options, `${scope} create ${key}`); + assert.notDeepEqual(options[key], baseCreate[scope][key], `${scope} create ${key}`); + } + + const updateKeys = rows(scope, (row) => row.mutability === 'updatable'); + assert.deepEqual(Object.keys(typedUpdate[scope]).sort(), updateKeys, `${scope} update`); + for (const [key, block] of Object.entries(typedUpdate[scope])) + assert.deepEqual(Object.keys(block()), [key], `${scope} update ${key}`); + + const resetKeys = rows(scope, (row) => (row.flags & constants.option_flags.resettable) !== 0); + for (const key of resetKeys) { + assert.ok(resetMembers[scope].includes(key), `${scope} reset member ${key}`); + assert.ok(typedReset[scope](key).includes(Buffer.from(key)), `${scope} reset ${key}`); + } + + const readOnlyKeys = rows(scope, (row) => row.visibility === 'read_only'); + assert.deepEqual(Object.keys(readAccessors[scope]).sort(), readOnlyKeys, `${scope} read`); + for (const accessor of Object.values(readAccessors[scope])) + assert.ok(accessor in stream, `${scope} accessor ${accessor}`); + } + }); + + it('encodes create commands byte for byte', () => { + for (const vector of fixture.create_stream) { + const { name, encryption_key, raw = {} } = vector.input; + const options: OptionEntry[] = Object.entries(raw) + .map(([key, value]) => ({ key, value: HeaderValue.String(value) })); + const encoded = CREATE_STREAM.serialize({ + name, + encryptionKey: encryption_key === undefined + ? undefined : StreamEncryptionKey.from(keyInput(encryption_key)), + options + }); + assert.equal(encoded.toString('hex'), vector.body_hex, vector.case); + } + for (const { input, body_hex, case: name } of fixture.create_topic) { + const encoded = CREATE_TOPIC.serialize({ + streamId: input.stream_id, + name: input.name, + partitionCount: input.partitions_count, + compressionAlgorithm: input.compression_algorithm === 'gzip' + ? CompressionAlgorithm.Gzip : CompressionAlgorithm.None, + messageExpiry: BigInt(input.message_expiry), + maxTopicSize: BigInt(input.max_topic_size), + segmentSize: BigInt(input.segment_size), + durability: input.durability, + consumerOffsetDurability: input.consumer_offset_durability + }); + assert.equal(encoded.toString('hex'), body_hex, name); + } + for (const { input, body_hex, case: name } of fixture.create_user) { + const encoded = CREATE_USER.serialize({ + username: input.username, password: input.password, + status: userStatus(input.status), options: [] + }); + assert.equal(encoded.toString('hex'), body_hex, name); + } + }); + + it('decodes stream responses into their projection without secrets', () => { + for (const { case: name, response_hex, expected } of fixture.stream_responses) { + const response = fromHex(response_hex); + const { bytesRead, data: stream } = deserializeToStream(response); + assert.equal(bytesRead, response.length, name); + assert.deepEqual({ + id: stream.id, + created_at: stream.createdAt.getTime() * 1000, + name: stream.name, + topics_count: stream.topicsCount, + size_bytes: Number(stream.sizeBytes), + messages_count: Number(stream.messagesCount), + options: withProvenance(stream.options, stream.derivedOptions), + encryption_key_source: stream.encryptionKeySource, + encryption_key_fingerprint: stream.encryptionKeyFingerprint, + encryption_kek_fingerprints: stream.encryptionKekFingerprints + }, expected, name); + assertNoSecrets(stream.options, stream.derivedOptions); + } + }); + + it('decodes topic responses with the settings read from the option blocks', () => { + for (const { case: name, response_hex, expected } of fixture.topic_responses) { + const response = fromHex(response_hex); + const { bytesRead, data: topic } = deserializeTopic(response); + assert.equal(bytesRead, response.length, name); + assert.deepEqual({ + id: topic.id, + created_at: topic.createdAt.getTime() * 1000, + name: topic.name, + partitions_count: topic.partitionsCount, + size_bytes: Number(topic.sizeBytes), + messages_count: Number(topic.messagesCount), + options: withProvenance(topic.options, topic.derivedOptions), + compression_algorithm: compressionAlgorithmName(topic.compressionAlgorithm), + message_expiry: topic.messageExpiry.toString(), + max_topic_size: topic.maxTopicSize.toString() + }, expected, name); + } + }); + + it('decodes user responses with both option blocks, keeping u64 precision', () => { + for (const { case: name, response_hex, expected } of fixture.user_responses) { + const user = deserializeUser(fromHex(response_hex)); + assert.deepEqual({ + id: user.id, + created_at: user.createdAt.getTime() * 1000, + status: user.status.toLowerCase(), + username: user.userName, + options: withProvenance(user.options, user.derivedOptions) + }, expected, name); + assertNoSecrets(user.options, user.derivedOptions); + } + }); + + it('describes every scope with the catalog flags', () => { + for (const { scope, request_hex, response_hex, entries } of fixture.describe_options) { + assert.equal( + DESCRIBE_OPTIONS.serialize({ scope: constants.scope_codes[scope] as OptionsScope }) + .toString('hex'), + request_hex + ); + const data = fromHex(response_hex); + const specs = DESCRIBE_OPTIONS.deserialize({ status: 0, length: data.length, data }); + assert.deepEqual(specs.map((spec) => ({ + key: spec.key, + kind: kindNames.get(spec.kind), + flags: specFlags(spec), + default: defaultText(spec) + })), entries, scope); + } + }); + + it('mirrors the catalog in its key constants and reset rules', () => { + assert.deepEqual([...OPTION_CATALOG.keys()], fixture.catalog.map((row) => row.key)); + for (const row of fixture.catalog) { + const descriptor = OPTION_CATALOG.get(row.key)!; + assert.deepEqual( + descriptor.scopes.map((code) => Object.entries(constants.scope_codes) + .find(([, value]) => value === code)![0]).sort(), + [...row.scopes].sort(), + row.key + ); + assert.equal(descriptor.flags, row.flags, row.key); + assert.equal(descriptor.internal, row.visibility === 'internal', row.key); + } + const described = (scope: Scope) => fixture.catalog + .filter((row) => row.described && row.scopes.includes(scope)).map((row) => row.key); + assert.deepEqual(Object.values(StreamOptionKey), described('stream')); + assert.deepEqual(Object.values(TopicOptionKey), described('topic')); + assert.deepEqual(Object.values(UserOptionKey), described('user')); + }); + + it('mirrors command codes, operations, scopes and the protocol version', () => { + assert.equal(IGGY_PROTOCOL_VERSION, constants.protocol_version.packed); + assert.equal(translateErrorCode(constants.error_codes.cannot_modify_root_user), 'Cannot modify root user'); + assert.deepEqual( + { topic: OptionsScope.Topic, stream: OptionsScope.Stream, user: OptionsScope.User }, + constants.scope_codes + ); + assert.deepEqual( + { active: UserStatus.Active, inactive: UserStatus.Inactive }, + constants.user_status_codes + ); + const codes = new Set<number>(Object.values(COMMAND_CODE)); + for (const [name, { code, operation }] of Object.entries(constants.command_codes)) { + assert.ok(codes.has(code), name); + assert.equal(operationForCode(code), operation ?? Operation.NonReplicated, name); + } + for (const [name, code] of Object.entries(constants.retired_command_codes)) + assert.equal(codes.has(code), false, `${name} code ${code} is retired`); + for (const [name, operation] of Object.entries(constants.operations)) { + assert.ok(isKnownOperation(operation.value), name); + assert.equal(isInternal(operation.value), operation.internal, name); + assert.equal(requestOperation(operation.value), operation.request_operation, name); + } + for (const [name, operation] of Object.entries(constants.retired_operations)) + assert.equal(isKnownOperation(operation), false, `${name} operation ${operation} is retired`); + }); + + it('computes the stream key fingerprint from raw bytes and from base64', () => { + const { key_hex, key_base64, fingerprint } = fixture.stream_key; + assert.equal(fromHex(key_hex).length, constants.stream_key_length); + assert.equal(encryptionKeyFingerprint(fromHex(key_hex)), fingerprint); + assert.equal(encryptionKeyFingerprint(key_base64), fingerprint); + }); +}); + +describe('given invalid key when encoding should reject before network io', () => { + const { key_hex, key_base64, invalid_keys } = fixture.stream_key; + const urlSafe = Buffer.alloc(32, 0xff).toString('base64url') + '='; + const unexpectedNetwork: ClientProvider = () => { + throw new Error('an invalid key reached the network'); + }; + + const refusals = [ + ...invalid_keys.map(({ case: name, input, error }) => ({ + name, input: keyInput(input), code: error.code + })), + { name: 'base64_url_safe_alphabet', input: urlSafe, code: INVALID_ENCRYPTION_KEY }, + { name: 'base64_leading_space', input: ` ${key_base64}`, code: INVALID_ENCRYPTION_KEY }, + { name: 'empty_string', input: '', code: INVALID_ENCRYPTION_KEY } + ]; + + for (const { name, input, code } of refusals) { + it(`refuses ${name}`, async () => { + const redacted = (error: unknown) => { + assert.ok(error instanceof ResponseError); + assert.equal(error.errorCode, code); + assert.equal(error.fromServer, false); + const text = `${error.message} ${String(error.stack)}`; + const hex = Buffer.isBuffer(input) ? input.toString('hex') : input.trim(); + assert.ok(hex.length === 0 || !text.includes(hex), 'diagnostic carries key material'); + return true; + }; + assert.throws(() => encryptionKeyFingerprint(input), redacted); + assert.throws(() => StreamEncryptionKey.from(input), redacted); + await assert.rejects( + createStream(unexpectedNetwork)({ + name: 'keyed', encryptionKey: input as unknown as StreamEncryptionKey + }), + redacted + ); + }); + } + + it('accepts exactly 32 raw bytes and canonical base64 as the same key', () => { + const raw = CREATE_STREAM.serialize({ + name: 'keyed', encryptionKey: StreamEncryptionKey.from(fromHex(key_hex)) + }); + const text = CREATE_STREAM.serialize({ + name: 'keyed', encryptionKey: StreamEncryptionKey.from(key_base64) + }); + assert.deepEqual(raw, text); + assert.ok(raw.subarray(raw.length - 32).equals(fromHex(key_hex))); + }); +}); + +describe('given resolved update reply when correlating should accept only matching request identity', () => { + const receiptFor = (request: Buffer, operation: number): Buffer => { + const receipt = Buffer.alloc(HEADER_SIZE); + receipt.writeUInt32LE(HEADER_SIZE, REPLY_OFFSET.size); + receipt[REPLY_OFFSET.command] = Command.Reply; + receipt[REPLY_OFFSET.operation] = operation; + receipt[REPLY_OFFSET.reserved] = COMMITTED_RESULT_MARKER; + receipt.writeBigUInt64LE(1n, REPLY_OFFSET.op); + receipt.writeBigUInt64LE(1n, REPLY_OFFSET.commit); + receipt.writeBigUInt64LE(request.readBigUInt64LE(REQUEST_OFFSET.request), REPLY_OFFSET.request); + request.copy(receipt, REPLY_OFFSET.client, REQUEST_OFFSET.client, REQUEST_OFFSET.client + 16); + request.copy(receipt, REPLY_OFFSET.requestChecksum, + REQUEST_OFFSET.requestChecksum, REQUEST_OFFSET.requestChecksum + 16); + return receipt; + }; + const refused = (error: unknown) => + error instanceof ResponseError && error.errorCode === 3 && error.commandCode === UPDATE_RESOURCE_CODE; + + for (const resolved of [ + Operation.UpdateStreamResolved, Operation.UpdateTopicResolved, Operation.UpdateUserResolved + ]) { + it(`accepts operation ${resolved} only for its own update request`, () => { + const session = new VsrSession(7n); + session.bind(42n); + const vector = fixture.update_resource[0]; + const request = session.encode(UPDATE_RESOURCE_CODE, fromHex(vector.body_hex)); + assert.equal(request[REQUEST_OFFSET.operation], Operation.UpdateResource); + const receipt = receiptFor(request, resolved); + assert.doesNotThrow(() => validateReplyForRequest(receipt, request, UPDATE_RESOURCE_CODE)); + + const otherClient = Buffer.from(receipt); + otherClient[REPLY_OFFSET.client] ^= 1; + const otherChecksum = Buffer.from(receipt); + otherChecksum[REPLY_OFFSET.requestChecksum] ^= 1; + const otherOperation = Buffer.from(receipt); + otherOperation[REPLY_OFFSET.operation] = Operation.CreateTopicWithAssignments; + for (const mismatch of [otherClient, otherChecksum, otherOperation]) + assert.throws(() => validateReplyForRequest(mismatch, request, UPDATE_RESOURCE_CODE), refused); + + const create = session.encode(COMMAND_CODE.CreateStream, Buffer.from([1, 0x61])); + assert.throws( + () => validateReplyForRequest(receiptFor(create, resolved), create, COMMAND_CODE.CreateStream), + (error: unknown) => error instanceof ResponseError && error.errorCode === 3 + ); + }); + } +});
diff --git a/foreign/node/src/wire/system/update-resource.command.ts b/foreign/node/src/wire/system/update-resource.command.ts new file mode 100644 index 0000000..0f8071a --- /dev/null +++ b/foreign/node/src/wire/system/update-resource.command.ts
@@ -0,0 +1,173 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { deserializeVoidResponse } from '../../client/client.utils.js'; +import { COMMAND_CODE } from '../command.code.js'; +import { responseError } from '../error.utils.js'; +import { serializeIdentifier, type Id } from '../identifier.utils.js'; +import { uint8ToBuf } from '../number.utils.js'; +import { HeaderKind } from '../message/header.type.js'; +import { HeaderValue } from '../message/header.utils.js'; +import { + OPTION_CATALOG, dedupeOptions, serializeOptions, sortOptions, + type OptionDescriptor, type OptionEntry +} from '../options.utils.js'; +import { OptionFlag, OptionsScope } from './describe-options.command.js'; + +const INVALID_COMMAND = 3; +const INVALID_USERNAME = 43; +const INVALID_USER_STATUS = 45; +const INVALID_STREAM_NAME = 1013; +const INVALID_TOPIC_NAME = 2014; +const UNSUPPORTED_OPTION_KEY = 4041; +const OPTION_NOT_UPDATABLE = 4103; +const OPTION_NOT_RESETTABLE = 4104; +const READ_ONLY_OPTION = 4105; +const MAX_NAME_LENGTH = 255; +const MAX_RESET_KEYS = 255; +const KEY_PREVIEW_LENGTH = 64; +const MIN_USERNAME_LENGTH = 3; +const MAX_USERNAME_LENGTH = 50; + +/** The resource an update patches, tagged by its options scope. */ +export type ResourceTarget = + | { scope: typeof OptionsScope.Topic, streamId: Id, topicId: Id } + | { scope: typeof OptionsScope.Stream, streamId: Id } + | { scope: typeof OptionsScope.User, userId: Id }; + +/** + * Parameters for the generic update: the keys to set, the keys to reset to + * their defaults. Every other key keeps its stored value. + * + * Typed keys are checked here with the category and in the order the server + * admits them. A patch carrying any raw entry or raw reset name gets only the + * structural checks, because the server catalog may know keys or flags this + * build does not, so the server decides the rest. + */ +export type UpdateResource = { + target: ResourceTarget, + /** Keys set through typed fields. A later entry for a key wins */ + options?: OptionEntry[], + /** Keys reset through typed members */ + reset?: readonly string[], + /** Entries with no typed field. A typed entry of the same key wins */ + rawOptions?: OptionEntry[], + /** Reset names with no typed member */ + rawReset?: readonly string[] +}; + +/** + * Update resource command definition, the one update for streams, topics and + * users. Layout: `[scope:u8][identifiers][reset_count:u8][reset_key:name]*` + * followed by the options block running to the end of the payload. Reset + * names and set entries are both written in ascending name byte order. + */ +export const UPDATE_RESOURCE = { + code: COMMAND_CODE.UpdateResource, + + serialize: ({ + target, options = [], reset = [], rawOptions = [], rawReset = [] + }: UpdateResource) => { + const set = sortOptions(dedupeOptions([...rawOptions, ...options])); + const resetKeys = [...new Set([...reset, ...rawReset])] + .sort((left, right) => Buffer.compare(Buffer.from(left), Buffer.from(right))); + checkPatch(target.scope, set, resetKeys, rawOptions.length > 0 || rawReset.length > 0); + + const identifiers = target.scope === OptionsScope.Topic + ? [serializeIdentifier(target.streamId), serializeIdentifier(target.topicId)] + : [serializeIdentifier(target.scope === OptionsScope.Stream ? target.streamId : target.userId)]; + + return Buffer.concat([ + uint8ToBuf(target.scope), + ...identifiers, + uint8ToBuf(resetKeys.length), + ...resetKeys.map((key) => { + const name = Buffer.from(key); + return Buffer.concat([uint8ToBuf(name.length), name]); + }), + serializeOptions(set) + ]); + }, + + deserialize: deserializeVoidResponse +}; + +/** + * A typed column value as an option entry. Text no header value can carry + * travels as a bool, a kind the column never accepts, so the refusal is the + * column's own error on either side. + */ +export const columnOption = (key: string, text: string): OptionEntry => { + const length = Buffer.byteLength(text); + return { + key, + value: length >= 1 && length <= MAX_NAME_LENGTH ? HeaderValue.String(text) : HeaderValue.Bool(false) + }; +}; + +/** Refuses a patch with the category and in the order the server admits it. */ +const checkPatch = (scope: OptionsScope, set: OptionEntry[], reset: string[], raw: boolean): void => { + const refuse = (code: number, detail: string): never => { + throw responseError(COMMAND_CODE.UpdateResource, code, detail); + }; + if (set.length === 0 && reset.length === 0) + refuse(INVALID_COMMAND, 'the update sets and resets nothing'); + const conflict = reset.find((key) => set.some((entry) => entry.key === key)); + if (conflict !== undefined) + refuse(INVALID_COMMAND, `key '${conflict}' is both set and reset`); + const nameLength = (key: string) => Buffer.byteLength(key); + // The catalog is far smaller than the count byte, so an overlong list always + // names a key admission would refuse. + const unwritable = reset.find((key) => nameLength(key) < 1 || nameLength(key) > MAX_NAME_LENGTH) + ?? (reset.length > MAX_RESET_KEYS + ? reset.find((key) => !OPTION_CATALOG.get(key)?.scopes.includes(scope)) + : undefined); + if (unwritable !== undefined) + refuse(UNSUPPORTED_OPTION_KEY, `reset of '${unwritable.slice(0, KEY_PREVIEW_LENGTH)}'`); + if (raw) + return; + + const keys = [...set.map((entry) => entry.key), ...reset]; + const outOfScope = keys.find((key) => !OPTION_CATALOG.get(key)?.scopes.includes(scope)); + if (outOfScope !== undefined) + refuse(UNSUPPORTED_OPTION_KEY, `key '${outOfScope}'`); + const first = (candidates: string[], refused: (descriptor: OptionDescriptor) => boolean) => + candidates.find((key) => { + const descriptor = OPTION_CATALOG.get(key); + return descriptor !== undefined && refused(descriptor); + }); + const readOnly = first(keys, ({ flags, internal }) => internal || (flags & OptionFlag.ReadOnly) !== 0); + if (readOnly !== undefined) + refuse(READ_ONLY_OPTION, `key '${readOnly}'`); + const createOnly = first(keys, ({ flags }) => (flags & OptionFlag.Updatable) === 0); + if (createOnly !== undefined) + refuse(OPTION_NOT_UPDATABLE, `key '${createOnly}'`); + const fixed = first(reset, ({ flags }) => (flags & OptionFlag.Resettable) === 0); + if (fixed !== undefined) + refuse(OPTION_NOT_RESETTABLE, `reset of '${fixed}'`); + + for (const { key, value } of set) { + const text = value.kind === HeaderKind.String ? value.value : undefined; + const length = text === undefined ? 0 : Buffer.byteLength(text); + if (key === 'name' && (length < 1 || length > MAX_NAME_LENGTH)) + refuse(scope === OptionsScope.Topic ? INVALID_TOPIC_NAME : INVALID_STREAM_NAME, 'name'); + if (key === 'username' && (length < MIN_USERNAME_LENGTH || length > MAX_USERNAME_LENGTH)) + refuse(INVALID_USERNAME, 'username'); + if (key === 'status' && text !== 'active' && text !== 'inactive') + refuse(INVALID_USER_STATUS, 'status'); + } +};
diff --git a/foreign/node/src/wire/token/create-token.command.test.ts b/foreign/node/src/wire/token/create-token.command.test.ts index f6cfe08..dbd8206 100644 --- a/foreign/node/src/wire/token/create-token.command.test.ts +++ b/foreign/node/src/wire/token/create-token.command.test.ts
@@ -57,5 +57,19 @@ ); }); + it('writes the expiry as microseconds in a u64 right after the name', () => { + assert.deepEqual( + CREATE_TOKEN.serialize({ name: 'token', expiry: 0x0102030405060708n }), + Buffer.from([5, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 8, 7, 6, 5, 4, 3, 2, 1]) + ); + }); + + it('defaults the expiry to ten minutes in microseconds', () => { + assert.equal( + CREATE_TOKEN.serialize({ name: 'token' }).readBigUInt64LE(6), + 600_000_000n + ); + }); + }); });
diff --git a/foreign/node/src/wire/token/create-token.command.ts b/foreign/node/src/wire/token/create-token.command.ts index d87d5c4..20f9044 100644 --- a/foreign/node/src/wire/token/create-token.command.ts +++ b/foreign/node/src/wire/token/create-token.command.ts
@@ -26,7 +26,7 @@ export type CreateToken = { /** Token name (1-255 bytes) */ name: string, - /** Token expiry time in seconds (default: 600) */ + /** Token lifetime in microseconds, 0 never expires (default: 10 minutes) */ expiry?: bigint } @@ -37,7 +37,7 @@ export const CREATE_TOKEN = { code: COMMAND_CODE.CreateAccessToken, - serialize: ({ name, expiry = 600n }: CreateToken): Buffer => { + serialize: ({ name, expiry = 600_000_000n }: CreateToken): Buffer => { const bName = Buffer.from(name); if (bName.length < 1 || bName.length > 255)
diff --git a/foreign/node/src/wire/topic/create-topic.command.ts b/foreign/node/src/wire/topic/create-topic.command.ts index cc57d39..1d3592e 100644 --- a/foreign/node/src/wire/topic/create-topic.command.ts +++ b/foreign/node/src/wire/topic/create-topic.command.ts
@@ -19,10 +19,10 @@ import { serializeIdentifier, type Id } from '../identifier.utils.js'; import { wrapCommand } from '../command.utils.js'; import { COMMAND_CODE } from '../command.code.js'; -import { dedupeOptions, serializeOptions, type OptionEntry } from '../options.utils.js'; +import { dedupeOptions, serializeOptions, sortOptions, type OptionEntry } from '../options.utils.js'; import { HeaderValue } from '../message/header.utils.js'; import { - isValidCompressionAlgorithm, CompressionAlgorithm, Durability, + isValidCompressionAlgorithm, CompressionAlgorithm, Durability, PartitionResizePolicy, compressionAlgorithmName, deserializeTopic, type Topic, @@ -31,17 +31,12 @@ /** - * Parameters for the create topic command. + * Settings a topic takes at create. Only the message expiry and the size cap + * can change later, through `updateTopic`. */ -export type CreateTopic = { - /** Stream identifier (ID or name) */ - streamId: Id, - /** Topic name (1-255 bytes) */ - name: string, - /** Number of partitions to create */ - partitionCount: number, - /** Compression algorithm (None or Gzip) */ - compressionAlgorithm: CompressionAlgorithmT, +export type TopicOptions = { + /** Compression algorithm label (None or Gzip), stored and reported only */ + compressionAlgorithm?: CompressionAlgorithmT, /** * Message expiry time in microseconds. `0n` is not "unlimited": it omits the * key so the server resolves its own default, reported back as a derived @@ -59,6 +54,8 @@ /** Message completion policy. Defaults to replicated, independently of offsets. */ durability?: Durability, consumerOffsetDurability?: Durability, + /** Whether partitions can be added or deleted later. The server default is mutable. */ + partitionResizePolicy?: PartitionResizePolicy, /** Message count that triggers a save (must be non-zero) */ messagesRequiredToSave?: number, /** Accumulated message bytes that trigger a save */ @@ -75,6 +72,18 @@ }; /** + * Parameters for the create topic command. + */ +export type CreateTopic = { + /** Stream identifier (ID or name) */ + streamId: Id, + /** Topic name (1-255 bytes) */ + name: string, + /** Number of partitions to create */ + partitionCount: number +} & TopicOptions; + +/** * Create topic command definition. * Creates a new topic within a stream. * Layout: `[stream_id][partitions_count:u32_le][name_len:u8][name]` @@ -93,6 +102,7 @@ segmentSize, durability = Durability.Replicated, consumerOffsetDurability = Durability.Replicated, + partitionResizePolicy, messagesRequiredToSave, sizeOfMessagesRequiredToSave, preallocateSegments, @@ -145,6 +155,13 @@ } options.push({ key, value: expected }); } + if (partitionResizePolicy !== undefined) { + if (!Object.values<string>(PartitionResizePolicy).includes(partitionResizePolicy)) + throw new Error(`Invalid partition_resize_policy: ${partitionResizePolicy}`); + options.push({ + key: 'partition_resize_policy', value: HeaderValue.String(partitionResizePolicy) + }); + } if (messagesRequiredToSave !== undefined) options.push({ key: 'messages_required_to_save', @@ -168,7 +185,7 @@ streamIdentifier, b, bName, - serializeOptions(dedupeOptions(options)), + serializeOptions(sortOptions(dedupeOptions(options))), ]); },
diff --git a/foreign/node/src/wire/topic/index.ts b/foreign/node/src/wire/topic/index.ts index b92a2cd..036c265 100644 --- a/foreign/node/src/wire/topic/index.ts +++ b/foreign/node/src/wire/topic/index.ts
@@ -21,4 +21,4 @@ export * from './get-topics.command.js'; export * from './update-topic.command.js'; export * from './ensure-topic.virtual.command.js'; -export { CompressionAlgorithm, Durability } from './topic.utils.js'; +export { CompressionAlgorithm, Durability, PartitionResizePolicy } from './topic.utils.js';
diff --git a/foreign/node/src/wire/topic/topic.utils.test.ts b/foreign/node/src/wire/topic/topic.utils.test.ts index 6d620e3..fc781cd 100644 --- a/foreign/node/src/wire/topic/topic.utils.test.ts +++ b/foreign/node/src/wire/topic/topic.utils.test.ts
@@ -22,18 +22,15 @@ import { HeaderValue } from '../message/header.utils.js'; describe('deserializeBaseTopic', () => { - it('uses the 50-byte fixed layout with two options blocks', () => { - const fixed = Buffer.alloc(51); + it('uses the 33-byte fixed layout with settings in two options blocks', () => { + const fixed = Buffer.alloc(34); fixed.writeUInt32LE(7, 0); fixed.writeBigUInt64LE(1_710_000_000_000n, 4); fixed.writeUInt32LE(3, 12); - fixed.writeBigUInt64LE(86_400_000_000n, 16); - fixed.writeUInt8(2, 24); - fixed.writeBigUInt64LE(1_000_000n, 25); - fixed.writeBigUInt64LE(4096n, 33); - fixed.writeBigUInt64LE(12n, 41); - fixed.writeUInt8(1, 49); - fixed.write('t', 50); + fixed.writeBigUInt64LE(4096n, 16); + fixed.writeBigUInt64LE(12n, 24); + fixed.writeUInt8(1, 32); + fixed.write('t', 33); const explicit = serializeOptions([ { key: 'message_expiry', value: HeaderValue.Uint64(86_400_000_000n) } @@ -67,12 +64,12 @@ }); }); - it('skips empty options blocks', () => { - const fixed = Buffer.alloc(51); + it('skips empty options blocks and falls back to the catalog defaults', () => { + const fixed = Buffer.alloc(34); fixed.writeUInt32LE(1, 0); fixed.writeUInt32LE(2, 12); - fixed.writeUInt8(1, 49); - fixed.write('t', 50); + fixed.writeUInt8(1, 32); + fixed.write('t', 33); const data = Buffer.concat([fixed, Buffer.alloc(8)]); const { bytesRead, data: topic } = deserializeBaseTopic(data); @@ -81,5 +78,8 @@ assert.equal(topic.partitionsCount, 2); assert.deepEqual(topic.options, {}); assert.deepEqual(topic.derivedOptions, {}); + assert.equal(topic.compressionAlgorithm, 1); + assert.equal(topic.messageExpiry, 0xffff_ffff_ffff_ffffn); + assert.equal(topic.maxTopicSize, 0xffff_ffff_ffff_ffffn); }); });
diff --git a/foreign/node/src/wire/topic/topic.utils.ts b/foreign/node/src/wire/topic/topic.utils.ts index 14b4ba2..c136cdc 100644 --- a/foreign/node/src/wire/topic/topic.utils.ts +++ b/foreign/node/src/wire/topic/topic.utils.ts
@@ -20,6 +20,9 @@ import type { ValueOf } from '../../type.utils.js'; import { deserializePrefixedOptions, type ParsedOptions } from '../options.utils.js'; +/** Default of `message_expiry` and `max_topic_size`: never expire, no cap. */ +const UNLIMITED = 0xffff_ffff_ffff_ffffn; + /** * Basic topic information without partition details. */ @@ -32,11 +35,11 @@ createdAt: Date, /** Number of partitions */ partitionsCount: number, - /** Compression algorithm used */ + /** Compression algorithm label, read from the options */ compressionAlgorithm: number, - /** Message expiry time in microseconds (0 = unlimited) */ + /** Message expiry in microseconds, read from the options (u64 max = never expire) */ messageExpiry: bigint, - /** Maximum topic size in bytes (0 = unlimited) */ + /** Maximum topic size in bytes, read from the options (u64 max = unlimited) */ maxTopicSize: bigint, /** Total size of the topic in bytes */ sizeBytes: bigint, @@ -128,8 +131,8 @@ /** * Deserializes a topic header from a buffer. - * Layout: 50 fixed bytes, name, then two length-prefixed options blocks - * (client-explicit and server-derived). + * Layout: 33 fixed bytes, name, then two length-prefixed options blocks + * (client-explicit and server-derived) holding every setting. * * @param p - Buffer containing serialized topic data * @param pos - Starting position in the buffer @@ -140,21 +143,28 @@ const id = p.readUInt32LE(pos); const createdAt = toDate(p.readBigUint64LE(pos + 4)); const partitionsCount = p.readUInt32LE(pos + 12); - const messageExpiry = p.readBigUInt64LE(pos + 16); - const compressionAlgorithm = p.readUInt8(pos + 24); - const maxTopicSize = p.readBigUInt64LE(pos + 25); - const sizeBytes = p.readBigUInt64LE(pos + 33); - const messagesCount = p.readBigUInt64LE(pos + 41); + const sizeBytes = p.readBigUInt64LE(pos + 16); + const messagesCount = p.readBigUInt64LE(pos + 24); - const nameLength = p.readUInt8(pos + 49); - const name = p.subarray(pos + 50, pos + 50 + nameLength).toString(); - pos += 50 + nameLength; + const nameLength = p.readUInt8(pos + 32); + const name = p.subarray(pos + 33, pos + 33 + nameLength).toString(); + pos += 33 + nameLength; const explicit = deserializePrefixedOptions(p, pos); pos += explicit.bytesRead; const derived = deserializePrefixedOptions(p, pos); pos += derived.bytesRead; + const setting = (key: string) => explicit.options[key] ?? derived.options[key]; + const limit = (key: string) => { + const value = setting(key); + return typeof value === 'bigint' ? value : UNLIMITED; + }; + const compressionAlgorithm = setting('compression_algorithm') === 'gzip' + ? CompressionAlgorithm.Gzip : CompressionAlgorithm.None; + const messageExpiry = limit('message_expiry'); + const maxTopicSize = limit('max_topic_size'); + return { bytesRead: pos - start, data: { @@ -251,3 +261,11 @@ } as const; export type Durability = typeof Durability[keyof typeof Durability]; + +/** Whether a topic may add and delete partitions after create. */ +export const PartitionResizePolicy = { + Mutable: 'mutable', + Fixed: 'fixed' +} as const; + +export type PartitionResizePolicy = typeof PartitionResizePolicy[keyof typeof PartitionResizePolicy];
diff --git a/foreign/node/src/wire/topic/update-topic.command.ts b/foreign/node/src/wire/topic/update-topic.command.ts index e8255ef..b389986 100644 --- a/foreign/node/src/wire/topic/update-topic.command.ts +++ b/foreign/node/src/wire/topic/update-topic.command.ts
@@ -15,22 +15,17 @@ // specific language governing permissions and limitations // under the License. -import { serializeIdentifier, type Id } from '../identifier.utils.js'; -import { dedupeOptions, serializeOptions, type OptionEntry } from '../options.utils.js'; +import type { Id } from '../identifier.utils.js'; +import type { OptionEntry, TopicOptionKey } from '../options.utils.js'; import { HeaderValue } from '../message/header.utils.js'; -import { deserializeVoidResponse } from '../../client/client.utils.js'; import { wrapCommand } from '../command.utils.js'; -import { COMMAND_CODE } from '../command.code.js'; -import { - type CompressionAlgorithm as CompressionAlgorithmT, - CompressionAlgorithm, - compressionAlgorithmName, - isValidCompressionAlgorithm -} from './topic.utils.js'; +import { OptionsScope } from '../system/describe-options.command.js'; +import { UPDATE_RESOURCE, columnOption } from '../system/update-resource.command.js'; /** - * Parameters for the update topic command. + * Parameters for the update topic command. Keys left out keep their values. + * Settings fixed at create, compression among them, have no field here. */ export type UpdateTopic = { /** Stream identifier (ID or name) */ @@ -38,62 +33,55 @@ /** Topic identifier (ID or name) */ topicId: Id, /** New topic name (1-255 bytes) */ - name: string, - /** Compression algorithm (None or Gzip) */ - compressionAlgorithm?: CompressionAlgorithmT, + name?: string, /** * Message expiry time in microseconds. `0n` is not "unlimited": it omits the - * key, so the topic keeps whatever expiry it currently has. Resetting a key - * back to the server default is not expressible on an update. + * key, so the topic keeps whatever expiry it currently has. Reset the key to + * return to the server default. */ messageExpiry?: bigint, /** * Maximum topic size in bytes. `0n` is not "unlimited": it omits the key, so - * the topic keeps whatever cap it currently has. Resetting a key back to the - * server default is not expressible on an update. + * the topic keeps whatever cap it currently has. Reset the key to return to + * the server default. */ maxTopicSize?: bigint, + /** Keys to reset to their defaults: `message_expiry` or `max_topic_size` */ + reset?: TopicOptionKey[], /** - * Option keys with no field of their own. The server refuses any key an update - * may not change, by name; a key left out keeps its current value. + * Reset names with no typed member, for a resettable key the server catalog + * gained after this build shipped. The server decides on them. + */ + rawReset?: readonly string[], + /** + * Option keys with no field of their own, for an updatable key the server + * catalog gained after this build shipped. A field above wins on collision. + * The server decides on them, so a create-only or unknown key comes back as + * a server error. */ options?: OptionEntry[] }; /** * Update topic command definition. - * Updates a topic's configuration. + * Changes only the keys it names, through the generic update command. */ export const UPDATE_TOPIC = { - code: COMMAND_CODE.UpdateTopic, + code: UPDATE_RESOURCE.code, serialize: ({ streamId, topicId, name, - compressionAlgorithm = CompressionAlgorithm.None, messageExpiry = 0n, maxTopicSize = 0n, - options: extraOptions = [] + reset, + rawReset, + options: rawOptions }: UpdateTopic) => { - const streamIdentifier = serializeIdentifier(streamId); - const topicIdentifier = serializeIdentifier(topicId); - const bName = Buffer.from(name) - - if (bName.length < 1 || bName.length > 255) - throw new Error('Topic name should be between 1 and 255 bytes'); - if(!isValidCompressionAlgorithm(compressionAlgorithm)) - throw new Error(`updateTopic: invalid compressionAlgorithm (${compressionAlgorithm})`); - - // Settings ride the options block. A default value means the caller did not - // set the key, so it is omitted and the server leaves the current value be. - // Caller keys first so a typed field below overwrites one of them. - const options: OptionEntry[] = [...extraOptions]; - if (compressionAlgorithm !== CompressionAlgorithm.None) - options.push({ - key: 'compression_algorithm', - value: HeaderValue.String(compressionAlgorithmName(compressionAlgorithm)) - }); + const options: OptionEntry[] = []; + if (name !== undefined) + options.push(columnOption('name', name)); if (messageExpiry !== 0n) options.push({ key: 'message_expiry', @@ -105,19 +93,12 @@ value: HeaderValue.Uint64(maxTopicSize) }); - const b = Buffer.allocUnsafe(1); - b.writeUInt8(bName.length, 0); - - return Buffer.concat([ - streamIdentifier, - topicIdentifier, - b, - bName, - serializeOptions(dedupeOptions(options)), - ]); + return UPDATE_RESOURCE.serialize({ + target: { scope: OptionsScope.Topic, streamId, topicId }, options, reset, rawOptions, rawReset + }); }, - deserialize: deserializeVoidResponse + deserialize: UPDATE_RESOURCE.deserialize };
diff --git a/foreign/node/src/wire/user/create-user.command.ts b/foreign/node/src/wire/user/create-user.command.ts index 316cdaf..55b4249 100644 --- a/foreign/node/src/wire/user/create-user.command.ts +++ b/foreign/node/src/wire/user/create-user.command.ts
@@ -21,9 +21,21 @@ import { deserializeUser, type User, type UserStatus } from './user.utils.js'; import { uint8ToBuf, uint32ToBuf, boolToBuf } from '../number.utils.js'; import { serializePermissions, type UserPermissions } from './permissions.utils.js'; +import { + dedupeOptions, serializeOptions, sortOptions, type OptionEntry +} from '../options.utils.js'; /** + * Options a user takes at create. The server catalog has no user key yet, so + * only keys newer than this build can ride here. + */ +export type UserOptions = { + /** Option keys with no field of their own */ + options?: OptionEntry[] +}; + +/** * Parameters for the create user command. */ export type CreateUser = { @@ -35,16 +47,17 @@ status: UserStatus /** Optional user permissions */ permissions?: UserPermissions -}; +} & UserOptions; /** * Create user command definition. * Creates a new user with the specified credentials and permissions. + * The options block runs to the end of the payload. */ export const CREATE_USER = { code: COMMAND_CODE.CreateUser, - serialize: ({ username, password, status, permissions }: CreateUser) => { + serialize: ({ username, password, status, permissions, options = [] }: CreateUser) => { const bUsername = Buffer.from(username); const bPassword = Buffer.from(password); @@ -62,15 +75,17 @@ uint8ToBuf(status), ]); + const bOptions = serializeOptions(sortOptions(dedupeOptions(options))); if (!permissions) - return Buffer.concat([bUser, boolToBuf(false)]); + return Buffer.concat([bUser, boolToBuf(false), bOptions]); const bPermissions = serializePermissions(permissions); return Buffer.concat([ bUser, boolToBuf(true), uint32ToBuf(bPermissions.length), - bPermissions + bPermissions, + bOptions ]); },
diff --git a/foreign/node/src/wire/user/index.ts b/foreign/node/src/wire/user/index.ts index 0de300a..7b94796 100644 --- a/foreign/node/src/wire/user/index.ts +++ b/foreign/node/src/wire/user/index.ts
@@ -22,3 +22,4 @@ export * from './get-users.command.js'; export * from './update-permissions.command.js'; export * from './update-user.command.js'; +export { UserStatus } from './user.utils.js';
diff --git a/foreign/node/src/wire/user/update-user.command.ts b/foreign/node/src/wire/user/update-user.command.ts index 4317237..9ea2a11 100644 --- a/foreign/node/src/wire/user/update-user.command.ts +++ b/foreign/node/src/wire/user/update-user.command.ts
@@ -16,64 +16,64 @@ // under the License. import { wrapCommand } from '../command.utils.js'; -import { COMMAND_CODE } from '../command.code.js'; -import { deserializeVoidResponse } from '../../client/client.utils.js'; -import type { UserStatus } from './user.utils.js'; -import { serializeIdentifier, type Id } from '../identifier.utils.js'; -import { uint8ToBuf } from '../number.utils.js'; +import type { Id } from '../identifier.utils.js'; +import { HeaderValue } from '../message/header.utils.js'; +import type { OptionEntry, UserOptionKey } from '../options.utils.js'; +import { OptionsScope } from '../system/describe-options.command.js'; +import { UPDATE_RESOURCE, columnOption } from '../system/update-resource.command.js'; +import { UserStatus } from './user.utils.js'; /** - * Parameters for the update user command. + * Parameters for the update user command. Keys left out keep their values. */ export type UpdateUser = { /** User identifier (ID or username) */ userId: Id, - /** Optional new username (1-255 bytes) */ + /** Optional new username (3-50 bytes) */ username?: string, - /** Optional new status */ - status?: UserStatus + /** Optional new status. Deactivating a user ends its sessions. */ + status?: UserStatus, + /** + * Keys to reset to their defaults. No user key is resettable yet, so any key + * is refused before the request is sent. + */ + reset?: UserOptionKey[], + /** + * Reset names with no typed member, for a resettable key the server catalog + * gained after this build shipped. The server decides on them. + */ + rawReset?: readonly string[], + /** + * Option keys with no field of their own, for an updatable key the server + * catalog gained after this build shipped. A field above wins on collision. + * The server decides on them, so a create-only or unknown key comes back as + * a server error. + */ + options?: OptionEntry[] }; /** * Update user command definition. - * Updates a user's username and/or status. + * Changes only the keys it names, through the generic update command. */ export const UPDATE_USER = { - code: COMMAND_CODE.UpdateUser, + code: UPDATE_RESOURCE.code, - serialize: ({ userId, username, status }: UpdateUser) => { - - const bId = serializeIdentifier(userId); - let bUsername, bStatus; - - if (username) { - const bn = Buffer.from(username); - - if (bn.length < 1 || bn.length > 255) - throw new Error('User username should be between 1 and 255 bytes'); - - bUsername = Buffer.concat([ - uint8ToBuf(1), - uint8ToBuf(bn.length), - bn - ]); - } else - bUsername = uint8ToBuf(0); - - if (status) - bStatus = Buffer.concat([uint8ToBuf(1), uint8ToBuf(status)]); - else - bStatus = uint8ToBuf(0); - - return Buffer.concat([ - bId, - bUsername, - bStatus, - ]); + serialize: ({ userId, username, status, reset, rawReset, options: rawOptions }: UpdateUser) => { + const options: OptionEntry[] = []; + if (username !== undefined) + options.push(columnOption('username', username)); + if (status !== undefined) + options.push(status === UserStatus.Active || status === UserStatus.Inactive + ? { key: 'status', value: HeaderValue.String(status === UserStatus.Active ? 'active' : 'inactive') } + : { key: 'status', value: HeaderValue.Bool(false) }); + return UPDATE_RESOURCE.serialize({ + target: { scope: OptionsScope.User, userId }, options, reset, rawOptions, rawReset + }); }, - deserialize: deserializeVoidResponse + deserialize: UPDATE_RESOURCE.deserialize };
diff --git a/foreign/node/src/wire/user/user.utils.ts b/foreign/node/src/wire/user/user.utils.ts index a7eaa71..4a6fd2c 100644 --- a/foreign/node/src/wire/user/user.utils.ts +++ b/foreign/node/src/wire/user/user.utils.ts
@@ -32,7 +32,9 @@ /** Username */ userName: string, /** Options the client explicitly sent at create */ - options: ParsedOptions + options: ParsedOptions, + /** Options the server filled */ + derivedOptions: ParsedOptions }; /** @@ -90,17 +92,18 @@ const status = statusString(p.readUInt8(pos + 12)); const userNameLength = p.readUInt8(pos + 13); const userName = p.subarray(pos + 14, pos + 14 + userNameLength).toString(); - const { bytesRead: optionsBytes, options } = - deserializePrefixedOptions(p, pos + 14 + userNameLength); + const explicit = deserializePrefixedOptions(p, pos + 14 + userNameLength); + const derived = deserializePrefixedOptions(p, pos + 14 + userNameLength + explicit.bytesRead); return { - bytesRead: 14 + userNameLength + optionsBytes, + bytesRead: 14 + userNameLength + explicit.bytesRead + derived.bytesRead, data: { id, createdAt, status, userName, - options, + options: explicit.options, + derivedOptions: derived.options, } } };
diff --git a/foreign/node/src/wire/vsr/index.ts b/foreign/node/src/wire/vsr/index.ts index a080e7f..4a59022 100644 --- a/foreign/node/src/wire/vsr/index.ts +++ b/foreign/node/src/wire/vsr/index.ts
@@ -16,7 +16,7 @@ // under the License. import { createRequire } from 'node:module'; -import { xxh3 } from '@node-rs/xxhash'; +import { randomFillSync } from 'node:crypto'; import type { CommandResponse } from '../../client/client.type.js'; import { COMMAND_CODE } from '../command.code.js'; import { responseError } from '../error.utils.js'; @@ -123,8 +123,14 @@ nonReplicatedCode: operation === Operation.NonReplicated ? command : undefined, }); - if (operation !== Operation.NonReplicated && operation < Operation.SendMessages) - header.writeBigUInt64LE(xxh3.xxh64(payload), REQUEST_OFFSET.requestChecksum); + if (operation === Operation.Register) { + this.state.registerRequestIdentifier.copy(header, REQUEST_OFFSET.requestChecksum); + } else if (operation !== Operation.NonReplicated && operation < Operation.SendMessages) { + do { + randomFillSync(header, REQUEST_OFFSET.requestChecksum, 16); + } while (header.subarray(REQUEST_OFFSET.requestChecksum, REQUEST_OFFSET.requestChecksum + 16) + .every(byte => byte === 0)); + } return payload.length === 0 ? header : Buffer.concat([header, payload]); } }
diff --git a/foreign/node/src/wire/vsr/operation.test.ts b/foreign/node/src/wire/vsr/operation.test.ts index 37bc09e..e6f4149 100644 --- a/foreign/node/src/wire/vsr/operation.test.ts +++ b/foreign/node/src/wire/vsr/operation.test.ts
@@ -24,15 +24,14 @@ isMetadata, isResultFramed, Operation, - operationForCode + operationForCode, + requestOperation } from './operation.js'; const replicated = new Map<number, number>([ [COMMAND_CODE.CreateStream, Operation.CreateStream], - [COMMAND_CODE.UpdateStream, Operation.UpdateStream], [COMMAND_CODE.DeleteStream, Operation.DeleteStream], [COMMAND_CODE.CreateTopic, Operation.CreateTopic], - [COMMAND_CODE.UpdateTopic, Operation.UpdateTopic], [COMMAND_CODE.DeleteTopic, Operation.DeleteTopic], [COMMAND_CODE.CreatePartitions, Operation.CreatePartitions], [COMMAND_CODE.DeletePartitions, Operation.DeletePartitions], @@ -40,7 +39,6 @@ [COMMAND_CODE.CreateGroup, Operation.CreateConsumerGroup], [COMMAND_CODE.DeleteGroup, Operation.DeleteConsumerGroup], [COMMAND_CODE.CreateUser, Operation.CreateUser], - [COMMAND_CODE.UpdateUser, Operation.UpdateUser], [COMMAND_CODE.DeleteUser, Operation.DeleteUser], [COMMAND_CODE.ChangePassword, Operation.ChangePassword], [COMMAND_CODE.UpdatePermissions, Operation.UpdatePermissions], @@ -48,6 +46,7 @@ [COMMAND_CODE.DeleteAccessToken, Operation.DeletePersonalAccessToken], [COMMAND_CODE.JoinGroup, Operation.JoinConsumerGroup], [COMMAND_CODE.LeaveGroup, Operation.LeaveConsumerGroup], + [COMMAND_CODE.UpdateResource, Operation.UpdateResource], [COMMAND_CODE.SendMessages, Operation.SendMessages], [COMMAND_CODE.StoreOffset, Operation.StoreConsumerOffset], [COMMAND_CODE.DeleteConsumerOffset, Operation.DeleteConsumerOffset] @@ -86,12 +85,34 @@ assert.equal(isMetadata(Operation.CreateStream), true); assert.equal(isMetadata(Operation.LeaveConsumerGroup), true); assert.equal(isMetadata(Operation.DeleteSegments), false); - assert.equal(isMetadata(150), false); + assert.equal(isMetadata(Operation.UpdateResource), true); + assert.equal(isResultFramed(Operation.UpdateResource), true); + for (const resolved of [Operation.UpdateStreamResolved, Operation.UpdateTopicResolved, + Operation.UpdateUserResolved, Operation.RewrapStreamKeys]) { + assert.equal(isInternal(resolved), true); + assert.equal(isMetadata(resolved), true); + assert.equal(isResultFramed(resolved), true); + } + assert.equal(isMetadata(151), false); assert.equal(isResultFramed(Operation.StoreConsumerOffset), true); assert.equal(isResultFramed(Operation.DeleteConsumerOffset), true); assert.equal(isResultFramed(Operation.SendMessages), false); }); + it('maps a resolved committed operation back to the request operation', () => { + for (const [committed, request] of [ + [Operation.CreateTopicWithAssignments, Operation.CreateTopic], + [Operation.CreatePartitionsWithAssignments, Operation.CreatePartitions], + [Operation.TruncatePartition, Operation.DeleteSegments], + [Operation.UpdateStreamResolved, Operation.UpdateResource], + [Operation.UpdateTopicResolved, Operation.UpdateResource], + [Operation.UpdateUserResolved, Operation.UpdateResource], + [Operation.RewrapStreamKeys, Operation.RewrapStreamKeys], + [Operation.CreateStream, Operation.CreateStream] + ]) + assert.equal(requestOperation(committed), request); + }); + it('recognizes only declared operation discriminants', () => { assert.equal(isKnownOperation(Operation.Register), true); assert.equal(isKnownOperation(Operation.SendMessages), true); @@ -99,7 +120,7 @@ assert.equal(isKnownOperation(Operation.FinalizeSession), true); assert.equal(isKnownOperation(Operation.RetireConsumerGroupOwners), true); assert.equal(isInternal(Operation.RetireConsumerGroupOwners), true); - for (const undeclared of [69, 127, 131, 135, 150, 159, 163, 164, 165, 170, 255]) + for (const undeclared of [69, 76, 127, 129, 131, 133, 135, 142, 151, 159, 163, 164, 165, 170, 255]) assert.equal(isKnownOperation(undeclared), false); }); });
diff --git a/foreign/node/src/wire/vsr/operation.ts b/foreign/node/src/wire/vsr/operation.ts index 202109f..6005777 100644 --- a/foreign/node/src/wire/vsr/operation.ts +++ b/foreign/node/src/wire/vsr/operation.ts
@@ -36,11 +36,15 @@ TruncatePartition: 68, FinalizeSession: 70, CompleteLifecycle: 71, + UpdateStreamResolved: 72, + UpdateTopicResolved: 73, + UpdateUserResolved: 74, + RewrapStreamKeys: 75, CreateStream: 128, - UpdateStream: 129, + // 129 was UpdateStream. Never reuse it. DeleteStream: 130, CreateTopic: 132, - UpdateTopic: 133, + // 133 was UpdateTopic. Never reuse it. DeleteTopic: 134, CreatePartitions: 136, DeletePartitions: 137, @@ -48,7 +52,7 @@ CreateConsumerGroup: 139, DeleteConsumerGroup: 140, CreateUser: 141, - UpdateUser: 142, + // 142 was UpdateUser. Never reuse it. DeleteUser: 143, ChangePassword: 144, UpdatePermissions: 145, @@ -56,6 +60,7 @@ DeletePersonalAccessToken: 147, JoinConsumerGroup: 148, LeaveConsumerGroup: 149, + UpdateResource: 150, SendMessages: 160, StoreConsumerOffset: 161, DeleteConsumerOffset: 162, @@ -76,7 +81,6 @@ const REPLICATED_OPERATION: ReadonlyMap<number, number> = new Map([ [COMMAND_CODE.CreateUser, Operation.CreateUser], [COMMAND_CODE.DeleteUser, Operation.DeleteUser], - [COMMAND_CODE.UpdateUser, Operation.UpdateUser], [COMMAND_CODE.UpdatePermissions, Operation.UpdatePermissions], [COMMAND_CODE.ChangePassword, Operation.ChangePassword], [COMMAND_CODE.CreateAccessToken, Operation.CreatePersonalAccessToken], @@ -86,17 +90,16 @@ [COMMAND_CODE.DeleteConsumerOffset, Operation.DeleteConsumerOffset], [COMMAND_CODE.CreateStream, Operation.CreateStream], [COMMAND_CODE.DeleteStream, Operation.DeleteStream], - [COMMAND_CODE.UpdateStream, Operation.UpdateStream], [COMMAND_CODE.CreateTopic, Operation.CreateTopic], [COMMAND_CODE.DeleteTopic, Operation.DeleteTopic], - [COMMAND_CODE.UpdateTopic, Operation.UpdateTopic], [COMMAND_CODE.CreatePartitions, Operation.CreatePartitions], [COMMAND_CODE.DeletePartitions, Operation.DeletePartitions], [COMMAND_CODE.DeleteSegments, Operation.DeleteSegments], [COMMAND_CODE.CreateGroup, Operation.CreateConsumerGroup], [COMMAND_CODE.DeleteGroup, Operation.DeleteConsumerGroup], [COMMAND_CODE.JoinGroup, Operation.JoinConsumerGroup], - [COMMAND_CODE.LeaveGroup, Operation.LeaveConsumerGroup] + [COMMAND_CODE.LeaveGroup, Operation.LeaveConsumerGroup], + [COMMAND_CODE.UpdateResource, Operation.UpdateResource] ]); const KNOWN_OPERATIONS: ReadonlySet<number> = @@ -128,7 +131,29 @@ if (isInternal(operation) && operation < Operation.SendMessages) return true; if (operation === Operation.DeleteSegments) return false; return operation >= METADATA_START && - operation <= Operation.LeaveConsumerGroup; + operation <= Operation.UpdateResource; +}; + +/** + * The client operation a committed reply answers. Preparation rewrites some + * requests into an internal operation, so retry identity maps it back. Port of + * `Operation::request_operation`. + */ +export const requestOperation = (operation: number): number => { + switch (operation) { + case Operation.CreateTopicWithAssignments: + return Operation.CreateTopic; + case Operation.CreatePartitionsWithAssignments: + return Operation.CreatePartitions; + case Operation.UpdateStreamResolved: + case Operation.UpdateTopicResolved: + case Operation.UpdateUserResolved: + return Operation.UpdateResource; + case Operation.TruncatePartition: + return Operation.DeleteSegments; + default: + return operation; + } }; /** Whether a reply body leads with a committed result section. */
diff --git a/foreign/node/src/wire/vsr/register.ts b/foreign/node/src/wire/vsr/register.ts index afcab86..af22915 100644 --- a/foreign/node/src/wire/vsr/register.ts +++ b/foreign/node/src/wire/vsr/register.ts
@@ -30,11 +30,11 @@ /** * Packed protocol semver of the wire contract this port implements, - * `pack(0, 11, 1)` per `core/binary_protocol/src/version.rs`. Update with the + * `pack(0, 12, 0)` per `core/binary_protocol/src/version.rs`. Update with the * Rust constant during stable release preparation. Edge builds sharing this * number are not guaranteed to have compatible layouts. */ -export const IGGY_PROTOCOL_VERSION = (0 << 20) | (11 << 10) | 1; +export const IGGY_PROTOCOL_VERSION = (0 << 20) | (12 << 10) | 0; const SDK_NAME = 'node-sdk';
diff --git a/foreign/node/src/wire/vsr/reply.test.ts b/foreign/node/src/wire/vsr/reply.test.ts index 33c8099..532c849 100644 --- a/foreign/node/src/wire/vsr/reply.test.ts +++ b/foreign/node/src/wire/vsr/reply.test.ts
@@ -75,7 +75,7 @@ }); it('rejects an undeclared reply operation discriminant', () => { - for (const operation of [69, 150, 163, 255]) + for (const operation of [69, 129, 133, 142, 163, 255]) assert.throws( () => decodeResponse(reply(operation)), (error: unknown) => @@ -120,6 +120,20 @@ ); }); + it('strips the result section of every resolved update reply', () => { + for (const operation of [Operation.UpdateStreamResolved, Operation.UpdateTopicResolved, + Operation.UpdateUserResolved]) { + assert.deepEqual(decodeResponse(reply(operation, Buffer.alloc(4))), Buffer.alloc(0)); + const refused = Buffer.alloc(12); + refused.writeUInt32LE(1, 0); + refused.writeUInt32LE(4103, 8); + assert.throws( + () => decodeResponse(reply(operation, refused), COMMAND_CODE.UpdateResource), + (error: unknown) => error instanceof ResponseError && error.errorCode === 4103 + ); + } + }); + it('strips a non-empty successful result section', () => { const body = Buffer.alloc(12 + 6); body.writeUInt32LE(1, 0);
diff --git a/foreign/node/src/wire/vsr/reply.ts b/foreign/node/src/wire/vsr/reply.ts index 0516dac..27c1d8e 100644 --- a/foreign/node/src/wire/vsr/reply.ts +++ b/foreign/node/src/wire/vsr/reply.ts
@@ -29,7 +29,7 @@ peekCommand, readEviction, readReplyOperation, readSize, readStatus, hasCommittedResult, REQUEST_OFFSET, REPLY_OFFSET } from './header.js'; -import { Operation, isKnownOperation, isResultFramed } from './operation.js'; +import { Operation, isKnownOperation, isResultFramed, requestOperation } from './operation.js'; /** `IggyError` codes this funnel raises client-side. */ const INVALID_COMMAND = 3; @@ -65,11 +65,8 @@ export const validateReplyForRequest = (reply: Buffer, request: Buffer, command: number): void => { const operation = request[REQUEST_OFFSET.operation]; const actual = readReplyOperation(reply); - const matches = actual === operation || - (operation === Operation.CreateTopic && actual === Operation.CreateTopicWithAssignments) || - (operation === Operation.CreatePartitions && actual === Operation.CreatePartitionsWithAssignments) || - (operation === Operation.DeleteSegments && actual === Operation.TruncatePartition); - if (reply.readBigUInt64LE(REPLY_OFFSET.request) !== request.readBigUInt64LE(REQUEST_OFFSET.request) || !matches) + if (reply.readBigUInt64LE(REPLY_OFFSET.request) !== request.readBigUInt64LE(REQUEST_OFFSET.request) || + requestOperation(actual) !== operation) throw responseError(command, INVALID_COMMAND, 'reply does not match request'); if (!hasCommittedResult(reply)) return;
diff --git a/foreign/node/src/wire/vsr/session.ts b/foreign/node/src/wire/vsr/session.ts index 2d1a8da..6013bb6 100644 --- a/foreign/node/src/wire/vsr/session.ts +++ b/foreign/node/src/wire/vsr/session.ts
@@ -36,6 +36,7 @@ private registerConsumed: boolean; private sessionRequestStart = 1n; readonly bindSecret: Buffer; + readonly registerRequestIdentifier: Buffer; constructor(clientId?: bigint) { this._clientId = clientId ?? generateClientId(); @@ -43,6 +44,7 @@ this.requestCounter = 1n; this.registerConsumed = false; this.bindSecret = randomBytes(BIND_SECRET_BYTES); + this.registerRequestIdentifier = nonzeroIdentifier(); } reset(): void { @@ -51,6 +53,7 @@ this.registerConsumed = false; this.sessionRequestStart = this.requestCounter; randomBytes(BIND_SECRET_BYTES).copy(this.bindSecret); + nonzeroIdentifier().copy(this.registerRequestIdentifier); } get clientId(): bigint { @@ -105,6 +108,14 @@ } } +function nonzeroIdentifier(): Buffer { + let identifier: Buffer; + do { + identifier = randomBytes(16); + } while (identifier.every(byte => byte === 0)); + return identifier; +} + /** Keep the top bit clear so generated IDs cannot enter the server-only range. */ const generateClientId = (): bigint => { for (;;) {
diff --git a/foreign/node/src/wire/vsr/vsr.test.ts b/foreign/node/src/wire/vsr/vsr.test.ts index 99bc7c5..7dfd911 100644 --- a/foreign/node/src/wire/vsr/vsr.test.ts +++ b/foreign/node/src/wire/vsr/vsr.test.ts
@@ -29,21 +29,22 @@ const BIND_SECRET = Buffer.alloc(32, 0x5a); describe('VSR custom request framing', () => { - it('stamps metadata and session payloads with the Rust XXH3-64 checksum', () => { - const vectors = [ - [Buffer.alloc(0), 0x2d06800538d394c2n], - [Buffer.from([1, 2, 3]), 0xebce9b7632ae733bn], - [Buffer.from('abc'), 0x78af5f94892f3950n] - ] as const; + it('stamps distinct random identifiers for metadata and session requests', () => { + const payloads = [Buffer.alloc(0), Buffer.from([1, 2, 3]), Buffer.from('abc')]; for (const command of [COMMAND_CODE.LoginRegister, COMMAND_CODE.LoginRegisterWithAccessToken, COMMAND_CODE.LogoutUser, COMMAND_CODE.CreateStream, COMMAND_CODE.DeleteSegments]) { const session = new VsrSession(7n); session.bind(42n); - for (const [payload, checksum] of vectors) { + for (const payload of payloads) { const frame = session.encode(command, payload); - assert.equal(frame.readBigUInt64LE(REQUEST_OFFSET.requestChecksum), checksum, - `command ${command}, payload ${payload.toString('hex')}`); - assert.equal(frame.readBigUInt64LE(REQUEST_OFFSET.requestChecksum + 8), 0n); + const another = session.encode(command, payload); + const identifier = frame.subarray(REQUEST_OFFSET.requestChecksum, REQUEST_OFFSET.requestChecksum + 16); + assert.notDeepEqual(identifier, Buffer.alloc(16)); + const repeated = another.subarray(REQUEST_OFFSET.requestChecksum, REQUEST_OFFSET.requestChecksum + 16); + if (command === COMMAND_CODE.LoginRegister || command === COMMAND_CODE.LoginRegisterWithAccessToken) + assert.deepEqual(identifier, repeated); + else + assert.notDeepEqual(identifier, repeated); assert.deepEqual(frame.subarray(HEADER_SIZE), payload); } }
diff --git a/foreign/php/Cargo.toml b/foreign/php/Cargo.toml index 2eebe53..e0d0d5a 100644 --- a/foreign/php/Cargo.toml +++ b/foreign/php/Cargo.toml
@@ -17,7 +17,7 @@ [package] name = "iggy-php" -version = "0.1.3" +version = "0.1.4" edition = "2024" authors = ["Iggy Committers <dev@iggy.apache.org>"] license = "Apache-2.0" @@ -33,6 +33,7 @@ futures = "0.3.34" ext-php-rs = "=0.16.3" iggy = { path = "../../core/sdk" } +libc = "0.2.191" tokio = "1.53.1" [profile.release]
diff --git a/foreign/php/README.md b/foreign/php/README.md index 1c64e7b..74c8910 100644 --- a/foreign/php/README.md +++ b/foreign/php/README.md
@@ -37,10 +37,14 @@ cargo php stubs target/debug/libiggy_php.so -o /tmp/iggy-php.stubs.php ``` -Stub generation requires a debug build. On macOS, use `libiggy_php.dylib`. -Preserve the existing Apache license header when updating `iggy-php.stubs.php`. -The CI lint job regenerates this file and fails if the checked-in stubs drift -from the Rust signatures. +Stub generation requires a debug build and cargo-php 0.2.1, the version CI installs. Older +cargo-php releases cannot load the extension. On macOS, use `libiggy_php.dylib`. +The checked-in `iggy-php.stubs.php` is the generated file with the Apache license header +in place of its first line and the `= null` default removed from every `public readonly +?string` property, which PHP refuses on readonly properties. The CI lint job +(`.github/actions/php/pre-merge/action.yml`) applies the same post-processing and fails +if the checked-in stubs drift from the Rust signatures, so copy its commands when +regenerating locally. ## Install @@ -204,6 +208,36 @@ iggy+tcp://iggy:iggy@127.0.0.1:8090?tls=true&tls_domain=localhost&tls_ca_file=/path/to/ca.pem ``` +## Resource Options + +Streams and topics carry options. A read returns two arrays of option name to value: `options` holds what the creating or updating client set, `derived_options` what the server filled in. Integers above `PHP_INT_MAX`, such as the never-expire and unlimited value 18446744073709551615, are decimal strings, and the 64-bit topic arguments take an int or a decimal string. + +```php +$client->createStream('orders', base64_encode(random_bytes(32))); +$stream = $client->getStream('orders'); +echo $stream->encryption_key_source, ' ', $stream->encryption_key_fingerprint, PHP_EOL; + +$client->createTopic('orders', 'events', 1, 'gzip', 3_600_000_000, null, null, + partition_resize_policy: 'fixed'); +$client->updateTopic('orders', 'events', message_expiry_micros: '18446744073709551615', + reset: [Iggy\TopicOptionKey::MaxTopicSize]); +print_r($client->getTopic('orders', 'events')->options); + +foreach ($client->describeOptions(Iggy\OptionsScope::Topic) as $spec) { + echo $spec->key, $spec->updatable ? ' updatable' : '', $spec->resettable ? ' resettable' : '', PHP_EOL; +} +``` + +- The stream encryption key is the stream's own data key: 32 random bytes, passed as canonical padded standard base64. Raw bytes or any other text are refused as `Invalid encryption key` (code 70) before anything is sent. The key travels in the create request, so protecting it in transit with TLS is the caller's job. The server stores it wrapped and never returns it. + + The key, the `loginUser()` password and the `fromConnectionString()` string are marked `#[\SensitiveParameter]`, so exception traces hold a `SensitiveParameterValue` in their place. The `raw` arrays of `createStream()`, `updateStream()`, `createTopic()` and `updateTopic()` are marked as well, because an unknown key may hold a secret. + + Reads report only `encryption_key_source` (`stream`, `server`, or `none` for a plaintext stream), `encryption_key_fingerprint` and `encryption_kek_fingerprint`, comma-separated during a server key rotation. `Iggy\Client::encryptionKeyFingerprint($key)` computes the fingerprint of a key you hold. Without a key, a server with encryption enabled generates one. +- `updateStream()` renames a stream and `updateTopic()` renames a topic or changes `message_expiry_micros` and `max_topic_size`. Null arguments keep stored values. `reset` takes `Iggy\StreamOptionKey` or `Iggy\TopicOptionKey` cases and returns keys to their server defaults. Only the topic's message expiry and size cap are resettable, every other case is refused with the server's error. +- `raw` on create and update sends string-valued keys this build does not know, and `raw_reset` resets keys by name. The server decides whether each one is accepted. A name listed twice across `reset` and `raw_reset` is refused as `Invalid command`. +- `describeOptions(Iggy\OptionsScope)` returns the server catalog as `Iggy\OptionSpec` objects with `key`, `kind`, `updatable`, `resettable`, `secret`, `read_only`, `critical`, `default_value` and `description`. +- SDK exceptions carry the server error code in `getCode()`, for example 4103 (option cannot be updated), 4104 (option cannot be reset), 4105 (read-only option), 4108 (root user cannot be changed by this caller) and 4041 (unsupported option key). + ## API Notes - `new Iggy\Client($address, new Iggy\PendingOperationsConfig(256, 64 * 1024 * 1024))`
diff --git a/foreign/php/docker-compose.test.yml b/foreign/php/docker-compose.test.yml index c80c759..4831bbc 100644 --- a/foreign/php/docker-compose.test.yml +++ b/foreign/php/docker-compose.test.yml
@@ -36,6 +36,8 @@ - IGGY_WEBSOCKET_ADDRESS=0.0.0.0:8092 - IGGY_ROOT_USERNAME=iggy - IGGY_ROOT_PASSWORD=iggy + - IGGY_ENCRYPTION_ENABLED=true + - IGGY_ENCRYPTION_KEY=/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE= networks: - php-test-network ports: @@ -65,6 +67,8 @@ - IGGY_TCP_ADDRESS=iggy-server:8090 - IGGY_ROOT_USERNAME=iggy - IGGY_ROOT_PASSWORD=iggy + - IGGY_ENCRYPTION_ENABLED=true + - IGGY_ENCRYPTION_KEY=/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE= volumes: - ./test-results:/workspace/foreign/php/test-results
diff --git a/foreign/php/iggy-php.stubs.php b/foreign/php/iggy-php.stubs.php index c387404..b00d5eb 100644 --- a/foreign/php/iggy-php.stubs.php +++ b/foreign/php/iggy-php.stubs.php
@@ -127,15 +127,31 @@ /** * Creates a new stream. * + * `encryption_key` is the stream's own data key: 32 random bytes as canonical padded + * standard base64, for example `base64_encode(random_bytes(32))`. Raw bytes or any other + * text are refused before anything is sent. The key travels in the request, so protecting + * it in transit with TLS is the caller's responsibility. It is marked + * `#[\SensitiveParameter]`, so exception traces never hold it. The server stores it wrapped + * and never returns it, reads report only its source and fingerprints. Without a key, a + * server with encryption enabled generates one. `raw` sends string-valued keys this + * build does not know, and the server decides whether to accept them. It is marked + * `#[\SensitiveParameter]` too, because an unknown key may hold a secret. + * * @param string $name + * @param string|null $encryption_key + * @param array|null $raw * @return void */ - public function createStream(string $name): void {} + public function createStream(string $name, ?string $encryption_key = null, ?array $raw = null): void {} /** * Creates a topic. * - * Every option left null resolves against the server default at admission. + * Every option left null resolves against the server default at admission. The 64-bit + * sizes and durations take an int or a decimal string, so never-expire and unlimited + * (18446744073709551615) can be sent. `raw` sends string-valued keys this build does not + * know, and the server decides whether to accept them. It is marked + * `#[\SensitiveParameter]`, because an unknown key may hold a secret. * * @param mixed $stream * @param string $name @@ -149,9 +165,11 @@ * @param mixed $messages_required_to_save * @param mixed $size_of_messages_required_to_save * @param bool|null $preallocate_segments + * @param string|null $partition_resize_policy + * @param array|null $raw * @return void */ - public function createTopic(mixed $stream, string $name, int $partitions_count, ?string $compression_algorithm = null, mixed $message_expiry_micros = null, mixed $max_topic_size = null, mixed $segment_size = null, ?\Iggy\Durability $durability = null, ?\Iggy\Durability $consumer_offset_durability = null, mixed $messages_required_to_save = null, mixed $size_of_messages_required_to_save = null, ?bool $preallocate_segments = null): void {} + public function createTopic(mixed $stream, string $name, int $partitions_count, ?string $compression_algorithm = null, mixed $message_expiry_micros = null, mixed $max_topic_size = null, mixed $segment_size = null, ?\Iggy\Durability $durability = null, ?\Iggy\Durability $consumer_offset_durability = null, mixed $messages_required_to_save = null, mixed $size_of_messages_required_to_save = null, ?bool $preallocate_segments = null, ?string $partition_resize_policy = null, ?array $raw = null): void {} /** * Deletes a stream by id or name. @@ -171,8 +189,29 @@ public function deleteTopic(mixed $stream_id, mixed $topic_id): void {} /** + * The server's option catalog for one resource: each key's kind and whether an update + * may set or reset it, whether it is secret, read-only or critical. + * + * @param \Iggy\OptionsScope $scope + * @return array + */ + public function describeOptions(\Iggy\OptionsScope $scope): array {} + + /** + * Fingerprint of a stream key given as canonical padded standard base64 of 32 bytes, the + * value `StreamDetails::$encryption_key_fingerprint` reports for a stream created with it. + * The key is marked `#[\SensitiveParameter]`, so exception traces never hold it. + * + * @param string $key + * @return string + */ + public static function encryptionKeyFingerprint(string $key): string {} + + /** * Constructs a new IggyClient from a connection string. * + * The string is marked `#[\SensitiveParameter]`, so exception traces never hold it. + * * @param string $connection_string * @return \Iggy\Client */ @@ -198,6 +237,8 @@ /** * Logs in the user with the given credentials. * + * The password is marked `#[\SensitiveParameter]`, so exception traces never hold it. + * * @param string $username * @param string $password * @return void @@ -264,6 +305,47 @@ * @return \Iggy\SendMessagesResponse */ public function sendMessages(mixed $stream, mixed $topic, int $partition_id, array $messages): \Iggy\SendMessagesResponse {} + + /** + * Renames a stream. + * + * A null name keeps the current one. No stream option is resettable, so a non-empty + * `reset` list, or an update that changes nothing, is refused before anything is sent. + * `raw` and `raw_reset` reach keys this build does not know, and the server decides + * whether each one may be set or reset. `raw` is marked `#[\SensitiveParameter]`, + * because an unknown key may hold a secret. Renaming keeps the stream's id, key and + * messages. + * + * @param mixed $stream + * @param string|null $name + * @param array $reset + * @param array|null $raw + * @param array|null $raw_reset + * @return void + */ + public function updateStream(mixed $stream, ?string $name = null, array $reset = [], ?array $raw = null, ?array $raw_reset = null): void {} + + /** + * Renames a topic or changes its message expiry and size cap. + * + * Null arguments keep the stored values. `message_expiry_micros` and `max_topic_size` + * take an int or a decimal string, 18446744073709551615 meaning never expire and + * unlimited, and 0 is refused: list the key in `reset` to return it to the server + * default. `raw` and `raw_reset` reach keys this build does not know, and the server + * decides whether each one may be set or reset. `raw` is marked + * `#[\SensitiveParameter]`, because an unknown key may hold a secret. + * + * @param mixed $stream + * @param mixed $topic + * @param string|null $name + * @param mixed $message_expiry_micros + * @param mixed $max_topic_size + * @param array $reset + * @param array|null $raw + * @param array|null $raw_reset + * @return void + */ + public function updateTopic(mixed $stream, mixed $topic, ?string $name = null, mixed $message_expiry_micros = null, mixed $max_topic_size = null, array $reset = [], ?array $raw = null, ?array $raw_reset = null): void {} } /** @@ -464,6 +546,75 @@ } /** + * One option key as the server's catalog describes it. + */ + class OptionSpec { + /** + * A node that does not understand the key must not serve the resource. + * + * @var bool + */ + public readonly bool $critical; + + /** + * The default in the kind's binary encoding, an empty string when the key has none. + * + * @var string + */ + public readonly string $default_value; + + public readonly string $description; + + public readonly string $key; + + /** + * The value kind, for example `string`, `uint64` or `raw`. + * + * @var string + */ + public readonly string $kind; + + /** + * Computed by the server and returned on reads, never accepted. + * + * @var bool + */ + public readonly bool $read_only; + + /** + * An update may reset the key to its default. + * + * @var bool + */ + public readonly bool $resettable; + + /** + * Accepted on create, never returned by a read. + * + * @var bool + */ + public readonly bool $secret; + + /** + * An update may set the key. + * + * @var bool + */ + public readonly bool $updatable; + + public function __construct() {} + } + + /** + * The resource whose option catalog `Client::describeOptions()` reads. + */ + enum OptionsScope: string { + case Topic = 'topic'; + case Stream = 'stream'; + case User = 'user'; + } + + /** * A PHP class representing the partition incarnation and owner that a poll captured. * * Keep it with an offset that is continued or committed later. The server then refuses the @@ -743,28 +894,117 @@ } class StreamDetails { + /** + * Options the server filled in, including the read-only encryption projections. + * + * @var array + */ + public readonly array $derived_options; + + /** + * Fingerprints of the server keys that wrap the stream's key, comma-separated: one + * outside a server key rotation. Null for a plaintext stream. + * + * @var string|null + */ + public readonly ?string $encryption_kek_fingerprint; + + /** + * Fingerprint of the stream's key, 16 lowercase hex characters. Compare it with + * `Client::encryptionKeyFingerprint()` of a key you hold. Null for a plaintext stream. + * + * @var string|null + */ + public readonly ?string $encryption_key_fingerprint; + + /** + * Who chose the stream's at-rest key: `stream`, `server`, or `none` for a plaintext + * stream. A source a newer server reports is returned as its text. + * + * @var string|null + */ + public readonly ?string $encryption_key_source; + public readonly int $id; public readonly int $messages_count; public readonly string $name; + /** + * Options the creating client set, as option name to scalar value. Key material is + * never returned. + * + * @var array + */ + public readonly array $options; + public readonly int $topics_count; public function __construct() {} } + /** + * Stream option keys, as the cases an update's reset list takes. + * + * No stream option is resettable in this release, so `Client::updateStream()` refuses every + * case with the error the server would return. + */ + enum StreamOptionKey: string { + case Name = 'name'; + case EncryptionKey = 'encryption_key'; + case EncryptionKeySource = 'encryption_key_source'; + case EncryptionKeyFingerprint = 'encryption_key_fingerprint'; + case EncryptionKekFingerprint = 'encryption_kek_fingerprint'; + } + class TopicDetails { + /** + * Options the server filled in from its catalog defaults. + * + * @var array + */ + public readonly array $derived_options; + public readonly int $id; public readonly int $messages_count; public readonly string $name; + /** + * Options the creating or updating client set, as option name to scalar value. + * Integers above `PHP_INT_MAX`, such as never-expire and unlimited + * (18446744073709551615), are decimal strings. + * + * @var array + */ + public readonly array $options; + public readonly int $partitions_count; public function __construct() {} } + + /** + * Topic option keys, as the cases an update's reset list takes. + * + * Only `MessageExpiry` and `MaxTopicSize` are resettable in this release. The server refuses + * every other case with its own category. + */ + enum TopicOptionKey: string { + case Name = 'name'; + case CompressionAlgorithm = 'compression_algorithm'; + case MessageExpiry = 'message_expiry'; + case MaxTopicSize = 'max_topic_size'; + case SegmentSize = 'segment_size'; + case Durability = 'durability'; + case ConsumerOffsetDurability = 'consumer_offset_durability'; + case PartitionResizePolicy = 'partition_resize_policy'; + case MessagesRequiredToSave = 'messages_required_to_save'; + case SizeOfMessagesRequiredToSave = 'size_of_messages_required_to_save'; + case PreallocateSegments = 'preallocate_segments'; + } } namespace Iggy\Exception {
diff --git a/foreign/php/src/client.rs b/foreign/php/src/client.rs index c1d792f..8745fdf 100644 --- a/foreign/php/src/client.rs +++ b/foreign/php/src/client.rs
@@ -16,6 +16,7 @@ // under the License. use std::{ + collections::BTreeMap, num::NonZeroUsize, str::FromStr, sync::Arc, @@ -35,6 +36,7 @@ use crate::durability::Durability as PhpDurability; use crate::error::to_php_exception; use crate::identifier::PhpIdentifier; +use crate::options::{OptionSpec, OptionsScope, StreamOptionKey, TopicOptionKey, reset_names}; use crate::receive_message::{PollingStrategy, ReceiveMessage}; use crate::runtime::runtime; use crate::send_message::{SendMessage, SendMessagesResponse}; @@ -119,6 +121,8 @@ } /// Constructs a new IggyClient from a connection string. + /// + /// The string is marked `#[\SensitiveParameter]`, so exception traces never hold it. pub fn from_connection_string(connection_string: String) -> PhpResult<Self> { let client = RustIggyClient::from_connection_string(&connection_string).map_err(to_php_exception)?; @@ -135,6 +139,8 @@ } /// Logs in the user with the given credentials. + /// + /// The password is marked `#[\SensitiveParameter]`, so exception traces never hold it. pub fn login_user(&self, username: String, password: String) -> PhpResult { let inner = self.inner.clone(); @@ -188,18 +194,79 @@ } /// Creates a new stream. - pub fn create_stream(&self, name: String) -> PhpResult { + /// + /// `encryption_key` is the stream's own data key: 32 random bytes as canonical padded + /// standard base64, for example `base64_encode(random_bytes(32))`. Raw bytes or any other + /// text are refused before anything is sent. The key travels in the request, so protecting + /// it in transit with TLS is the caller's responsibility. It is marked + /// `#[\SensitiveParameter]`, so exception traces never hold it. The server stores it wrapped + /// and never returns it, reads report only its source and fingerprints. Without a key, a + /// server with encryption enabled generates one. `raw` sends string-valued keys this + /// build does not know, and the server decides whether to accept them. It is marked + /// `#[\SensitiveParameter]` too, because an unknown key may hold a secret. + pub fn create_stream( + &self, + name: String, + encryption_key: Option<Binary<u8>>, + raw: Option<BTreeMap<String, String>>, + ) -> PhpResult { + let options = StreamCreateOptions { + encryption_key: encryption_key.map(stream_key).transpose()?, + raw: raw.unwrap_or_default(), + }; let inner = self.inner.clone(); runtime().block_on(async move { inner - .create_stream(&name) + .create_stream(&name, &options) .await .map(|_| ()) .map_err(to_php_exception) }) } + /// Renames a stream. + /// + /// A null name keeps the current one. No stream option is resettable, so a non-empty + /// `reset` list, or an update that changes nothing, is refused before anything is sent. + /// `raw` and `raw_reset` reach keys this build does not know, and the server decides + /// whether each one may be set or reset. `raw` is marked `#[\SensitiveParameter]`, + /// because an unknown key may hold a secret. Renaming keeps the stream's id, key and + /// messages. + #[php(defaults(reset = Vec::<StreamOptionKey>::new()))] + pub fn update_stream( + &self, + stream: PhpIdentifier, + name: Option<String>, + reset: Vec<StreamOptionKey>, + raw: Option<BTreeMap<String, String>>, + raw_reset: Option<Vec<String>>, + ) -> PhpResult { + let stream: Identifier = stream.try_into()?; + let (reset, raw_reset) = reset_names(reset, raw_reset)?; + let options = StreamUpdateOptions { + name, + reset, + raw_reset, + raw: raw.unwrap_or_default(), + }; + let inner = self.inner.clone(); + + runtime().block_on(async move { + inner + .update_stream(&stream, &options) + .await + .map_err(to_php_exception) + }) + } + + /// Fingerprint of a stream key given as canonical padded standard base64 of 32 bytes, the + /// value `StreamDetails::$encryption_key_fingerprint` reports for a stream created with it. + /// The key is marked `#[\SensitiveParameter]`, so exception traces never hold it. + pub fn encryption_key_fingerprint(key: Binary<u8>) -> PhpResult<String> { + stream_key(key).map(|key| key.fingerprint()) + } + /// Gets a stream by id or name. pub fn get_stream(&self, stream_id: PhpIdentifier) -> PhpResult<Option<StreamDetails>> { let stream_id: Identifier = stream_id.try_into()?; @@ -216,7 +283,11 @@ /// Creates a topic. /// - /// Every option left null resolves against the server default at admission. + /// Every option left null resolves against the server default at admission. The 64-bit + /// sizes and durations take an int or a decimal string, so never-expire and unlimited + /// (18446744073709551615) can be sent. `raw` sends string-valued keys this build does not + /// know, and the server decides whether to accept them. It is marked + /// `#[\SensitiveParameter]`, because an unknown key may hold a secret. #[allow(clippy::too_many_arguments)] pub fn create_topic( &self, @@ -232,14 +303,15 @@ messages_required_to_save: Option<&Zval>, size_of_messages_required_to_save: Option<&Zval>, preallocate_segments: Option<bool>, + partition_resize_policy: Option<String>, + raw: Option<BTreeMap<String, String>>, ) -> PhpResult { - let message_expiry_micros = - optional_unsigned::<u64>(message_expiry_micros, "message_expiry_micros")?; - let max_topic_size = optional_unsigned::<u64>(max_topic_size, "max_topic_size")?; - let segment_size = optional_unsigned::<u64>(segment_size, "segment_size")?; + let message_expiry_micros = optional_u64(message_expiry_micros, "message_expiry_micros")?; + let max_topic_size = optional_u64(max_topic_size, "max_topic_size")?; + let segment_size = optional_u64(segment_size, "segment_size")?; let messages_required_to_save = optional_unsigned::<u32>(messages_required_to_save, "messages_required_to_save")?; - let size_of_messages_required_to_save = optional_unsigned::<u64>( + let size_of_messages_required_to_save = optional_u64( size_of_messages_required_to_save, "size_of_messages_required_to_save", )?; @@ -247,9 +319,14 @@ Some(value) => CompressionAlgorithm::from_str(&value).map_err(to_php_exception)?, None => CompressionAlgorithm::default(), }; - let expiry = message_expiry_micros.map_or(IggyExpiry::ServerDefault, |micros| { - IggyExpiry::ExpireDuration(IggyDuration::from(micros)) - }); + let partition_resize_policy = partition_resize_policy + .as_deref() + .map(PartitionResizePolicy::from_str) + .transpose() + .map_err(|_| { + to_php_exception("'partition_resize_policy' must be 'mutable' or 'fixed'") + })?; + let expiry = message_expiry_micros.map_or(IggyExpiry::ServerDefault, IggyExpiry::from); let max_size = max_topic_size.map_or(MaxTopicSize::ServerDefault, MaxTopicSize::from); let stream: Identifier = stream.try_into()?; let inner = self.inner.clone(); @@ -265,11 +342,12 @@ segment_size: segment_size.map(IggyByteSize::from), durability: durability.unwrap_or_default().into(), consumer_offset_durability: consumer_offset_durability.unwrap_or_default().into(), + partition_resize_policy, messages_required_to_save, size_of_messages_required_to_save: size_of_messages_required_to_save .map(IggyByteSize::from), preallocate_segments, - ..TopicCreateOptions::default() + raw: raw.unwrap_or_default(), }; runtime().block_on(async move { @@ -281,6 +359,68 @@ }) } + /// Renames a topic or changes its message expiry and size cap. + /// + /// Null arguments keep the stored values. `message_expiry_micros` and `max_topic_size` + /// take an int or a decimal string, 18446744073709551615 meaning never expire and + /// unlimited, and 0 is refused: list the key in `reset` to return it to the server + /// default. `raw` and `raw_reset` reach keys this build does not know, and the server + /// decides whether each one may be set or reset. `raw` is marked + /// `#[\SensitiveParameter]`, because an unknown key may hold a secret. + #[allow(clippy::too_many_arguments)] + #[php(defaults(reset = Vec::<TopicOptionKey>::new()))] + pub fn update_topic( + &self, + stream: PhpIdentifier, + topic: PhpIdentifier, + name: Option<String>, + message_expiry_micros: Option<&Zval>, + max_topic_size: Option<&Zval>, + reset: Vec<TopicOptionKey>, + raw: Option<BTreeMap<String, String>>, + raw_reset: Option<Vec<String>>, + ) -> PhpResult { + let message_expiry = optional_u64(message_expiry_micros, "message_expiry_micros")? + .map(|micros| positive_update(micros, "message_expiry_micros").map(IggyExpiry::from)) + .transpose()?; + let max_topic_size = optional_u64(max_topic_size, "max_topic_size")? + .map(|bytes| positive_update(bytes, "max_topic_size").map(MaxTopicSize::from)) + .transpose()?; + let stream: Identifier = stream.try_into()?; + let topic: Identifier = topic.try_into()?; + let (reset, raw_reset) = reset_names(reset, raw_reset)?; + let options = TopicUpdateOptions { + name, + message_expiry, + max_topic_size, + reset, + raw_reset, + raw: raw.unwrap_or_default(), + }; + let inner = self.inner.clone(); + + runtime().block_on(async move { + inner + .update_topic(&stream, &topic, &options) + .await + .map_err(to_php_exception) + }) + } + + /// The server's option catalog for one resource: each key's kind and whether an update + /// may set or reset it, whether it is secret, read-only or critical. + pub fn describe_options(&self, scope: OptionsScope) -> PhpResult<Vec<OptionSpec>> { + let inner = self.inner.clone(); + + runtime().block_on(async move { + inner + .describe_options(scope.into()) + .await + .map(|specs| specs.into_iter().map(OptionSpec::from).collect()) + .map_err(to_php_exception) + }) + } + /// Gets a topic by stream and topic id/name. pub fn get_topic( &self, @@ -555,6 +695,44 @@ }) } +// u64 values above PHP_INT_MAX, such as the never-expire sentinel, only fit a decimal string. +fn optional_u64(value: Option<&Zval>, field: &str) -> PhpResult<Option<u64>> { + let Some(value) = value.filter(|value| !value.is_null()) else { + return Ok(None); + }; + let number = match (value.long(), value.str()) { + (Some(number), _) => u64::try_from(number).ok(), + (None, Some(text)) + if !text.is_empty() && text.bytes().all(|byte| byte.is_ascii_digit()) => + { + text.parse::<u64>().ok() + } + _ => None, + }; + number.map(Some).ok_or_else(|| { + to_php_exception(format!( + "'{field}' must be a non-negative integer, a decimal string or null" + )) + }) +} + +fn positive_update(value: u64, field: &str) -> PhpResult<u64> { + if value == 0 { + return Err(to_php_exception(format!( + "'{field}' must be greater than 0, reset it to return to the server default" + ))); + } + Ok(value) +} + +fn stream_key(key: Binary<u8>) -> PhpResult<StreamEncryptionKey> { + let bytes: Vec<u8> = key.into(); + str::from_utf8(&bytes) + .map_err(|_| IggyError::InvalidEncryptionKey) + .and_then(StreamEncryptionKey::from_str) + .map_err(to_php_exception) +} + fn optional_unsigned<T: TryFrom<i64>>(value: Option<&Zval>, field: &str) -> PhpResult<Option<T>> { optional_integer(value, field)? .map(|value| {
diff --git a/foreign/php/src/error.rs b/foreign/php/src/error.rs index df3e551..d095779 100644 --- a/foreign/php/src/error.rs +++ b/foreign/php/src/error.rs
@@ -18,6 +18,7 @@ use std::ptr; use ext_php_rs::{ + class::RegisteredClass, convert::IntoZval, error::Error, exception::PhpException, @@ -66,7 +67,6 @@ operation_id: String, kind: String, } - #[php_impl] impl PendingOperationException { /// Exact unsigned 64-bit operation ID, represented as a decimal string. @@ -138,17 +138,14 @@ }; } - match PhpExceptionKind::from(&self) { - PhpExceptionKind::Connection => { - PhpException::from_class::<ConnectionException>(message) - } - PhpExceptionKind::Authentication => { - PhpException::from_class::<AuthenticationException>(message) - } - PhpExceptionKind::NotFound => PhpException::from_class::<NotFoundException>(message), - PhpExceptionKind::Transient => PhpException::from_class::<TransientException>(message), - PhpExceptionKind::Base => PhpException::from_class::<IggyException>(message), - } + let class = match PhpExceptionKind::from(&self) { + PhpExceptionKind::Connection => ConnectionException::get_metadata().ce(), + PhpExceptionKind::Authentication => AuthenticationException::get_metadata().ce(), + PhpExceptionKind::NotFound => NotFoundException::get_metadata().ce(), + PhpExceptionKind::Transient => TransientException::get_metadata().ce(), + PhpExceptionKind::Base => IggyException::get_metadata().ce(), + }; + PhpException::new(message, self.as_code() as i32, class) } } @@ -216,6 +213,7 @@ | IggyError::InvalidUsername | IggyError::InvalidPassword | IggyError::InvalidUserStatus + | IggyError::CannotModifyRootUser | IggyError::UserInactive | IggyError::InvalidPersonalAccessToken | IggyError::PersonalAccessTokenExpired(_, _)
diff --git a/foreign/php/src/lib.rs b/foreign/php/src/lib.rs index 39475c0..9cc6c5f 100644 --- a/foreign/php/src/lib.rs +++ b/foreign/php/src/lib.rs
@@ -21,9 +21,11 @@ pub mod error; pub mod identifier; pub mod message_iterator; +pub mod options; pub mod receive_message; pub mod runtime; pub mod send_message; +pub mod sensitive_parameters; pub mod stream; pub mod topic; @@ -36,6 +38,7 @@ PendingOperationException, TransientException, }; use crate::message_iterator::MessageIterator; +use crate::options::{OptionSpec, OptionsScope, StreamOptionKey, TopicOptionKey}; use crate::receive_message::{ConsumerPosition, PartitionContext, PollingStrategy, ReceiveMessage}; use crate::send_message::{SendMessage, SendMessagesConfirmation, SendMessagesResponse}; use crate::stream::StreamDetails; @@ -44,7 +47,11 @@ #[php_module] pub fn get_module(module: ModuleBuilder) -> ModuleBuilder { module + .request_startup_function(sensitive_parameters::mark_sensitive_parameters) .enumeration::<durability::Durability>() + .enumeration::<StreamOptionKey>() + .enumeration::<TopicOptionKey>() + .enumeration::<OptionsScope>() // Parent classes must be registered before subclasses because ext-php-rs resolves // the parent ClassEntry during child registration. .class::<IggyException>() @@ -68,4 +75,5 @@ .class::<SendMessagesResponse>() .class::<StreamDetails>() .class::<TopicDetails>() + .class::<OptionSpec>() }
diff --git a/foreign/php/src/options.rs b/foreign/php/src/options.rs new file mode 100644 index 0000000..411db32 --- /dev/null +++ b/foreign/php/src/options.rs
@@ -0,0 +1,338 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::BTreeSet; + +use ext_php_rs::{ + binary::Binary, + boxed::ZBox, + convert::{IntoZval, StubLiteral}, + exception::PhpResult, + php_class, php_enum, php_impl, + types::{ZendHashTable, Zval}, +}; +use iggy::prelude::{ + HeaderKind, HeaderValue, IggyError, OptionKey, OptionSpec as RustOptionSpec, + OptionsScope as RustOptionsScope, ResourceOptions, +}; + +use crate::error::to_php_exception; + +/// Stream option keys, as the cases an update's reset list takes. +/// +/// No stream option is resettable in this release, so `Client::updateStream()` refuses every +/// case with the error the server would return. +#[php_enum] +#[php(name = "Iggy\\StreamOptionKey")] +#[derive(Clone, Copy)] +pub enum StreamOptionKey { + #[php(value = "name")] + Name, + #[php(value = "encryption_key")] + EncryptionKey, + #[php(value = "encryption_key_source")] + EncryptionKeySource, + #[php(value = "encryption_key_fingerprint")] + EncryptionKeyFingerprint, + #[php(value = "encryption_kek_fingerprint")] + EncryptionKekFingerprint, +} + +impl From<StreamOptionKey> for OptionKey { + fn from(key: StreamOptionKey) -> Self { + match key { + StreamOptionKey::Name => Self::Name, + StreamOptionKey::EncryptionKey => Self::EncryptionKey, + StreamOptionKey::EncryptionKeySource => Self::EncryptionKeySource, + StreamOptionKey::EncryptionKeyFingerprint => Self::EncryptionKeyFingerprint, + StreamOptionKey::EncryptionKekFingerprint => Self::EncryptionKekFingerprint, + } + } +} + +// Lets a reset list default to `[]` in the generated stub. +impl StubLiteral for StreamOptionKey { + fn stub_literal(&self) -> String { + let case = match self { + Self::Name => "Name", + Self::EncryptionKey => "EncryptionKey", + Self::EncryptionKeySource => "EncryptionKeySource", + Self::EncryptionKeyFingerprint => "EncryptionKeyFingerprint", + Self::EncryptionKekFingerprint => "EncryptionKekFingerprint", + }; + format!("\\Iggy\\StreamOptionKey::{case}") + } +} + +/// Topic option keys, as the cases an update's reset list takes. +/// +/// Only `MessageExpiry` and `MaxTopicSize` are resettable in this release. The server refuses +/// every other case with its own category. +#[php_enum] +#[php(name = "Iggy\\TopicOptionKey")] +#[derive(Clone, Copy)] +pub enum TopicOptionKey { + #[php(value = "name")] + Name, + #[php(value = "compression_algorithm")] + CompressionAlgorithm, + #[php(value = "message_expiry")] + MessageExpiry, + #[php(value = "max_topic_size")] + MaxTopicSize, + #[php(value = "segment_size")] + SegmentSize, + #[php(value = "durability")] + Durability, + #[php(value = "consumer_offset_durability")] + ConsumerOffsetDurability, + #[php(value = "partition_resize_policy")] + PartitionResizePolicy, + #[php(value = "messages_required_to_save")] + MessagesRequiredToSave, + #[php(value = "size_of_messages_required_to_save")] + SizeOfMessagesRequiredToSave, + #[php(value = "preallocate_segments")] + PreallocateSegments, +} + +impl From<TopicOptionKey> for OptionKey { + fn from(key: TopicOptionKey) -> Self { + match key { + TopicOptionKey::Name => Self::Name, + TopicOptionKey::CompressionAlgorithm => Self::CompressionAlgorithm, + TopicOptionKey::MessageExpiry => Self::MessageExpiry, + TopicOptionKey::MaxTopicSize => Self::MaxTopicSize, + TopicOptionKey::SegmentSize => Self::SegmentSize, + TopicOptionKey::Durability => Self::Durability, + TopicOptionKey::ConsumerOffsetDurability => Self::ConsumerOffsetDurability, + TopicOptionKey::PartitionResizePolicy => Self::PartitionResizePolicy, + TopicOptionKey::MessagesRequiredToSave => Self::MessagesRequiredToSave, + TopicOptionKey::SizeOfMessagesRequiredToSave => Self::SizeOfMessagesRequiredToSave, + TopicOptionKey::PreallocateSegments => Self::PreallocateSegments, + } + } +} + +// Lets a reset list default to `[]` in the generated stub. +impl StubLiteral for TopicOptionKey { + fn stub_literal(&self) -> String { + let case = match self { + Self::Name => "Name", + Self::CompressionAlgorithm => "CompressionAlgorithm", + Self::MessageExpiry => "MessageExpiry", + Self::MaxTopicSize => "MaxTopicSize", + Self::SegmentSize => "SegmentSize", + Self::Durability => "Durability", + Self::ConsumerOffsetDurability => "ConsumerOffsetDurability", + Self::PartitionResizePolicy => "PartitionResizePolicy", + Self::MessagesRequiredToSave => "MessagesRequiredToSave", + Self::SizeOfMessagesRequiredToSave => "SizeOfMessagesRequiredToSave", + Self::PreallocateSegments => "PreallocateSegments", + }; + format!("\\Iggy\\TopicOptionKey::{case}") + } +} + +/// The resource whose option catalog `Client::describeOptions()` reads. +#[php_enum] +#[php(name = "Iggy\\OptionsScope")] +#[derive(Clone, Copy)] +pub enum OptionsScope { + #[php(value = "topic")] + Topic, + #[php(value = "stream")] + Stream, + #[php(value = "user")] + User, +} + +impl From<OptionsScope> for RustOptionsScope { + fn from(scope: OptionsScope) -> Self { + match scope { + OptionsScope::Topic => Self::Topic, + OptionsScope::Stream => Self::Stream, + OptionsScope::User => Self::User, + } + } +} + +/// One option key as the server's catalog describes it. +#[php_class] +#[php(name = "Iggy\\OptionSpec")] +pub struct OptionSpec { + inner: RustOptionSpec, +} + +impl From<RustOptionSpec> for OptionSpec { + fn from(inner: RustOptionSpec) -> Self { + Self { inner } + } +} + +#[php_impl] +impl OptionSpec { + #[php(getter)] + pub fn key(&self) -> String { + self.inner.key.clone() + } + + /// The value kind, for example `string`, `uint64` or `raw`. + #[php(getter)] + pub fn kind(&self) -> String { + self.inner.kind.to_string() + } + + /// An update may set the key. + #[php(getter)] + pub fn updatable(&self) -> bool { + self.inner.updatable + } + + /// An update may reset the key to its default. + #[php(getter)] + pub fn resettable(&self) -> bool { + self.inner.resettable + } + + /// Accepted on create, never returned by a read. + #[php(getter)] + pub fn secret(&self) -> bool { + self.inner.secret + } + + /// Computed by the server and returned on reads, never accepted. + #[php(getter, change_case = "snake_case")] + pub fn read_only(&self) -> bool { + self.inner.read_only + } + + /// A node that does not understand the key must not serve the resource. + #[php(getter)] + pub fn critical(&self) -> bool { + self.inner.critical + } + + /// The default in the kind's binary encoding, an empty string when the key has none. + #[php(getter, change_case = "snake_case")] + pub fn default_value(&self) -> Binary<u8> { + Binary::new(self.inner.default_value.clone()) + } + + #[php(getter)] + pub fn description(&self) -> String { + self.inner.description.clone() + } +} + +/// The typed and raw reset lists of an update. +/// +/// A name listed twice, typed or raw, is refused as `InvalidCommand`, as server admission +/// refuses a repeated reset name. +pub(crate) fn reset_names<K: Into<OptionKey>>( + typed: Vec<K>, + raw: Option<Vec<String>>, +) -> PhpResult<(BTreeSet<OptionKey>, BTreeSet<String>)> { + let mut reset = BTreeSet::new(); + let mut names = BTreeSet::new(); + for key in typed { + let key = key.into(); + let name: &'static str = key.into(); + if !reset.insert(key) || !names.insert(name.to_owned()) { + return Err(to_php_exception(IggyError::InvalidCommand)); + } + } + let mut raw_reset = BTreeSet::new(); + for name in raw.unwrap_or_default() { + if !names.insert(name.clone()) { + return Err(to_php_exception(IggyError::InvalidCommand)); + } + raw_reset.insert(name); + } + Ok((reset, raw_reset)) +} + +/// The options of one provenance as a PHP array of option name to scalar value. +/// +/// Integers that do not fit a PHP int, and 128-bit integers, are decimal strings. Raw values +/// are binary strings. +pub(crate) fn options_array( + options: &ResourceOptions, + explicit: bool, +) -> PhpResult<ZBox<ZendHashTable>> { + let mut array = ZendHashTable::new(); + for (key, option) in options + .iter() + .filter(|(_, option)| option.explicit == explicit) + { + let name = key.as_str().map_err(to_php_exception)?; + array + .insert(name, option_scalar(&option.value)?) + .map_err(to_php_exception)?; + } + Ok(array) +} + +fn option_scalar(value: &HeaderValue) -> PhpResult<Zval> { + let zval = match value.kind() { + HeaderKind::Raw => Binary::new(value.as_bytes().to_vec()).into_zval(false), + HeaderKind::String => value.as_str().map_err(to_php_exception)?.into_zval(false), + HeaderKind::Bool => value.as_bool().map_err(to_php_exception)?.into_zval(false), + HeaderKind::Int8 => i64::from(value.as_int8().map_err(to_php_exception)?).into_zval(false), + HeaderKind::Int16 => { + i64::from(value.as_int16().map_err(to_php_exception)?).into_zval(false) + } + HeaderKind::Int32 => { + i64::from(value.as_int32().map_err(to_php_exception)?).into_zval(false) + } + HeaderKind::Int64 => value.as_int64().map_err(to_php_exception)?.into_zval(false), + HeaderKind::Int128 => value + .as_int128() + .map_err(to_php_exception)? + .to_string() + .into_zval(false), + HeaderKind::Uint8 => { + i64::from(value.as_uint8().map_err(to_php_exception)?).into_zval(false) + } + HeaderKind::Uint16 => { + i64::from(value.as_uint16().map_err(to_php_exception)?).into_zval(false) + } + HeaderKind::Uint32 => { + i64::from(value.as_uint32().map_err(to_php_exception)?).into_zval(false) + } + HeaderKind::Uint64 => { + let number = value.as_uint64().map_err(to_php_exception)?; + match i64::try_from(number) { + Ok(number) => number.into_zval(false), + Err(_) => number.to_string().into_zval(false), + } + } + HeaderKind::Uint128 => value + .as_uint128() + .map_err(to_php_exception)? + .to_string() + .into_zval(false), + HeaderKind::Float32 => { + f64::from(value.as_float32().map_err(to_php_exception)?).into_zval(false) + } + HeaderKind::Float64 => value + .as_float64() + .map_err(to_php_exception)? + .into_zval(false), + }; + zval.map_err(to_php_exception) +}
diff --git a/foreign/php/src/sensitive_parameters.rs b/foreign/php/src/sensitive_parameters.rs new file mode 100644 index 0000000..12735e9 --- /dev/null +++ b/foreign/php/src/sensitive_parameters.rs
@@ -0,0 +1,135 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::{ + ffi::{CStr, c_char, c_void}, + mem, + sync::{Mutex, PoisonError}, +}; + +use ext_php_rs::{ + ffi::{HashTable, zend_function, zend_hash_str_find_ptr_lc, zend_string}, + types::ZendStr, + zend::ClassEntry, +}; + +type AddAttribute = unsafe extern "C" fn( + attributes: *mut *mut HashTable, + name: *mut zend_string, + argc: u32, + flags: u32, + offset: u32, + lineno: u32, +) -> *mut c_void; +type GetParameterAttribute = unsafe extern "C" fn( + attributes: *mut HashTable, + name: *const c_char, + length: usize, + offset: u32, +) -> *mut c_void; + +const SENSITIVE_PARAMETER: &str = "SensitiveParameter"; +const SENSITIVE_PARAMETER_LOWERCASE: &str = "sensitiveparameter"; +const ZEND_ATTRIBUTE_PERSISTENT: u32 = 1; +const SUCCESS: i32 = 0; + +/// Class, method and zero-based position of every parameter that carries a secret. +// Every `raw` array is included, since this build cannot tell which unknown keys are secret. +const SENSITIVE_PARAMETERS: &[(&str, &str, u32)] = &[ + ("Iggy\\Client", "fromConnectionString", 0), + ("Iggy\\Client", "loginUser", 1), + ("Iggy\\Client", "sendBinaryRequest", 1), + ("Iggy\\Client", "createStream", 1), + ("Iggy\\Client", "createStream", 2), + ("Iggy\\Client", "updateStream", 3), + ("Iggy\\Client", "createTopic", 13), + ("Iggy\\Client", "updateTopic", 6), + ("Iggy\\Client", "encryptionKeyFingerprint", 0), +]; + +static MARKING: Mutex<()> = Mutex::new(()); + +/// Marks secret parameters `#[\SensitiveParameter]`, so traces hold a `SensitiveParameterValue`. +// Request startup, because the module startup hook runs before the classes are registered. The +// lock holds every request until the first one has marked them, so no script reads them mid-change. +pub extern "C" fn mark_sensitive_parameters(_type: i32, _module_number: i32) -> i32 { + let _marking = MARKING.lock().unwrap_or_else(PoisonError::into_inner); + let Some((add_attribute, get_parameter_attribute)) = attribute_functions() else { + return SUCCESS; + }; + let mut name = ZendStr::new(SENSITIVE_PARAMETER, true); + + for (class, method, offset) in SENSITIVE_PARAMETERS { + let Some(class) = ClassEntry::try_find(class) else { + continue; + }; + // SAFETY: the function table belongs to a registered class and outlives the request, + // and the returned pointer is either null or a live function of that table. + unsafe { + let function = zend_hash_str_find_ptr_lc( + &raw const class.function_table, + method.as_ptr().cast::<c_char>(), + method.len(), + ) + .cast::<zend_function>(); + let Some(function) = function.as_mut() else { + continue; + }; + let attributes = &raw mut function.common.attributes; + let marked = !get_parameter_attribute( + *attributes, + SENSITIVE_PARAMETER_LOWERCASE.as_ptr().cast::<c_char>(), + SENSITIVE_PARAMETER_LOWERCASE.len(), + *offset, + ) + .is_null(); + if !marked { + // Offset 0 addresses the function itself, parameters start at 1. + add_attribute( + attributes, + &raw mut *name, + 0, + ZEND_ATTRIBUTE_PERSISTENT, + offset + 1, + 0, + ); + } + } + } + + SUCCESS +} + +// Resolved at run time because `cargo php stubs` loads the module with stand-ins for the +// ext-php-rs bindings only, and an unresolved Zend symbol would fail that load. +fn attribute_functions() -> Option<(AddAttribute, GetParameterAttribute)> { + let add_attribute = resolve(c"zend_add_attribute")?; + let get_parameter_attribute = resolve(c"zend_get_parameter_attribute_str")?; + // SAFETY: both symbols are the engine's exported functions with the signatures above. + unsafe { + Some(( + mem::transmute::<*mut c_void, AddAttribute>(add_attribute), + mem::transmute::<*mut c_void, GetParameterAttribute>(get_parameter_attribute), + )) + } +} + +fn resolve(symbol: &CStr) -> Option<*mut c_void> { + // SAFETY: dlsym only reads the NUL-terminated name. + let address = unsafe { libc::dlsym(libc::RTLD_DEFAULT, symbol.as_ptr()) }; + (!address.is_null()).then_some(address) +}
diff --git a/foreign/php/src/stream.rs b/foreign/php/src/stream.rs index 2d13c35..eab1f23 100644 --- a/foreign/php/src/stream.rs +++ b/foreign/php/src/stream.rs
@@ -15,8 +15,10 @@ // specific language governing permissions and limitations // under the License. -use ext_php_rs::{php_class, php_impl}; -use iggy::prelude::StreamDetails as RustStreamDetails; +use ext_php_rs::{boxed::ZBox, exception::PhpResult, php_class, php_impl, types::ZendHashTable}; +use iggy::prelude::{HeaderKey, OptionKey, StreamDetails as RustStreamDetails}; + +use crate::options::options_array; #[php_class] #[php(name = "Iggy\\StreamDetails")] @@ -53,4 +55,42 @@ pub fn topics_count(&self) -> u32 { self.inner.topics_count } + + /// Options the creating client set, as option name to scalar value. Key material is + /// never returned. + #[php(getter)] + pub fn options(&self) -> PhpResult<ZBox<ZendHashTable>> { + options_array(&self.inner.options, true) + } + + /// Options the server filled in, including the read-only encryption projections. + #[php(getter, change_case = "snake_case")] + pub fn derived_options(&self) -> PhpResult<ZBox<ZendHashTable>> { + options_array(&self.inner.options, false) + } + + /// Who chose the stream's at-rest key: `stream`, `server`, or `none` for a plaintext + /// stream. A source a newer server reports is returned as its text. + #[php(getter, change_case = "snake_case")] + pub fn encryption_key_source(&self) -> Option<String> { + self.inner + .options + .get(&HeaderKey::from(OptionKey::EncryptionKeySource)) + .and_then(|option| option.value.as_str().ok().map(str::to_owned)) + } + + /// Fingerprint of the stream's key, 16 lowercase hex characters. Compare it with + /// `Client::encryptionKeyFingerprint()` of a key you hold. Null for a plaintext stream. + #[php(getter, change_case = "snake_case")] + pub fn encryption_key_fingerprint(&self) -> Option<String> { + self.inner.encryption_key_fingerprint() + } + + /// Fingerprints of the server keys that wrap the stream's key, comma-separated: one + /// outside a server key rotation. Null for a plaintext stream. + #[php(getter, change_case = "snake_case")] + pub fn encryption_kek_fingerprint(&self) -> Option<String> { + let fingerprints = self.inner.encryption_kek_fingerprints(); + (!fingerprints.is_empty()).then(|| fingerprints.join(",")) + } }
diff --git a/foreign/php/src/topic.rs b/foreign/php/src/topic.rs index d8f1772..7708b7a 100644 --- a/foreign/php/src/topic.rs +++ b/foreign/php/src/topic.rs
@@ -15,9 +15,11 @@ // specific language governing permissions and limitations // under the License. -use ext_php_rs::{php_class, php_impl}; +use ext_php_rs::{boxed::ZBox, exception::PhpResult, php_class, php_impl, types::ZendHashTable}; use iggy::prelude::TopicDetails as RustTopicDetails; +use crate::options::options_array; + #[php_class] #[php(name = "Iggy\\TopicDetails")] pub struct TopicDetails { @@ -51,4 +53,18 @@ pub fn partitions_count(&self) -> u32 { self.inner.partitions_count } + + /// Options the creating or updating client set, as option name to scalar value. + /// Integers above `PHP_INT_MAX`, such as never-expire and unlimited + /// (18446744073709551615), are decimal strings. + #[php(getter)] + pub fn options(&self) -> PhpResult<ZBox<ZendHashTable>> { + options_array(&self.inner.options, true) + } + + /// Options the server filled in from its catalog defaults. + #[php(getter, change_case = "snake_case")] + pub fn derived_options(&self) -> PhpResult<ZBox<ZendHashTable>> { + options_array(&self.inner.options, false) + } }
diff --git a/foreign/php/tests/ErrorTest.php b/foreign/php/tests/ErrorTest.php index 25af60d..6179df7 100644 --- a/foreign/php/tests/ErrorTest.php +++ b/foreign/php/tests/ErrorTest.php
@@ -33,6 +33,24 @@ final class ErrorTest extends TestCase { + #[TestDox('Root policy refusal keeps its typed exception and shared code')] + public function testRootPolicyRefusalKeepsTypedCode(): void + { + $fixture = json_decode((string) file_get_contents(__DIR__ . '/fixtures/resource_options.json'), true, 512, JSON_THROW_ON_ERROR); + $vector = array_values(array_filter( + $fixture['update_resource'], + static fn (array $entry): bool => $entry['case'] === 'user_username_and_status', + ))[0]; + $body = hex2bin($vector['body_hex']); + assert_true($body !== false); + $body[3] = "\0"; + $client = new_client(); + + $error = assert_throws(static fn () => $client->sendBinaryRequest(16, $body)); + assert_instance_of(AuthenticationException::class, $error); + assert_same($fixture['constants']['error_codes']['cannot_modify_root_user'], $error->getCode()); + } + #[TestDox('Connection configuration errors use the typed connection exception')] public function testConnectionConfigurationErrorsUseTypedException(): void {
diff --git a/foreign/php/tests/ResourceOptionsTest.php b/foreign/php/tests/ResourceOptionsTest.php new file mode 100644 index 0000000..beb2d04 --- /dev/null +++ b/foreign/php/tests/ResourceOptionsTest.php
@@ -0,0 +1,640 @@ +<?php +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +declare(strict_types=1); + +use Iggy\Client as IggyClient; +use Iggy\Durability; +use Iggy\Exception\IggyException; +use Iggy\OptionSpec; +use Iggy\OptionsScope; +use Iggy\PollingStrategy; +use Iggy\SendMessage; +use Iggy\StreamDetails; +use Iggy\StreamOptionKey; +use Iggy\TopicDetails; +use Iggy\TopicOptionKey; +use PHPUnit\Framework\Assert; +use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\Attributes\TestDox; +use PHPUnit\Framework\TestCase; + +final class ResourceOptionsTest extends TestCase +{ + private const FINGERPRINT_PATTERN = '/^[0-9a-f]{16}$/'; + private const CATEGORY_MESSAGES = [ + 'invalid_command' => 'Invalid command', + 'invalid_encryption_key' => 'Invalid encryption key', + 'option_not_updatable' => 'Option cannot be updated after creation', + 'option_not_resettable' => 'Option cannot be reset', + 'read_only_option' => 'Option is read-only', + ]; + private const FLAG_UPDATABLE = 1; + private const FLAG_RESETTABLE = 2; + private const FLAG_SECRET = 4; + private const FLAG_READ_ONLY = 8; + private const FLAG_CRITICAL = 16; + private const NEVER = '18446744073709551615'; + // Both durabilities are always sent, so a topic created without options stores them as explicit. + private const DURABILITY_DEFAULTS = ['consumer_offset_durability' => 'replicated', 'durability' => 'replicated']; + private const STREAM_DERIVED_KEYS = ['encryption_kek_fingerprint', 'encryption_key_fingerprint', 'encryption_key_source']; + // Create and update arguments named after the unit they take rather than the option key. + private const PARAMETER_NAMES = ['message_expiry' => 'message_expiry_micros']; + + private array $createdStreams = []; + + public static function transports(): array + { + return [ + 'tcp' => ['tcp'], + 'quic' => ['quic'], + 'http' => ['http'], + 'websocket' => ['websocket'], + ]; + } + + protected function tearDown(): void + { + if ($this->createdStreams !== []) { + $client = new_client(); + foreach ($this->createdStreams as $stream) { + cleanup_stream($client, $stream); + } + } + + parent::tearDown(); + } + + #[DataProvider('transports')] + #[TestDox('A stream created with its own key reports source stream and the key fingerprint on $transport')] + public function testGivenStreamKeyWhenCreatingShouldReadMatchingProjection(string $transport): void + { + $fixture = self::fixture()['stream_key']; + $client = new_transport_client($transport); + $stream = $this->createStream($client, unique_name('keyed-stream'), $fixture['key_base64']); + + self::assertProjection($stream, 'stream'); + assert_same($fixture['fingerprint'], $stream->encryption_key_fingerprint); + assert_same(IggyClient::encryptionKeyFingerprint($fixture['key_base64']), $stream->encryption_key_fingerprint); + } + + #[DataProvider('transports')] + #[TestDox('A stream created without a key reports a server-generated key on $transport')] + public function testGivenKeylessStreamWhenReadingShouldReportServerGeneratedKey(string $transport): void + { + $client = new_transport_client($transport); + $stream = $this->createStream($client, unique_name('keyless-stream')); + + self::assertProjection($stream, 'server'); + } + + #[DataProvider('transports')] + #[TestDox('Renaming a keyed stream keeps its id, key projection and readable messages on $transport')] + public function testGivenStreamRenameWhenUpdatingShouldPreserveEncryptionProjection(string $transport): void + { + $fixture = self::fixture()['stream_key']; + $client = new_transport_client($transport); + $created = $this->createStream($client, unique_name('rename-stream'), $fixture['key_base64']); + $client->createTopic($created->id, 'events', 1, null, null, null, null); + $client->sendMessages($created->id, 'events', 0, [new SendMessage('before-rename')]); + $renamed = unique_name('renamed-stream'); + + $client->updateStream($created->id, name: $renamed); + + $stream = $client->getStream($created->id); + assert_not_null($stream); + assert_same($created->id, $stream->id); + assert_same($renamed, $stream->name); + self::assertProjection($stream, 'stream'); + assert_same($fixture['fingerprint'], $stream->encryption_key_fingerprint); + assert_same($created->encryption_kek_fingerprint, $stream->encryption_kek_fingerprint); + assert_same( + ['before-rename'], + collect_payloads($client->pollMessages($created->id, 'events', 0, PollingStrategy::offset(0), 10, false)), + ); + } + + #[DataProvider('transports')] + #[TestDox('Resetting any stream option is refused with the server category and changes nothing on $transport')] + public function testGivenStreamResetWhenUpdatingShouldRejectWithoutChangingStream(string $transport): void + { + $fixture = self::fixture()['stream_key']; + $client = new_transport_client($transport); + $created = $this->createStream($client, unique_name('reset-stream'), $fixture['key_base64']); + $catalog = self::catalogRows('stream'); + + foreach (StreamOptionKey::cases() as $key) { + self::assertRefused( + static fn () => $client->updateStream($created->id, reset: [$key]), + self::resetCategory($catalog[$key->value]), + ); + } + self::assertRefused(static fn () => $client->updateStream($created->id), 'invalid_command'); + self::assertRefused( + static fn () => $client->updateStream($created->id, name: unique_name('never'), reset: [StreamOptionKey::Name]), + 'invalid_command', + ); + + $stream = $client->getStream($created->id); + assert_not_null($stream); + assert_same($created->name, $stream->name); + assert_same($fixture['fingerprint'], $stream->encryption_key_fingerprint); + } + + #[TestDox('A repeated reset name is refused as an invalid command, as server admission refuses it')] + public function testGivenDuplicateResetWhenUpdatingShouldRejectAsInvalidCommand(): void + { + $client = new_client(); + $created = $this->createStream($client, unique_name('duplicate-reset')); + + self::assertRefused( + static fn () => $client->updateStream($created->id, reset: [StreamOptionKey::Name, StreamOptionKey::Name]), + 'invalid_command', + ); + self::assertRefused( + static fn () => $client->updateStream($created->id, reset: [StreamOptionKey::Name], raw_reset: ['name']), + 'invalid_command', + ); + self::assertRefused( + static fn () => $client->updateStream($created->id, raw_reset: ['future_option', 'future_option']), + 'invalid_command', + ); + } + + #[TestDox('Invalid keys are refused before any request and their text never reaches the error')] + public function testGivenInvalidKeyWhenEncodingShouldRejectBeforeNetworkIo(): void + { + $fixture = self::fixture()['stream_key']; + $neverConnected = new IggyClient(server_address()); + + foreach ($fixture['invalid_keys'] as $vector) { + $input = $vector['input']; + $key = array_key_exists('raw_hex', $input) ? base64_encode(hex2bin($input['raw_hex'])) : $input['base64']; + assert_same('invalid_encryption_key', $vector['error']['name']); + + $fingerprintError = self::assertRefused( + static fn () => IggyClient::encryptionKeyFingerprint($key), + 'invalid_encryption_key', + ); + $createError = self::assertRefused( + static fn () => $neverConnected->createStream('never-sent', $key), + 'invalid_encryption_key', + ); + foreach ([$fingerprintError, $createError] as $error) { + Assert::assertStringNotContainsString(trim($key), $error->getMessage(), $vector['case']); + Assert::assertStringNotContainsString($fixture['key_base64'], $error->getMessage(), $vector['case']); + } + } + + assert_same($fixture['fingerprint'], IggyClient::encryptionKeyFingerprint($fixture['key_base64'])); + $accepted = assert_throws(static fn () => $neverConnected->createStream('never-sent', $fixture['key_base64'])); + Assert::assertStringNotContainsString(self::CATEGORY_MESSAGES['invalid_encryption_key'], $accepted->getMessage()); + } + + #[TestDox('Raw key bytes passed instead of base64 are refused as an invalid key and never reach the exception trace')] + public function testGivenRawKeyBytesWhenCreatingShouldRejectWithoutLeakingKey(): void + { + $neverConnected = new IggyClient(server_address()); + $key = "\xff" . random_bytes(31); + + $errors = [ + self::assertRefused(static fn () => $neverConnected->createStream('never-sent', $key), 'invalid_encryption_key'), + self::assertRefused(static fn () => IggyClient::encryptionKeyFingerprint($key), 'invalid_encryption_key'), + ]; + + foreach ($errors as $error) { + Assert::assertStringNotContainsString(substr($key, 0, 15), $error->getTraceAsString()); + Assert::assertStringNotContainsString($key, print_r($error->getTrace(), true)); + Assert::assertStringNotContainsString(substr($key, 0, 15), $error->getMessage()); + } + } + + #[TestDox('Raw create and update entries reach the server, which decides keys this build does not know')] + public function testGivenRawEntriesWhenCreatingAndUpdatingShouldLetServerDecide(): void + { + $client = new_client(); + foreach (self::fixture()['create_stream'] as $vector) { + if (!array_key_exists('raw', $vector['input'])) { + continue; + } + $name = unique_name($vector['input']['name']); + self::assertRefused( + static fn () => $client->createStream($name, null, $vector['input']['raw']), + $vector['admission_error']['name'], + ); + assert_null($client->getStream($name)); + } + + $created = $this->createStream($client, unique_name('raw-update')); + self::assertRefused( + static fn () => $client->updateStream($created->id, raw: ['future_option' => 'enabled']), + 'unsupported_option_key', + ); + self::assertRefused( + static fn () => $client->updateStream($created->id, raw_reset: ['future_option']), + 'unsupported_option_key', + ); + self::assertRefused( + static fn () => $client->updateStream($created->id, raw_reset: ['encryption_key_source']), + 'read_only_option', + ); + } + + #[TestDox('Stream vectors match the shared fixture: fingerprint, key constants, create admission and update refusals')] + public function testGivenSharedVectorsWhenEncodingShouldMatchServerContract(): void + { + $fixture = self::fixture(); + assert_same(1, $fixture['schema_version']); + assert_same( + $fixture['stream_key']['fingerprint'], + IggyClient::encryptionKeyFingerprint($fixture['stream_key']['key_base64']), + ); + + $client = new_client(); + foreach ($fixture['create_stream'] as $vector) { + $input = $vector['input']; + if (array_key_exists('raw', $input)) { + continue; + } + $key = array_key_exists('encryption_key', $input) + ? base64_encode(hex2bin($input['encryption_key']['raw_hex'])) + : null; + assert_null($vector['admission_error'], $vector['case']); + $stream = $this->createStream($client, unique_name($input['name']), $key); + self::assertProjection($stream, $key === null ? 'server' : 'stream'); + if ($key !== null) { + assert_same($fixture['stream_key']['fingerprint'], $stream->encryption_key_fingerprint); + } + } + + $target = $this->createStream($client, unique_name('update-errors')); + foreach ($fixture['update_resource_errors'] as $vector) { + if ($vector['scope'] !== 'stream') { + continue; + } + $input = $vector['input']; + $reset = array_map(static fn (string $key): StreamOptionKey => StreamOptionKey::from($key), $input['reset'] ?? []); + self::assertRefused( + static fn () => $client->updateStream($target->id, name: $input['name'] ?? null, reset: $reset), + $vector['error']['name'], + ); + } + } + + #[DataProvider('transports')] + #[TestDox('A topic created with options reads them back with provenance, and an update renames, sets and resets on $transport')] + public function testGivenTopicOptionsWhenCreatingAndUpdatingShouldReadThemWithProvenance(string $transport): void + { + $vector = self::fixture()['create_topic'][0]; + $input = $vector['input']; + $client = new_transport_client($transport); + $stream = $this->createStream($client, unique_name('topic-options')); + $topicName = unique_name($input['name']); + + $client->createTopic( + $stream->id, + $topicName, + $input['partitions_count'], + $input['compression_algorithm'], + $input['message_expiry'], + $input['max_topic_size'], + $input['segment_size'], + Durability::from($input['durability']), + Durability::from($input['consumer_offset_durability']), + partition_resize_policy: 'fixed', + ); + + $expected = self::phpOptions($vector['wire_options']); + $expected['partition_resize_policy'] = 'fixed'; + ksort($expected); + $topic = $client->getTopic($stream->id, $topicName); + assert_not_null($topic); + self::assertTopicOptions($topic, $expected); + + $renamed = unique_name('renamed-topic'); + $client->updateTopic( + $stream->id, + $topicName, + name: $renamed, + message_expiry_micros: self::NEVER, + reset: [TopicOptionKey::MaxTopicSize], + ); + + $updated = $client->getTopic($stream->id, $renamed); + assert_not_null($updated); + assert_same($topic->id, $updated->id); + $expected['message_expiry'] = self::NEVER; + unset($expected['max_topic_size']); + self::assertTopicOptions($updated, $expected); + assert_same(self::NEVER, $updated->derived_options['max_topic_size']); + + $client->updateTopic($stream->id, $renamed, max_topic_size: $input['max_topic_size']); + $client->updateTopic($stream->id, $renamed, raw_reset: ['max_topic_size']); + $reset = $client->getTopic($stream->id, $renamed); + assert_not_null($reset); + self::assertTopicOptions($reset, $expected); + } + + #[TestDox('Topic update vectors are refused with the server category and the topic keeps its options')] + public function testGivenTopicUpdateVectorsWhenUpdatingShouldRejectWithServerCategory(): void + { + $client = new_client(); + $stream = $this->createStream($client, unique_name('topic-errors')); + $client->createTopic($stream->id, 'events', 1, null, 604800000000, null, null); + $before = $client->getTopic($stream->id, 'events'); + assert_not_null($before); + + foreach (self::fixture()['update_resource_errors'] as $vector) { + if ($vector['scope'] !== 'topic') { + continue; + } + $input = $vector['input']; + $reset = array_map(static fn (string $key): TopicOptionKey => TopicOptionKey::from($key), $input['reset'] ?? []); + self::assertRefused( + static fn () => $client->updateTopic( + $stream->id, + 'events', + name: $input['name'] ?? null, + message_expiry_micros: $input['message_expiry'] ?? null, + reset: $reset, + ), + $vector['error']['name'], + ); + } + self::assertRefused( + static fn () => $client->updateTopic($stream->id, 'events', reset: [TopicOptionKey::MaxTopicSize, TopicOptionKey::MaxTopicSize]), + 'invalid_command', + ); + + $after = $client->getTopic($stream->id, 'events'); + assert_not_null($after); + assert_same($before->name, $after->name); + assert_same($before->options, $after->options); + assert_same($before->derived_options, $after->derived_options); + } + + #[TestDox('A topic created without options reads the catalog defaults as derived, and never-expire and unlimited travel exactly')] + public function testGivenTopicDefaultsWhenReadingShouldReportDerivedCatalogValues(): void + { + $client = new_client(); + $stream = $this->createStream($client, unique_name('topic-defaults')); + $client->createTopic($stream->id, 'defaults', 1, null, null, null, null); + $client->createTopic($stream->id, 'unbounded', 1, null, self::NEVER, self::NEVER, null); + + $defaults = $client->getTopic($stream->id, 'defaults'); + assert_not_null($defaults); + $explicit = self::DURABILITY_DEFAULTS; + assert_same($explicit, self::sorted($defaults->options)); + $expected = []; + foreach (self::catalogRows('topic') as $key => $row) { + if ($row['storage'] === 'map' && !array_key_exists($key, $explicit)) { + $expected[$key] = self::phpScalar($row['default'], $row['kind']); + } + } + ksort($expected); + $derived = $defaults->derived_options; + ksort($derived); + assert_same($expected, $derived); + + $unbounded = $client->getTopic($stream->id, 'unbounded'); + assert_not_null($unbounded); + assert_same( + self::sorted([...self::DURABILITY_DEFAULTS, 'max_topic_size' => self::NEVER, 'message_expiry' => self::NEVER]), + self::sorted($unbounded->options), + ); + } + + #[TestDox('describeOptions reports every described catalog row with its kind and flags')] + public function testGivenEveryScopeWhenDescribingOptionsShouldMatchCatalog(): void + { + $client = new_client(); + foreach (OptionsScope::cases() as $scope) { + $described = []; + foreach ($client->describeOptions($scope) as $spec) { + Assert::assertInstanceOf(OptionSpec::class, $spec); + $described[$spec->key] = [ + $spec->kind, + $spec->updatable, + $spec->resettable, + $spec->secret, + $spec->read_only, + $spec->critical, + ]; + } + $expected = []; + foreach (self::catalogRows($scope->value) as $key => $row) { + $expected[$key] = [ + $row['kind'], + ($row['flags'] & self::FLAG_UPDATABLE) !== 0, + ($row['flags'] & self::FLAG_RESETTABLE) !== 0, + ($row['flags'] & self::FLAG_SECRET) !== 0, + ($row['flags'] & self::FLAG_READ_ONLY) !== 0, + ($row['flags'] & self::FLAG_CRITICAL) !== 0, + ]; + } + assert_same(self::sorted($expected), self::sorted($described), $scope->value); + } + } + + #[TestDox('The typed surface covers every catalog row: create arguments, update arguments, reset cases and accessors')] + public function testGivenFixtureCatalogWhenInspectingTypedSurfaceShouldCoverEveryRow(): void + { + $surface = [ + 'stream' => [StreamOptionKey::class, 'createStream', 'updateStream'], + 'topic' => [TopicOptionKey::class, 'createTopic', 'updateTopic'], + ]; + $client = new_client(); + $stream = $this->createStream($client, unique_name('surface')); + $client->createTopic($stream->id, 'events', 1, null, null, null, null); + $details = [ + 'stream' => $client->getStream($stream->id), + 'topic' => $client->getTopic($stream->id, 'events'), + ]; + + foreach ($surface as $scope => [$enum, $create, $update]) { + $rows = self::catalogRows($scope); + $cases = array_map(static fn ($key): string => $key->value, $enum::cases()); + assert_same(self::sorted(array_keys($rows)), self::sorted($cases), "$scope reset cases"); + $createParameters = self::parameterNames($create); + $updateParameters = self::parameterNames($update); + Assert::assertContains('raw', $createParameters, "$scope create raw"); + Assert::assertContains('raw', $updateParameters, "$scope update raw"); + Assert::assertContains('raw_reset', $updateParameters, "$scope update raw_reset"); + foreach ($rows as $key => $row) { + $parameter = self::PARAMETER_NAMES[$key] ?? $key; + if ($row['storage'] === 'map' && in_array($row['mutability'], ['create_only', 'updatable'], true)) { + Assert::assertContains($parameter, $createParameters, "$scope create $key"); + } + if ($row['mutability'] === 'updatable') { + Assert::assertContains($parameter, $updateParameters, "$scope update $key"); + } + if ($row['mutability'] === 'read_only') { + Assert::assertTrue(property_exists($details[$scope], $key), "$scope accessor $key"); + } + } + } + } + + private function createStream(IggyClient $client, string $name, ?string $key = null): StreamDetails + { + $client->createStream($name, $key); + $stream = $client->getStream($name); + assert_not_null($stream); + $this->createdStreams[] = $stream->id; + + return $stream; + } + + private static function assertRefused(callable $callback, string $category): Throwable + { + $error = assert_throws($callback, self::CATEGORY_MESSAGES[$category] ?? null); + Assert::assertInstanceOf(IggyException::class, $error, $error->getMessage()); + assert_same(self::errorCodes()[$category], $error->getCode(), $category); + + return $error; + } + + private static function assertProjection(StreamDetails $stream, string $source): void + { + assert_same([], $stream->options); + assert_same(self::STREAM_DERIVED_KEYS, array_keys(self::sorted($stream->derived_options))); + assert_same($source, $stream->encryption_key_source); + assert_same($source, $stream->derived_options['encryption_key_source']); + assert_same($stream->encryption_key_fingerprint, $stream->derived_options['encryption_key_fingerprint']); + assert_same($stream->encryption_kek_fingerprint, $stream->derived_options['encryption_kek_fingerprint']); + Assert::assertMatchesRegularExpression(self::FINGERPRINT_PATTERN, (string) $stream->encryption_key_fingerprint); + Assert::assertNotNull($stream->encryption_kek_fingerprint); + foreach (explode(',', $stream->encryption_kek_fingerprint) as $kekFingerprint) { + Assert::assertMatchesRegularExpression(self::FINGERPRINT_PATTERN, $kekFingerprint); + } + foreach (self::fixture()['constants']['never_returned_keys'] as $secret) { + Assert::assertArrayNotHasKey($secret, $stream->options); + Assert::assertArrayNotHasKey($secret, $stream->derived_options); + } + } + + private static function assertTopicOptions(TopicDetails $topic, array $explicit): void + { + assert_same($explicit, self::sorted($topic->options)); + $mapKeys = []; + foreach (self::catalogRows('topic') as $key => $row) { + if ($row['storage'] === 'map') { + $mapKeys[] = $key; + } + } + $readKeys = [...array_keys($topic->options), ...array_keys($topic->derived_options)]; + assert_same(self::sorted($mapKeys), self::sorted($readKeys)); + } + + private static function phpOptions(array $wireOptions): array + { + $options = []; + foreach ($wireOptions as $key => $option) { + $options[$key] = self::phpScalar($option['value'], $option['kind']); + } + ksort($options); + + return $options; + } + + // The projection the SDK documents: integers above PHP_INT_MAX stay decimal strings. + private static function phpScalar(string $value, string $kind): mixed + { + return match ($kind) { + 'bool' => $value === 'true', + 'uint8', 'uint16', 'uint32', 'uint64', 'int8', 'int16', 'int32', 'int64' => self::integerOrString($value), + default => $value, + }; + } + + private static function integerOrString(string $value): int|string + { + $integer = filter_var($value, FILTER_VALIDATE_INT); + + return $integer === false ? $value : $integer; + } + + private static function parameterNames(string $method): array + { + return array_map( + static fn (ReflectionParameter $parameter): string => $parameter->getName(), + (new ReflectionMethod(IggyClient::class, $method))->getParameters(), + ); + } + + private static function sorted(array $values): array + { + if (array_is_list($values)) { + sort($values); + } else { + ksort($values); + } + + return $values; + } + + private static function errorCodes(): array + { + static $codes = null; + if ($codes !== null) { + return $codes; + } + $fixture = self::fixture(); + $errors = [ + ...array_column($fixture['update_resource_errors'], 'error'), + ...array_column($fixture['stream_key']['invalid_keys'], 'error'), + ...array_filter(array_column($fixture['create_stream'], 'admission_error')), + ]; + $codes = array_column($errors, 'code', 'name'); + + return $codes; + } + + private static function catalogRows(string $scope): array + { + $rows = []; + foreach (self::fixture()['catalog'] as $row) { + if ($row['described'] && in_array($scope, $row['scopes'], true)) { + $rows[$row['key']] = $row; + } + } + + return $rows; + } + + private static function resetCategory(array $row): string + { + return match (true) { + in_array($row['visibility'], ['read_only', 'internal'], true) => 'read_only_option', + $row['mutability'] === 'create_only' => 'option_not_updatable', + default => 'option_not_resettable', + }; + } + + private static function fixture(): array + { + static $fixture = null; + + return $fixture ??= json_decode( + (string) file_get_contents(__DIR__ . '/fixtures/resource_options.json'), + true, + 512, + JSON_THROW_ON_ERROR | JSON_BIGINT_AS_STRING, + ); + } +}
diff --git a/foreign/php/tests/SensitiveParameterTest.php b/foreign/php/tests/SensitiveParameterTest.php new file mode 100644 index 0000000..42142b2 --- /dev/null +++ b/foreign/php/tests/SensitiveParameterTest.php
@@ -0,0 +1,166 @@ +<?php +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +declare(strict_types=1); + +use Iggy\Client as IggyClient; +use PHPUnit\Framework\Assert; +use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\Attributes\TestDox; +use PHPUnit\Framework\TestCase; + +final class SensitiveParameterTest extends TestCase +{ + private array $previousIni = []; + + public static function sensitiveParameters(): array + { + return [ + 'connection string' => ['fromConnectionString', 'connection_string'], + 'password' => ['loginUser', 'password'], + 'raw binary payload' => ['sendBinaryRequest', 'payload'], + 'stream key' => ['createStream', 'encryption_key'], + 'fingerprinted key' => ['encryptionKeyFingerprint', 'key'], + 'raw stream options' => ['createStream', 'raw'], + 'raw stream update' => ['updateStream', 'raw'], + 'raw topic options' => ['createTopic', 'raw'], + 'raw topic update' => ['updateTopic', 'raw'], + ]; + } + + // A raw entry may name a secret this build does not know, so every raw array is hidden. + public static function rawOptionCalls(): array + { + return [ + 'createStream' => ['createStream', 2, static fn (IggyClient $client, array $raw) => $client->createStream('never-sent', null, $raw, 'unexpected')], + 'updateStream' => ['updateStream', 3, static fn (IggyClient $client, array $raw) => $client->updateStream('never-sent', null, [], $raw, null, 'unexpected')], + 'createTopic' => ['createTopic', 13, static fn (IggyClient $client, array $raw) => $client->createTopic('never-sent', 'never-sent', 1, null, null, null, null, null, null, null, null, null, null, $raw, 'unexpected')], + 'updateTopic' => ['updateTopic', 6, static fn (IggyClient $client, array $raw) => $client->updateTopic('never-sent', 'never-sent', null, null, null, [], $raw, null, 'unexpected')], + ]; + } + + protected function setUp(): void + { + parent::setUp(); + // Distribution php.ini files turn argument capture off, which would hide a leak. + foreach (['zend.exception_ignore_args' => '0', 'zend.exception_string_param_max_len' => '15'] as $name => $value) { + $previous = ini_set($name, $value); + if ($previous !== false) { + $this->previousIni[$name] = $previous; + } + } + } + + protected function tearDown(): void + { + foreach ($this->previousIni as $name => $value) { + ini_set($name, $value); + } + + parent::tearDown(); + } + + #[DataProvider('sensitiveParameters')] + #[TestDox('The $parameter parameter of $method is marked SensitiveParameter')] + public function testGivenSecretParameterWhenReflectingShouldCarrySensitiveParameter(string $method, string $parameter): void + { + $parameters = (new ReflectionMethod(IggyClient::class, $method))->getParameters(); + $matching = array_values(array_filter( + $parameters, + static fn (ReflectionParameter $candidate): bool => $candidate->getName() === $parameter, + )); + + assert_count(1, $matching, "{$method} has no parameter {$parameter}"); + assert_count(1, $matching[0]->getAttributes(SensitiveParameter::class), "{$method}({$parameter})"); + } + + // Engine argument errors are raised inside the native frame, so its arguments land in the trace. + #[TestDox('A stream key never reaches the trace when createStream rejects another argument')] + public function testGivenStreamKeyWhenCreateRejectsArgumentShouldHideKeyInTrace(): void + { + $key = base64_encode(random_bytes(32)); + $client = new IggyClient(server_address()); + + $error = assert_throws(static fn () => $client->createStream('never-sent', $key, 'not-an-array')); + + assert_instance_of(TypeError::class, $error); + self::assertHidden($error, 'createStream', 1, $key); + } + + #[DataProvider('rawOptionCalls')] + #[TestDox('A raw stream key never reaches the trace when $method rejects the call')] + public function testGivenRawStreamKeyWhenCallRejectsArgumentsShouldHideRawOptionsInTrace(string $method, int $position, callable $call): void + { + $key = base64_encode(random_bytes(32)); + $client = new IggyClient(server_address()); + + $error = assert_throws(static fn () => $call($client, ['encryption_key' => $key])); + + assert_instance_of(ArgumentCountError::class, $error); + self::assertHidden($error, $method, $position, $key); + } + + #[TestDox('A stream key never reaches the trace when encryptionKeyFingerprint rejects the call')] + public function testGivenStreamKeyWhenFingerprintRejectsCallShouldHideKeyInTrace(): void + { + $key = base64_encode(random_bytes(32)); + + $error = assert_throws(static fn () => IggyClient::encryptionKeyFingerprint($key, 'unexpected')); + + assert_instance_of(ArgumentCountError::class, $error); + self::assertHidden($error, 'encryptionKeyFingerprint', 0, $key); + } + + #[TestDox('A password never reaches the trace when loginUser rejects the call')] + public function testGivenPasswordWhenLoginRejectsCallShouldHidePasswordInTrace(): void + { + $password = bin2hex(random_bytes(16)); + $client = new IggyClient(server_address()); + + $error = assert_throws(static fn () => $client->loginUser('iggy', $password, 'unexpected')); + + assert_instance_of(ArgumentCountError::class, $error); + self::assertHidden($error, 'loginUser', 1, $password); + } + + #[TestDox('Connection string credentials never reach the trace when fromConnectionString rejects the call')] + public function testGivenConnectionStringWhenFactoryRejectsCallShouldHideCredentialsInTrace(): void + { + $connectionString = 'iggy+tcp://iggy:' . bin2hex(random_bytes(16)) . '@' . server_address(); + + $error = assert_throws(static fn () => IggyClient::fromConnectionString($connectionString, 'unexpected')); + + assert_instance_of(ArgumentCountError::class, $error); + self::assertHidden($error, 'fromConnectionString', 0, $connectionString); + } + + private static function assertHidden(Throwable $error, string $method, int $position, string $secret): void + { + $frames = array_values(array_filter( + $error->getTrace(), + static fn (array $frame): bool => ($frame['class'] ?? null) === IggyClient::class + && $frame['function'] === $method, + )); + + assert_count(1, $frames, "{$method} frame in the trace"); + Assert::assertArrayHasKey('args', $frames[0], 'exception arguments are captured'); + assert_instance_of(SensitiveParameterValue::class, $frames[0]['args'][$position] ?? null, $method); + Assert::assertStringNotContainsString(substr($secret, 0, 15), $error->getTraceAsString()); + Assert::assertStringNotContainsString($secret, print_r($frames[0]['args'], true)); + } +}
diff --git a/foreign/php/tests/bootstrap.php b/foreign/php/tests/bootstrap.php index 5bf4a0f..20ce827 100644 --- a/foreign/php/tests/bootstrap.php +++ b/foreign/php/tests/bootstrap.php
@@ -141,6 +141,34 @@ return $client; } +/** + * Logged-in client for one transport. QUIC, HTTP and WebSocket addresses default to the + * server's default ports on the TCP host. + */ +function new_transport_client(string $transport): IggyClient +{ + if ($transport === 'tcp') { + return new_client(); + } + + $schemes = ['quic' => 'iggy+quic', 'http' => 'iggy+http', 'websocket' => 'iggy+ws']; + $addresses = [ + 'quic' => env_or_default('IGGY_QUIC_ADDRESS', server_host() . ':8080'), + 'http' => env_or_default('IGGY_HTTP_ADDRESS', server_host() . ':3000'), + 'websocket' => env_or_default('IGGY_WEBSOCKET_ADDRESS', server_host() . ':8092'), + ]; + $username = rawurlencode(env_or_default('IGGY_ROOT_USERNAME', 'iggy')); + $password = rawurlencode(env_or_default('IGGY_ROOT_PASSWORD', 'iggy')); + + $client = IggyClient::fromConnectionString("{$schemes[$transport]}://{$username}:{$password}@{$addresses[$transport]}"); + $client->connect(); + if ($transport === 'http') { + $client->loginUser(env_or_default('IGGY_ROOT_USERNAME', 'iggy'), env_or_default('IGGY_ROOT_PASSWORD', 'iggy')); + } + + return $client; +} + function create_stream_and_topic(object $client, string $stream, string $topic, int $partitions = 1): void { $client->createStream($stream);
diff --git a/foreign/php/tests/fixtures/resource_options.json b/foreign/php/tests/fixtures/resource_options.json new file mode 100644 index 0000000..c5cc2ac --- /dev/null +++ b/foreign/php/tests/fixtures/resource_options.json
@@ -0,0 +1,1619 @@ +{ + "schema_version": 1, + "generated_by": "core/common/examples/resource_options_fixtures.rs", + "conventions": { + "byte_strings": "Every *_hex field is lowercase hex without separators.", + "errors": "name is the snake_case IggyError name and code its numeric code.", + "max_topic_size": "Bytes. 18446744073709551615 means unlimited.", + "message_expiry": "Microseconds. 18446744073709551615 means never expire.", + "never_returned_keys": "No response, decoded projection or accessor may contain these keys or their values.", + "option_maps": "Option maps use the HTTP JSON form: value as reversible text, kind, and explicit provenance. Request blocks mark every entry explicit.", + "options_block_order": "Options block entries are encoded in ascending key byte order.", + "request_bodies": "body_hex and update_resource_hex are command payloads without framing. constants.command_codes maps each command to its code and VSR operation.", + "reset_order": "The typed encoders and the HTTP handler write reset keys in ascending key byte order.", + "stream_state": "state is the stored server state a stream response was projected from. SDKs need only response_hex and expected.", + "u64_values": "u64 option values and typed u64 inputs are decimal strings, so values above 2^53 keep their precision." + }, + "constants": { + "protocol_version": { + "packed": 12288, + "text": "0.12.0", + "major": 0, + "minor": 12, + "patch": 0 + }, + "protocol_version_min": { + "packed": 12288, + "text": "0.12.0", + "major": 0, + "minor": 12, + "patch": 0 + }, + "command_codes": { + "create_stream": { + "code": 202, + "operation": 128 + }, + "create_topic": { + "code": 302, + "operation": 132 + }, + "create_user": { + "code": 33, + "operation": 141 + }, + "describe_options": { + "code": 13, + "operation": null + }, + "get_stream": { + "code": 200, + "operation": null + }, + "get_streams": { + "code": 201, + "operation": null + }, + "get_topic": { + "code": 300, + "operation": null + }, + "get_topics": { + "code": 301, + "operation": null + }, + "get_user": { + "code": 31, + "operation": null + }, + "get_users": { + "code": 32, + "operation": null + }, + "update_resource": { + "code": 16, + "operation": 150 + } + }, + "error_codes": { + "cannot_modify_root_user": 4108 + }, + "retired_command_codes": { + "update_stream": 204, + "update_topic": 304, + "update_user": 35 + }, + "operations": { + "rewrap_stream_keys": { + "value": 75, + "internal": true, + "request_operation": 75 + }, + "update_resource": { + "value": 150, + "internal": false, + "request_operation": 150 + }, + "update_stream_resolved": { + "value": 72, + "internal": true, + "request_operation": 150 + }, + "update_topic_resolved": { + "value": 73, + "internal": true, + "request_operation": 150 + }, + "update_user_resolved": { + "value": 74, + "internal": true, + "request_operation": 150 + } + }, + "retired_operations": { + "update_stream": 129, + "update_topic": 133, + "update_user": 142 + }, + "scope_codes": { + "stream": 2, + "topic": 1, + "user": 3 + }, + "header_kind_codes": { + "raw": 1, + "string": 2, + "bool": 3, + "uint32": 11, + "uint64": 12 + }, + "option_flags": { + "critical": 16, + "read_only": 8, + "resettable": 2, + "secret": 4, + "updatable": 1 + }, + "user_status_codes": { + "active": 1, + "inactive": 2 + }, + "stream_key_length": 32, + "never_returned_keys": [ + "encryption_key", + "encryption_key_wrapped", + "critical_keys" + ] + }, + "catalog": [ + { + "key": "name", + "scopes": [ + "stream", + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "column", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 1, + "described": true, + "default": null + }, + { + "key": "username", + "scopes": [ + "user" + ], + "kind": "string", + "kind_code": 2, + "storage": "column", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 1, + "described": true, + "default": null + }, + { + "key": "status", + "scopes": [ + "user" + ], + "kind": "string", + "kind_code": 2, + "storage": "column", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 1, + "described": true, + "default": null + }, + { + "key": "compression_algorithm", + "scopes": [ + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "none" + }, + { + "key": "message_expiry", + "scopes": [ + "topic" + ], + "kind": "uint64", + "kind_code": 12, + "storage": "map", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 3, + "described": true, + "default": "18446744073709551615" + }, + { + "key": "max_topic_size", + "scopes": [ + "topic" + ], + "kind": "uint64", + "kind_code": 12, + "storage": "map", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 3, + "described": true, + "default": "18446744073709551615" + }, + { + "key": "segment_size", + "scopes": [ + "topic" + ], + "kind": "uint64", + "kind_code": 12, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "1073741824" + }, + { + "key": "durability", + "scopes": [ + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "replicated" + }, + { + "key": "consumer_offset_durability", + "scopes": [ + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "replicated" + }, + { + "key": "partition_resize_policy", + "scopes": [ + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "mutable" + }, + { + "key": "messages_required_to_save", + "scopes": [ + "topic" + ], + "kind": "uint32", + "kind_code": 11, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "1024" + }, + { + "key": "size_of_messages_required_to_save", + "scopes": [ + "topic" + ], + "kind": "uint64", + "kind_code": 12, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "1048576" + }, + { + "key": "preallocate_segments", + "scopes": [ + "topic" + ], + "kind": "bool", + "kind_code": 3, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "false" + }, + { + "key": "encryption_key", + "scopes": [ + "stream" + ], + "kind": "raw", + "kind_code": 1, + "storage": "map", + "mutability": "create_only", + "visibility": "secret", + "critical": true, + "flags": 20, + "described": true, + "default": null + }, + { + "key": "encryption_key_wrapped", + "scopes": [ + "stream" + ], + "kind": "raw", + "kind_code": 1, + "storage": "map", + "mutability": "create_only", + "visibility": "internal", + "critical": true, + "flags": 16, + "described": false, + "default": null + }, + { + "key": "encryption_key_source", + "scopes": [ + "stream" + ], + "kind": "string", + "kind_code": 2, + "storage": "computed", + "mutability": "read_only", + "visibility": "read_only", + "critical": false, + "flags": 8, + "described": true, + "default": null + }, + { + "key": "encryption_key_fingerprint", + "scopes": [ + "stream" + ], + "kind": "string", + "kind_code": 2, + "storage": "computed", + "mutability": "read_only", + "visibility": "read_only", + "critical": false, + "flags": 8, + "described": true, + "default": null + }, + { + "key": "encryption_kek_fingerprint", + "scopes": [ + "stream" + ], + "kind": "string", + "kind_code": 2, + "storage": "computed", + "mutability": "read_only", + "visibility": "read_only", + "critical": false, + "flags": 8, + "described": true, + "default": null + }, + { + "key": "critical_keys", + "scopes": [ + "stream", + "topic", + "user" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "internal", + "critical": false, + "flags": 0, + "described": false, + "default": null + } + ], + "describe_options": [ + { + "scope": "topic", + "request_hex": "01", + "response_hex": "0b000000046e616d650201000000007e0000005265736f75726365206e616d652c203120746f203235352062797465732c20756e697175652077697468696e2069747320706172656e742e20536574206279207468652063726561746520636f6d6d616e642773206e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e15636f6d7072657373696f6e5f616c676f726974686d0200040000006e6f6e6550000000436f6d7072657373696f6e20616c676f726974686d20286e6f6e652c20677a6970292e2053746f72656420616e64207265706f727465642c206e6f74206170706c69656420746f206d657373616765730e6d6573736167655f6578706972790c0308000000ffffffffffffffff430000004d6573736167652065787069727920696e206d6963726f7365636f6e64732c206f7220612068756d616e74696d6520737472696e672028652e672e20372064617973290e6d61785f746f7069635f73697a650c0308000000ffffffffffffffff9d000000546f7069632d77696465207365616c65642d7365676d656e742073697a65206361702c2073706c6974206163726f737320616c6c20706172746974696f6e732c20696e206279746573206f72206120627974652d73697a6520737472696e672028652e672e203120476942293b2066696e6974652076616c756573206d757374206265206174206c6561737420746865207365676d656e742073697a650c7365676d656e745f73697a650c000800000000000040000000006c0000005365676d656e742073697a6520696e2062797465732c206f72206120627974652d73697a6520737472696e672028652e672e20313238204d6942293b2061203531322d62797465206d756c7469706c652077697468696e20313034383537362e2e3d313037333734313832340a6475726162696c69747902000a0000007265706c6963617465643e0100004d65737361676520636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20412073696e676c65746f6e2071756f72756d20686173206f6e6520636f70792e20496e207265706c6963617465642067726f7570732c20706572736973746564206d65737361676573207573652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c2057414c207265636c616d6174696f6e2e205468652066756c6c20626f64792073697a65207374696c6c20636f756e747320616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e1a636f6e73756d65725f6f66667365745f6475726162696c69747902000a0000007265706c6963617465646e0100004578706c69636974206f666673657420636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20506f6c6c206175746f2d636f6d6d69742072656d61696e73206173796e6368726f6e6f75732e20496e207265706c6963617465642067726f7570732c20706572736973746564206f66667365747320616c736f20656e61626c652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c207265636c616d6174696f6e2c206576656e2077697468207265706c696361746564206d657373616765206475726162696c6974792e2046756c6c20626f64792073697a657320636f756e7420616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e17706172746974696f6e5f726573697a655f706f6c6963790200070000006d757461626c65c8000000496d6d757461626c65206166746572206372656174696f6e3a206d757461626c65207065726d69747320706172746974696f6e206164646974696f6e2f64656c6574696f6e20776974686f7574206120626f756e646564206375746f766572206f72207065722d6b6579206f72646572696e67206163726f7373206c61796f7574733b2066697865642072656a6563747320626f74682e20526574656e74696f6e20616e6420746f7069632064656c6574696f6e2072656d61696e2064657374727563746976652e196d657373616765735f72657175697265645f746f5f736176650b000400000000040000870000004f7264696e617279206d6573736167652d636f756e7420666c75736820747269676765723b20312e2e3d31363737373231362e2052657175697265642070657273697374656e63652c2063617061636974792070726573737572652c20616e64206c6966656379636c65206f7065726174696f6e732063616e20666c757368206561726c6965722173697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c0008000000000010000000000083000000466c75736820746865206a6f75726e616c206f6e636520697420686f6c64732074686973206d616e792062797465732c206f72206120627974652d73697a6520737472696e673b20776869636865766572207468726573686f6c6420747269707320666972737420666c75736865732e204174206d6f7374203130373337343138323414707265616c6c6f636174655f7365676d656e7473030001000000001201000052657175657374207365676d656e745f73697a65206279746573206f66206469736b207265736572766174696f6e207768656e2065616368207365676d656e74206f70656e732e20556e737570706f72746564206f72206661696c6564207265736572766174696f6e732066616c6c206261636b20746f206f7264696e61727920616c6c6f636174696f6e20776974682061207761726e696e672e205265736572766174696f6e2072756e7320696e6c696e65206f6e20746865206f776e696e672073686172642e207365676d656e745f73697a65202a20706172746974696f6e735f636f756e74206973206361707065642061742036383731393437363733362062797465732070657220637265617465", + "entries": [ + { + "key": "name", + "kind": "string", + "flags": 1, + "default": null + }, + { + "key": "compression_algorithm", + "kind": "string", + "flags": 0, + "default": "none" + }, + { + "key": "message_expiry", + "kind": "uint64", + "flags": 3, + "default": "18446744073709551615" + }, + { + "key": "max_topic_size", + "kind": "uint64", + "flags": 3, + "default": "18446744073709551615" + }, + { + "key": "segment_size", + "kind": "uint64", + "flags": 0, + "default": "1073741824" + }, + { + "key": "durability", + "kind": "string", + "flags": 0, + "default": "replicated" + }, + { + "key": "consumer_offset_durability", + "kind": "string", + "flags": 0, + "default": "replicated" + }, + { + "key": "partition_resize_policy", + "kind": "string", + "flags": 0, + "default": "mutable" + }, + { + "key": "messages_required_to_save", + "kind": "uint32", + "flags": 0, + "default": "1024" + }, + { + "key": "size_of_messages_required_to_save", + "kind": "uint64", + "flags": 0, + "default": "1048576" + }, + { + "key": "preallocate_segments", + "kind": "bool", + "flags": 0, + "default": "false" + } + ] + }, + { + "scope": "stream", + "request_hex": "02", + "response_hex": "05000000046e616d650201000000007e0000005265736f75726365206e616d652c203120746f203235352062797465732c20756e697175652077697468696e2069747320706172656e742e20536574206279207468652063726561746520636f6d6d616e642773206e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e0e656e6372797074696f6e5f6b6579011400000000a30100005468652073747265616d2773206f776e206461746120656e6372797074696f6e206b65793a2033322072616e646f6d2062797465732c20726177206f72207374616e6461726420706164646564206261736536342e204372656174652d6f6e6c7920616e642077726974652d6f6e6c793a20746865207365727665722073746f726573206974207772617070656420756e6465722061206b657920646572697665642066726f6d207468652073657276657220656e6372797074696f6e206b657920616e64206e657665722072657475726e732069742e205265717569726573207468652073657276657220656e6372797074696f6e206b657920746f20626520636f6e666967757265642e20576974686f75742069742c206120736572766572207769746820656e6372797074696f6e20656e61626c65642067656e6572617465732061206b657920666f72207468652073747265616d2e20546865206b65792074726176656c7320696e20746865206372656174652072657175657374206f76657220746865207472616e73706f72742074686520636c69656e7420757365732e15656e6372797074696f6e5f6b65795f736f757263650208000000009f00000057686f2063686f7365207468652073747265616d27732061742d72657374206b65793a2073747265616d207768656e2074686520636c69656e742073656e742069742c20736572766572207768656e20746865207365727665722067656e6572617465642069742c206e6f6e65207768656e207468652073747265616d2773206d65737361676573206172652073746f72656420756e656e637279707465641a656e6372797074696f6e5f6b65795f66696e6765727072696e740208000000009f0000005468652066697273742038206279746573206f66205348412d323536206f76657220746865206c6162656c20696767792f73747265616d2d6b65792d66696e6765727072696e742f763120616e64207468652073747265616d2773206b65792c206173203136206c6f776572636173652068657820636861726163746572732e20416273656e74207768656e2074686520736f75726365206973206e6f6e651a656e6372797074696f6e5f6b656b5f66696e6765727072696e74020800000000b800000046696e6765727072696e7473206f6620746865206b65792d656e6372797074696f6e206b65797320746861742077726170207468652073747265616d2773206b65792c20636f6d6d612d7365706172617465643a206f6e65206f7574736964652061206b657920726f746174696f6e2c206f6e652070657220636f6e6669677572656420736572766572206b657920647572696e672069742e20416273656e74207768656e2074686520736f75726365206973206e6f6e65", + "entries": [ + { + "key": "name", + "kind": "string", + "flags": 1, + "default": null + }, + { + "key": "encryption_key", + "kind": "raw", + "flags": 20, + "default": null + }, + { + "key": "encryption_key_source", + "kind": "string", + "flags": 8, + "default": null + }, + { + "key": "encryption_key_fingerprint", + "kind": "string", + "flags": 8, + "default": null + }, + { + "key": "encryption_kek_fingerprint", + "kind": "string", + "flags": 8, + "default": null + } + ] + }, + { + "scope": "user", + "request_hex": "03", + "response_hex": "0200000008757365726e616d650201000000006b00000055736572206e616d652c203320746f2035302062797465732c20756e697175652e20536574206279207468652063726561746520636f6d6d616e64277320757365726e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e067374617475730201000000008f00000055736572207374617475732c20616374697665206f7220696e6163746976652e20536574206279207468652063726561746520636f6d6d616e64277320737461747573206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e20446561637469766174696e672061207573657220656e6473206974732073657373696f6e732e", + "entries": [ + { + "key": "username", + "kind": "string", + "flags": 1, + "default": null + }, + { + "key": "status", + "kind": "string", + "flags": 1, + "default": null + } + ] + } + ], + "stream_key": { + "key_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "key_base64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", + "fingerprint_label": "iggy/stream-key-fingerprint/v1", + "fingerprint": "8441e38538cbfa57", + "invalid_keys": [ + { + "case": "raw_31_bytes", + "input": { + "raw_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + }, + { + "case": "raw_33_bytes", + "input": { + "raw_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + }, + { + "case": "base64_with_whitespace", + "input": { + "base64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=\n" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + }, + { + "case": "base64_missing_padding", + "input": { + "base64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + }, + { + "case": "base64_noncanonical_trailing_bits", + "input": { + "base64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh9=" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + } + ] + }, + "update_resource": [ + { + "case": "topic_set_expiry_without_name_reset_max_topic_size", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "message_expiry": "604800000000", + "reset": [ + "max_topic_size" + ] + }, + "wire_set": { + "message_expiry": { + "value": "604800000000", + "kind": "uint64", + "explicit": true + } + }, + "wire_reset": [ + "max_topic_size" + ], + "body_hex": "01010401000000010402000000010e6d61785f746f7069635f73697a65020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000" + }, + { + "case": "topic_rename_only_named_identifiers", + "scope": "topic", + "target": { + "stream_id": "orders", + "topic_id": "events" + }, + "input": { + "name": "events-v2" + }, + "wire_set": { + "name": { + "value": "events-v2", + "kind": "string", + "explicit": true + } + }, + "wire_reset": [], + "body_hex": "0102066f726465727302066576656e74730002040000006e616d6502090000006576656e74732d7632" + }, + { + "case": "topic_rename_set_expiry_reset_max_topic_size", + "scope": "topic", + "target": { + "stream_id": 7, + "topic_id": "events" + }, + "input": { + "name": "events-v3", + "message_expiry": "86400000000", + "reset": [ + "max_topic_size" + ] + }, + "wire_set": { + "message_expiry": { + "value": "86400000000", + "kind": "uint64", + "explicit": true + }, + "name": { + "value": "events-v3", + "kind": "string", + "explicit": true + } + }, + "wire_reset": [ + "max_topic_size" + ], + "body_hex": "0101040700000002066576656e7473010e6d61785f746f7069635f73697a65020e0000006d6573736167655f6578706972790c080000000060d71d1400000002040000006e616d6502090000006576656e74732d7633" + }, + { + "case": "topic_reset_both_defaults", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "reset": [ + "max_topic_size", + "message_expiry" + ] + }, + "wire_set": {}, + "wire_reset": [ + "max_topic_size", + "message_expiry" + ], + "body_hex": "01010401000000010402000000020e6d61785f746f7069635f73697a650e6d6573736167655f657870697279" + }, + { + "case": "topic_max_topic_size_above_2_pow_53", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "max_topic_size": "9007199254740993" + }, + "wire_set": { + "max_topic_size": { + "value": "9007199254740993", + "kind": "uint64", + "explicit": true + } + }, + "wire_reset": [], + "body_hex": "0101040100000001040200000000020e0000006d61785f746f7069635f73697a650c080000000100000000002000" + }, + { + "case": "stream_rename", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "input": { + "name": "payments-v2" + }, + "wire_set": { + "name": { + "value": "payments-v2", + "kind": "string", + "explicit": true + } + }, + "wire_reset": [], + "body_hex": "020104030000000002040000006e616d65020b0000007061796d656e74732d7632" + }, + { + "case": "user_username_and_status", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "username": "alice2", + "status": "inactive" + }, + "wire_set": { + "status": { + "value": "inactive", + "kind": "string", + "explicit": true + }, + "username": { + "value": "alice2", + "kind": "string", + "explicit": true + } + }, + "wire_reset": [], + "body_hex": "030104050000000002060000007374617475730208000000696e6163746976650208000000757365726e616d650206000000616c69636532" + } + ], + "update_resource_errors": [ + { + "case": "topic_empty_update", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": {}, + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_set_and_reset_same_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "message_expiry": "604800000000", + "reset": [ + "message_expiry" + ] + }, + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_reset_create_only_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "reset": [ + "compression_algorithm" + ] + }, + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "stream_reset_name", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "input": { + "reset": [ + "name" + ] + }, + "error": { + "name": "option_not_resettable", + "code": 4104 + } + }, + { + "case": "user_reset_status", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "reset": [ + "status" + ] + }, + "error": { + "name": "option_not_resettable", + "code": 4104 + } + }, + { + "case": "stream_reset_read_only_key", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "input": { + "reset": [ + "encryption_key_source" + ] + }, + "error": { + "name": "read_only_option", + "code": 4105 + } + }, + { + "case": "user_reset_internal_key", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "reset": [ + "critical_keys" + ] + }, + "error": { + "name": "read_only_option", + "code": 4105 + } + }, + { + "case": "topic_empty_name", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "name": "" + }, + "error": { + "name": "invalid_topic_name", + "code": 2014 + } + }, + { + "case": "topic_empty_name_and_create_only_reset", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "name": "", + "reset": [ + "segment_size" + ] + }, + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "stream_empty_name", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "input": { + "name": "" + }, + "error": { + "name": "invalid_stream_name", + "code": 1013 + } + }, + { + "case": "user_short_username", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "username": "a" + }, + "error": { + "name": "invalid_username", + "code": 43 + } + }, + { + "case": "user_short_username_and_status_reset", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "username": "a", + "reset": [ + "status" + ] + }, + "error": { + "name": "option_not_resettable", + "code": 4104 + } + } + ], + "create_stream": [ + { + "case": "stream_without_key", + "input": { + "name": "orders" + }, + "wire_options": {}, + "body_hex": "066f7264657273", + "http_body": "{\"name\":\"orders\"}", + "admission_error": null + }, + { + "case": "stream_with_key", + "input": { + "name": "orders", + "encryption_key": { + "raw_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f" + } + }, + "wire_options": { + "encryption_key": { + "value": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", + "kind": "raw", + "explicit": true + } + }, + "body_hex": "066f7264657273020e000000656e6372797074696f6e5f6b65790120000000000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "http_body": "{\"name\":\"orders\",\"options\":{\"encryption_key\":\"AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=\"}}", + "admission_error": null + }, + { + "case": "stream_with_raw_extra_key", + "input": { + "name": "orders", + "raw": { + "future_option": "enabled" + } + }, + "wire_options": { + "future_option": { + "value": "enabled", + "kind": "string", + "explicit": true + } + }, + "body_hex": "066f7264657273020d0000006675747572655f6f7074696f6e0207000000656e61626c6564", + "http_body": "{\"name\":\"orders\",\"options\":{\"future_option\":\"enabled\"}}", + "admission_error": { + "name": "unsupported_option_key", + "code": 4041 + } + } + ], + "create_topic": [ + { + "case": "topic_with_options", + "input": { + "stream_id": 1, + "name": "events", + "partitions_count": 3, + "compression_algorithm": "gzip", + "message_expiry": "604800000000", + "max_topic_size": "9007199254740993", + "segment_size": "134217728", + "durability": "persisted", + "consumer_offset_durability": "replicated" + }, + "wire_options": { + "compression_algorithm": { + "value": "gzip", + "kind": "string", + "explicit": true + }, + "consumer_offset_durability": { + "value": "replicated", + "kind": "string", + "explicit": true + }, + "durability": { + "value": "persisted", + "kind": "string", + "explicit": true + }, + "max_topic_size": { + "value": "9007199254740993", + "kind": "uint64", + "explicit": true + }, + "message_expiry": { + "value": "604800000000", + "kind": "uint64", + "explicit": true + }, + "segment_size": { + "value": "134217728", + "kind": "uint64", + "explicit": true + } + }, + "body_hex": "01040100000003000000066576656e74730215000000636f6d7072657373696f6e5f616c676f726974686d0204000000677a6970021a000000636f6e73756d65725f6f66667365745f6475726162696c697479020a0000007265706c696361746564020a0000006475726162696c6974790209000000706572736973746564020e0000006d61785f746f7069635f73697a650c080000000100000000002000020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000020c0000007365676d656e745f73697a650c080000000000000800000000", + "http_body": "{\"partitions_count\":3,\"name\":\"events\",\"options\":{\"compression_algorithm\":\"gzip\",\"consumer_offset_durability\":\"replicated\",\"durability\":\"persisted\",\"max_topic_size\":\"9007199254740993\",\"message_expiry\":604800000000,\"segment_size\":134217728}}", + "admission_error": null + } + ], + "create_user": [ + { + "case": "user_with_empty_options", + "input": { + "username": "alice", + "password": "s3cret-pass", + "status": "active" + }, + "wire_options": {}, + "body_hex": "05616c6963650b7333637265742d706173730100", + "http_body": "{\"username\":\"alice\",\"password\":\"s3cret-pass\",\"status\":\"active\",\"permissions\":null}", + "admission_error": null + } + ], + "http_update": [ + { + "case": "topic_flat_keys", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"name\":\"events-v2\",\"message_expiry\":\"7 days\"}", + "update_resource_hex": "0101040100000001040200000000020e0000006d6573736167655f657870697279020600000037206461797302040000006e616d6502090000006576656e74732d7632", + "outcome": "accepted", + "error": null + }, + { + "case": "topic_reset_array", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":604800000000,\"reset\":[\"max_topic_size\"]}", + "update_resource_hex": "01010401000000010402000000010e6d61785f746f7069635f73697a65020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000", + "outcome": "accepted", + "error": null + }, + { + "case": "topic_null_ignored", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"name\":null,\"max_topic_size\":null,\"message_expiry\":\"1 day\"}", + "update_resource_hex": "0101040100000001040200000000020e0000006d6573736167655f65787069727902050000003120646179", + "outcome": "accepted", + "error": null + }, + { + "case": "topic_u64_above_2_pow_53_as_string", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"max_topic_size\":\"9007199254740993\"}", + "update_resource_hex": "0101040100000001040200000000020e0000006d61785f746f7069635f73697a65021000000039303037313939323534373430393933", + "outcome": "accepted", + "error": null + }, + { + "case": "stream_rename", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "body": "{\"name\":\"payments-v2\"}", + "update_resource_hex": "020104030000000002040000006e616d65020b0000007061796d656e74732d7632", + "outcome": "accepted", + "error": null + }, + { + "case": "user_username_and_status", + "scope": "user", + "target": { + "user_id": 5 + }, + "body": "{\"username\":\"alice2\",\"status\":\"inactive\"}", + "update_resource_hex": "030104050000000002060000007374617475730208000000696e6163746976650208000000757365726e616d650206000000616c69636532", + "outcome": "accepted", + "error": null + }, + { + "case": "topic_duplicate_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":\"1 day\",\"message_expiry\":\"2 days\"}", + "update_resource_hex": null, + "outcome": "json_refused", + "error": null + }, + { + "case": "topic_nested_object", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":{\"value\":\"1 day\"}}", + "update_resource_hex": null, + "outcome": "json_refused", + "error": null + }, + { + "case": "topic_null_only", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"max_topic_size\":null}", + "update_resource_hex": "0101040100000001040200000000", + "outcome": "decode_refused", + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_zero_sentinel_only", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":0}", + "update_resource_hex": "0101040100000001040200000000020e0000006d6573736167655f6578706972790c080000000000000000000000", + "outcome": "admission_refused", + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_reset_names_set_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":\"1 day\",\"reset\":[\"message_expiry\"]}", + "update_resource_hex": "01010401000000010402000000010e6d6573736167655f657870697279020e0000006d6573736167655f65787069727902050000003120646179", + "outcome": "decode_refused", + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_unknown_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"retention\":\"1 day\"}", + "update_resource_hex": "01010401000000010402000000000209000000726574656e74696f6e02050000003120646179", + "outcome": "admission_refused", + "error": { + "name": "unsupported_option_key", + "code": 4041 + } + }, + { + "case": "topic_set_create_only_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"compression_algorithm\":\"gzip\"}", + "update_resource_hex": "01010401000000010402000000000215000000636f6d7072657373696f6e5f616c676f726974686d0204000000677a6970", + "outcome": "admission_refused", + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "topic_reset_create_only_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"reset\":[\"compression_algorithm\"]}", + "update_resource_hex": "010104010000000104020000000115636f6d7072657373696f6e5f616c676f726974686d", + "outcome": "admission_refused", + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "stream_reset_name", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "body": "{\"reset\":[\"name\"]}", + "update_resource_hex": "0201040300000001046e616d65", + "outcome": "admission_refused", + "error": { + "name": "option_not_resettable", + "code": 4104 + } + }, + { + "case": "topic_name_as_number", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"name\":5}", + "update_resource_hex": "010104010000000104020000000002040000006e616d650c080000000500000000000000", + "outcome": "admission_refused", + "error": { + "name": "invalid_topic_name", + "code": 2014 + } + }, + { + "case": "stream_empty_name", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "body": "{\"name\":\"\"}", + "update_resource_hex": "020104030000000002040000006e616d65030100000000", + "outcome": "admission_refused", + "error": { + "name": "invalid_stream_name", + "code": 1013 + } + }, + { + "case": "user_status_as_bool", + "scope": "user", + "target": { + "user_id": 5 + }, + "body": "{\"status\":true}", + "update_resource_hex": "03010405000000000206000000737461747573030100000001", + "outcome": "admission_refused", + "error": { + "name": "invalid_user_status", + "code": 45 + } + }, + { + "case": "topic_create_only_key_as_bool", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"segment_size\":true}", + "update_resource_hex": "0101040100000001040200000000020c0000007365676d656e745f73697a65030100000001", + "outcome": "admission_refused", + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "stream_read_only_key_as_number", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "body": "{\"encryption_key_source\":1}", + "update_resource_hex": "02010403000000000215000000656e6372797074696f6e5f6b65795f736f757263650c080000000100000000000000", + "outcome": "admission_refused", + "error": { + "name": "read_only_option", + "code": 4105 + } + }, + { + "case": "topic_invalid_value", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":\"soon\"}", + "update_resource_hex": "0101040100000001040200000000020e0000006d6573736167655f6578706972790204000000736f6f6e", + "outcome": "admission_refused", + "error": { + "name": "invalid_option_value", + "code": 4042 + } + } + ], + "stream_responses": [ + { + "case": "stream_source_none", + "state": { + "encryption_key_source": "none", + "stored_extra": {} + }, + "response_hex": "0100000000e090663c1306000000000000100000000000000c000000000000000c706c61696e2d73747265616d00000000230000000215000000656e6372797074696f6e5f6b65795f736f7572636502040000006e6f6e65", + "expected": { + "id": 1, + "created_at": 1710000000000000, + "name": "plain-stream", + "topics_count": 0, + "size_bytes": 4096, + "messages_count": 12, + "options": { + "encryption_key_source": { + "value": "none", + "kind": "string", + "explicit": false + } + }, + "encryption_key_source": "none", + "encryption_key_fingerprint": null, + "encryption_kek_fingerprints": [] + } + }, + { + "case": "stream_source_stream_with_unknown_explicit_key", + "state": { + "encryption_key_source": "stream", + "data_key": { + "raw_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f" + }, + "kek_config_keys": [ + "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=" + ], + "stored_extra": { + "future_option": { + "value": "enabled", + "kind": "string", + "explicit": true + } + } + }, + "response_hex": "0200000000e090663c1306000000000000100000000000000c0000000000000011636c69656e742d6b65792d73747265616d1e000000020d0000006675747572655f6f7074696f6e0207000000656e61626c65648d000000021a000000656e6372797074696f6e5f6b656b5f66696e6765727072696e74021000000033396534663066393563333639336337021a000000656e6372797074696f6e5f6b65795f66696e6765727072696e740210000000383434316533383533386362666135370215000000656e6372797074696f6e5f6b65795f736f75726365020600000073747265616d", + "expected": { + "id": 2, + "created_at": 1710000000000000, + "name": "client-key-stream", + "topics_count": 0, + "size_bytes": 4096, + "messages_count": 12, + "options": { + "encryption_kek_fingerprint": { + "value": "39e4f0f95c3693c7", + "kind": "string", + "explicit": false + }, + "encryption_key_fingerprint": { + "value": "8441e38538cbfa57", + "kind": "string", + "explicit": false + }, + "encryption_key_source": { + "value": "stream", + "kind": "string", + "explicit": false + }, + "future_option": { + "value": "enabled", + "kind": "string", + "explicit": true + } + }, + "encryption_key_source": "stream", + "encryption_key_fingerprint": "8441e38538cbfa57", + "encryption_kek_fingerprints": [ + "39e4f0f95c3693c7" + ] + } + }, + { + "case": "stream_source_server_two_keks", + "state": { + "encryption_key_source": "server", + "data_key": { + "raw_hex": "202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f" + }, + "kek_config_keys": [ + "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=", + "AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA=" + ], + "stored_extra": {} + }, + "response_hex": "0300000000e090663c1306000000000000100000000000000c00000000000000117365727665722d6b65792d73747265616d000000009e000000021a000000656e6372797074696f6e5f6b656b5f66696e6765727072696e740221000000333965346630663935633336393363372c64353564343739383239626332393336021a000000656e6372797074696f6e5f6b65795f66696e6765727072696e740210000000306563616666316261323931666530640215000000656e6372797074696f6e5f6b65795f736f757263650206000000736572766572", + "expected": { + "id": 3, + "created_at": 1710000000000000, + "name": "server-key-stream", + "topics_count": 0, + "size_bytes": 4096, + "messages_count": 12, + "options": { + "encryption_kek_fingerprint": { + "value": "39e4f0f95c3693c7,d55d479829bc2936", + "kind": "string", + "explicit": false + }, + "encryption_key_fingerprint": { + "value": "0ecaff1ba291fe0d", + "kind": "string", + "explicit": false + }, + "encryption_key_source": { + "value": "server", + "kind": "string", + "explicit": false + } + }, + "encryption_key_source": "server", + "encryption_key_fingerprint": "0ecaff1ba291fe0d", + "encryption_kek_fingerprints": [ + "39e4f0f95c3693c7", + "d55d479829bc2936" + ] + } + } + ], + "topic_responses": [ + { + "case": "topic_explicit_and_derived", + "response_hex": "0400000000e090663c1306000000000000080000000000000700000000000000066576656e7473430000000215000000636f6d7072657373696f6e5f616c676f726974686d0204000000677a6970020e0000006d6573736167655f6578706972790c0800000000a0e3d08c0000002b010000021a000000636f6e73756d65725f6f66667365745f6475726162696c697479020a0000007265706c696361746564020a0000006475726162696c697479020a0000007265706c696361746564020e0000006d61785f746f7069635f73697a650c08000000ffffffffffffffff02190000006d657373616765735f72657175697265645f746f5f736176650b04000000000400000217000000706172746974696f6e5f726573697a655f706f6c69637902070000006d757461626c650214000000707265616c6c6f636174655f7365676d656e7473030100000000020c0000007365676d656e745f73697a650c080000000000004000000000022100000073697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c080000000000100000000000", + "expected": { + "id": 4, + "created_at": 1710000000000000, + "name": "events", + "partitions_count": 0, + "size_bytes": 2048, + "messages_count": 7, + "options": { + "compression_algorithm": { + "value": "gzip", + "kind": "string", + "explicit": true + }, + "consumer_offset_durability": { + "value": "replicated", + "kind": "string", + "explicit": false + }, + "durability": { + "value": "replicated", + "kind": "string", + "explicit": false + }, + "max_topic_size": { + "value": "18446744073709551615", + "kind": "uint64", + "explicit": false + }, + "message_expiry": { + "value": "604800000000", + "kind": "uint64", + "explicit": true + }, + "messages_required_to_save": { + "value": "1024", + "kind": "uint32", + "explicit": false + }, + "partition_resize_policy": { + "value": "mutable", + "kind": "string", + "explicit": false + }, + "preallocate_segments": { + "value": "false", + "kind": "bool", + "explicit": false + }, + "segment_size": { + "value": "1073741824", + "kind": "uint64", + "explicit": false + }, + "size_of_messages_required_to_save": { + "value": "1048576", + "kind": "uint64", + "explicit": false + } + }, + "compression_algorithm": "gzip", + "message_expiry": "604800000000", + "max_topic_size": "18446744073709551615" + } + } + ], + "user_responses": [ + { + "case": "user_empty_blocks", + "response_hex": "0500000000e090663c1306000105616c696365000000000000000000000000", + "expected": { + "id": 5, + "created_at": 1710000000000000, + "status": "active", + "username": "alice", + "options": {} + } + }, + { + "case": "user_populated_blocks", + "response_hex": "0600000000e090663c1306000203626f621e000000020d0000006675747572655f6f7074696f6e0207000000656e61626c65641e000000020c0000006675747572655f71756f74610c08000000010000000000200000000000", + "expected": { + "id": 6, + "created_at": 1710000000000000, + "status": "inactive", + "username": "bob", + "options": { + "future_option": { + "value": "enabled", + "kind": "string", + "explicit": true + }, + "future_quota": { + "value": "9007199254740993", + "kind": "uint64", + "explicit": false + } + } + } + } + ] +}
diff --git a/foreign/python/Cargo.toml b/foreign/python/Cargo.toml index 7205dde..390e87b 100644 --- a/foreign/python/Cargo.toml +++ b/foreign/python/Cargo.toml
@@ -17,7 +17,7 @@ [package] name = "apache-iggy" -version = "0.9.1-dev1" +version = "0.9.1-dev2" edition = "2024" authors = ["Iggy Committers <dev@iggy.apache.org>"] license = "Apache-2.0" @@ -37,7 +37,7 @@ [dependencies] bytes = "1.12.1" futures = "0.3.34" -iggy = { path = "../../core/sdk", version = "0.11.1-edge.1" } +iggy = { path = "../../core/sdk", version = "0.11.1-edge.2" } paste = "1" pyo3 = "0.29.3" pyo3-async-runtimes = { version = "0.29.0", features = [ @@ -46,4 +46,10 @@ ] } pyo3-stub-gen = "0.23.1" secrecy = "0.10" +serde = { version = "1.0.229", features = ["derive"] } +serde_json = "1.0.154" tokio = "1.53.3" + +[dev-dependencies] +iggy_binary_protocol = { path = "../../core/binary_protocol" } +iggy_common = { path = "../../core/common" }
diff --git a/foreign/python/README.md b/foreign/python/README.md index 4094c5c..c51d2d4 100644 --- a/foreign/python/README.md +++ b/foreign/python/README.md
@@ -214,6 +214,65 @@ asyncio.run(main()) ``` +## Stream, Topic and User Options + +`create_stream`, `create_topic` and `create_user` take typed options: `StreamOptions`, `TopicOptions` and `UserOptions`. Updates are patches: `update_stream`, `update_topic` and `update_user` change only the fields you pass, and `reset` returns keys to their server defaults. + +A topic's `message_expiry` and `max_topic_size` are the only resettable keys. Settings such as the compression algorithm are fixed at creation and have no update field. `describe_options("stream" | "topic" | "user")` reports which keys are updatable, resettable, secret or read-only. + +The update methods also take `options` and `raw_reset`, which send keys and reset names verbatim, so a key newer than this build can be set or reset. The server decides those. A refused option raises `OptionError`, a `RuntimeError` whose `code` and `kind` name the refusal, for example `4103` and `"option_not_updatable"`, the same on every transport. + +Root user rename and status refusals also raise `OptionError` with code `4108` and kind `"cannot_modify_root_user"`. + +```python +import asyncio +import secrets +from datetime import timedelta + +from apache_iggy import ( + IggyClient, + IggyExpiry, + OptionKey, + StreamOptions, + TopicOptions, + encryption_key_fingerprint, +) + + +async def main(): + client = IggyClient.from_connection_string("iggy+tcp://iggy:iggy@127.0.0.1:8090") + await client.connect() + + key = secrets.token_bytes(32) + stream = await client.create_stream("orders", StreamOptions(encryption_key=key)) + assert stream.encryption_key_source == "stream" + assert stream.encryption_key_fingerprint == encryption_key_fingerprint(key) + + await client.create_topic( + "orders", + "created", + TopicOptions( + partitions_count=3, + compression_algorithm="gzip", + message_expiry=IggyExpiry.ExpireDuration(timedelta(days=7)), + ), + ) + await client.update_topic( + "orders", + "created", + message_expiry=IggyExpiry.ExpireDuration(timedelta(days=1)), + reset=[OptionKey.MaxTopicSize], + ) + await client.update_stream("orders", name="orders-v2") + + +asyncio.run(main()) +``` + +A stream encryption key must be 32 random bytes, given as `bytes` or as their canonical base64 text. The key travels inside the create request over whatever transport the client uses, so protecting it in transit with TLS is your responsibility. + +The server never returns the key, only `encryption_key_source`, `encryption_key_fingerprint` and `encryption_kek_fingerprints`. Without a key, a server with encryption enabled generates one for the stream and reports the source `"server"`. An invalid key raises `ValueError` before anything is sent, and no repr or error message contains it. + ## High-Level Producer The Python high-level producer API is a port of the Rust high-level producer @@ -306,7 +365,7 @@ - 57 (`TransientNotCommitted`): the outcome is unknown, and the server possibly applied the write. Read the state before you write it again. - 40 (`Unauthenticated`): the request needs a session. Sign in, then retry. -`PendingOperationError` and `ProducerSendError` are `IggyError` subclasses. An error that the binding raises itself, such as a `ValueError`, a `TypeError`, or a plain `RuntimeError` for an invalid argument or a shut-down producer, is not an `IggyError` and carries no code. +`OptionError`, `PendingOperationError` and `ProducerSendError` are `IggyError` subclasses. An error that the binding raises itself, such as a `ValueError`, a `TypeError`, or a plain `RuntimeError` for an invalid argument or a shut-down producer, is not an `IggyError` and carries no code. `PendingOperationError` exposes the exact integer `operation_id`, the numeric `code`, and `kind` (`"uncertain"`, `"lane_blocked"`, or `"not_retained"`). On the
diff --git a/foreign/python/apache_iggy.pyi b/foreign/python/apache_iggy.pyi index d48d7ef..f807d68 100644 --- a/foreign/python/apache_iggy.pyi +++ b/foreign/python/apache_iggy.pyi
@@ -52,9 +52,12 @@ "IggyExpiry", "IggyProducer", "MaxTopicSize", + "OptionError", + "OptionKey", "OptionSpec", "Partition", "PartitionContext", + "PartitionResizePolicy", "Partitioning", "PendingOperationError", "PendingOperationsConfig", @@ -71,20 +74,24 @@ "Stats", "Stream", "StreamDetails", + "StreamOptions", "StreamPermissions", "TcpConfig", "TcpReconnectionConfig", "TokenInfo", "Topic", "TopicDetails", + "TopicOptions", "TopicPermissions", "UserHeaders", "UserInfo", "UserInfoDetails", + "UserOptions", "UserStatus", "WebSocketConfig", "WebSocketFramingConfig", "WebSocketReconnectionConfig", + "encryption_key_fingerprint", ] class AutoCommit: @@ -1298,6 +1305,7 @@ password: builtins.str, status: UserStatus | None = None, permissions: Permissions | None = None, + options: UserOptions | None = None, ) -> collections.abc.Awaitable[UserInfoDetails]: r""" Create a new user. @@ -1307,6 +1315,7 @@ password: Password as `str`. status: User status as `UserStatus | None`; defaults to `UserStatus.Active`. permissions: Permissions as `Permissions | None`; the user has none when `None`. + options: User options as `UserOptions | None`. Returns: An awaitable that resolves to the created `UserInfoDetails`. @@ -1319,25 +1328,41 @@ user_id: builtins.str | builtins.int, username: builtins.str | None = None, status: UserStatus | None = None, + reset: builtins.list[OptionKey] | None = None, options: builtins.dict[builtins.str, builtins.str] | None = None, + raw_reset: builtins.list[builtins.str] | None = None, ) -> collections.abc.Awaitable[None]: r""" Update a user by unique ID or username. + A patch: only the fields passed change, every other setting keeps its + value. Renaming the user this client is signed in as also updates the + credentials a reconnect replays. + Args: user_id: User identifier as `str | int`. username: New username as `str | None`; unchanged when `None`. status: New status as `UserStatus | None`; unchanged when `None`. - options: Reserved for future use. Additional option keys as - `dict[str, str] | None`, forwarded to the server. No user update - option key exists yet, so a current server rejects every key. + Deactivating a user ends its sessions. + reset: Keys to return to their defaults as `list[OptionKey] | None`. + No user key is resettable yet, so any key is refused. + options: Additional option keys as `dict[str, str] | None`, + forwarded to the server. No other user key exists yet, so a + current server rejects every key. + raw_reset: Additional reset names as `list[str] | None`, sent + verbatim so a key newer than this build can be reset. The + server decides them. Returns: An awaitable that resolves to `None` when the user is updated. Raises: ValueError: If a string identifier is invalid. - RuntimeError: If the request fails. + OptionError: A `RuntimeError` carrying the error `code` and + `kind` when the client or the server refuses an option. + RuntimeError: Before anything is sent, if the update changes + nothing, names a key both set and reset, or resets a key that + cannot be reset. Otherwise if the request fails. """ def delete_user( self, user_id: builtins.str | builtins.int @@ -1456,10 +1481,26 @@ Raises: RuntimeError: If the client cannot be shut down. """ - def create_stream(self, name: builtins.str) -> collections.abc.Awaitable[None]: + def create_stream( + self, name: builtins.str, options: StreamOptions | None = None + ) -> collections.abc.Awaitable[StreamDetails]: r""" - Creates a new stream with the provided ID and name. - Raises `RuntimeError` if the stream cannot be created. + Create a new stream. + + On a server with encryption enabled every stream is stored under its + own key: the one `options` carries, or one the server generates. The + returned details report only the key's source and fingerprints. + + Args: + name: Stream name as `str`, between 1 and 255 UTF-8 bytes. + options: Stream options as `StreamOptions | None`. + + Returns: + An awaitable that resolves to the created `StreamDetails`. + + Raises: + RuntimeError: If the stream cannot be created, for example when + `options` carries a key and the server has no encryption key. """ def get_stream( self, stream_id: builtins.str | builtins.int @@ -1483,23 +1524,32 @@ def update_stream( self, stream_id: builtins.str | builtins.int, - name: builtins.str, + name: builtins.str | None = None, + reset: builtins.list[OptionKey] | None = None, options: builtins.dict[builtins.str, builtins.str] | None = None, + raw_reset: builtins.list[builtins.str] | None = None, ) -> collections.abc.Awaitable[None]: r""" - Rename a stream selected by name or numeric ID. + Update a stream selected by name or numeric ID. - `stream_id` accepts a stream name as `str` or numeric ID as `int`. A - decimal-only string is interpreted as a numeric ID. `name` must be unique - and contain between 1 and 255 UTF-8 bytes. Renaming a stream to its current - name succeeds without changing it. + A patch: only the fields passed change. The stream keeps its id, its + encryption key and its data. `stream_id` accepts a stream name as `str` + or numeric ID as `int`. A decimal-only string is interpreted as a + numeric ID. Renaming a stream to its current name succeeds without + changing it. Args: stream_id: Stream identifier as `str | int`. - name: New stream name as `str`. + name: New stream name as `str | None`, unique and between 1 and 255 + UTF-8 bytes, unchanged when `None`. + reset: Keys to return to their defaults as `list[OptionKey] | None`. + No stream key is resettable, so any key is refused. options: Additional option keys as `dict[str, str] | None`, forwarded to the server. Current server versions reject all stream update option keys. + raw_reset: Additional reset names as `list[str] | None`, sent + verbatim so a key newer than this build can be reset. The + server decides them. Returns: None. @@ -1508,10 +1558,14 @@ TypeError: If `stream_id` is not `str` or an integer in `0..=2**32 - 1`, or `name` is not `str`. ValueError: If a string identifier is empty or exceeds 255 UTF-8 bytes. - RuntimeError: If the client is not authenticated, the user lacks global - `manage_streams` or per-stream `manage_stream` permission, the - stream does not exist, the new name is invalid or already used, or - the request fails. + OptionError: A `RuntimeError` carrying the error `code` and + `kind` when the client or the server refuses an option. + RuntimeError: Before anything is sent, if the update changes + nothing, the name does not fit, or a reset key cannot be reset. + Otherwise if the client is not authenticated, the user lacks + global `manage_streams` or per-stream `manage_stream` permission, + the stream does not exist, the new name is already used, or the + request fails. """ def delete_stream( self, stream_id: builtins.str | builtins.int @@ -1538,48 +1592,23 @@ self, stream: builtins.str | builtins.int, name: builtins.str, - partitions_count: builtins.int, - compression_algorithm: builtins.str | None = None, - message_expiry: IggyExpiry | None = None, - max_topic_size: MaxTopicSize | None = None, - segment_size: builtins.int | None = None, - durability: Durability | None = None, - consumer_offset_durability: Durability | None = None, - messages_required_to_save: builtins.int | None = None, - size_of_messages_required_to_save: builtins.int | None = None, - preallocate_segments: builtins.bool | None = None, - options: builtins.dict[builtins.str, builtins.str] | None = None, + options: TopicOptions | None = None, ) -> collections.abc.Awaitable[None]: r""" - Creates a new topic with the given parameters. + Creates a new topic. Args: stream: Stream identifier as `str | int`. name: Topic name as `str`. - partitions_count: Number of partitions as `int`, at most 1000. - compression_algorithm: Compression algorithm as `str | None`. - message_expiry: Message expiry as `IggyExpiry | None`. - max_topic_size: Maximum topic size as `MaxTopicSize | None`. - segment_size: Per-topic segment size in bytes as `int | None`. - durability: Message completion policy, defaulting to replicated. - consumer_offset_durability: Independent offset policy, defaulting to replicated. - messages_required_to_save: Message-count flush threshold as `int | None`. - size_of_messages_required_to_save: Byte flush threshold as `int | None`. - preallocate_segments: Reserve segment bytes on open as `bool | None`. - options: Additional option keys as `dict[str, str] | None`, sent - verbatim so a newer server key can be set from this build. - `{"partition_resize_policy": "fixed"}` makes the server reject - later partition creation and deletion on this topic. - - Every option left as `None` resolves against the server default at - admission. + options: Topic options as `TopicOptions | None`. Without options the + topic gets one partition and every setting resolves against the + server default at admission. Returns: An awaitable that resolves to `None` when the topic is created. Raises: - ValueError: If `message_expiry` or `max_topic_size` is out of range. - PyRuntimeError: If another argument is invalid or the request fails. + RuntimeError: If an argument is invalid or the request fails. """ def get_topic( self, @@ -1610,36 +1639,50 @@ self, stream_id: builtins.str | builtins.int, topic_id: builtins.str | builtins.int, - name: builtins.str, - compression_algorithm: builtins.str | None = None, + name: builtins.str | None = None, message_expiry: IggyExpiry | None = None, max_topic_size: MaxTopicSize | None = None, + reset: builtins.list[OptionKey] | None = None, options: builtins.dict[builtins.str, builtins.str] | None = None, + raw_reset: builtins.list[builtins.str] | None = None, ) -> collections.abc.Awaitable[None]: r""" Update an existing topic. - A patch, not a replacement: every setting rides the options block, so a - field left unset keeps the topic's current value rather than resetting - it to a server default. + A patch, not a replacement: only the fields passed change, every other + setting keeps its value. Settings fixed at creation, such as the + compression algorithm, have no field here. Args: stream_id: Stream identifier as `str | int`. topic_id: Topic identifier as `str | int`. - name: New topic name as `str`. - compression_algorithm: Compression algorithm as `str | None`. - message_expiry: Message expiry as `IggyExpiry | None`. - max_topic_size: Maximum topic size as `MaxTopicSize | None`. + name: New topic name as `str | None`, unchanged when `None`. + message_expiry: Message expiry as `IggyExpiry | None`, unchanged + when `None`. `IggyExpiry.ServerDefault()` changes nothing either, + reset `OptionKey.MessageExpiry` to return to the default. + max_topic_size: Maximum topic size as `MaxTopicSize | None`, + unchanged when `None`. `MaxTopicSize.ServerDefault()` changes + nothing either, reset `OptionKey.MaxTopicSize` to return to the + default. + reset: Keys to return to their defaults as `list[OptionKey] | None`: + `OptionKey.MessageExpiry` or `OptionKey.MaxTopicSize`. options: Additional option keys as `dict[str, str] | None`, sent verbatim so an updatable server key can be set from this build. A create-only key is refused by name. + raw_reset: Additional reset names as `list[str] | None`, sent + verbatim so a key newer than this build can be reset. The + server decides them. Returns: An awaitable that resolves to `None` when the topic is updated. Raises: ValueError: If `message_expiry` or `max_topic_size` is out of range. - PyRuntimeError: If another argument is invalid or the request fails. + OptionError: A `RuntimeError` carrying the error `code` and + `kind` when the client or the server refuses an option. + RuntimeError: Before anything is sent, if the update changes + nothing, names a key both set and reset, the name does not fit, + or a reset key cannot be reset. Otherwise if the request fails. """ def delete_topic( self, @@ -2169,7 +2212,7 @@ microsecond count a `u64` can hold (about 584,542 years): those two values are reserved on the wire for `ServerDefault` and `NeverExpire` respectively, so a `duration` at either boundary raises `ValueError` - when passed to `create_topic`/`update_topic`. A negative `timedelta` + when passed to `TopicOptions`/`update_topic`. A negative `timedelta` also raises `ValueError`. """ @@ -2275,7 +2318,7 @@ an unsigned 64-bit integer: those two values are reserved on the wire for `ServerDefault` and `Unlimited` respectively, so a `Custom` size at either boundary raises `ValueError` when passed to - `create_topic`/`update_topic`. + `TopicOptions`/`update_topic`. """ __match_args__ = ("bytes",) @@ -2297,6 +2340,14 @@ ... @typing.final +class OptionError(IggyError): + r""" + An option refusal, with the server error code and its snake_case name as `kind`. + """ + @property + def kind(self) -> builtins.str: ... + +@typing.final class OptionSpec: r""" One entry of a resource's option catalog, as served by `describe_options`. @@ -2310,10 +2361,9 @@ def kind(self) -> builtins.str: r""" Name of this key's canonical kind: what the server encodes its default - under, and what a value set by `create_topic` is stored as whatever kind - it was sent in, since create admission re-encodes the block from its own - parse. `update_topic` stores what the client sent verbatim and is the - exception. + under, and what a value set by a create or an update is stored as + whatever kind it was sent in, since admission re-encodes the block from + its own parse. """ @property def default_value(self) -> HeaderValue | None: @@ -2329,6 +2379,33 @@ r""" What the option does, including the bounds its value is checked against. """ + @property + def updatable(self) -> builtins.bool: + r""" + Whether an update may set the key. + """ + @property + def resettable(self) -> builtins.bool: + r""" + Whether an update may reset the key to its default. + """ + @property + def secret(self) -> builtins.bool: + r""" + Whether the key is write-only: accepted on create, never returned. + """ + @property + def read_only(self) -> builtins.bool: + r""" + Whether the server computes the key and returns it on reads, never + accepting it. + """ + @property + def critical(self) -> builtins.bool: + r""" + Whether a node that does not understand the key must refuse to serve + the resource. + """ def __repr__(self) -> builtins.str: ... @typing.final @@ -3088,6 +3165,43 @@ r""" Number of topics in the stream. """ + @property + def options(self) -> UserHeaders: + r""" + Options the creating client set explicitly. The stream's encryption + key is write-only and never appears here. + + The same `dict[HeaderKey, HeaderValue]` that `ReceiveMessage.user_headers` + returns, since options ride that codec. `to_scalar_dict()` gives the + plain-scalar form. + """ + @property + def derived_options(self) -> UserHeaders: + r""" + Options the server resolved or computed, including the encryption key + projections. Same shape as `options`. + """ + @property + def encryption_key_source(self) -> builtins.str | None: + r""" + Who chose the key the stream's data is stored under: `"stream"` for a + key the client sent, `"server"` for a key the server generated, or + `"none"` for a plaintext stream. `None` when the server reported no + source. + """ + @property + def encryption_key_fingerprint(self) -> builtins.str | None: + r""" + Fingerprint of the stream's key, 16 lowercase hex characters. `None` + for a plaintext stream. Compare it with `encryption_key_fingerprint()` + of the key the stream was created with. + """ + @property + def encryption_kek_fingerprints(self) -> builtins.list[builtins.str]: + r""" + Fingerprints of the server keys that wrap the stream's key: one outside + a key rotation, empty for a plaintext stream. + """ @typing.final class StreamDetails: @@ -3114,6 +3228,85 @@ r""" Returns the topics in the stream. """ + @property + def options(self) -> UserHeaders: + r""" + Options the creating client set explicitly. The stream's encryption + key is write-only and never appears here. + + The same `dict[HeaderKey, HeaderValue]` that `ReceiveMessage.user_headers` + returns, since options ride that codec. `to_scalar_dict()` gives the + plain-scalar form. + """ + @property + def derived_options(self) -> UserHeaders: + r""" + Options the server resolved or computed, including the encryption key + projections. Same shape as `options`. + """ + @property + def encryption_key_source(self) -> builtins.str | None: + r""" + Who chose the key the stream's data is stored under: `"stream"` for a + key the client sent, `"server"` for a key the server generated, or + `"none"` for a plaintext stream. `None` when the server reported no + source. + """ + @property + def encryption_key_fingerprint(self) -> builtins.str | None: + r""" + Fingerprint of the stream's key, 16 lowercase hex characters. `None` + for a plaintext stream. Compare it with `encryption_key_fingerprint()` + of the key the stream was created with. + """ + @property + def encryption_kek_fingerprints(self) -> builtins.list[builtins.str]: + r""" + Fingerprints of the server keys that wrap the stream's key: one outside + a key rotation, empty for a plaintext stream. + """ + +@typing.final +class StreamOptions: + r""" + Options for `create_stream`. + + The encryption key is write-only: the server stores it wrapped and returns + only its source and fingerprints, and this object never shows it. + """ + @property + def has_encryption_key(self) -> builtins.bool: + r""" + Whether an encryption key is set, without exposing the key itself. + """ + def __new__( + cls, + *, + encryption_key: builtins.bytes | builtins.str | None = None, + raw: builtins.dict[builtins.str, builtins.str] | None = None, + ) -> StreamOptions: + r""" + Constructs stream create options. + + Args: + encryption_key: The stream's own data key: exactly 32 bytes, or + their canonical standard padded base64 text. Use 32 random + bytes, for example `secrets.token_bytes(32)`. The key travels + inside the create request over whatever transport the client + uses, so protecting it in transit (TLS) is the caller's + responsibility. The server never returns it, only its source + and fingerprints. Without a key, a server with encryption + enabled generates one for the stream. + raw: Additional option keys as `dict[str, str]`, sent as string + values so a key newer than this build can be set. A typed field + wins on collision. + + Raises: + ValueError: If `encryption_key` is not exactly 32 bytes or their + canonical base64 text. The message never contains the key. + TypeError: If `encryption_key` is neither `bytes` nor `str`. + """ + def __repr__(self) -> builtins.str: ... @typing.final class StreamPermissions: @@ -3368,17 +3561,19 @@ @property def message_expiry(self) -> IggyExpiry: r""" - The expiry of the messages in the topic. + The expiry of the messages in the topic, the server default when the + topic does not set one. """ @property def compression_algorithm(self) -> builtins.str: r""" - Compression algorithm for the topic. + Compression algorithm for the topic, fixed at creation. """ @property def max_topic_size(self) -> MaxTopicSize: r""" - The maximum size of the topic. + The maximum size of the topic, the server default when the topic does + not set one. """ @property def options(self) -> UserHeaders: @@ -3433,17 +3628,19 @@ @property def message_expiry(self) -> IggyExpiry: r""" - The expiry of the messages in the topic. + The expiry of the messages in the topic, the server default when the + topic does not set one. """ @property def compression_algorithm(self) -> builtins.str: r""" - Compression algorithm for the topic. + Compression algorithm for the topic, fixed at creation. """ @property def max_topic_size(self) -> MaxTopicSize: r""" - The maximum size of the topic. + The maximum size of the topic, the server default when the topic does + not set one. """ @property def options(self) -> UserHeaders: @@ -3472,6 +3669,64 @@ """ @typing.final +class TopicOptions: + r""" + Options for `create_topic`. + """ + def __new__( + cls, + *, + partitions_count: builtins.int = 1, + compression_algorithm: builtins.str | None = None, + message_expiry: IggyExpiry | None = None, + max_topic_size: MaxTopicSize | None = None, + segment_size: builtins.int | None = None, + durability: Durability | None = None, + consumer_offset_durability: Durability | None = None, + partition_resize_policy: PartitionResizePolicy | None = None, + messages_required_to_save: builtins.int | None = None, + size_of_messages_required_to_save: builtins.int | None = None, + preallocate_segments: builtins.bool | None = None, + raw: builtins.dict[builtins.str, builtins.str] | None = None, + ) -> TopicOptions: + r""" + Constructs topic create options. + + Every setting left as `None` resolves against the server default at + admission. Only `message_expiry`, `max_topic_size` and the name can + change after creation, through `update_topic`. + + Args: + partitions_count: Number of partitions as `int`, at most 1000. + Defaults to 1. + compression_algorithm: Compression algorithm as `str | None`. + message_expiry: Message expiry as `IggyExpiry | None`. + `IggyExpiry.ServerDefault()` is the same as `None`. + max_topic_size: Maximum topic size as `MaxTopicSize | None`. + `MaxTopicSize.ServerDefault()` is the same as `None`. + segment_size: Per-topic segment size in bytes as `int | None`. + durability: Message completion policy, defaulting to replicated. + consumer_offset_durability: Independent offset policy, defaulting + to replicated. + partition_resize_policy: `PartitionResizePolicy.Fixed` makes the + server reject later partition creation and deletion on this + topic. + messages_required_to_save: Message-count flush threshold as + `int | None`. + size_of_messages_required_to_save: Byte flush threshold as + `int | None`. + preallocate_segments: Reserve segment bytes on open as `bool | None`. + raw: Additional option keys as `dict[str, str]`, sent as string + values so a key newer than this build can be set. A typed field + wins on collision. + + Raises: + ValueError: If `message_expiry` or `max_topic_size` is out of range. + RuntimeError: If `compression_algorithm` names no known algorithm. + TypeError: If a durability is not a `Durability`. + """ + +@typing.final class TopicPermissions: r""" Permissions for a specific topic of a stream. The lowest level of permissions. @@ -3579,6 +3834,21 @@ r""" The username of the user. """ + @property + def options(self) -> UserHeaders: + r""" + Options the creating client set explicitly. No user key exists yet, + so this is empty unless a newer server added one. + + The same `dict[HeaderKey, HeaderValue]` that `ReceiveMessage.user_headers` + returns, since options ride that codec. + """ + @property + def derived_options(self) -> UserHeaders: + r""" + Options the server resolved for the keys the client did not send. + Same shape as `options`. + """ @typing.final class UserInfoDetails: @@ -3607,6 +3877,39 @@ r""" The permissions of the user, or `None` when the user has none assigned. """ + @property + def options(self) -> UserHeaders: + r""" + Options the creating client set explicitly. No user key exists yet, + so this is empty unless a newer server added one. + + The same `dict[HeaderKey, HeaderValue]` that `ReceiveMessage.user_headers` + returns, since options ride that codec. + """ + @property + def derived_options(self) -> UserHeaders: + r""" + Options the server resolved for the keys the client did not send. + Same shape as `options`. + """ + +@typing.final +class UserOptions: + r""" + Options for `create_user`. No user key exists yet, so only the raw escape + hatch is available. + """ + def __new__( + cls, *, raw: builtins.dict[builtins.str, builtins.str] | None = None + ) -> UserOptions: + r""" + Constructs user create options. + + Args: + raw: Option keys as `dict[str, str]`, sent as string values so a + key newer than this build can be set. A current server rejects + every key. + """ @typing.final class WebSocketConfig: @@ -3791,6 +4094,100 @@ def __repr__(self) -> builtins.str: ... @typing.final +class OptionKey(enum.Enum): + r""" + A catalog option key, as `update_stream`, `update_topic` and `update_user` + take it in `reset`. `str()` gives the key as it travels on the wire. + """ + + Name = ... + r""" + Stream or topic name. + """ + Username = ... + r""" + User name. + """ + Status = ... + r""" + User status. + """ + CompressionAlgorithm = ... + r""" + Topic compression algorithm, fixed at creation. + """ + MessageExpiry = ... + r""" + Topic message expiry, resettable to never expire. + """ + MaxTopicSize = ... + r""" + Topic size cap, resettable to unlimited. + """ + SegmentSize = ... + r""" + Topic segment size, fixed at creation. + """ + Durability = ... + r""" + Topic message durability, fixed at creation. + """ + ConsumerOffsetDurability = ... + r""" + Topic consumer offset durability, fixed at creation. + """ + PartitionResizePolicy = ... + r""" + Topic partition resize policy, fixed at creation. + """ + MessagesRequiredToSave = ... + r""" + Topic message-count flush threshold, fixed at creation. + """ + SizeOfMessagesRequiredToSave = ... + r""" + Topic byte flush threshold, fixed at creation. + """ + PreallocateSegments = ... + r""" + Topic segment preallocation, fixed at creation. + """ + EncryptionKey = ... + r""" + Stream encryption key, write-only and fixed at creation. + """ + EncryptionKeySource = ... + r""" + Who chose the stream's key, computed by the server. + """ + EncryptionKeyFingerprint = ... + r""" + Fingerprint of the stream's key, computed by the server. + """ + EncryptionKekFingerprint = ... + r""" + Fingerprints of the server keys wrapping the stream's key, computed by + the server. + """ + + def __str__(self) -> builtins.str: ... + +@typing.final +class PartitionResizePolicy(enum.Enum): + r""" + Whether partitions can be added to or deleted from a topic after creation. + """ + + Mutable = ... + r""" + Partitions can be added and deleted. + """ + Fixed = ... + r""" + The partition set is fixed at topic creation. + """ + +@typing.final class ProducerSharding(enum.Enum): r""" How a background producer distributes sends among its workers. @@ -3814,6 +4211,23 @@ The user account is inactive and cannot be used. """ +def encryption_key_fingerprint(key: builtins.bytes | builtins.str) -> builtins.str: + r""" + Fingerprint of a stream encryption key: the first 8 bytes of SHA-256 over + the label `iggy/stream-key-fingerprint/v1` followed by the key, as 16 + lowercase hex characters. Streams report the same value in + `encryption_key_fingerprint`, so a caller can match a key to its stream + without the server ever returning the key. + + Args: + key: Exactly 32 bytes, or their canonical standard padded base64 text. + + Raises: + ValueError: If `key` is not exactly 32 bytes or their canonical base64 + text. The message never contains the key. + TypeError: If `key` is neither `bytes` nor `str`. + """ + class Durability(str, enum.Enum): # noqa: UP042 REPLICATED = "replicated" PERSISTED = "persisted"
diff --git a/foreign/python/docker-compose.test.yml b/foreign/python/docker-compose.test.yml index c16eded..68312f5 100644 --- a/foreign/python/docker-compose.test.yml +++ b/foreign/python/docker-compose.test.yml
@@ -42,6 +42,9 @@ - IGGY_WEBSOCKET_ADDRESS=0.0.0.0:8092 - IGGY_ROOT_USERNAME=iggy - IGGY_ROOT_PASSWORD=iggy + # Stream key tests need server encryption. Test-only key. + - IGGY_ENCRYPTION_ENABLED=true + - IGGY_ENCRYPTION_KEY=/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE= healthcheck: test: [ "CMD", "/usr/local/bin/iggy", "--tcp-server-address", "127.0.0.1:8090", "ping" ] interval: 5s
diff --git a/foreign/python/pyproject.toml b/foreign/python/pyproject.toml index bc8ae6e..3e41184 100644 --- a/foreign/python/pyproject.toml +++ b/foreign/python/pyproject.toml
@@ -22,7 +22,7 @@ [project] name = "apache-iggy" requires-python = ">=3.11" -version = "0.9.1.dev1" +version = "0.9.1.dev2" description = "Apache Iggy is the persistent message streaming platform written in Rust, supporting QUIC, TCP and HTTP transport protocols, capable of processing millions of messages per second." readme = "README.md" license = { file = "LICENSE" } @@ -150,6 +150,8 @@ [tool.uv] exclude-newer = "7 days" +# certifi ships CA removals that cannot wait out the full cooldown. +exclude-newer-package = { certifi = "2 days" } cache-keys = [ { file = "pyproject.toml" }, { file = "Cargo.toml" },
diff --git a/foreign/python/src/client.rs b/foreign/python/src/client.rs index e6a4f2d..04b0aef 100644 --- a/foreign/python/src/client.rs +++ b/foreign/python/src/client.rs
@@ -19,7 +19,8 @@ use iggy::prelude::{ AutoCommit as RustAutoCommit, Consumer as RustConsumer, IggyClient as RustIggyClient, IggyExpiry as RustIggyExpiry, IggyMessage as RustMessage, MaxTopicSize as RustMaxTopicSize, - Partitioning as RustPartitioning, PollingStrategy as RustPollingStrategy, *, + OptionKey as RustOptionKey, Partitioning as RustPartitioning, + PollingStrategy as RustPollingStrategy, *, }; use pyo3::PyRef; use pyo3::exceptions::{PyRuntimeError, PyValueError}; @@ -28,9 +29,9 @@ use pyo3_async_runtimes::tokio::future_into_py; use pyo3_stub_gen::define_stub_info_gatherer; use pyo3_stub_gen::derive::{gen_stub_pyclass, gen_stub_pymethods}; -use std::collections::BTreeMap; +use serde::Deserialize; +use std::collections::{BTreeMap, BTreeSet}; use std::fmt::Display; -use std::str::FromStr; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -46,6 +47,7 @@ use crate::permissions::Permissions as PyPermissions; use crate::producer::{IggyProducer, ProducerMode, u32_param as producer_u32_param}; use crate::receive_message::{PartitionContext, PollingStrategy, ReceiveMessage}; +use crate::resource_options::{OptionKey, StreamOptions, TopicOptions, UserOptions}; use crate::send_message::{SendMessage, SendMessagesResponse as PySendMessagesResponse}; use crate::stats::Stats as PyStats; use crate::stream::{Stream, StreamDetails}; @@ -93,6 +95,21 @@ kind: &'static str, } +/// An option refusal, with the server error code and its snake_case name as `kind`. +#[gen_stub_pyclass] +#[pyclass(frozen, extends=PyIggyError)] +pub struct OptionError { + #[pyo3(get)] + kind: &'static str, +} + +/// The JSON error body the HTTP API returns. +#[derive(Deserialize)] +struct HttpErrorBody { + id: u32, + reason: String, +} + /// Raises every SDK error with its code, and an operation with its recovery /// identity, rather than requiring callers to parse an error message. pub(crate) fn to_runtime_error(error: IggyError) -> PyErr { @@ -100,7 +117,12 @@ IggyError::OperationUncertain(operation_id) => Some((operation_id, "uncertain")), IggyError::LaneBlocked(operation_id) => Some((operation_id, "lane_blocked")), IggyError::OperationNotRetained(operation_id) => Some((operation_id, "not_retained")), - _ => None, + _ => { + if let Some(refusal) = to_option_error(&error) { + return refusal; + } + None + } }; Python::attach(|py| { let coded = PyIggyError::from(&error); @@ -124,33 +146,60 @@ }) } +/// Every transport reports an option refusal with the same code: binary by the error itself, +/// HTTP by the `id` of its JSON error body. +fn to_option_error(error: &IggyError) -> Option<PyErr> { + let (code, message) = match error { + IggyError::HttpResponseError(_, body) => { + let body = serde_json::from_str::<HttpErrorBody>(body).ok()?; + (body.id, body.reason) + } + _ => (error.as_code(), error.to_string()), + }; + if !is_option_error(&IggyError::from_code(code)) { + return None; + } + let kind = IggyError::from_code_as_string(code); + Some(Python::attach(|py| { + let instance = match Bound::new( + py, + PyClassInitializer::from(PyIggyError { code }).add_subclass(OptionError { kind }), + ) { + Ok(instance) => instance, + Err(error) => return error, + }; + if let Err(error) = instance.setattr("args", (message,)) { + return error; + } + PyErr::from_value(instance.into_any()) + })) +} + +fn is_option_error(error: &IggyError) -> bool { + matches!( + error, + IggyError::UnsupportedOptionKey(_) + | IggyError::InvalidOptionValue(_) + | IggyError::OptionsBlockTooLarge(_) + | IggyError::OptionNotUpdatable(_) + | IggyError::OptionNotResettable(_) + | IggyError::ReadOnlyOption(_) + | IggyError::StreamEncryptionUnavailable + | IggyError::UnsupportedCriticalOption(_) + | IggyError::CannotModifyRootUser + ) +} + fn to_value_error<E: Display>(error: E) -> PyErr { PyErr::new::<pyo3::exceptions::PyValueError, _>(error.to_string()) } -/// Resolves the shared `create_topic`/`update_topic` parameters, applying -/// server defaults where the caller left them unset. -fn resolve_topic_params( - compression_algorithm: Option<String>, - message_expiry: Option<&IggyExpiry>, - max_topic_size: Option<&MaxTopicSize>, -) -> PyResult<(CompressionAlgorithm, RustIggyExpiry, RustMaxTopicSize)> { - let compression_algorithm = match compression_algorithm { - Some(algo) => CompressionAlgorithm::from_str(&algo).map_err(PyRuntimeError::new_err)?, - None => CompressionAlgorithm::default(), - }; - - let expiry = message_expiry - .map(RustIggyExpiry::try_from) - .transpose()? - .unwrap_or(RustIggyExpiry::ServerDefault); - - let max_size = max_topic_size - .map(RustMaxTopicSize::try_from) - .transpose()? - .unwrap_or(RustMaxTopicSize::ServerDefault); - - Ok((compression_algorithm, expiry, max_size)) +fn reset_keys(reset: Option<Vec<OptionKey>>) -> BTreeSet<RustOptionKey> { + reset + .into_iter() + .flatten() + .map(RustOptionKey::from) + .collect() } #[gen_stub_pymethods] @@ -394,13 +443,14 @@ /// password: Password as `str`. /// status: User status as `UserStatus | None`; defaults to `UserStatus.Active`. /// permissions: Permissions as `Permissions | None`; the user has none when `None`. + /// options: User options as `UserOptions | None`. /// /// Returns: /// An awaitable that resolves to the created `UserInfoDetails`. /// /// Raises: /// RuntimeError: If an argument is invalid or the request fails. - #[pyo3(signature = (username, password, status=None, permissions=None))] + #[pyo3(signature = (username, password, status=None, permissions=None, options=None))] #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[UserInfoDetails]", imports=("collections.abc")))] fn create_user<'a>( &self, @@ -411,14 +461,16 @@ #[gen_stub(override_type(type_repr = "Permissions | None"))] permissions: Option< PyPermissions, >, + #[gen_stub(override_type(type_repr = "UserOptions | None"))] options: Option<UserOptions>, ) -> PyResult<Bound<'a, PyAny>> { let status = status.map_or(UserStatus::Active, UserStatus::from); let permissions = permissions.map(|permissions| permissions.inner); + let options = options.map(|options| options.inner).unwrap_or_default(); let inner = self.inner.clone(); future_into_py(py, async move { let user = inner - .create_user(&username, &password, status, permissions) + .create_user(&username, &password, status, permissions, &options) .await .map_err(to_runtime_error)?; Ok(PyUserInfoDetails::from(user)) @@ -427,21 +479,36 @@ /// Update a user by unique ID or username. /// + /// A patch: only the fields passed change, every other setting keeps its + /// value. Renaming the user this client is signed in as also updates the + /// credentials a reconnect replays. + /// /// Args: /// user_id: User identifier as `str | int`. /// username: New username as `str | None`; unchanged when `None`. /// status: New status as `UserStatus | None`; unchanged when `None`. - /// options: Reserved for future use. Additional option keys as - /// `dict[str, str] | None`, forwarded to the server. No user update - /// option key exists yet, so a current server rejects every key. + /// Deactivating a user ends its sessions. + /// reset: Keys to return to their defaults as `list[OptionKey] | None`. + /// No user key is resettable yet, so any key is refused. + /// options: Additional option keys as `dict[str, str] | None`, + /// forwarded to the server. No other user key exists yet, so a + /// current server rejects every key. + /// raw_reset: Additional reset names as `list[str] | None`, sent + /// verbatim so a key newer than this build can be reset. The + /// server decides them. /// /// Returns: /// An awaitable that resolves to `None` when the user is updated. /// /// Raises: /// ValueError: If a string identifier is invalid. - /// RuntimeError: If the request fails. - #[pyo3(signature = (user_id, username=None, status=None, options=None))] + /// OptionError: A `RuntimeError` carrying the error `code` and + /// `kind` when the client or the server refuses an option. + /// RuntimeError: Before anything is sent, if the update changes + /// nothing, names a key both set and reset, or resets a key that + /// cannot be reset. Otherwise if the request fails. + #[pyo3(signature = (user_id, username=None, status=None, reset=None, options=None, raw_reset=None))] + #[allow(clippy::too_many_arguments)] #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn update_user<'a>( &self, @@ -449,19 +516,28 @@ user_id: PyIdentifier, #[gen_stub(override_type(type_repr = "builtins.str | None"))] username: Option<String>, #[gen_stub(override_type(type_repr = "UserStatus | None"))] status: Option<PyUserStatus>, + #[gen_stub(override_type(type_repr = "builtins.list[OptionKey] | None"))] reset: Option< + Vec<OptionKey>, + >, #[gen_stub(override_type(type_repr = "builtins.dict[builtins.str, builtins.str] | None"))] options: Option<BTreeMap<String, String>>, + #[gen_stub(override_type(type_repr = "builtins.list[builtins.str] | None"))] + raw_reset: Option<Vec<String>>, ) -> PyResult<Bound<'a, PyAny>> { let user_id = Identifier::try_from(user_id)?; - let status = status.map(UserStatus::from); let update_options = UserUpdateOptions { + username, + status: status.map(UserStatus::from), + reset: reset_keys(reset), + raw_reset: raw_reset.into_iter().flatten().collect(), raw: options.unwrap_or_default(), }; + update_options.to_patch().map_err(to_runtime_error)?; let inner = self.inner.clone(); future_into_py(py, async move { inner - .update_user(&user_id, username.as_deref(), status, &update_options) + .update_user(&user_id, &update_options) .await .map_err(to_runtime_error)?; Ok(()) @@ -684,15 +760,40 @@ }) } - /// Creates a new stream with the provided ID and name. - /// Raises `RuntimeError` if the stream cannot be created. - #[pyo3(signature = (name))] - #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] - fn create_stream<'a>(&self, py: Python<'a>, name: String) -> PyResult<Bound<'a, PyAny>> { + /// Create a new stream. + /// + /// On a server with encryption enabled every stream is stored under its + /// own key: the one `options` carries, or one the server generates. The + /// returned details report only the key's source and fingerprints. + /// + /// Args: + /// name: Stream name as `str`, between 1 and 255 UTF-8 bytes. + /// options: Stream options as `StreamOptions | None`. + /// + /// Returns: + /// An awaitable that resolves to the created `StreamDetails`. + /// + /// Raises: + /// RuntimeError: If the stream cannot be created, for example when + /// `options` carries a key and the server has no encryption key. + #[pyo3(signature = (name, options=None))] + #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[StreamDetails]", imports=("collections.abc")))] + fn create_stream<'a>( + &self, + py: Python<'a>, + name: String, + #[gen_stub(override_type(type_repr = "StreamOptions | None"))] options: Option< + StreamOptions, + >, + ) -> PyResult<Bound<'a, PyAny>> { + let options = options.map(|options| options.inner).unwrap_or_default(); let inner = self.inner.clone(); future_into_py(py, async move { - inner.create_stream(&name).await.map_err(to_runtime_error)?; - Ok(()) + let stream = inner + .create_stream(&name, &options) + .await + .map_err(to_runtime_error)?; + Ok(StreamDetails::from(stream)) }) } @@ -734,19 +835,26 @@ }) } - /// Rename a stream selected by name or numeric ID. + /// Update a stream selected by name or numeric ID. /// - /// `stream_id` accepts a stream name as `str` or numeric ID as `int`. A - /// decimal-only string is interpreted as a numeric ID. `name` must be unique - /// and contain between 1 and 255 UTF-8 bytes. Renaming a stream to its current - /// name succeeds without changing it. + /// A patch: only the fields passed change. The stream keeps its id, its + /// encryption key and its data. `stream_id` accepts a stream name as `str` + /// or numeric ID as `int`. A decimal-only string is interpreted as a + /// numeric ID. Renaming a stream to its current name succeeds without + /// changing it. /// /// Args: /// stream_id: Stream identifier as `str | int`. - /// name: New stream name as `str`. + /// name: New stream name as `str | None`, unique and between 1 and 255 + /// UTF-8 bytes, unchanged when `None`. + /// reset: Keys to return to their defaults as `list[OptionKey] | None`. + /// No stream key is resettable, so any key is refused. /// options: Additional option keys as `dict[str, str] | None`, forwarded /// to the server. Current server versions reject all stream update /// option keys. + /// raw_reset: Additional reset names as `list[str] | None`, sent + /// verbatim so a key newer than this build can be reset. The + /// server decides them. /// /// Returns: /// None. @@ -755,28 +863,41 @@ /// TypeError: If `stream_id` is not `str` or an integer in /// `0..=2**32 - 1`, or `name` is not `str`. /// ValueError: If a string identifier is empty or exceeds 255 UTF-8 bytes. - /// RuntimeError: If the client is not authenticated, the user lacks global - /// `manage_streams` or per-stream `manage_stream` permission, the - /// stream does not exist, the new name is invalid or already used, or - /// the request fails. - #[pyo3(signature = (stream_id, name, options=None))] + /// OptionError: A `RuntimeError` carrying the error `code` and + /// `kind` when the client or the server refuses an option. + /// RuntimeError: Before anything is sent, if the update changes + /// nothing, the name does not fit, or a reset key cannot be reset. + /// Otherwise if the client is not authenticated, the user lacks + /// global `manage_streams` or per-stream `manage_stream` permission, + /// the stream does not exist, the new name is already used, or the + /// request fails. + #[pyo3(signature = (stream_id, name=None, reset=None, options=None, raw_reset=None))] #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn update_stream<'a>( &self, py: Python<'a>, stream_id: PyIdentifier, - name: String, + #[gen_stub(override_type(type_repr = "builtins.str | None"))] name: Option<String>, + #[gen_stub(override_type(type_repr = "builtins.list[OptionKey] | None"))] reset: Option< + Vec<OptionKey>, + >, #[gen_stub(override_type(type_repr = "builtins.dict[builtins.str, builtins.str] | None"))] options: Option<BTreeMap<String, String>>, + #[gen_stub(override_type(type_repr = "builtins.list[builtins.str] | None"))] + raw_reset: Option<Vec<String>>, ) -> PyResult<Bound<'a, PyAny>> { let stream_id = Identifier::try_from(stream_id)?; let update_options = StreamUpdateOptions { + name, + reset: reset_keys(reset), + raw_reset: raw_reset.into_iter().flatten().collect(), raw: options.unwrap_or_default(), }; + update_options.to_patch().map_err(to_runtime_error)?; let inner = self.inner.clone(); future_into_py(py, async move { inner - .update_stream(&stream_id, &name, &update_options) + .update_stream(&stream_id, &update_options) .await .map_err(to_runtime_error)?; Ok(()) @@ -816,96 +937,30 @@ }) } - /// Creates a new topic with the given parameters. + /// Creates a new topic. /// /// Args: /// stream: Stream identifier as `str | int`. /// name: Topic name as `str`. - /// partitions_count: Number of partitions as `int`, at most 1000. - /// compression_algorithm: Compression algorithm as `str | None`. - /// message_expiry: Message expiry as `IggyExpiry | None`. - /// max_topic_size: Maximum topic size as `MaxTopicSize | None`. - /// segment_size: Per-topic segment size in bytes as `int | None`. - /// durability: Message completion policy, defaulting to replicated. - /// consumer_offset_durability: Independent offset policy, defaulting to replicated. - /// messages_required_to_save: Message-count flush threshold as `int | None`. - /// size_of_messages_required_to_save: Byte flush threshold as `int | None`. - /// preallocate_segments: Reserve segment bytes on open as `bool | None`. - /// options: Additional option keys as `dict[str, str] | None`, sent - /// verbatim so a newer server key can be set from this build. - /// `{"partition_resize_policy": "fixed"}` makes the server reject - /// later partition creation and deletion on this topic. - /// - /// Every option left as `None` resolves against the server default at - /// admission. + /// options: Topic options as `TopicOptions | None`. Without options the + /// topic gets one partition and every setting resolves against the + /// server default at admission. /// /// Returns: /// An awaitable that resolves to `None` when the topic is created. /// /// Raises: - /// ValueError: If `message_expiry` or `max_topic_size` is out of range. - /// PyRuntimeError: If another argument is invalid or the request fails. - #[pyo3( - signature = (stream, name, partitions_count, compression_algorithm = None, message_expiry = None, max_topic_size = None, segment_size = None, durability = None, consumer_offset_durability = None, messages_required_to_save = None, size_of_messages_required_to_save = None, preallocate_segments = None, options = None) - )] - #[allow(clippy::too_many_arguments)] + /// RuntimeError: If an argument is invalid or the request fails. + #[pyo3(signature = (stream, name, options=None))] #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn create_topic<'a>( &self, py: Python<'a>, stream: PyIdentifier, name: String, - partitions_count: u32, - #[gen_stub(override_type(type_repr = "builtins.str | None"))] compression_algorithm: Option< - String, - >, - #[gen_stub(override_type(type_repr = "IggyExpiry | None"))] message_expiry: Option< - &IggyExpiry, - >, - #[gen_stub(override_type(type_repr = "MaxTopicSize | None"))] max_topic_size: Option< - &MaxTopicSize, - >, - #[gen_stub(override_type(type_repr = "builtins.int | None"))] segment_size: Option<u64>, - #[gen_stub(override_type(type_repr = "Durability | None"))] durability: Option< - &Bound<'_, PyAny>, - >, - #[gen_stub(override_type(type_repr = "Durability | None"))] - consumer_offset_durability: Option<&Bound<'_, PyAny>>, - #[gen_stub(override_type(type_repr = "builtins.int | None"))] - messages_required_to_save: Option<u32>, - #[gen_stub(override_type(type_repr = "builtins.int | None"))] - size_of_messages_required_to_save: Option<u64>, - #[gen_stub(override_type(type_repr = "builtins.bool | None"))] preallocate_segments: Option< - bool, - >, - #[gen_stub(override_type(type_repr = "builtins.dict[builtins.str, builtins.str] | None"))] - options: Option<BTreeMap<String, String>>, + #[gen_stub(override_type(type_repr = "TopicOptions | None"))] options: Option<TopicOptions>, ) -> PyResult<Bound<'a, PyAny>> { - let (compression_algorithm, expiry, max_size) = - resolve_topic_params(compression_algorithm, message_expiry, max_topic_size)?; - - let topic_options = TopicCreateOptions { - partitions_count: Some(partitions_count), - // `None` is what tells admission to resolve the server default, so - // the sentinels the resolver returns must collapse back to it. - compression_algorithm: (compression_algorithm != CompressionAlgorithm::default()) - .then_some(compression_algorithm), - message_expiry: (expiry != RustIggyExpiry::ServerDefault).then_some(expiry), - max_topic_size: (max_size != RustMaxTopicSize::ServerDefault).then_some(max_size), - segment_size: segment_size.map(IggyByteSize::from), - durability: crate::durability::Durability::try_from(durability)?.0, - consumer_offset_durability: crate::durability::Durability::try_from( - consumer_offset_durability, - )? - .0, - messages_required_to_save, - size_of_messages_required_to_save: size_of_messages_required_to_save - .map(IggyByteSize::from), - preallocate_segments, - raw: options.unwrap_or_default(), - partition_resize_policy: None, - }; - + let topic_options = options.map(|options| options.inner).unwrap_or_default(); let stream = Identifier::try_from(stream)?; let inner = self.inner.clone(); @@ -971,29 +1026,42 @@ /// Update an existing topic. /// - /// A patch, not a replacement: every setting rides the options block, so a - /// field left unset keeps the topic's current value rather than resetting - /// it to a server default. + /// A patch, not a replacement: only the fields passed change, every other + /// setting keeps its value. Settings fixed at creation, such as the + /// compression algorithm, have no field here. /// /// Args: /// stream_id: Stream identifier as `str | int`. /// topic_id: Topic identifier as `str | int`. - /// name: New topic name as `str`. - /// compression_algorithm: Compression algorithm as `str | None`. - /// message_expiry: Message expiry as `IggyExpiry | None`. - /// max_topic_size: Maximum topic size as `MaxTopicSize | None`. + /// name: New topic name as `str | None`, unchanged when `None`. + /// message_expiry: Message expiry as `IggyExpiry | None`, unchanged + /// when `None`. `IggyExpiry.ServerDefault()` changes nothing either, + /// reset `OptionKey.MessageExpiry` to return to the default. + /// max_topic_size: Maximum topic size as `MaxTopicSize | None`, + /// unchanged when `None`. `MaxTopicSize.ServerDefault()` changes + /// nothing either, reset `OptionKey.MaxTopicSize` to return to the + /// default. + /// reset: Keys to return to their defaults as `list[OptionKey] | None`: + /// `OptionKey.MessageExpiry` or `OptionKey.MaxTopicSize`. /// options: Additional option keys as `dict[str, str] | None`, sent /// verbatim so an updatable server key can be set from this build. /// A create-only key is refused by name. + /// raw_reset: Additional reset names as `list[str] | None`, sent + /// verbatim so a key newer than this build can be reset. The + /// server decides them. /// /// Returns: /// An awaitable that resolves to `None` when the topic is updated. /// /// Raises: /// ValueError: If `message_expiry` or `max_topic_size` is out of range. - /// PyRuntimeError: If another argument is invalid or the request fails. + /// OptionError: A `RuntimeError` carrying the error `code` and + /// `kind` when the client or the server refuses an option. + /// RuntimeError: Before anything is sent, if the update changes + /// nothing, names a key both set and reset, the name does not fit, + /// or a reset key cannot be reset. Otherwise if the request fails. #[pyo3( - signature = (stream_id, topic_id, name, compression_algorithm = None, message_expiry = None, max_topic_size = None, options = None) + signature = (stream_id, topic_id, name = None, message_expiry = None, max_topic_size = None, reset = None, options = None, raw_reset = None) )] #[allow(clippy::too_many_arguments)] #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] @@ -1002,30 +1070,30 @@ py: Python<'a>, stream_id: PyIdentifier, topic_id: PyIdentifier, - name: String, - #[gen_stub(override_type(type_repr = "builtins.str | None"))] compression_algorithm: Option< - String, - >, + #[gen_stub(override_type(type_repr = "builtins.str | None"))] name: Option<String>, #[gen_stub(override_type(type_repr = "IggyExpiry | None"))] message_expiry: Option< &IggyExpiry, >, #[gen_stub(override_type(type_repr = "MaxTopicSize | None"))] max_topic_size: Option< &MaxTopicSize, >, + #[gen_stub(override_type(type_repr = "builtins.list[OptionKey] | None"))] reset: Option< + Vec<OptionKey>, + >, #[gen_stub(override_type(type_repr = "builtins.dict[builtins.str, builtins.str] | None"))] options: Option<BTreeMap<String, String>>, + #[gen_stub(override_type(type_repr = "builtins.list[builtins.str] | None"))] + raw_reset: Option<Vec<String>>, ) -> PyResult<Bound<'a, PyAny>> { - // Absent stays absent: a key the caller did not pass is left alone - // server-side rather than reset to a default. - let compression_algorithm = compression_algorithm - .map(|algo| CompressionAlgorithm::from_str(&algo).map_err(PyRuntimeError::new_err)) - .transpose()?; let update_options = TopicUpdateOptions { - compression_algorithm, + name, message_expiry: message_expiry.map(RustIggyExpiry::try_from).transpose()?, max_topic_size: max_topic_size.map(RustMaxTopicSize::try_from).transpose()?, + reset: reset_keys(reset), + raw_reset: raw_reset.into_iter().flatten().collect(), raw: options.unwrap_or_default(), }; + update_options.to_patch().map_err(to_runtime_error)?; let stream_id = Identifier::try_from(stream_id)?; let topic_id = Identifier::try_from(topic_id)?; @@ -1033,7 +1101,7 @@ future_into_py(py, async move { inner - .update_topic(&stream_id, &topic_id, &name, &update_options) + .update_topic(&stream_id, &topic_id, &update_options) .await .map_err(to_runtime_error)?; Ok(()) @@ -1827,6 +1895,24 @@ use super::*; #[test] + fn given_root_policy_refusal_when_mapping_should_preserve_code_and_kind() { + Python::initialize(); + Python::attach(|py| { + let error = to_runtime_error(IggyError::CannotModifyRootUser); + assert!(error.is_instance_of::<OptionError>(py)); + let value = error.value(py); + assert_eq!( + value.getattr("code").unwrap().extract::<u32>().unwrap(), + 4108 + ); + assert_eq!( + value.getattr("kind").unwrap().extract::<String>().unwrap(), + "cannot_modify_root_user" + ); + }); + } + + #[test] fn pending_operation_errors_preserve_exact_identity_and_kind() { Python::initialize(); Python::attach(|py| {
diff --git a/foreign/python/src/lib.rs b/foreign/python/src/lib.rs index 11c1ab4..b5d25ab 100644 --- a/foreign/python/src/lib.rs +++ b/foreign/python/src/lib.rs
@@ -26,6 +26,7 @@ mod permissions; mod producer; mod receive_message; +mod resource_options; mod send_message; mod stats; mod stream; @@ -33,7 +34,7 @@ mod user; mod user_headers; -use client::{IggyClient, PendingOperationError, PyIggyError}; +use client::{IggyClient, OptionError, PendingOperationError, PyIggyError}; use config::{ AutoLogin, HttpConfig, PendingOperationsConfig, QuicConfig, QuicReconnectionConfig, TcpConfig, TcpReconnectionConfig, WebSocketConfig, WebSocketFramingConfig, WebSocketReconnectionConfig, @@ -51,6 +52,10 @@ }; use pyo3::prelude::*; use receive_message::{ConsumerPosition, PartitionContext, PollingStrategy, ReceiveMessage}; +use resource_options::{ + OptionKey, PartitionResizePolicy, StreamOptions, TopicOptions, UserOptions, + encryption_key_fingerprint, +}; use send_message::{SendMessage, SendMessagesConfirmation, SendMessagesResponse}; use stats::{CacheMetrics, CacheMetricsKey, Stats}; use stream::{Stream, StreamDetails}; @@ -71,6 +76,7 @@ m.add_class::<IggyClient>()?; m.add_class::<PyIggyError>()?; m.add_class::<PendingOperationError>()?; + m.add_class::<OptionError>()?; m.add_class::<IggyProducer>()?; m.add_class::<ProducerSendError>()?; m.add_class::<DirectProducerConfig>()?; @@ -97,6 +103,12 @@ m.add_class::<IggyExpiry>()?; m.add_class::<MaxTopicSize>()?; m.add_class::<OptionSpec>()?; + m.add_class::<OptionKey>()?; + m.add_class::<StreamOptions>()?; + m.add_class::<TopicOptions>()?; + m.add_class::<UserOptions>()?; + m.add_class::<PartitionResizePolicy>()?; + m.add_function(wrap_pyfunction!(encryption_key_fingerprint, m)?)?; m.add_class::<Partitioning>()?; m.add_class::<Partition>()?; m.add_class::<Consumer>()?;
diff --git a/foreign/python/src/options.rs b/foreign/python/src/options.rs index a0d4fa8..ad2414a 100644 --- a/foreign/python/src/options.rs +++ b/foreign/python/src/options.rs
@@ -17,13 +17,16 @@ use iggy::prelude::{ HeaderValue as RustHeaderValue, OptionSpec as RustOptionSpec, OptionsScope as RustOptionsScope, + ResourceOptions, }; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use pyo3_stub_gen::derive::{gen_stub_pyclass, gen_stub_pymethods}; use std::str::FromStr; -use crate::user_headers::{HeaderValue, rust_header_value_to_py}; +use crate::user_headers::{ + HeaderValue, UserHeaders, rust_header_value_to_py, rust_user_headers_to_py, +}; /// One entry of a resource's option catalog, as served by `describe_options`. #[gen_stub_pyclass] @@ -48,10 +51,9 @@ } /// Name of this key's canonical kind: what the server encodes its default - /// under, and what a value set by `create_topic` is stored as whatever kind - /// it was sent in, since create admission re-encodes the block from its own - /// parse. `update_topic` stores what the client sent verbatim and is the - /// exception. + /// under, and what a value set by a create or an update is stored as + /// whatever kind it was sent in, since admission re-encodes the block from + /// its own parse. #[getter] pub fn kind(&self) -> String { self.inner.kind.to_string() @@ -86,6 +88,38 @@ self.inner.description.clone() } + /// Whether an update may set the key. + #[getter] + pub fn updatable(&self) -> bool { + self.inner.updatable + } + + /// Whether an update may reset the key to its default. + #[getter] + pub fn resettable(&self) -> bool { + self.inner.resettable + } + + /// Whether the key is write-only: accepted on create, never returned. + #[getter] + pub fn secret(&self) -> bool { + self.inner.secret + } + + /// Whether the server computes the key and returns it on reads, never + /// accepting it. + #[getter] + pub fn read_only(&self) -> bool { + self.inner.read_only + } + + /// Whether a node that does not understand the key must refuse to serve + /// the resource. + #[getter] + pub fn critical(&self) -> bool { + self.inner.critical + } + fn __repr__(&self) -> String { format!( "OptionSpec(key='{}', kind='{}')", @@ -109,3 +143,23 @@ )) }) } + +/// The entries of one provenance, as the dictionary message user headers come +/// back as. +/// +/// Options ride the user-headers codec, so they are handed back through the +/// same `HeaderKey`/`HeaderValue` types rather than a second shape meaning the +/// same thing: `to_scalar_dict()` works on the result exactly as it does on +/// `ReceiveMessage.user_headers`. +pub(crate) fn options_by_provenance<'a>( + py: Python<'a>, + options: &ResourceOptions, + explicit: bool, +) -> PyResult<Bound<'a, UserHeaders>> { + let selected = options + .iter() + .filter(|(_, option)| option.explicit == explicit) + .map(|(key, option)| (key.clone(), option.value.clone())) + .collect(); + rust_user_headers_to_py(py, selected) +}
diff --git a/foreign/python/src/resource_options.rs b/foreign/python/src/resource_options.rs new file mode 100644 index 0000000..a5fd650 --- /dev/null +++ b/foreign/python/src/resource_options.rs
@@ -0,0 +1,614 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use iggy::prelude::{ + CompressionAlgorithm, IggyByteSize, IggyError, IggyExpiry as RustIggyExpiry, + MaxTopicSize as RustMaxTopicSize, OptionKey as RustOptionKey, + PartitionResizePolicy as RustPartitionResizePolicy, StreamCreateOptions, StreamEncryptionKey, + TopicCreateOptions, UserCreateOptions, +}; +use pyo3::exceptions::{PyRuntimeError, PyValueError}; +use pyo3::prelude::*; +use pyo3::types::{PyBytes, PyString}; +use pyo3_stub_gen::derive::{ + gen_stub_pyclass, gen_stub_pyclass_enum, gen_stub_pyfunction, gen_stub_pymethods, +}; +use std::collections::BTreeMap; +use std::str::FromStr; + +use crate::durability::Durability; +use crate::topic::{IggyExpiry, MaxTopicSize}; + +/// Options for `create_stream`. +/// +/// The encryption key is write-only: the server stores it wrapped and returns +/// only its source and fingerprints, and this object never shows it. +#[gen_stub_pyclass] +#[pyclass(from_py_object)] +#[derive(Clone)] +pub struct StreamOptions { + pub(crate) inner: StreamCreateOptions, +} + +/// Options for `create_topic`. +#[gen_stub_pyclass] +#[pyclass(from_py_object)] +#[derive(Clone)] +pub struct TopicOptions { + pub(crate) inner: TopicCreateOptions, +} + +/// Options for `create_user`. No user key exists yet, so only the raw escape +/// hatch is available. +#[gen_stub_pyclass] +#[pyclass(from_py_object)] +#[derive(Clone)] +pub struct UserOptions { + pub(crate) inner: UserCreateOptions, +} + +/// A catalog option key, as `update_stream`, `update_topic` and `update_user` +/// take it in `reset`. `str()` gives the key as it travels on the wire. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[gen_stub_pyclass_enum] +#[pyclass(eq, from_py_object)] +pub enum OptionKey { + /// Stream or topic name. + Name, + /// User name. + Username, + /// User status. + Status, + /// Topic compression algorithm, fixed at creation. + CompressionAlgorithm, + /// Topic message expiry, resettable to never expire. + MessageExpiry, + /// Topic size cap, resettable to unlimited. + MaxTopicSize, + /// Topic segment size, fixed at creation. + SegmentSize, + /// Topic message durability, fixed at creation. + Durability, + /// Topic consumer offset durability, fixed at creation. + ConsumerOffsetDurability, + /// Topic partition resize policy, fixed at creation. + PartitionResizePolicy, + /// Topic message-count flush threshold, fixed at creation. + MessagesRequiredToSave, + /// Topic byte flush threshold, fixed at creation. + SizeOfMessagesRequiredToSave, + /// Topic segment preallocation, fixed at creation. + PreallocateSegments, + /// Stream encryption key, write-only and fixed at creation. + EncryptionKey, + /// Who chose the stream's key, computed by the server. + EncryptionKeySource, + /// Fingerprint of the stream's key, computed by the server. + EncryptionKeyFingerprint, + /// Fingerprints of the server keys wrapping the stream's key, computed by + /// the server. + EncryptionKekFingerprint, +} + +/// Whether partitions can be added to or deleted from a topic after creation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[gen_stub_pyclass_enum] +#[pyclass(eq, from_py_object)] +pub enum PartitionResizePolicy { + /// Partitions can be added and deleted. + Mutable, + /// The partition set is fixed at topic creation. + Fixed, +} + +#[derive(FromPyObject)] +pub(crate) enum PyStreamKey<'py> { + #[pyo3(transparent, annotation = "bytes")] + Raw(Bound<'py, PyBytes>), + #[pyo3(transparent, annotation = "str")] + Base64(Bound<'py, PyString>), +} + +#[gen_stub_pymethods] +#[pymethods] +impl StreamOptions { + /// Constructs stream create options. + /// + /// Args: + /// encryption_key: The stream's own data key: exactly 32 bytes, or + /// their canonical standard padded base64 text. Use 32 random + /// bytes, for example `secrets.token_bytes(32)`. The key travels + /// inside the create request over whatever transport the client + /// uses, so protecting it in transit (TLS) is the caller's + /// responsibility. The server never returns it, only its source + /// and fingerprints. Without a key, a server with encryption + /// enabled generates one for the stream. + /// raw: Additional option keys as `dict[str, str]`, sent as string + /// values so a key newer than this build can be set. A typed field + /// wins on collision. + /// + /// Raises: + /// ValueError: If `encryption_key` is not exactly 32 bytes or their + /// canonical base64 text. The message never contains the key. + /// TypeError: If `encryption_key` is neither `bytes` nor `str`. + #[new] + #[pyo3(signature = (*, encryption_key=None, raw=None))] + fn new( + #[gen_stub(override_type(type_repr = "builtins.bytes | builtins.str | None"))] + encryption_key: Option<PyStreamKey<'_>>, + #[gen_stub(override_type(type_repr = "builtins.dict[builtins.str, builtins.str] | None"))] + raw: Option<BTreeMap<String, String>>, + ) -> PyResult<Self> { + Ok(Self { + inner: StreamCreateOptions { + encryption_key: encryption_key + .map(StreamEncryptionKey::try_from) + .transpose()?, + raw: raw.unwrap_or_default(), + }, + }) + } + + /// Whether an encryption key is set, without exposing the key itself. + #[getter] + fn has_encryption_key(&self) -> bool { + self.inner.encryption_key.is_some() + } + + fn __repr__(&self) -> String { + let encryption_key = if self.inner.encryption_key.is_some() { + "..." + } else { + "None" + }; + let secret_key: &str = RustOptionKey::EncryptionKey.into(); + let raw = self + .inner + .raw + .iter() + .map(|(key, value)| { + let shown = if key == secret_key { + "...".to_owned() + } else { + format!("{value:?}") + }; + format!("{key:?}: {shown}") + }) + .collect::<Vec<_>>() + .join(", "); + format!("StreamOptions(encryption_key={encryption_key}, raw={{{raw}}})") + } +} + +#[gen_stub_pymethods] +#[pymethods] +impl TopicOptions { + /// Constructs topic create options. + /// + /// Every setting left as `None` resolves against the server default at + /// admission. Only `message_expiry`, `max_topic_size` and the name can + /// change after creation, through `update_topic`. + /// + /// Args: + /// partitions_count: Number of partitions as `int`, at most 1000. + /// Defaults to 1. + /// compression_algorithm: Compression algorithm as `str | None`. + /// message_expiry: Message expiry as `IggyExpiry | None`. + /// `IggyExpiry.ServerDefault()` is the same as `None`. + /// max_topic_size: Maximum topic size as `MaxTopicSize | None`. + /// `MaxTopicSize.ServerDefault()` is the same as `None`. + /// segment_size: Per-topic segment size in bytes as `int | None`. + /// durability: Message completion policy, defaulting to replicated. + /// consumer_offset_durability: Independent offset policy, defaulting + /// to replicated. + /// partition_resize_policy: `PartitionResizePolicy.Fixed` makes the + /// server reject later partition creation and deletion on this + /// topic. + /// messages_required_to_save: Message-count flush threshold as + /// `int | None`. + /// size_of_messages_required_to_save: Byte flush threshold as + /// `int | None`. + /// preallocate_segments: Reserve segment bytes on open as `bool | None`. + /// raw: Additional option keys as `dict[str, str]`, sent as string + /// values so a key newer than this build can be set. A typed field + /// wins on collision. + /// + /// Raises: + /// ValueError: If `message_expiry` or `max_topic_size` is out of range. + /// RuntimeError: If `compression_algorithm` names no known algorithm. + /// TypeError: If a durability is not a `Durability`. + #[new] + #[pyo3( + signature = (*, partitions_count = 1, compression_algorithm = None, message_expiry = None, max_topic_size = None, segment_size = None, durability = None, consumer_offset_durability = None, partition_resize_policy = None, messages_required_to_save = None, size_of_messages_required_to_save = None, preallocate_segments = None, raw = None) + )] + #[allow(clippy::too_many_arguments)] + fn new( + partitions_count: u32, + #[gen_stub(override_type(type_repr = "builtins.str | None"))] compression_algorithm: Option< + String, + >, + #[gen_stub(override_type(type_repr = "IggyExpiry | None"))] message_expiry: Option< + &IggyExpiry, + >, + #[gen_stub(override_type(type_repr = "MaxTopicSize | None"))] max_topic_size: Option< + &MaxTopicSize, + >, + #[gen_stub(override_type(type_repr = "builtins.int | None"))] segment_size: Option<u64>, + #[gen_stub(override_type(type_repr = "Durability | None"))] durability: Option< + &Bound<'_, PyAny>, + >, + #[gen_stub(override_type(type_repr = "Durability | None"))] + consumer_offset_durability: Option<&Bound<'_, PyAny>>, + #[gen_stub(override_type(type_repr = "PartitionResizePolicy | None"))] + partition_resize_policy: Option<PartitionResizePolicy>, + #[gen_stub(override_type(type_repr = "builtins.int | None"))] + messages_required_to_save: Option<u32>, + #[gen_stub(override_type(type_repr = "builtins.int | None"))] + size_of_messages_required_to_save: Option<u64>, + #[gen_stub(override_type(type_repr = "builtins.bool | None"))] preallocate_segments: Option< + bool, + >, + #[gen_stub(override_type(type_repr = "builtins.dict[builtins.str, builtins.str] | None"))] + raw: Option<BTreeMap<String, String>>, + ) -> PyResult<Self> { + let compression_algorithm = compression_algorithm + .map(|name| CompressionAlgorithm::from_str(&name).map_err(PyRuntimeError::new_err)) + .transpose()?; + // `None` is what tells admission to resolve the server default, so the + // explicit sentinels collapse back to it. + let message_expiry = message_expiry + .map(RustIggyExpiry::try_from) + .transpose()? + .filter(|expiry| *expiry != RustIggyExpiry::ServerDefault); + let max_topic_size = max_topic_size + .map(RustMaxTopicSize::try_from) + .transpose()? + .filter(|size| *size != RustMaxTopicSize::ServerDefault); + Ok(Self { + inner: TopicCreateOptions { + partitions_count: Some(partitions_count), + compression_algorithm, + message_expiry, + max_topic_size, + segment_size: segment_size.map(IggyByteSize::from), + durability: Durability::try_from(durability)?.0, + consumer_offset_durability: Durability::try_from(consumer_offset_durability)?.0, + partition_resize_policy: partition_resize_policy + .map(RustPartitionResizePolicy::from), + messages_required_to_save, + size_of_messages_required_to_save: size_of_messages_required_to_save + .map(IggyByteSize::from), + preallocate_segments, + raw: raw.unwrap_or_default(), + }, + }) + } +} + +#[gen_stub_pymethods] +#[pymethods] +impl UserOptions { + /// Constructs user create options. + /// + /// Args: + /// raw: Option keys as `dict[str, str]`, sent as string values so a + /// key newer than this build can be set. A current server rejects + /// every key. + #[new] + #[pyo3(signature = (*, raw=None))] + fn new( + #[gen_stub(override_type(type_repr = "builtins.dict[builtins.str, builtins.str] | None"))] + raw: Option<BTreeMap<String, String>>, + ) -> Self { + Self { + inner: UserCreateOptions { + raw: raw.unwrap_or_default(), + }, + } + } +} + +#[gen_stub_pymethods] +#[pymethods] +impl OptionKey { + fn __str__(&self) -> &'static str { + RustOptionKey::from(*self).into() + } +} + +/// Fingerprint of a stream encryption key: the first 8 bytes of SHA-256 over +/// the label `iggy/stream-key-fingerprint/v1` followed by the key, as 16 +/// lowercase hex characters. Streams report the same value in +/// `encryption_key_fingerprint`, so a caller can match a key to its stream +/// without the server ever returning the key. +/// +/// Args: +/// key: Exactly 32 bytes, or their canonical standard padded base64 text. +/// +/// Raises: +/// ValueError: If `key` is not exactly 32 bytes or their canonical base64 +/// text. The message never contains the key. +/// TypeError: If `key` is neither `bytes` nor `str`. +#[gen_stub_pyfunction] +#[pyfunction] +pub fn encryption_key_fingerprint( + #[gen_stub(override_type(type_repr = "builtins.bytes | builtins.str"))] key: PyStreamKey<'_>, +) -> PyResult<String> { + StreamEncryptionKey::try_from(key).map(|key| key.fingerprint()) +} + +impl From<OptionKey> for RustOptionKey { + fn from(key: OptionKey) -> Self { + match key { + OptionKey::Name => Self::Name, + OptionKey::Username => Self::Username, + OptionKey::Status => Self::Status, + OptionKey::CompressionAlgorithm => Self::CompressionAlgorithm, + OptionKey::MessageExpiry => Self::MessageExpiry, + OptionKey::MaxTopicSize => Self::MaxTopicSize, + OptionKey::SegmentSize => Self::SegmentSize, + OptionKey::Durability => Self::Durability, + OptionKey::ConsumerOffsetDurability => Self::ConsumerOffsetDurability, + OptionKey::PartitionResizePolicy => Self::PartitionResizePolicy, + OptionKey::MessagesRequiredToSave => Self::MessagesRequiredToSave, + OptionKey::SizeOfMessagesRequiredToSave => Self::SizeOfMessagesRequiredToSave, + OptionKey::PreallocateSegments => Self::PreallocateSegments, + OptionKey::EncryptionKey => Self::EncryptionKey, + OptionKey::EncryptionKeySource => Self::EncryptionKeySource, + OptionKey::EncryptionKeyFingerprint => Self::EncryptionKeyFingerprint, + OptionKey::EncryptionKekFingerprint => Self::EncryptionKekFingerprint, + } + } +} + +impl From<PartitionResizePolicy> for RustPartitionResizePolicy { + fn from(policy: PartitionResizePolicy) -> Self { + match policy { + PartitionResizePolicy::Mutable => Self::Mutable, + PartitionResizePolicy::Fixed => Self::Fixed, + } + } +} + +impl TryFrom<PyStreamKey<'_>> for StreamEncryptionKey { + type Error = PyErr; + + fn try_from(key: PyStreamKey<'_>) -> PyResult<Self> { + match key { + PyStreamKey::Raw(bytes) => Self::try_from(bytes.as_bytes()), + PyStreamKey::Base64(text) => Self::from_str(text.to_str()?), + } + .map_err(|error: IggyError| PyValueError::new_err(error.to_string())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::options::OptionSpec; + use crate::stream::StreamDetails; + use crate::topic::TopicDetails; + use crate::user::UserInfoDetails; + use iggy::prelude::{ + StreamDetails as RustStreamDetails, TopicDetails as RustTopicDetails, + UserInfoDetails as RustUserInfoDetails, + }; + use iggy_binary_protocol::WireDecode; + use iggy_binary_protocol::responses::streams::GetStreamResponse; + use iggy_binary_protocol::responses::system::DescribeOptionsResponse; + use iggy_binary_protocol::responses::topics::GetTopicResponse; + use iggy_binary_protocol::responses::users::UserDetailsResponse; + use iggy_common::wire_conversions::option_specs_from_wire; + use pyo3::types::PyDict; + use serde_json::Value; + + const FIXTURE: &str = include_str!("../tests/fixtures/resource_options.json"); + + #[test] + fn given_stream_response_vectors_when_reading_should_match_projections() { + Python::initialize(); + Python::attach(|py| { + for vector in vectors("stream_responses") { + let bytes = decode_hex(&vector["response_hex"]); + let (response, _) = GetStreamResponse::decode(&bytes).expect("stream decodes"); + let details = RustStreamDetails::try_from(response).expect("stream converts"); + let stream = Bound::new(py, StreamDetails::from(details)).expect("stream wraps"); + let expected = &vector["expected"]; + + assert_eq!( + attribute::<Option<String>>(stream.as_any(), "encryption_key_source"), + expected["encryption_key_source"] + .as_str() + .map(str::to_owned) + ); + assert_eq!( + attribute::<Option<String>>(stream.as_any(), "encryption_key_fingerprint"), + expected["encryption_key_fingerprint"] + .as_str() + .map(str::to_owned) + ); + assert_eq!( + attribute::<Vec<String>>(stream.as_any(), "encryption_kek_fingerprints"), + serde_json::from_value::<Vec<String>>( + expected["encryption_kek_fingerprints"].clone() + ) + .expect("fingerprint list") + ); + assert_options(stream.as_any(), &expected["options"]); + } + }); + } + + #[test] + fn given_topic_and_user_response_vectors_when_reading_should_match_projections() { + Python::initialize(); + Python::attach(|py| { + for vector in vectors("topic_responses") { + let bytes = decode_hex(&vector["response_hex"]); + let (response, _) = GetTopicResponse::decode(&bytes).expect("topic decodes"); + let details = RustTopicDetails::try_from(response).expect("topic converts"); + let topic = Bound::new(py, TopicDetails::from(details)).expect("topic wraps"); + let expected = &vector["expected"]; + + assert_eq!( + attribute::<String>(topic.as_any(), "compression_algorithm"), + expected["compression_algorithm"] + .as_str() + .expect("compression") + ); + let message_expiry = topic.getattr("message_expiry").expect("expiry"); + let duration = message_expiry + .getattr("duration") + .expect("custom expiry") + .extract::<std::time::Duration>() + .expect("duration"); + assert_eq!( + duration.as_micros().to_string(), + expected["message_expiry"].as_str().expect("expiry") + ); + let max_topic_size = topic.getattr("max_topic_size").expect("max size"); + assert!(max_topic_size.is_instance_of::<crate::topic::MaxTopicSize>()); + assert_eq!( + expected["max_topic_size"].as_str(), + Some(u64::MAX.to_string().as_str()) + ); + assert!( + max_topic_size.getattr("bytes").is_err(), + "the unlimited cap carries no byte count" + ); + assert_options(topic.as_any(), &expected["options"]); + } + + for vector in vectors("user_responses") { + let bytes = decode_hex(&vector["response_hex"]); + let (response, _) = UserDetailsResponse::decode(&bytes).expect("user decodes"); + let details = RustUserInfoDetails::try_from(response).expect("user converts"); + let user = Bound::new(py, UserInfoDetails::from(details)).expect("user wraps"); + let expected = &vector["expected"]; + + assert_eq!( + attribute::<String>(user.as_any(), "username"), + expected["username"].as_str().expect("username") + ); + assert_options(user.as_any(), &expected["options"]); + } + }); + } + + #[test] + fn given_describe_options_vectors_when_reading_should_expose_every_flag() { + Python::initialize(); + let fixture: Value = serde_json::from_str(FIXTURE).expect("fixture parses"); + let flags = &fixture["constants"]["option_flags"]; + Python::attach(|py| { + for vector in vectors("describe_options") { + let bytes = decode_hex(&vector["response_hex"]); + let (response, _) = + DescribeOptionsResponse::decode(&bytes).expect("describe decodes"); + let specs = option_specs_from_wire(response).expect("specs convert"); + let entries = vector["entries"].as_array().expect("entries"); + assert_eq!(specs.len(), entries.len()); + + for (spec, entry) in specs.into_iter().zip(entries) { + let spec = Bound::new(py, OptionSpec::from(spec)).expect("spec wraps"); + assert_eq!( + attribute::<String>(spec.as_any(), "key"), + entry["key"].as_str().expect("key") + ); + let bits = entry["flags"].as_u64().expect("flags"); + for flag in ["updatable", "resettable", "secret", "read_only", "critical"] { + let mask = flags[flag].as_u64().expect("flag bit"); + assert_eq!( + attribute::<bool>(spec.as_any(), flag), + bits & mask != 0, + "{flag}" + ); + } + } + } + }); + } + + fn vectors(section: &str) -> Vec<Value> { + let fixture: Value = serde_json::from_str(FIXTURE).expect("fixture parses"); + fixture[section].as_array().expect("vector list").clone() + } + + fn attribute<'py, T: FromPyObjectOwned<'py>>(object: &Bound<'py, PyAny>, name: &str) -> T { + object + .getattr(name) + .expect("attribute exists") + .extract::<T>() + .map_err(Into::<PyErr>::into) + .expect("attribute extracts") + } + + /// Compare both provenance getters with the fixture's option map. + fn assert_options(object: &Bound<'_, PyAny>, expected: &Value) { + let py = object.py(); + let expected = expected.as_object().expect("option map"); + for (getter, explicit) in [("options", true), ("derived_options", false)] { + let actual = object + .getattr(getter) + .and_then(|headers| headers.call_method0("to_scalar_dict")) + .expect("scalar options"); + let actual = actual.cast::<PyDict>().expect("dict"); + let wanted = PyDict::new(py); + for (key, option) in expected + .iter() + .filter(|(_, option)| option["explicit"].as_bool() == Some(explicit)) + { + let text = option["value"].as_str().expect("value text"); + let value: Bound<'_, PyAny> = match option["kind"].as_str() { + Some("uint64") => text + .parse::<u64>() + .expect("u64") + .into_pyobject(py) + .expect("int") + .into_any(), + Some("uint32") => text + .parse::<u32>() + .expect("u32") + .into_pyobject(py) + .expect("int") + .into_any(), + Some("bool") => text + .parse::<bool>() + .expect("bool") + .into_pyobject(py) + .expect("bool") + .to_owned() + .into_any(), + _ => PyString::new(py, text).into_any(), + }; + wanted.set_item(key, value).expect("set item"); + } + assert!( + actual.eq(&wanted).expect("comparable"), + "{getter}: {actual} != {wanted}" + ); + } + } + + fn decode_hex(text: &Value) -> Vec<u8> { + let text = text.as_str().expect("hex text"); + (0..text.len()) + .step_by(2) + .map(|index| u8::from_str_radix(&text[index..index + 2], 16).expect("hex byte")) + .collect() + } +}
diff --git a/foreign/python/src/stream.rs b/foreign/python/src/stream.rs index d5f6a8e..ad4b623 100644 --- a/foreign/python/src/stream.rs +++ b/foreign/python/src/stream.rs
@@ -19,7 +19,9 @@ use pyo3::prelude::*; use pyo3_stub_gen::derive::{gen_stub_pyclass, gen_stub_pymethods}; +use crate::options::options_by_provenance; use crate::topic::Topic; +use crate::user_headers::UserHeaders; #[pyclass] #[gen_stub_pyclass] @@ -75,6 +77,50 @@ pub fn topics(&self) -> Vec<Topic> { self.inner.topics.iter().map(Topic::from).collect() } + + /// Options the creating client set explicitly. The stream's encryption + /// key is write-only and never appears here. + /// + /// The same `dict[HeaderKey, HeaderValue]` that `ReceiveMessage.user_headers` + /// returns, since options ride that codec. `to_scalar_dict()` gives the + /// plain-scalar form. + #[getter] + pub fn options<'a>(&self, py: Python<'a>) -> PyResult<Bound<'a, UserHeaders>> { + options_by_provenance(py, &self.inner.options, true) + } + + /// Options the server resolved or computed, including the encryption key + /// projections. Same shape as `options`. + #[getter] + pub fn derived_options<'a>(&self, py: Python<'a>) -> PyResult<Bound<'a, UserHeaders>> { + options_by_provenance(py, &self.inner.options, false) + } + + /// Who chose the key the stream's data is stored under: `"stream"` for a + /// key the client sent, `"server"` for a key the server generated, or + /// `"none"` for a plaintext stream. `None` when the server reported no + /// source. + #[getter] + pub fn encryption_key_source(&self) -> Option<String> { + self.inner + .encryption_key_source() + .map(|source| source.to_string()) + } + + /// Fingerprint of the stream's key, 16 lowercase hex characters. `None` + /// for a plaintext stream. Compare it with `encryption_key_fingerprint()` + /// of the key the stream was created with. + #[getter] + pub fn encryption_key_fingerprint(&self) -> Option<String> { + self.inner.encryption_key_fingerprint() + } + + /// Fingerprints of the server keys that wrap the stream's key: one outside + /// a key rotation, empty for a plaintext stream. + #[getter] + pub fn encryption_kek_fingerprints(&self) -> Vec<String> { + self.inner.encryption_kek_fingerprints() + } } /// Summary information returned by `IggyClient.get_streams()`. @@ -131,4 +177,48 @@ pub fn topics_count(&self) -> u32 { self.inner.topics_count } + + /// Options the creating client set explicitly. The stream's encryption + /// key is write-only and never appears here. + /// + /// The same `dict[HeaderKey, HeaderValue]` that `ReceiveMessage.user_headers` + /// returns, since options ride that codec. `to_scalar_dict()` gives the + /// plain-scalar form. + #[getter] + pub fn options<'a>(&self, py: Python<'a>) -> PyResult<Bound<'a, UserHeaders>> { + options_by_provenance(py, &self.inner.options, true) + } + + /// Options the server resolved or computed, including the encryption key + /// projections. Same shape as `options`. + #[getter] + pub fn derived_options<'a>(&self, py: Python<'a>) -> PyResult<Bound<'a, UserHeaders>> { + options_by_provenance(py, &self.inner.options, false) + } + + /// Who chose the key the stream's data is stored under: `"stream"` for a + /// key the client sent, `"server"` for a key the server generated, or + /// `"none"` for a plaintext stream. `None` when the server reported no + /// source. + #[getter] + pub fn encryption_key_source(&self) -> Option<String> { + self.inner + .encryption_key_source() + .map(|source| source.to_string()) + } + + /// Fingerprint of the stream's key, 16 lowercase hex characters. `None` + /// for a plaintext stream. Compare it with `encryption_key_fingerprint()` + /// of the key the stream was created with. + #[getter] + pub fn encryption_key_fingerprint(&self) -> Option<String> { + self.inner.encryption_key_fingerprint() + } + + /// Fingerprints of the server keys that wrap the stream's key: one outside + /// a key rotation, empty for a plaintext stream. + #[getter] + pub fn encryption_kek_fingerprints(&self) -> Vec<String> { + self.inner.encryption_kek_fingerprints() + } }
diff --git a/foreign/python/src/topic.rs b/foreign/python/src/topic.rs index 8d455c4..e7c6815 100644 --- a/foreign/python/src/topic.rs +++ b/foreign/python/src/topic.rs
@@ -17,8 +17,7 @@ use iggy::prelude::{ IggyByteSize, IggyExpiry as RustIggyExpiry, MaxTopicSize as RustMaxTopicSize, - Partition as RustPartition, ResourceOptions, Topic as RustTopic, - TopicDetails as RustTopicDetails, + Partition as RustPartition, Topic as RustTopic, TopicDetails as RustTopicDetails, }; use pyo3::exceptions::PyValueError; @@ -27,28 +26,9 @@ use pyo3_stub_gen::derive::{gen_stub_pyclass, gen_stub_pyclass_complex_enum, gen_stub_pymethods}; use crate::duration::{iggy_duration_to_py_delta, py_delta_to_iggy_duration}; +use crate::options::options_by_provenance; use crate::receive_message::PartitionContext; -use crate::user_headers::{UserHeaders, rust_user_headers_to_py}; - -/// The entries of one provenance, as the dictionary message user headers come -/// back as. -/// -/// Options ride the user-headers codec, so they are handed back through the -/// same `HeaderKey`/`HeaderValue` types rather than a second shape meaning the -/// same thing: `to_scalar_dict()` works on the result exactly as it does on -/// `ReceiveMessage.user_headers`. -fn options_by_provenance<'a>( - py: Python<'a>, - options: &ResourceOptions, - explicit: bool, -) -> PyResult<Bound<'a, UserHeaders>> { - let selected = options - .iter() - .filter(|(_, option)| option.explicit == explicit) - .map(|(key, option)| (key.clone(), option.value.clone())) - .collect(); - rust_user_headers_to_py(py, selected) -} +use crate::user_headers::UserHeaders; /// The expiry of the messages in a topic. #[gen_stub_pyclass_complex_enum] @@ -63,7 +43,7 @@ /// microsecond count a `u64` can hold (about 584,542 years): those two /// values are reserved on the wire for `ServerDefault` and `NeverExpire` /// respectively, so a `duration` at either boundary raises `ValueError` - /// when passed to `create_topic`/`update_topic`. A negative `timedelta` + /// when passed to `TopicOptions`/`update_topic`. A negative `timedelta` /// also raises `ValueError`. ExpireDuration { duration: Py<PyDelta> }, /// Retain messages indefinitely; they never expire. @@ -137,7 +117,7 @@ /// an unsigned 64-bit integer: those two values are reserved on the wire /// for `ServerDefault` and `Unlimited` respectively, so a `Custom` size /// at either boundary raises `ValueError` when passed to - /// `create_topic`/`update_topic`. + /// `TopicOptions`/`update_topic`. Custom { bytes: u64 }, /// Do not cap the topic size; it may grow without bound. Unlimited(), @@ -194,9 +174,6 @@ created_at: topic.created_at, name: topic.name.clone(), size: topic.size, - message_expiry: topic.message_expiry, - compression_algorithm: topic.compression_algorithm, - max_topic_size: topic.max_topic_size, messages_count: topic.messages_count, partitions_count: topic.partitions_count, options: topic.options.clone(), @@ -244,22 +221,27 @@ self.inner.size.as_bytes_u64() } - /// The expiry of the messages in the topic. + /// The expiry of the messages in the topic, the server default when the + /// topic does not set one. #[getter] pub fn message_expiry(&self) -> PyResult<IggyExpiry> { - self.inner.message_expiry.try_into() + self.inner.resolved_options().message_expiry.try_into() } - /// Compression algorithm for the topic. + /// Compression algorithm for the topic, fixed at creation. #[getter] pub fn compression_algorithm(&self) -> String { - self.inner.compression_algorithm.to_string() + self.inner + .resolved_options() + .compression_algorithm + .to_string() } - /// The maximum size of the topic. + /// The maximum size of the topic, the server default when the topic does + /// not set one. #[getter] pub fn max_topic_size(&self) -> MaxTopicSize { - self.inner.max_topic_size.into() + self.inner.resolved_options().max_topic_size.into() } /// Options the creating client set explicitly. @@ -335,22 +317,27 @@ self.inner.size.as_bytes_u64() } - /// The expiry of the messages in the topic. + /// The expiry of the messages in the topic, the server default when the + /// topic does not set one. #[getter] pub fn message_expiry(&self) -> PyResult<IggyExpiry> { - self.inner.message_expiry.try_into() + self.inner.resolved_options().message_expiry.try_into() } - /// Compression algorithm for the topic. + /// Compression algorithm for the topic, fixed at creation. #[getter] pub fn compression_algorithm(&self) -> String { - self.inner.compression_algorithm.to_string() + self.inner + .resolved_options() + .compression_algorithm + .to_string() } - /// The maximum size of the topic. + /// The maximum size of the topic, the server default when the topic does + /// not set one. #[getter] pub fn max_topic_size(&self) -> MaxTopicSize { - self.inner.max_topic_size.into() + self.inner.resolved_options().max_topic_size.into() } /// Options the creating client set explicitly.
diff --git a/foreign/python/src/user.rs b/foreign/python/src/user.rs index b000e89..96cb0ef 100644 --- a/foreign/python/src/user.rs +++ b/foreign/python/src/user.rs
@@ -22,7 +22,9 @@ use pyo3::prelude::*; use pyo3_stub_gen::derive::{gen_stub_pyclass, gen_stub_pyclass_enum, gen_stub_pymethods}; +use crate::options::options_by_provenance; use crate::permissions::Permissions; +use crate::user_headers::UserHeaders; /// The status of a user account. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -159,6 +161,23 @@ pub fn username(&self) -> String { self.inner.username.to_string() } + + /// Options the creating client set explicitly. No user key exists yet, + /// so this is empty unless a newer server added one. + /// + /// The same `dict[HeaderKey, HeaderValue]` that `ReceiveMessage.user_headers` + /// returns, since options ride that codec. + #[getter] + pub fn options<'a>(&self, py: Python<'a>) -> PyResult<Bound<'a, UserHeaders>> { + options_by_provenance(py, &self.inner.options, true) + } + + /// Options the server resolved for the keys the client did not send. + /// Same shape as `options`. + #[getter] + pub fn derived_options<'a>(&self, py: Python<'a>) -> PyResult<Bound<'a, UserHeaders>> { + options_by_provenance(py, &self.inner.options, false) + } } #[gen_stub_pyclass] @@ -206,4 +225,21 @@ pub fn permissions(&self) -> Option<Permissions> { self.inner.permissions.clone().map(Permissions::from) } + + /// Options the creating client set explicitly. No user key exists yet, + /// so this is empty unless a newer server added one. + /// + /// The same `dict[HeaderKey, HeaderValue]` that `ReceiveMessage.user_headers` + /// returns, since options ride that codec. + #[getter] + pub fn options<'a>(&self, py: Python<'a>) -> PyResult<Bound<'a, UserHeaders>> { + options_by_provenance(py, &self.inner.options, true) + } + + /// Options the server resolved for the keys the client did not send. + /// Same shape as `options`. + #[getter] + pub fn derived_options<'a>(&self, py: Python<'a>) -> PyResult<Bound<'a, UserHeaders>> { + options_by_provenance(py, &self.inner.options, false) + } }
diff --git a/foreign/python/tests/conftest.py b/foreign/python/tests/conftest.py index 73507fb..c3a9fc1 100644 --- a/foreign/python/tests/conftest.py +++ b/foreign/python/tests/conftest.py
@@ -33,7 +33,14 @@ from apache_iggy import IggyClient, PendingOperationError -from .utils import get_server_config, wait_for_ping, wait_for_server +from .utils import ( + get_http_server_config, + get_quic_server_config, + get_server_config, + get_websocket_server_config, + wait_for_ping, + wait_for_server, +) # A request and its retained replay can each spend 30 seconds. STREAM_CLEANUP_TIMEOUT_SECONDS = 60 @@ -72,6 +79,32 @@ return client +@pytest.fixture(params=["tcp", "quic", "http", "websocket"]) +async def transport_client(request: pytest.FixtureRequest) -> IggyClient: + """ + Return a client logged in as root over each transport the SDK exposes. + + Every transport is reached through its connection string, so a test that + takes this fixture runs once per transport against the same server. + """ + scheme, (host, port), query = { + "tcp": ("tcp", get_server_config(), ""), + "quic": ("quic", get_quic_server_config(), "?reconnection_max_retries=0"), + "http": ("http", get_http_server_config(), ""), + "websocket": ("ws", get_websocket_server_config(), ""), + }[request.param] + # QUIC listens on UDP, which a TCP probe cannot reach. + if scheme != "quic": + wait_for_server(host, port) + client = IggyClient.from_connection_string( + f"iggy+{scheme}://iggy:iggy@{host}:{port}{query}" + ) + await client.connect() + await wait_for_ping(client) + await client.login_user("iggy", "iggy") + return client + + @pytest.fixture def unique_name(): """Return a factory for generating unique test names."""
diff --git a/foreign/python/tests/fixtures/resource_options.json b/foreign/python/tests/fixtures/resource_options.json new file mode 100644 index 0000000..c5cc2ac --- /dev/null +++ b/foreign/python/tests/fixtures/resource_options.json
@@ -0,0 +1,1619 @@ +{ + "schema_version": 1, + "generated_by": "core/common/examples/resource_options_fixtures.rs", + "conventions": { + "byte_strings": "Every *_hex field is lowercase hex without separators.", + "errors": "name is the snake_case IggyError name and code its numeric code.", + "max_topic_size": "Bytes. 18446744073709551615 means unlimited.", + "message_expiry": "Microseconds. 18446744073709551615 means never expire.", + "never_returned_keys": "No response, decoded projection or accessor may contain these keys or their values.", + "option_maps": "Option maps use the HTTP JSON form: value as reversible text, kind, and explicit provenance. Request blocks mark every entry explicit.", + "options_block_order": "Options block entries are encoded in ascending key byte order.", + "request_bodies": "body_hex and update_resource_hex are command payloads without framing. constants.command_codes maps each command to its code and VSR operation.", + "reset_order": "The typed encoders and the HTTP handler write reset keys in ascending key byte order.", + "stream_state": "state is the stored server state a stream response was projected from. SDKs need only response_hex and expected.", + "u64_values": "u64 option values and typed u64 inputs are decimal strings, so values above 2^53 keep their precision." + }, + "constants": { + "protocol_version": { + "packed": 12288, + "text": "0.12.0", + "major": 0, + "minor": 12, + "patch": 0 + }, + "protocol_version_min": { + "packed": 12288, + "text": "0.12.0", + "major": 0, + "minor": 12, + "patch": 0 + }, + "command_codes": { + "create_stream": { + "code": 202, + "operation": 128 + }, + "create_topic": { + "code": 302, + "operation": 132 + }, + "create_user": { + "code": 33, + "operation": 141 + }, + "describe_options": { + "code": 13, + "operation": null + }, + "get_stream": { + "code": 200, + "operation": null + }, + "get_streams": { + "code": 201, + "operation": null + }, + "get_topic": { + "code": 300, + "operation": null + }, + "get_topics": { + "code": 301, + "operation": null + }, + "get_user": { + "code": 31, + "operation": null + }, + "get_users": { + "code": 32, + "operation": null + }, + "update_resource": { + "code": 16, + "operation": 150 + } + }, + "error_codes": { + "cannot_modify_root_user": 4108 + }, + "retired_command_codes": { + "update_stream": 204, + "update_topic": 304, + "update_user": 35 + }, + "operations": { + "rewrap_stream_keys": { + "value": 75, + "internal": true, + "request_operation": 75 + }, + "update_resource": { + "value": 150, + "internal": false, + "request_operation": 150 + }, + "update_stream_resolved": { + "value": 72, + "internal": true, + "request_operation": 150 + }, + "update_topic_resolved": { + "value": 73, + "internal": true, + "request_operation": 150 + }, + "update_user_resolved": { + "value": 74, + "internal": true, + "request_operation": 150 + } + }, + "retired_operations": { + "update_stream": 129, + "update_topic": 133, + "update_user": 142 + }, + "scope_codes": { + "stream": 2, + "topic": 1, + "user": 3 + }, + "header_kind_codes": { + "raw": 1, + "string": 2, + "bool": 3, + "uint32": 11, + "uint64": 12 + }, + "option_flags": { + "critical": 16, + "read_only": 8, + "resettable": 2, + "secret": 4, + "updatable": 1 + }, + "user_status_codes": { + "active": 1, + "inactive": 2 + }, + "stream_key_length": 32, + "never_returned_keys": [ + "encryption_key", + "encryption_key_wrapped", + "critical_keys" + ] + }, + "catalog": [ + { + "key": "name", + "scopes": [ + "stream", + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "column", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 1, + "described": true, + "default": null + }, + { + "key": "username", + "scopes": [ + "user" + ], + "kind": "string", + "kind_code": 2, + "storage": "column", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 1, + "described": true, + "default": null + }, + { + "key": "status", + "scopes": [ + "user" + ], + "kind": "string", + "kind_code": 2, + "storage": "column", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 1, + "described": true, + "default": null + }, + { + "key": "compression_algorithm", + "scopes": [ + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "none" + }, + { + "key": "message_expiry", + "scopes": [ + "topic" + ], + "kind": "uint64", + "kind_code": 12, + "storage": "map", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 3, + "described": true, + "default": "18446744073709551615" + }, + { + "key": "max_topic_size", + "scopes": [ + "topic" + ], + "kind": "uint64", + "kind_code": 12, + "storage": "map", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 3, + "described": true, + "default": "18446744073709551615" + }, + { + "key": "segment_size", + "scopes": [ + "topic" + ], + "kind": "uint64", + "kind_code": 12, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "1073741824" + }, + { + "key": "durability", + "scopes": [ + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "replicated" + }, + { + "key": "consumer_offset_durability", + "scopes": [ + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "replicated" + }, + { + "key": "partition_resize_policy", + "scopes": [ + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "mutable" + }, + { + "key": "messages_required_to_save", + "scopes": [ + "topic" + ], + "kind": "uint32", + "kind_code": 11, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "1024" + }, + { + "key": "size_of_messages_required_to_save", + "scopes": [ + "topic" + ], + "kind": "uint64", + "kind_code": 12, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "1048576" + }, + { + "key": "preallocate_segments", + "scopes": [ + "topic" + ], + "kind": "bool", + "kind_code": 3, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "false" + }, + { + "key": "encryption_key", + "scopes": [ + "stream" + ], + "kind": "raw", + "kind_code": 1, + "storage": "map", + "mutability": "create_only", + "visibility": "secret", + "critical": true, + "flags": 20, + "described": true, + "default": null + }, + { + "key": "encryption_key_wrapped", + "scopes": [ + "stream" + ], + "kind": "raw", + "kind_code": 1, + "storage": "map", + "mutability": "create_only", + "visibility": "internal", + "critical": true, + "flags": 16, + "described": false, + "default": null + }, + { + "key": "encryption_key_source", + "scopes": [ + "stream" + ], + "kind": "string", + "kind_code": 2, + "storage": "computed", + "mutability": "read_only", + "visibility": "read_only", + "critical": false, + "flags": 8, + "described": true, + "default": null + }, + { + "key": "encryption_key_fingerprint", + "scopes": [ + "stream" + ], + "kind": "string", + "kind_code": 2, + "storage": "computed", + "mutability": "read_only", + "visibility": "read_only", + "critical": false, + "flags": 8, + "described": true, + "default": null + }, + { + "key": "encryption_kek_fingerprint", + "scopes": [ + "stream" + ], + "kind": "string", + "kind_code": 2, + "storage": "computed", + "mutability": "read_only", + "visibility": "read_only", + "critical": false, + "flags": 8, + "described": true, + "default": null + }, + { + "key": "critical_keys", + "scopes": [ + "stream", + "topic", + "user" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "internal", + "critical": false, + "flags": 0, + "described": false, + "default": null + } + ], + "describe_options": [ + { + "scope": "topic", + "request_hex": "01", + "response_hex": "0b000000046e616d650201000000007e0000005265736f75726365206e616d652c203120746f203235352062797465732c20756e697175652077697468696e2069747320706172656e742e20536574206279207468652063726561746520636f6d6d616e642773206e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e15636f6d7072657373696f6e5f616c676f726974686d0200040000006e6f6e6550000000436f6d7072657373696f6e20616c676f726974686d20286e6f6e652c20677a6970292e2053746f72656420616e64207265706f727465642c206e6f74206170706c69656420746f206d657373616765730e6d6573736167655f6578706972790c0308000000ffffffffffffffff430000004d6573736167652065787069727920696e206d6963726f7365636f6e64732c206f7220612068756d616e74696d6520737472696e672028652e672e20372064617973290e6d61785f746f7069635f73697a650c0308000000ffffffffffffffff9d000000546f7069632d77696465207365616c65642d7365676d656e742073697a65206361702c2073706c6974206163726f737320616c6c20706172746974696f6e732c20696e206279746573206f72206120627974652d73697a6520737472696e672028652e672e203120476942293b2066696e6974652076616c756573206d757374206265206174206c6561737420746865207365676d656e742073697a650c7365676d656e745f73697a650c000800000000000040000000006c0000005365676d656e742073697a6520696e2062797465732c206f72206120627974652d73697a6520737472696e672028652e672e20313238204d6942293b2061203531322d62797465206d756c7469706c652077697468696e20313034383537362e2e3d313037333734313832340a6475726162696c69747902000a0000007265706c6963617465643e0100004d65737361676520636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20412073696e676c65746f6e2071756f72756d20686173206f6e6520636f70792e20496e207265706c6963617465642067726f7570732c20706572736973746564206d65737361676573207573652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c2057414c207265636c616d6174696f6e2e205468652066756c6c20626f64792073697a65207374696c6c20636f756e747320616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e1a636f6e73756d65725f6f66667365745f6475726162696c69747902000a0000007265706c6963617465646e0100004578706c69636974206f666673657420636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20506f6c6c206175746f2d636f6d6d69742072656d61696e73206173796e6368726f6e6f75732e20496e207265706c6963617465642067726f7570732c20706572736973746564206f66667365747320616c736f20656e61626c652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c207265636c616d6174696f6e2c206576656e2077697468207265706c696361746564206d657373616765206475726162696c6974792e2046756c6c20626f64792073697a657320636f756e7420616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e17706172746974696f6e5f726573697a655f706f6c6963790200070000006d757461626c65c8000000496d6d757461626c65206166746572206372656174696f6e3a206d757461626c65207065726d69747320706172746974696f6e206164646974696f6e2f64656c6574696f6e20776974686f7574206120626f756e646564206375746f766572206f72207065722d6b6579206f72646572696e67206163726f7373206c61796f7574733b2066697865642072656a6563747320626f74682e20526574656e74696f6e20616e6420746f7069632064656c6574696f6e2072656d61696e2064657374727563746976652e196d657373616765735f72657175697265645f746f5f736176650b000400000000040000870000004f7264696e617279206d6573736167652d636f756e7420666c75736820747269676765723b20312e2e3d31363737373231362e2052657175697265642070657273697374656e63652c2063617061636974792070726573737572652c20616e64206c6966656379636c65206f7065726174696f6e732063616e20666c757368206561726c6965722173697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c0008000000000010000000000083000000466c75736820746865206a6f75726e616c206f6e636520697420686f6c64732074686973206d616e792062797465732c206f72206120627974652d73697a6520737472696e673b20776869636865766572207468726573686f6c6420747269707320666972737420666c75736865732e204174206d6f7374203130373337343138323414707265616c6c6f636174655f7365676d656e7473030001000000001201000052657175657374207365676d656e745f73697a65206279746573206f66206469736b207265736572766174696f6e207768656e2065616368207365676d656e74206f70656e732e20556e737570706f72746564206f72206661696c6564207265736572766174696f6e732066616c6c206261636b20746f206f7264696e61727920616c6c6f636174696f6e20776974682061207761726e696e672e205265736572766174696f6e2072756e7320696e6c696e65206f6e20746865206f776e696e672073686172642e207365676d656e745f73697a65202a20706172746974696f6e735f636f756e74206973206361707065642061742036383731393437363733362062797465732070657220637265617465", + "entries": [ + { + "key": "name", + "kind": "string", + "flags": 1, + "default": null + }, + { + "key": "compression_algorithm", + "kind": "string", + "flags": 0, + "default": "none" + }, + { + "key": "message_expiry", + "kind": "uint64", + "flags": 3, + "default": "18446744073709551615" + }, + { + "key": "max_topic_size", + "kind": "uint64", + "flags": 3, + "default": "18446744073709551615" + }, + { + "key": "segment_size", + "kind": "uint64", + "flags": 0, + "default": "1073741824" + }, + { + "key": "durability", + "kind": "string", + "flags": 0, + "default": "replicated" + }, + { + "key": "consumer_offset_durability", + "kind": "string", + "flags": 0, + "default": "replicated" + }, + { + "key": "partition_resize_policy", + "kind": "string", + "flags": 0, + "default": "mutable" + }, + { + "key": "messages_required_to_save", + "kind": "uint32", + "flags": 0, + "default": "1024" + }, + { + "key": "size_of_messages_required_to_save", + "kind": "uint64", + "flags": 0, + "default": "1048576" + }, + { + "key": "preallocate_segments", + "kind": "bool", + "flags": 0, + "default": "false" + } + ] + }, + { + "scope": "stream", + "request_hex": "02", + "response_hex": "05000000046e616d650201000000007e0000005265736f75726365206e616d652c203120746f203235352062797465732c20756e697175652077697468696e2069747320706172656e742e20536574206279207468652063726561746520636f6d6d616e642773206e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e0e656e6372797074696f6e5f6b6579011400000000a30100005468652073747265616d2773206f776e206461746120656e6372797074696f6e206b65793a2033322072616e646f6d2062797465732c20726177206f72207374616e6461726420706164646564206261736536342e204372656174652d6f6e6c7920616e642077726974652d6f6e6c793a20746865207365727665722073746f726573206974207772617070656420756e6465722061206b657920646572697665642066726f6d207468652073657276657220656e6372797074696f6e206b657920616e64206e657665722072657475726e732069742e205265717569726573207468652073657276657220656e6372797074696f6e206b657920746f20626520636f6e666967757265642e20576974686f75742069742c206120736572766572207769746820656e6372797074696f6e20656e61626c65642067656e6572617465732061206b657920666f72207468652073747265616d2e20546865206b65792074726176656c7320696e20746865206372656174652072657175657374206f76657220746865207472616e73706f72742074686520636c69656e7420757365732e15656e6372797074696f6e5f6b65795f736f757263650208000000009f00000057686f2063686f7365207468652073747265616d27732061742d72657374206b65793a2073747265616d207768656e2074686520636c69656e742073656e742069742c20736572766572207768656e20746865207365727665722067656e6572617465642069742c206e6f6e65207768656e207468652073747265616d2773206d65737361676573206172652073746f72656420756e656e637279707465641a656e6372797074696f6e5f6b65795f66696e6765727072696e740208000000009f0000005468652066697273742038206279746573206f66205348412d323536206f76657220746865206c6162656c20696767792f73747265616d2d6b65792d66696e6765727072696e742f763120616e64207468652073747265616d2773206b65792c206173203136206c6f776572636173652068657820636861726163746572732e20416273656e74207768656e2074686520736f75726365206973206e6f6e651a656e6372797074696f6e5f6b656b5f66696e6765727072696e74020800000000b800000046696e6765727072696e7473206f6620746865206b65792d656e6372797074696f6e206b65797320746861742077726170207468652073747265616d2773206b65792c20636f6d6d612d7365706172617465643a206f6e65206f7574736964652061206b657920726f746174696f6e2c206f6e652070657220636f6e6669677572656420736572766572206b657920647572696e672069742e20416273656e74207768656e2074686520736f75726365206973206e6f6e65", + "entries": [ + { + "key": "name", + "kind": "string", + "flags": 1, + "default": null + }, + { + "key": "encryption_key", + "kind": "raw", + "flags": 20, + "default": null + }, + { + "key": "encryption_key_source", + "kind": "string", + "flags": 8, + "default": null + }, + { + "key": "encryption_key_fingerprint", + "kind": "string", + "flags": 8, + "default": null + }, + { + "key": "encryption_kek_fingerprint", + "kind": "string", + "flags": 8, + "default": null + } + ] + }, + { + "scope": "user", + "request_hex": "03", + "response_hex": "0200000008757365726e616d650201000000006b00000055736572206e616d652c203320746f2035302062797465732c20756e697175652e20536574206279207468652063726561746520636f6d6d616e64277320757365726e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e067374617475730201000000008f00000055736572207374617475732c20616374697665206f7220696e6163746976652e20536574206279207468652063726561746520636f6d6d616e64277320737461747573206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e20446561637469766174696e672061207573657220656e6473206974732073657373696f6e732e", + "entries": [ + { + "key": "username", + "kind": "string", + "flags": 1, + "default": null + }, + { + "key": "status", + "kind": "string", + "flags": 1, + "default": null + } + ] + } + ], + "stream_key": { + "key_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "key_base64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", + "fingerprint_label": "iggy/stream-key-fingerprint/v1", + "fingerprint": "8441e38538cbfa57", + "invalid_keys": [ + { + "case": "raw_31_bytes", + "input": { + "raw_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + }, + { + "case": "raw_33_bytes", + "input": { + "raw_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + }, + { + "case": "base64_with_whitespace", + "input": { + "base64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=\n" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + }, + { + "case": "base64_missing_padding", + "input": { + "base64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + }, + { + "case": "base64_noncanonical_trailing_bits", + "input": { + "base64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh9=" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + } + ] + }, + "update_resource": [ + { + "case": "topic_set_expiry_without_name_reset_max_topic_size", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "message_expiry": "604800000000", + "reset": [ + "max_topic_size" + ] + }, + "wire_set": { + "message_expiry": { + "value": "604800000000", + "kind": "uint64", + "explicit": true + } + }, + "wire_reset": [ + "max_topic_size" + ], + "body_hex": "01010401000000010402000000010e6d61785f746f7069635f73697a65020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000" + }, + { + "case": "topic_rename_only_named_identifiers", + "scope": "topic", + "target": { + "stream_id": "orders", + "topic_id": "events" + }, + "input": { + "name": "events-v2" + }, + "wire_set": { + "name": { + "value": "events-v2", + "kind": "string", + "explicit": true + } + }, + "wire_reset": [], + "body_hex": "0102066f726465727302066576656e74730002040000006e616d6502090000006576656e74732d7632" + }, + { + "case": "topic_rename_set_expiry_reset_max_topic_size", + "scope": "topic", + "target": { + "stream_id": 7, + "topic_id": "events" + }, + "input": { + "name": "events-v3", + "message_expiry": "86400000000", + "reset": [ + "max_topic_size" + ] + }, + "wire_set": { + "message_expiry": { + "value": "86400000000", + "kind": "uint64", + "explicit": true + }, + "name": { + "value": "events-v3", + "kind": "string", + "explicit": true + } + }, + "wire_reset": [ + "max_topic_size" + ], + "body_hex": "0101040700000002066576656e7473010e6d61785f746f7069635f73697a65020e0000006d6573736167655f6578706972790c080000000060d71d1400000002040000006e616d6502090000006576656e74732d7633" + }, + { + "case": "topic_reset_both_defaults", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "reset": [ + "max_topic_size", + "message_expiry" + ] + }, + "wire_set": {}, + "wire_reset": [ + "max_topic_size", + "message_expiry" + ], + "body_hex": "01010401000000010402000000020e6d61785f746f7069635f73697a650e6d6573736167655f657870697279" + }, + { + "case": "topic_max_topic_size_above_2_pow_53", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "max_topic_size": "9007199254740993" + }, + "wire_set": { + "max_topic_size": { + "value": "9007199254740993", + "kind": "uint64", + "explicit": true + } + }, + "wire_reset": [], + "body_hex": "0101040100000001040200000000020e0000006d61785f746f7069635f73697a650c080000000100000000002000" + }, + { + "case": "stream_rename", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "input": { + "name": "payments-v2" + }, + "wire_set": { + "name": { + "value": "payments-v2", + "kind": "string", + "explicit": true + } + }, + "wire_reset": [], + "body_hex": "020104030000000002040000006e616d65020b0000007061796d656e74732d7632" + }, + { + "case": "user_username_and_status", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "username": "alice2", + "status": "inactive" + }, + "wire_set": { + "status": { + "value": "inactive", + "kind": "string", + "explicit": true + }, + "username": { + "value": "alice2", + "kind": "string", + "explicit": true + } + }, + "wire_reset": [], + "body_hex": "030104050000000002060000007374617475730208000000696e6163746976650208000000757365726e616d650206000000616c69636532" + } + ], + "update_resource_errors": [ + { + "case": "topic_empty_update", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": {}, + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_set_and_reset_same_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "message_expiry": "604800000000", + "reset": [ + "message_expiry" + ] + }, + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_reset_create_only_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "reset": [ + "compression_algorithm" + ] + }, + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "stream_reset_name", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "input": { + "reset": [ + "name" + ] + }, + "error": { + "name": "option_not_resettable", + "code": 4104 + } + }, + { + "case": "user_reset_status", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "reset": [ + "status" + ] + }, + "error": { + "name": "option_not_resettable", + "code": 4104 + } + }, + { + "case": "stream_reset_read_only_key", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "input": { + "reset": [ + "encryption_key_source" + ] + }, + "error": { + "name": "read_only_option", + "code": 4105 + } + }, + { + "case": "user_reset_internal_key", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "reset": [ + "critical_keys" + ] + }, + "error": { + "name": "read_only_option", + "code": 4105 + } + }, + { + "case": "topic_empty_name", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "name": "" + }, + "error": { + "name": "invalid_topic_name", + "code": 2014 + } + }, + { + "case": "topic_empty_name_and_create_only_reset", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "name": "", + "reset": [ + "segment_size" + ] + }, + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "stream_empty_name", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "input": { + "name": "" + }, + "error": { + "name": "invalid_stream_name", + "code": 1013 + } + }, + { + "case": "user_short_username", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "username": "a" + }, + "error": { + "name": "invalid_username", + "code": 43 + } + }, + { + "case": "user_short_username_and_status_reset", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "username": "a", + "reset": [ + "status" + ] + }, + "error": { + "name": "option_not_resettable", + "code": 4104 + } + } + ], + "create_stream": [ + { + "case": "stream_without_key", + "input": { + "name": "orders" + }, + "wire_options": {}, + "body_hex": "066f7264657273", + "http_body": "{\"name\":\"orders\"}", + "admission_error": null + }, + { + "case": "stream_with_key", + "input": { + "name": "orders", + "encryption_key": { + "raw_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f" + } + }, + "wire_options": { + "encryption_key": { + "value": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", + "kind": "raw", + "explicit": true + } + }, + "body_hex": "066f7264657273020e000000656e6372797074696f6e5f6b65790120000000000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "http_body": "{\"name\":\"orders\",\"options\":{\"encryption_key\":\"AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=\"}}", + "admission_error": null + }, + { + "case": "stream_with_raw_extra_key", + "input": { + "name": "orders", + "raw": { + "future_option": "enabled" + } + }, + "wire_options": { + "future_option": { + "value": "enabled", + "kind": "string", + "explicit": true + } + }, + "body_hex": "066f7264657273020d0000006675747572655f6f7074696f6e0207000000656e61626c6564", + "http_body": "{\"name\":\"orders\",\"options\":{\"future_option\":\"enabled\"}}", + "admission_error": { + "name": "unsupported_option_key", + "code": 4041 + } + } + ], + "create_topic": [ + { + "case": "topic_with_options", + "input": { + "stream_id": 1, + "name": "events", + "partitions_count": 3, + "compression_algorithm": "gzip", + "message_expiry": "604800000000", + "max_topic_size": "9007199254740993", + "segment_size": "134217728", + "durability": "persisted", + "consumer_offset_durability": "replicated" + }, + "wire_options": { + "compression_algorithm": { + "value": "gzip", + "kind": "string", + "explicit": true + }, + "consumer_offset_durability": { + "value": "replicated", + "kind": "string", + "explicit": true + }, + "durability": { + "value": "persisted", + "kind": "string", + "explicit": true + }, + "max_topic_size": { + "value": "9007199254740993", + "kind": "uint64", + "explicit": true + }, + "message_expiry": { + "value": "604800000000", + "kind": "uint64", + "explicit": true + }, + "segment_size": { + "value": "134217728", + "kind": "uint64", + "explicit": true + } + }, + "body_hex": "01040100000003000000066576656e74730215000000636f6d7072657373696f6e5f616c676f726974686d0204000000677a6970021a000000636f6e73756d65725f6f66667365745f6475726162696c697479020a0000007265706c696361746564020a0000006475726162696c6974790209000000706572736973746564020e0000006d61785f746f7069635f73697a650c080000000100000000002000020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000020c0000007365676d656e745f73697a650c080000000000000800000000", + "http_body": "{\"partitions_count\":3,\"name\":\"events\",\"options\":{\"compression_algorithm\":\"gzip\",\"consumer_offset_durability\":\"replicated\",\"durability\":\"persisted\",\"max_topic_size\":\"9007199254740993\",\"message_expiry\":604800000000,\"segment_size\":134217728}}", + "admission_error": null + } + ], + "create_user": [ + { + "case": "user_with_empty_options", + "input": { + "username": "alice", + "password": "s3cret-pass", + "status": "active" + }, + "wire_options": {}, + "body_hex": "05616c6963650b7333637265742d706173730100", + "http_body": "{\"username\":\"alice\",\"password\":\"s3cret-pass\",\"status\":\"active\",\"permissions\":null}", + "admission_error": null + } + ], + "http_update": [ + { + "case": "topic_flat_keys", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"name\":\"events-v2\",\"message_expiry\":\"7 days\"}", + "update_resource_hex": "0101040100000001040200000000020e0000006d6573736167655f657870697279020600000037206461797302040000006e616d6502090000006576656e74732d7632", + "outcome": "accepted", + "error": null + }, + { + "case": "topic_reset_array", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":604800000000,\"reset\":[\"max_topic_size\"]}", + "update_resource_hex": "01010401000000010402000000010e6d61785f746f7069635f73697a65020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000", + "outcome": "accepted", + "error": null + }, + { + "case": "topic_null_ignored", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"name\":null,\"max_topic_size\":null,\"message_expiry\":\"1 day\"}", + "update_resource_hex": "0101040100000001040200000000020e0000006d6573736167655f65787069727902050000003120646179", + "outcome": "accepted", + "error": null + }, + { + "case": "topic_u64_above_2_pow_53_as_string", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"max_topic_size\":\"9007199254740993\"}", + "update_resource_hex": "0101040100000001040200000000020e0000006d61785f746f7069635f73697a65021000000039303037313939323534373430393933", + "outcome": "accepted", + "error": null + }, + { + "case": "stream_rename", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "body": "{\"name\":\"payments-v2\"}", + "update_resource_hex": "020104030000000002040000006e616d65020b0000007061796d656e74732d7632", + "outcome": "accepted", + "error": null + }, + { + "case": "user_username_and_status", + "scope": "user", + "target": { + "user_id": 5 + }, + "body": "{\"username\":\"alice2\",\"status\":\"inactive\"}", + "update_resource_hex": "030104050000000002060000007374617475730208000000696e6163746976650208000000757365726e616d650206000000616c69636532", + "outcome": "accepted", + "error": null + }, + { + "case": "topic_duplicate_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":\"1 day\",\"message_expiry\":\"2 days\"}", + "update_resource_hex": null, + "outcome": "json_refused", + "error": null + }, + { + "case": "topic_nested_object", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":{\"value\":\"1 day\"}}", + "update_resource_hex": null, + "outcome": "json_refused", + "error": null + }, + { + "case": "topic_null_only", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"max_topic_size\":null}", + "update_resource_hex": "0101040100000001040200000000", + "outcome": "decode_refused", + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_zero_sentinel_only", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":0}", + "update_resource_hex": "0101040100000001040200000000020e0000006d6573736167655f6578706972790c080000000000000000000000", + "outcome": "admission_refused", + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_reset_names_set_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":\"1 day\",\"reset\":[\"message_expiry\"]}", + "update_resource_hex": "01010401000000010402000000010e6d6573736167655f657870697279020e0000006d6573736167655f65787069727902050000003120646179", + "outcome": "decode_refused", + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_unknown_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"retention\":\"1 day\"}", + "update_resource_hex": "01010401000000010402000000000209000000726574656e74696f6e02050000003120646179", + "outcome": "admission_refused", + "error": { + "name": "unsupported_option_key", + "code": 4041 + } + }, + { + "case": "topic_set_create_only_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"compression_algorithm\":\"gzip\"}", + "update_resource_hex": "01010401000000010402000000000215000000636f6d7072657373696f6e5f616c676f726974686d0204000000677a6970", + "outcome": "admission_refused", + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "topic_reset_create_only_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"reset\":[\"compression_algorithm\"]}", + "update_resource_hex": "010104010000000104020000000115636f6d7072657373696f6e5f616c676f726974686d", + "outcome": "admission_refused", + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "stream_reset_name", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "body": "{\"reset\":[\"name\"]}", + "update_resource_hex": "0201040300000001046e616d65", + "outcome": "admission_refused", + "error": { + "name": "option_not_resettable", + "code": 4104 + } + }, + { + "case": "topic_name_as_number", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"name\":5}", + "update_resource_hex": "010104010000000104020000000002040000006e616d650c080000000500000000000000", + "outcome": "admission_refused", + "error": { + "name": "invalid_topic_name", + "code": 2014 + } + }, + { + "case": "stream_empty_name", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "body": "{\"name\":\"\"}", + "update_resource_hex": "020104030000000002040000006e616d65030100000000", + "outcome": "admission_refused", + "error": { + "name": "invalid_stream_name", + "code": 1013 + } + }, + { + "case": "user_status_as_bool", + "scope": "user", + "target": { + "user_id": 5 + }, + "body": "{\"status\":true}", + "update_resource_hex": "03010405000000000206000000737461747573030100000001", + "outcome": "admission_refused", + "error": { + "name": "invalid_user_status", + "code": 45 + } + }, + { + "case": "topic_create_only_key_as_bool", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"segment_size\":true}", + "update_resource_hex": "0101040100000001040200000000020c0000007365676d656e745f73697a65030100000001", + "outcome": "admission_refused", + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "stream_read_only_key_as_number", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "body": "{\"encryption_key_source\":1}", + "update_resource_hex": "02010403000000000215000000656e6372797074696f6e5f6b65795f736f757263650c080000000100000000000000", + "outcome": "admission_refused", + "error": { + "name": "read_only_option", + "code": 4105 + } + }, + { + "case": "topic_invalid_value", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":\"soon\"}", + "update_resource_hex": "0101040100000001040200000000020e0000006d6573736167655f6578706972790204000000736f6f6e", + "outcome": "admission_refused", + "error": { + "name": "invalid_option_value", + "code": 4042 + } + } + ], + "stream_responses": [ + { + "case": "stream_source_none", + "state": { + "encryption_key_source": "none", + "stored_extra": {} + }, + "response_hex": "0100000000e090663c1306000000000000100000000000000c000000000000000c706c61696e2d73747265616d00000000230000000215000000656e6372797074696f6e5f6b65795f736f7572636502040000006e6f6e65", + "expected": { + "id": 1, + "created_at": 1710000000000000, + "name": "plain-stream", + "topics_count": 0, + "size_bytes": 4096, + "messages_count": 12, + "options": { + "encryption_key_source": { + "value": "none", + "kind": "string", + "explicit": false + } + }, + "encryption_key_source": "none", + "encryption_key_fingerprint": null, + "encryption_kek_fingerprints": [] + } + }, + { + "case": "stream_source_stream_with_unknown_explicit_key", + "state": { + "encryption_key_source": "stream", + "data_key": { + "raw_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f" + }, + "kek_config_keys": [ + "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=" + ], + "stored_extra": { + "future_option": { + "value": "enabled", + "kind": "string", + "explicit": true + } + } + }, + "response_hex": "0200000000e090663c1306000000000000100000000000000c0000000000000011636c69656e742d6b65792d73747265616d1e000000020d0000006675747572655f6f7074696f6e0207000000656e61626c65648d000000021a000000656e6372797074696f6e5f6b656b5f66696e6765727072696e74021000000033396534663066393563333639336337021a000000656e6372797074696f6e5f6b65795f66696e6765727072696e740210000000383434316533383533386362666135370215000000656e6372797074696f6e5f6b65795f736f75726365020600000073747265616d", + "expected": { + "id": 2, + "created_at": 1710000000000000, + "name": "client-key-stream", + "topics_count": 0, + "size_bytes": 4096, + "messages_count": 12, + "options": { + "encryption_kek_fingerprint": { + "value": "39e4f0f95c3693c7", + "kind": "string", + "explicit": false + }, + "encryption_key_fingerprint": { + "value": "8441e38538cbfa57", + "kind": "string", + "explicit": false + }, + "encryption_key_source": { + "value": "stream", + "kind": "string", + "explicit": false + }, + "future_option": { + "value": "enabled", + "kind": "string", + "explicit": true + } + }, + "encryption_key_source": "stream", + "encryption_key_fingerprint": "8441e38538cbfa57", + "encryption_kek_fingerprints": [ + "39e4f0f95c3693c7" + ] + } + }, + { + "case": "stream_source_server_two_keks", + "state": { + "encryption_key_source": "server", + "data_key": { + "raw_hex": "202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f" + }, + "kek_config_keys": [ + "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=", + "AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA=" + ], + "stored_extra": {} + }, + "response_hex": "0300000000e090663c1306000000000000100000000000000c00000000000000117365727665722d6b65792d73747265616d000000009e000000021a000000656e6372797074696f6e5f6b656b5f66696e6765727072696e740221000000333965346630663935633336393363372c64353564343739383239626332393336021a000000656e6372797074696f6e5f6b65795f66696e6765727072696e740210000000306563616666316261323931666530640215000000656e6372797074696f6e5f6b65795f736f757263650206000000736572766572", + "expected": { + "id": 3, + "created_at": 1710000000000000, + "name": "server-key-stream", + "topics_count": 0, + "size_bytes": 4096, + "messages_count": 12, + "options": { + "encryption_kek_fingerprint": { + "value": "39e4f0f95c3693c7,d55d479829bc2936", + "kind": "string", + "explicit": false + }, + "encryption_key_fingerprint": { + "value": "0ecaff1ba291fe0d", + "kind": "string", + "explicit": false + }, + "encryption_key_source": { + "value": "server", + "kind": "string", + "explicit": false + } + }, + "encryption_key_source": "server", + "encryption_key_fingerprint": "0ecaff1ba291fe0d", + "encryption_kek_fingerprints": [ + "39e4f0f95c3693c7", + "d55d479829bc2936" + ] + } + } + ], + "topic_responses": [ + { + "case": "topic_explicit_and_derived", + "response_hex": "0400000000e090663c1306000000000000080000000000000700000000000000066576656e7473430000000215000000636f6d7072657373696f6e5f616c676f726974686d0204000000677a6970020e0000006d6573736167655f6578706972790c0800000000a0e3d08c0000002b010000021a000000636f6e73756d65725f6f66667365745f6475726162696c697479020a0000007265706c696361746564020a0000006475726162696c697479020a0000007265706c696361746564020e0000006d61785f746f7069635f73697a650c08000000ffffffffffffffff02190000006d657373616765735f72657175697265645f746f5f736176650b04000000000400000217000000706172746974696f6e5f726573697a655f706f6c69637902070000006d757461626c650214000000707265616c6c6f636174655f7365676d656e7473030100000000020c0000007365676d656e745f73697a650c080000000000004000000000022100000073697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c080000000000100000000000", + "expected": { + "id": 4, + "created_at": 1710000000000000, + "name": "events", + "partitions_count": 0, + "size_bytes": 2048, + "messages_count": 7, + "options": { + "compression_algorithm": { + "value": "gzip", + "kind": "string", + "explicit": true + }, + "consumer_offset_durability": { + "value": "replicated", + "kind": "string", + "explicit": false + }, + "durability": { + "value": "replicated", + "kind": "string", + "explicit": false + }, + "max_topic_size": { + "value": "18446744073709551615", + "kind": "uint64", + "explicit": false + }, + "message_expiry": { + "value": "604800000000", + "kind": "uint64", + "explicit": true + }, + "messages_required_to_save": { + "value": "1024", + "kind": "uint32", + "explicit": false + }, + "partition_resize_policy": { + "value": "mutable", + "kind": "string", + "explicit": false + }, + "preallocate_segments": { + "value": "false", + "kind": "bool", + "explicit": false + }, + "segment_size": { + "value": "1073741824", + "kind": "uint64", + "explicit": false + }, + "size_of_messages_required_to_save": { + "value": "1048576", + "kind": "uint64", + "explicit": false + } + }, + "compression_algorithm": "gzip", + "message_expiry": "604800000000", + "max_topic_size": "18446744073709551615" + } + } + ], + "user_responses": [ + { + "case": "user_empty_blocks", + "response_hex": "0500000000e090663c1306000105616c696365000000000000000000000000", + "expected": { + "id": 5, + "created_at": 1710000000000000, + "status": "active", + "username": "alice", + "options": {} + } + }, + { + "case": "user_populated_blocks", + "response_hex": "0600000000e090663c1306000203626f621e000000020d0000006675747572655f6f7074696f6e0207000000656e61626c65641e000000020c0000006675747572655f71756f74610c08000000010000000000200000000000", + "expected": { + "id": 6, + "created_at": 1710000000000000, + "status": "inactive", + "username": "bob", + "options": { + "future_option": { + "value": "enabled", + "kind": "string", + "explicit": true + }, + "future_quota": { + "value": "9007199254740993", + "kind": "uint64", + "explicit": false + } + } + } + } + ] +}
diff --git a/foreign/python/tests/test_client_config.py b/foreign/python/tests/test_client_config.py index 8621759..32c9f1d 100644 --- a/foreign/python/tests/test_client_config.py +++ b/foreign/python/tests/test_client_config.py
@@ -42,6 +42,7 @@ QuicConfig, TcpConfig, TcpReconnectionConfig, + TopicOptions, WebSocketConfig, ) @@ -311,8 +312,9 @@ client.create_topic( stream="stream", name="topic", - partitions_count=1, - message_expiry=IggyExpiry.ExpireDuration(timedelta(seconds=-1)), + options=TopicOptions( + message_expiry=IggyExpiry.ExpireDuration(timedelta(seconds=-1)) + ), ) @pytest.mark.parametrize(
diff --git a/foreign/python/tests/test_consumer_group.py b/foreign/python/tests/test_consumer_group.py index da7b5a8..f8c4472 100644 --- a/foreign/python/tests/test_consumer_group.py +++ b/foreign/python/tests/test_consumer_group.py
@@ -28,6 +28,7 @@ IggyClient, PollingStrategy, ReceiveMessage, + TopicOptions, ) from apache_iggy import SendMessage as Message @@ -52,11 +53,7 @@ assert stream is not None assert isinstance(stream.id, int) - await iggy_client.create_topic( - stream=stream.id, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream.id, name=topic_name) topic = await iggy_client.get_topic(stream.id, topic_name) assert topic is not None assert isinstance(topic.id, int) @@ -85,7 +82,7 @@ await iggy_client.create_topic( stream=stream_name, name=topic_name, - partitions_count=2, + options=TopicOptions(partitions_count=2), ) await iggy_client.create_consumer_group( stream_name, @@ -119,11 +116,7 @@ stream_id = stream.id assert isinstance(stream_id, int) - await iggy_client.create_topic( - stream=stream_id, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_id, name=topic_name) topic = await iggy_client.get_topic(stream_id, topic_name) assert topic is not None topic_id = topic.id @@ -156,11 +149,7 @@ group_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.create_consumer_group( stream_name, topic_name, @@ -212,11 +201,7 @@ group_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.create_consumer_group( stream_name, topic_name, @@ -255,11 +240,7 @@ topic_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) group_by_name = await iggy_client.get_consumer_group( stream_name, @@ -286,11 +267,7 @@ partition_id = 0 await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.consumer_group( consumer_name, @@ -354,11 +331,7 @@ topic_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) groups = await iggy_client.get_consumer_groups(stream_name, topic_name) assert groups == [] @@ -374,11 +347,7 @@ second_group_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.create_consumer_group( stream_name, topic_name, @@ -412,11 +381,7 @@ stream_id = stream.id assert isinstance(stream_id, int) - await iggy_client.create_topic( - stream=stream_id, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_id, name=topic_name) topic = await iggy_client.get_topic(stream_id, topic_name) assert topic is not None topic_id = topic.id @@ -440,11 +405,7 @@ group_names = [unique_name() for _ in range(2)] await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) for group_name in group_names: await iggy_client.create_consumer_group( stream_name, @@ -488,11 +449,7 @@ group_to_keep = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.create_consumer_group( stream_name, topic_name, @@ -553,11 +510,7 @@ topic_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) with pytest.raises(RuntimeError, match="Consumer group with ID:"): await iggy_client.delete_consumer_group( @@ -615,11 +568,7 @@ group_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.create_consumer_group(stream_name, topic_name, group_name) await iggy_client.join_consumer_group(stream_name, topic_name, group_name) @@ -682,11 +631,7 @@ topic_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) with pytest.raises(RuntimeError, match="Consumer group with ID:"): await iggy_client.join_consumer_group( @@ -709,11 +654,7 @@ group_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.create_consumer_group(stream_name, topic_name, group_name) await iggy_client.join_consumer_group(stream_name, topic_name, group_name) @@ -768,11 +709,7 @@ group_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.create_consumer_group(stream_name, topic_name, group_name) await iggy_client.join_consumer_group(stream_name, topic_name, group_name) @@ -792,11 +729,7 @@ topic_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) with pytest.raises(RuntimeError, match="Consumer group with ID:"): await iggy_client.leave_consumer_group( @@ -846,11 +779,7 @@ partition_id = 0 await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) consumer = await iggy_client.consumer_group( consumer_name, @@ -885,11 +814,7 @@ shutdown_event = asyncio.Event() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) consumer = await iggy_client.consumer_group( consumer_name, @@ -946,11 +871,7 @@ shutdown_event = asyncio.Event() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) consumer = await iggy_client.consumer_group( consumer_name, @@ -1004,11 +925,7 @@ second_shutdown = asyncio.Event() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) consumer = await iggy_client.consumer_group( consumer_name, @@ -1080,11 +997,7 @@ second_shutdown = asyncio.Event() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) consumer = await iggy_client.consumer_group( consumer_name, @@ -1151,11 +1064,7 @@ received_messages = [] await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) consumer = await iggy_client.consumer_group( consumer_name, @@ -1211,11 +1120,7 @@ shutdown_event = asyncio.Event() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) consumer = await iggy_client.consumer_group( consumer_name, @@ -1262,11 +1167,7 @@ shutdown_event = asyncio.Event() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) consumer = await iggy_client.consumer_group( consumer_name, @@ -1308,11 +1209,7 @@ received_messages = [] await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) consumer = await iggy_client.consumer_group( consumer_name, @@ -1354,11 +1251,7 @@ } await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) consumer = await iggy_client.consumer_group( consumer_name, @@ -1400,11 +1293,7 @@ received_messages = [] await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.send_messages( stream_name, @@ -1444,11 +1333,7 @@ received_messages = [] await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.send_messages( stream_name, @@ -1493,11 +1378,7 @@ received_messages = [] await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.send_messages( stream_name, @@ -1538,11 +1419,7 @@ shutdown_event = asyncio.Event() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.send_messages( stream_name, @@ -1589,11 +1466,7 @@ offset_messages = [] await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.send_messages( stream_name, @@ -1659,11 +1532,7 @@ shutdown_event = asyncio.Event() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.send_messages( stream_name, @@ -1732,11 +1601,7 @@ timestamp_messages = [] await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) for message in test_messages: await iggy_client.send_messages( @@ -1806,11 +1671,7 @@ shutdown_event = asyncio.Event() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) for message in test_messages: await iggy_client.send_messages( @@ -1879,11 +1740,7 @@ received_messages = [] await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) consumer = await iggy_client.consumer_group( consumer_name, @@ -1937,11 +1794,7 @@ received_messages = [] await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.send_messages( stream_name, @@ -1986,11 +1839,7 @@ replayed_messages = [] await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) consumer = await iggy_client.consumer_group( consumer_name, @@ -2075,11 +1924,7 @@ await asyncio.sleep(timedelta(milliseconds=50).total_seconds() * 2) await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) consumer = await consumer_task @@ -2098,11 +1943,7 @@ partition_id = 0 await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) consumer = await iggy_client.consumer_group( consumer_name, @@ -2131,11 +1972,7 @@ topic_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.consumer_group( consumer_name, @@ -2206,11 +2043,7 @@ topic_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) with pytest.raises(RuntimeError, match="Consumer group with name:"): await iggy_client.consumer_group( @@ -2247,11 +2080,7 @@ topic_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) with pytest.raises(RuntimeError, match=expected_error): await iggy_client.consumer_group( @@ -2316,11 +2145,7 @@ shutdown_event = asyncio.Event() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) consumer = await iggy_client.consumer_group( consumer_name,
diff --git a/foreign/python/tests/test_http_config.py b/foreign/python/tests/test_http_config.py index d2237aa..aefe11d 100644 --- a/foreign/python/tests/test_http_config.py +++ b/foreign/python/tests/test_http_config.py
@@ -251,9 +251,7 @@ await client.login_user("iggy", "iggy") await client.create_stream(stream_name) - await client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await client.create_topic(stream=stream_name, name=topic_name) await client.send_messages( stream=stream_name, topic=topic_name, @@ -353,9 +351,7 @@ await client.login_user("iggy", "iggy") await client.create_stream(stream_name) - await client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await client.create_topic(stream=stream_name, name=topic_name) with pytest.raises(RuntimeError, match="Feature is unavailable"): await client.consumer_group(
diff --git a/foreign/python/tests/test_message_operations.py b/foreign/python/tests/test_message_operations.py index a734a51..8817e16 100644 --- a/foreign/python/tests/test_message_operations.py +++ b/foreign/python/tests/test_message_operations.py
@@ -29,6 +29,7 @@ PartitionContext, Partitioning, PollingStrategy, + TopicOptions, UserHeaders, ) from apache_iggy import SendMessage as Message @@ -133,9 +134,7 @@ test_messages = [f"Test message {i} - {unique_id}" for i in range(1, 4)] await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) messages = [Message(msg) for msg in test_messages] await iggy_client.send_messages( @@ -169,9 +168,7 @@ partition_id = 0 await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) payloads = [f"Confirmation test {i}" for i in range(3)] response = await iggy_client.send_messages( @@ -211,7 +208,9 @@ await iggy_client.create_stream(stream_name) await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=3 + stream=stream_name, + name=topic_name, + options=TopicOptions(partitions_count=3), ) response = await iggy_client.send_messages( @@ -244,7 +243,7 @@ await iggy_client.create_topic( stream=stream_name, name=topic_name, - partitions_count=partitions_count, + options=TopicOptions(partitions_count=partitions_count), ) responses = [ @@ -273,7 +272,9 @@ await iggy_client.create_stream(stream_name) await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=3 + stream=stream_name, + name=topic_name, + options=TopicOptions(partitions_count=3), ) partitioning = Partitioning.messages_key(key) @@ -308,9 +309,7 @@ test_messages = [f"Test message {i} - {unique_id}" for i in range(1, 4)] await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) messages = [Message(msg.encode()) for msg in test_messages] await iggy_client.send_messages( @@ -342,9 +341,7 @@ partition_id = 0 await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) test_payload = f"Property test - {uuid.uuid4().hex[:8]}" message = Message(test_payload) @@ -395,9 +392,7 @@ } await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.send_messages( stream=stream_name, @@ -452,9 +447,7 @@ } await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.send_messages( stream=stream_name, @@ -495,9 +488,7 @@ } await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.send_messages( stream=stream_name, @@ -797,9 +788,7 @@ expected_payload = payload.encode("utf-8") await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.send_messages( stream=stream_name, @@ -839,9 +828,7 @@ expected_payload = payload.encode("utf-8") await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.send_messages( stream=stream_name, @@ -879,9 +866,7 @@ partition_id = 0 await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.send_messages( stream=stream_name, @@ -916,9 +901,7 @@ test_messages = [f"Count test {i} - {unique_name()}" for i in range(3)] await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.send_messages( stream=stream_name, @@ -951,9 +934,7 @@ test_messages = [f"Count test {i} - {unique_name()}" for i in range(3)] await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.send_messages( stream=stream_name, @@ -988,9 +969,7 @@ test_messages = [f"Count test {i} - {unique_name()}" for i in range(3)] await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.send_messages( stream=stream_name, @@ -1020,9 +999,7 @@ partition_id = 0 await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.send_messages( stream=stream_name, @@ -1059,9 +1036,7 @@ test_messages = [f"Polling test {i} - {unique_name()}" for i in range(5)] await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) messages = [Message(msg) for msg in test_messages] await iggy_client.send_messages( @@ -1094,9 +1069,7 @@ test_messages = [f"Polling test {i} - {unique_name()}" for i in range(5)] await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) messages = [Message(msg) for msg in test_messages] await iggy_client.send_messages( @@ -1131,9 +1104,7 @@ test_messages = [f"Polling test {i} - {unique_name()}" for i in range(5)] await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) messages = [Message(msg) for msg in test_messages] await iggy_client.send_messages( @@ -1180,9 +1151,7 @@ test_messages = [f"Polling test {i} - {unique_name()}" for i in range(3)] await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.send_messages( stream=stream_name, @@ -1225,9 +1194,7 @@ test_messages = [f"Polling test {i} - {unique_name()}" for i in range(5)] await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) for message in test_messages: await iggy_client.send_messages( @@ -1275,9 +1242,7 @@ test_messages = [f"Polling test {i} - {unique_name()}" for i in range(3)] await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) for message in test_messages: await iggy_client.send_messages( @@ -1325,9 +1290,7 @@ new_messages = [f"Polling test {i} - {unique_name()}" for i in range(2)] await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.send_messages( stream=stream_name, @@ -1385,9 +1348,7 @@ new_messages = [f"Polling test {i} - {unique_name()}" for i in range(2)] await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.send_messages( stream=stream_name, @@ -1442,9 +1403,7 @@ test_messages = [f"Consumer isolation {i} - {unique_name()}" for i in range(3)] await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.send_messages( stream=stream_name, topic=topic_name, @@ -1477,9 +1436,7 @@ test_messages = [f"Shared consumer {i} - {unique_name()}" for i in range(3)] await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.send_messages( stream=stream_name, topic=topic_name, @@ -1577,9 +1534,7 @@ topic_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) with pytest.raises(ValueError, match="Invalid identifier"): await iggy_client.poll_messages( @@ -1603,7 +1558,9 @@ await iggy_client.create_stream(stream_name) await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=partitions_count + stream=stream_name, + name=topic_name, + options=TopicOptions(partitions_count=partitions_count), ) for partition_id in range(partitions_count): await iggy_client.send_messages( @@ -1639,7 +1596,9 @@ await iggy_client.create_stream(stream_name) await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=partitions_count + stream=stream_name, + name=topic_name, + options=TopicOptions(partitions_count=partitions_count), ) await iggy_client.create_consumer_group(stream_name, topic_name, group_name) await iggy_client.join_consumer_group(stream_name, topic_name, group_name) @@ -1684,9 +1643,7 @@ group_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.create_consumer_group(stream_name, topic_name, group_name) await iggy_client.send_messages( stream=stream_name, @@ -1718,7 +1675,9 @@ await iggy_client.create_stream(stream_name) await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=partitions_count + stream=stream_name, + name=topic_name, + options=TopicOptions(partitions_count=partitions_count), ) await iggy_client.create_consumer_group(stream_name, topic_name, group_name) await iggy_client.join_consumer_group(stream_name, topic_name, group_name) @@ -1761,9 +1720,7 @@ test_messages = [f"Group polling {i} - {unique_name()}" for i in range(3)] await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.create_consumer_group(stream_name, topic_name, group_name) await iggy_client.join_consumer_group(stream_name, topic_name, group_name) await wait_for_consumer_group_assignment(
diff --git a/foreign/python/tests/test_partition.py b/foreign/python/tests/test_partition.py index 193b3cb..2a08ace 100644 --- a/foreign/python/tests/test_partition.py +++ b/foreign/python/tests/test_partition.py
@@ -24,6 +24,7 @@ Permissions, SendMessage, StreamPermissions, + TopicOptions, TopicPermissions, ) @@ -36,7 +37,7 @@ await iggy_client.create_stream(stream_name) await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=2 + stream=stream_name, name=topic_name, options=TopicOptions(partitions_count=2) ) return stream_name, topic_name @@ -182,7 +183,7 @@ stream_name, topic_name = await _create_topic(iggy_client, unique_name) other_topic_name = unique_name() await iggy_client.create_topic( - stream_name, other_topic_name, partitions_count=2 + stream_name, other_topic_name, TopicOptions(partitions_count=2) ) stream = await iggy_client.get_stream(stream_name) assert stream is not None
diff --git a/foreign/python/tests/test_permissions.py b/foreign/python/tests/test_permissions.py index e3b54f1..58f8675 100644 --- a/foreign/python/tests/test_permissions.py +++ b/foreign/python/tests/test_permissions.py
@@ -209,7 +209,7 @@ stream = await iggy_client.get_stream(stream_name) assert stream is not None topic_name = unique_name(prefix="perm-topic-") - await iggy_client.create_topic(stream_name, topic_name, partitions_count=1) + await iggy_client.create_topic(stream_name, topic_name) topic = await iggy_client.get_topic(stream_name, topic_name) assert topic is not None
diff --git a/foreign/python/tests/test_producer.py b/foreign/python/tests/test_producer.py index fd0d09b..9205f0c 100644 --- a/foreign/python/tests/test_producer.py +++ b/foreign/python/tests/test_producer.py
@@ -44,6 +44,7 @@ ProducerSharding, SendMessage, SendMessagesResponse, + TopicOptions, ) from .utils import wait_for_server @@ -467,7 +468,7 @@ stream_name = unique_name() topic_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic(stream_name, topic_name, 1) + await iggy_client.create_topic(stream_name, topic_name) producer = await iggy_client.producer( stream_name, @@ -646,7 +647,9 @@ destination_stream = unique_name() destination_topic = unique_name() await iggy_client.create_stream(destination_stream) - await iggy_client.create_topic(destination_stream, destination_topic, 2) + await iggy_client.create_topic( + destination_stream, destination_topic, TopicOptions(partitions_count=2) + ) stream = await iggy_client.get_stream(destination_stream) topic = await iggy_client.get_topic(destination_stream, destination_topic) assert stream is not None @@ -754,7 +757,7 @@ destination_stream = unique_name("to-stream-", min_bytes=20, max_bytes=20) destination_topic = unique_name("to-topic-", min_bytes=20, max_bytes=20) await iggy_client.create_stream(destination_stream) - await iggy_client.create_topic(destination_stream, destination_topic, 1) + await iggy_client.create_topic(destination_stream, destination_topic) producer = await iggy_client.producer( stream_name, @@ -1466,7 +1469,7 @@ await producer.shutdown() await iggy_client.create_stream(destination_stream) - await iggy_client.create_topic(destination_stream, destination_topic, 1) + await iggy_client.create_topic(destination_stream, destination_topic) assert ( await poll_payloads(iggy_client, destination_stream, destination_topic) == []
diff --git a/foreign/python/tests/test_resource_options.py b/foreign/python/tests/test_resource_options.py new file mode 100644 index 0000000..8e56337 --- /dev/null +++ b/foreign/python/tests/test_resource_options.py
@@ -0,0 +1,742 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Tests for typed stream, topic and user options. + +The server cases run once per transport through `transport_client`, against a +server with encryption enabled. The vector cases read the fixture the Rust +encoders generated and need no server. +""" + +import base64 +import inspect +import json +import re +import secrets +from datetime import timedelta +from pathlib import Path + +import pytest + +from apache_iggy import ( + AutoLogin, + Consumer, + Durability, + GlobalPermissions, + IggyClient, + IggyExpiry, + MaxTopicSize, + OptionError, + OptionKey, + PartitionResizePolicy, + Permissions, + PollingStrategy, + QuicConfig, + QuicReconnectionConfig, + Stream, + StreamDetails, + StreamOptions, + TcpConfig, + TcpReconnectionConfig, + Topic, + TopicDetails, + TopicOptions, + UserInfo, + UserInfoDetails, + UserOptions, + UserStatus, + WebSocketConfig, + encryption_key_fingerprint, +) +from apache_iggy import SendMessage as Message + +from .utils import ( + get_quic_server_config, + get_server_config, + get_websocket_server_config, + unique_credentials, + wait_for_ping, +) + +FIXTURE = json.loads( + (Path(__file__).parent / "fixtures" / "resource_options.json").read_text() +) +# The binding raises the server's error text, so each fixture error name maps +# to the text it carries. +ERROR_MESSAGES = { + "invalid_command": "Invalid command", + "invalid_encryption_key": "Invalid encryption key", + "invalid_stream_name": "Invalid stream name", + "invalid_topic_name": "Invalid topic name", + "invalid_username": "Invalid username", + "option_not_resettable": "Option cannot be reset", + "option_not_updatable": "Option cannot be updated after creation", + "read_only_option": "Option is read-only", + "unsupported_option_key": "Unsupported option key", +} +# Option refusals raise `OptionError`, which carries the code on every transport. +OPTION_ERROR_CODES = { + vector["error"]["name"]: vector["error"]["code"] + for section in ("update_resource_errors", "http_update") + for vector in FIXTURE[section] + if vector.get("error") and vector["error"]["code"] >= 4000 +} +STREAM_KEY_LENGTH = FIXTURE["constants"]["stream_key_length"] +NEVER_RETURNED_KEYS = set(FIXTURE["constants"]["never_returned_keys"]) +FINGERPRINT_LENGTH = 16 +U64_MAX = 2**64 - 1 +MICROS_PER_HOUR = 3_600_000_000 +GIB = 1024**3 + + +@pytest.mark.usefixtures("iggy_client") +class TestStreamOptions: + """Typed stream create options, encryption projections and updates.""" + + @pytest.mark.asyncio + async def test_given_stream_key_when_creating_should_read_matching_projection( + self, transport_client: IggyClient, unique_name + ): + """A client key comes back only as its source and fingerprints.""" + key = secrets.token_bytes(STREAM_KEY_LENGTH) + fingerprint = encryption_key_fingerprint(key) + + created = await transport_client.create_stream( + unique_name(), StreamOptions(encryption_key=key) + ) + fetched = await transport_client.get_stream(created.id) + listed = next( + stream + for stream in await transport_client.get_streams() + if stream.id == created.id + ) + + assert fetched is not None + for stream in (created, fetched, listed): + assert_encryption_projection(stream, "stream", fingerprint) + assert_no_key_material(stream, key) + assert listed.encryption_kek_fingerprints == fetched.encryption_kek_fingerprints + + @pytest.mark.asyncio + async def test_given_keyless_stream_when_reading_should_report_server_generated_key( # noqa: E501 + self, transport_client: IggyClient, unique_name + ): + """A server with encryption enabled generates a key for a keyless stream.""" + created = await transport_client.create_stream(unique_name()) + fetched = await transport_client.get_stream(created.id) + + assert fetched is not None + fingerprint = fetched.encryption_key_fingerprint + assert fingerprint is not None + assert re.fullmatch(f"[0-9a-f]{{{FINGERPRINT_LENGTH}}}", fingerprint) + for stream in (created, fetched): + assert_encryption_projection(stream, "server", fingerprint) + + @pytest.mark.asyncio + async def test_given_stream_rename_when_updating_should_preserve_encryption_projection( # noqa: E501 + self, transport_client: IggyClient, unique_name + ): + """A rename keeps the stream's key, so data written before still reads.""" + key = secrets.token_bytes(STREAM_KEY_LENGTH) + topic_name = unique_name() + new_name = unique_name() + payload = f"before-rename-{unique_name()}" + created = await transport_client.create_stream( + unique_name(), + StreamOptions(encryption_key=base64.b64encode(key).decode()), + ) + await transport_client.create_topic(created.id, topic_name) + await transport_client.send_messages( + stream=created.id, + topic=topic_name, + partitioning=0, + messages=[Message(payload)], + ) + + await transport_client.update_stream(created.id, name=new_name) + + renamed = await transport_client.get_stream(created.id) + assert renamed is not None + assert renamed.id == created.id + assert renamed.name == new_name + assert await transport_client.get_stream(created.name) is None + assert_encryption_projection(renamed, "stream", encryption_key_fingerprint(key)) + assert ( + renamed.encryption_kek_fingerprints == created.encryption_kek_fingerprints + ) + polled = await transport_client.poll_messages( + stream=new_name, + topic=topic_name, + consumer=Consumer.Single("stream-rename"), + partition_id=0, + polling_strategy=PollingStrategy.First(), + count=1, + auto_commit=False, + ) + assert [message.payload().decode() for message in polled] == [payload] + + @pytest.mark.asyncio + async def test_given_stream_key_reset_when_updating_should_reject_without_changing_state( # noqa: E501 + self, transport_client: IggyClient, unique_name + ): + """No stream key is resettable, so a reset is refused before it is sent.""" + key = secrets.token_bytes(STREAM_KEY_LENGTH) + created = await transport_client.create_stream( + unique_name(), StreamOptions(encryption_key=key) + ) + + with pytest.raises(RuntimeError, match=ERROR_MESSAGES["option_not_updatable"]): + await transport_client.update_stream( + created.id, reset=[OptionKey.EncryptionKey] + ) + with pytest.raises(RuntimeError, match=ERROR_MESSAGES["option_not_resettable"]): + await transport_client.update_stream(created.id, reset=[OptionKey.Name]) + + stream = await transport_client.get_stream(created.id) + assert stream is not None + assert stream.name == created.name + assert_encryption_projection(stream, "stream", encryption_key_fingerprint(key)) + + +@pytest.mark.usefixtures("iggy_client") +class TestTopicOptions: + """Typed topic create options and generic topic updates.""" + + @pytest.mark.asyncio + async def test_given_keyword_topic_options_when_creating_should_preserve_all_ten_settings( # noqa: E501 + self, transport_client: IggyClient, unique_name + ): + """Every known create setting has a typed field and reads back explicit.""" + stream_name = unique_name() + topic_name = unique_name() + await transport_client.create_stream(stream_name) + + await transport_client.create_topic( + stream_name, + topic_name, + TopicOptions( + partitions_count=2, + compression_algorithm="gzip", + message_expiry=IggyExpiry.ExpireDuration(duration=timedelta(hours=1)), + max_topic_size=MaxTopicSize.Custom(bytes=GIB), + segment_size=1024 * 1024, + durability=Durability.PERSISTED, + consumer_offset_durability=Durability.PERSISTED, + partition_resize_policy=PartitionResizePolicy.Fixed, + messages_required_to_save=7, + size_of_messages_required_to_save=4096, + preallocate_segments=True, + ), + ) + + topic = await transport_client.get_topic(stream_name, topic_name) + assert topic is not None + assert topic.partitions_count == 2 + assert topic.options.to_scalar_dict() == { + "compression_algorithm": "gzip", + "message_expiry": MICROS_PER_HOUR, + "max_topic_size": GIB, + "segment_size": 1024 * 1024, + "durability": "persisted", + "consumer_offset_durability": "persisted", + "partition_resize_policy": "fixed", + "messages_required_to_save": 7, + "size_of_messages_required_to_save": 4096, + "preallocate_segments": True, + } + assert topic.derived_options.to_scalar_dict() == {} + assert topic.compression_algorithm == "gzip" + assert isinstance(topic.max_topic_size, MaxTopicSize.Custom) + assert topic.max_topic_size.bytes == GIB + + @pytest.mark.asyncio + async def test_given_topic_reset_when_updating_should_preserve_unmentioned_values( # noqa: E501 + self, transport_client: IggyClient, unique_name + ): + """An update without a name changes only the keys it names or resets.""" + stream_name = unique_name() + topic_name = unique_name() + await transport_client.create_stream(stream_name) + await transport_client.create_topic( + stream_name, + topic_name, + TopicOptions( + compression_algorithm="gzip", + message_expiry=IggyExpiry.ExpireDuration(duration=timedelta(hours=1)), + max_topic_size=MaxTopicSize.Custom(bytes=GIB), + ), + ) + + await transport_client.update_topic( + stream_name, + topic_name, + message_expiry=IggyExpiry.ExpireDuration(duration=timedelta(hours=2)), + reset=[OptionKey.MaxTopicSize], + ) + + topic = await transport_client.get_topic(stream_name, topic_name) + assert topic is not None + assert topic.name == topic_name + explicit = topic.options.to_scalar_dict() + derived = topic.derived_options.to_scalar_dict() + assert explicit["message_expiry"] == 2 * MICROS_PER_HOUR + assert explicit["compression_algorithm"] == "gzip" + assert "max_topic_size" not in explicit + assert derived["max_topic_size"] == U64_MAX + assert topic.compression_algorithm == "gzip" + assert isinstance(topic.max_topic_size, MaxTopicSize.Unlimited) + # Compression is create-only, so the update has no field for it. + with pytest.raises(TypeError): + await transport_client.update_topic( + stream_name, + topic_name, + # pyrefly: ignore # unexpected-keyword + compression_algorithm="none", + ) + + +@pytest.mark.usefixtures("iggy_client") +class TestUserOptions: + """Typed user create options and generic user updates.""" + + @pytest.mark.asyncio + async def test_given_user_options_when_creating_and_updating_should_preserve_user_behavior( # noqa: E501 + self, transport_client: IggyClient, unique_name + ): + """A username and status patch applies together or not at all.""" + username, password = unique_credentials(unique_name) + other_username, other_password = unique_credentials(unique_name) + new_username, _ = unique_credentials(unique_name) + created = await transport_client.create_user( + username, password, options=UserOptions() + ) + other = await transport_client.create_user(other_username, other_password) + try: + assert created.options.to_scalar_dict() == {} + assert created.derived_options.to_scalar_dict() == {} + + await transport_client.update_user( + created.id, username=new_username, status=UserStatus.Inactive + ) + + user = await transport_client.get_user(created.id) + assert user is not None + assert user.username == new_username + assert user.status == UserStatus.Inactive + assert user.options.to_scalar_dict() == {} + listed = next( + entry + for entry in await transport_client.get_users() + if entry.id == created.id + ) + assert listed.username == new_username + assert listed.derived_options.to_scalar_dict() == {} + + with pytest.raises(RuntimeError): + await transport_client.update_user( + other.id, username=new_username, status=UserStatus.Inactive + ) + unchanged = await transport_client.get_user(other.id) + assert unchanged is not None + assert unchanged.username == other_username + assert unchanged.status == UserStatus.Active + finally: + await transport_client.delete_user(created.id) + await transport_client.delete_user(other.id) + + @pytest.mark.asyncio + @pytest.mark.parametrize("transport", ["tcp", "quic", "websocket"]) + async def test_given_logged_in_user_rename_when_reconnecting_should_keep_new_username( # noqa: E501 + self, iggy_client: IggyClient, unique_name, transport: str + ): + """A self-rename refreshes the credentials a reconnect replays. + + HTTP holds no session and replays no credentials, so only the binary + transports reconnect. + """ + username, password = unique_credentials(unique_name) + new_username, _ = unique_credentials(unique_name) + permissions = Permissions( + GlobalPermissions(manage_users=True, manage_streams=True) + ) + user = await iggy_client.create_user( + username, password, permissions=permissions + ) + client = IggyClient( + auto_login_config( + transport, AutoLogin.username_password(username, password) + ) + ) + try: + await client.connect() + await wait_for_ping(client) + + with pytest.raises(RuntimeError): + await client.update_user(user.id, username="iggy") + stream = await client.create_stream(unique_name()) + await client.update_stream(stream.id, name=unique_name()) + await reconnect(client) + + await client.update_user(user.id, username=new_username) + await reconnect(client) + + renamed = await iggy_client.get_user(user.id) + assert renamed is not None + assert renamed.username == new_username + finally: + await client.shutdown() + await iggy_client.delete_user(user.id) + + +@pytest.mark.usefixtures("iggy_client") +class TestServerOptionContract: + """Descriptors and refusals that come back from the server.""" + + @pytest.mark.asyncio + async def test_given_every_scope_when_describing_options_should_match_fixture( # noqa: E501 + self, transport_client: IggyClient + ): + """Every transport decodes the same descriptors and flags.""" + flags = FIXTURE["constants"]["option_flags"] + for vector in FIXTURE["describe_options"]: + specs = await transport_client.describe_options(vector["scope"]) + assert [spec.key for spec in specs] == [ + entry["key"] for entry in vector["entries"] + ] + for spec, entry in zip(specs, vector["entries"], strict=True): + for flag, mask in flags.items(): + assert getattr(spec, flag) == bool(entry["flags"] & mask), flag + + @pytest.mark.asyncio + async def test_given_raw_reset_of_internal_key_when_updating_should_get_server_refusal( # noqa: E501 + self, transport_client: IggyClient, unique_name + ): + """A raw reset name travels to the server, which decides it.""" + vector = fixture_case("update_resource_errors", "user_reset_internal_key") + username, password = unique_credentials(unique_name) + created = await transport_client.create_user(username, password) + try: + with pytest.raises( + OptionError, match=ERROR_MESSAGES[vector["error"]["name"]] + ) as refusal: + await transport_client.update_user( + created.id, raw_reset=vector["input"]["reset"] + ) + assert_option_error(refusal.value, vector["error"]["name"]) + finally: + await transport_client.delete_user(created.id) + + @pytest.mark.asyncio + async def test_given_raw_set_key_refused_by_admission_when_updating_should_get_server_refusal( # noqa: E501 + self, transport_client: IggyClient, unique_name + ): + """Raw set keys reach the server, which refuses them by category.""" + stream_name = unique_name() + topic_name = unique_name() + await transport_client.create_stream(stream_name) + await transport_client.create_topic(stream_name, topic_name) + for options, name in ( + ({"compression_algorithm": "gzip"}, "option_not_updatable"), + ({"future_key": "on"}, "unsupported_option_key"), + ): + with pytest.raises(OptionError, match=ERROR_MESSAGES[name]) as refusal: + await transport_client.update_topic( + stream_name, topic_name, options=options + ) + assert_option_error(refusal.value, name) + with pytest.raises( + OptionError, match=ERROR_MESSAGES["read_only_option"] + ) as refusal: + await transport_client.update_stream( + stream_name, raw_reset=["encryption_key_source"] + ) + assert_option_error(refusal.value, "read_only_option") + + +class TestSharedVectors: + """The binding against the vectors the Rust encoders generated.""" + + @pytest.mark.unit + def test_given_shared_vectors_when_encoding_should_match_server_contract( + self, + ): + """Fingerprints, key members and typed refusals match the server.""" + vectors = FIXTURE["stream_key"] + assert ( + encryption_key_fingerprint(bytes.fromhex(vectors["key_hex"])) + == vectors["fingerprint"] + ) + assert ( + encryption_key_fingerprint(vectors["key_base64"]) == vectors["fingerprint"] + ) + + described = {row["key"] for row in FIXTURE["catalog"] if row["described"]} + internal = {row["key"] for row in FIXTURE["catalog"] if not row["described"]} + members = { + str(value) + for value in vars(OptionKey).values() + if isinstance(value, OptionKey) + } + assert members == described + assert all(str(option_key(key)) == key for key in described) + assert not any(hasattr(OptionKey, member_name(key)) for key in internal) + + # A client that never connects proves the refusal happens before I/O: + # the call raises before it returns an awaitable. + client = IggyClient("127.0.0.1:1") + for vector in FIXTURE["update_resource_errors"]: + if raw_reset_names(vector): + # The server decides raw names, see the transport refusal tests. + continue + name = vector["error"]["name"] + with pytest.raises(RuntimeError, match=ERROR_MESSAGES[name]) as refusal: + typed_update(client, vector) + if name in OPTION_ERROR_CODES: + assert_option_error(refusal.value, name) + else: + assert not isinstance(refusal.value, OptionError), name + + @pytest.mark.unit + def test_given_fixture_catalog_when_reading_typed_surface_should_cover_every_key( # noqa: E501 + self, + ): + """Every public catalog key has its typed field, reset member or accessor.""" + create_fields = { + "stream": parameters(StreamOptions), + "topic": parameters(TopicOptions), + "user": parameters(UserOptions), + } + update_fields = { + "stream": parameters(IggyClient.update_stream), + "topic": parameters(IggyClient.update_topic), + "user": parameters(IggyClient.update_user), + } + readers = { + "stream": (Stream, StreamDetails), + "topic": (Topic, TopicDetails), + "user": (UserInfo, UserInfoDetails), + } + resettable = FIXTURE["constants"]["option_flags"]["resettable"] + for scope in create_fields: + assert "raw" in create_fields[scope], scope + assert {"reset", "raw_reset", "options"} <= update_fields[scope], scope + for row in FIXTURE["catalog"]: + if row["visibility"] == "internal": + continue + key = row["key"] + for scope in row["scopes"]: + if row["storage"] == "map" and row["mutability"] != "read_only": + assert key in create_fields[scope], f"{scope} create {key}" + if row["mutability"] == "updatable": + assert key in update_fields[scope], f"{scope} update {key}" + if row["flags"] & resettable: + assert hasattr(OptionKey, member_name(key)), f"reset {key}" + if row["mutability"] == "read_only": + for reader in readers[scope]: + # A multi-valued read-only key reads through a plural. + assert hasattr(reader, key) or hasattr(reader, f"{key}s"), ( + f"{reader.__name__} {key}" + ) + + @pytest.mark.unit + def test_given_invalid_key_when_encoding_should_reject_before_network_io( + self, + ): + """Only exactly 32 bytes or their canonical base64 form are accepted.""" + for vector in FIXTURE["stream_key"]["invalid_keys"]: + key = key_input(vector["input"]) + with pytest.raises( + ValueError, match=ERROR_MESSAGES[vector["error"]["name"]] + ) as refusal: + StreamOptions(encryption_key=key) + assert_secret_absent(str(refusal.value), key) + with pytest.raises( + ValueError, match=ERROR_MESSAGES[vector["error"]["name"]] + ): + encryption_key_fingerprint(key) + with pytest.raises(TypeError): + # pyrefly: ignore # bad-argument-type + StreamOptions(encryption_key=7) + + key = bytes.fromhex(FIXTURE["stream_key"]["key_hex"]) + for accepted in (key, FIXTURE["stream_key"]["key_base64"]): + options = StreamOptions(encryption_key=accepted, raw={"future": "on"}) + assert options.has_encryption_key + assert_secret_absent(repr(options), key) + assert "future" in repr(options) + assert not StreamOptions().has_encryption_key + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("iggy_client") +async def test_given_invalid_key_when_creating_should_send_no_request( + transport_client: IggyClient, unique_name +): + """An invalid key never reaches the server on any transport.""" + name = unique_name() + for vector in FIXTURE["stream_key"]["invalid_keys"]: + with pytest.raises(ValueError): + await transport_client.create_stream( + name, StreamOptions(encryption_key=key_input(vector["input"])) + ) + + assert await transport_client.get_stream(name) is None + + +def assert_encryption_projection( + stream: Stream | StreamDetails, source: str, fingerprint: str +) -> None: + """Check the typed accessors and the derived block report the same key.""" + assert stream.encryption_key_source == source + assert stream.encryption_key_fingerprint == fingerprint + kek_fingerprints = stream.encryption_kek_fingerprints + assert kek_fingerprints + assert all(len(kek) == FINGERPRINT_LENGTH for kek in kek_fingerprints) + derived = stream.derived_options.to_scalar_dict() + assert derived["encryption_key_source"] == source + assert derived["encryption_key_fingerprint"] == fingerprint + assert derived["encryption_kek_fingerprint"] == ",".join(kek_fingerprints) + returned = set(stream.options.to_scalar_dict()) | set(derived) + assert not NEVER_RETURNED_KEYS & returned + + +def assert_no_key_material(stream: Stream | StreamDetails, key: bytes) -> None: + """Check no returned option value carries the key, raw or encoded.""" + values = [ + *stream.options.to_scalar_dict().values(), + *stream.derived_options.to_scalar_dict().values(), + ] + assert key not in values + assert base64.b64encode(key).decode() not in values + assert key.hex() not in values + + +def assert_option_error(error: BaseException, name: str) -> None: + """Check a refusal carries the code and kind of a fixture error name.""" + assert isinstance(error, OptionError), repr(error) + assert (error.code, error.kind) == (OPTION_ERROR_CODES[name], name) + + +def assert_secret_absent(text: str, key: bytes | str) -> None: + """Check a diagnostic carries none of the forms a key can be given in.""" + raw = key if isinstance(key, bytes) else key.encode() + for secret in (raw.hex(), base64.b64encode(raw).decode(), repr(raw)): + assert secret not in text + if isinstance(key, str) and key.strip(): + assert key.strip() not in text + + +def key_input(value: dict) -> bytes | str: + """Return a fixture key input as the bytes or text the API takes.""" + if "raw_hex" in value: + return bytes.fromhex(value["raw_hex"]) + return value["base64"] + + +def fixture_case(section: str, case: str) -> dict: + """Return the fixture vector of a section by its case name.""" + return next(vector for vector in FIXTURE[section] if vector["case"] == case) + + +def parameters(callable_object) -> set[str]: + """Return the keyword names a binding constructor or method takes.""" + return set(inspect.signature(callable_object).parameters) + + +def raw_reset_names(vector: dict) -> list[str]: + """Return the reset names of a vector that have no `OptionKey` member.""" + return [ + key + for key in vector["input"].get("reset", []) + if not hasattr(OptionKey, member_name(key)) + ] + + +def member_name(key: str) -> str: + """Return the `OptionKey` member name for a catalog key.""" + return "".join(part.title() for part in key.split("_")) + + +def option_key(key: str) -> OptionKey: + """Return the `OptionKey` member for a catalog key.""" + return getattr(OptionKey, member_name(key)) + + +def typed_update(client: IggyClient, vector: dict): + """Issue the typed update a fixture vector describes.""" + target = vector["target"] + values = vector["input"] + reset = [option_key(key) for key in values.get("reset", [])] or None + if vector["scope"] == "topic": + expiry = values.get("message_expiry") + max_size = values.get("max_topic_size") + return client.update_topic( + target["stream_id"], + target["topic_id"], + name=values.get("name"), + message_expiry=None + if expiry is None + else IggyExpiry.ExpireDuration( + duration=timedelta(microseconds=int(expiry)) + ), + max_topic_size=None + if max_size is None + else MaxTopicSize.Custom(bytes=int(max_size)), + reset=reset, + ) + if vector["scope"] == "stream": + return client.update_stream( + target["stream_id"], name=values.get("name"), reset=reset + ) + status = values.get("status") + return client.update_user( + target["user_id"], + username=values.get("username"), + status=None if status is None else getattr(UserStatus, status.title()), + reset=reset, + ) + + +def auto_login_config( + transport: str, auto_login: AutoLogin +) -> TcpConfig | QuicConfig | WebSocketConfig: + """Return a binary transport config that signs in on every connect.""" + if transport == "tcp": + host, port = get_server_config() + return TcpConfig( + server_address=f"{host}:{port}", + auto_login=auto_login, + reconnection=TcpReconnectionConfig(reestablish_after=timedelta(0)), + ) + if transport == "quic": + host, port = get_quic_server_config() + return QuicConfig( + server_address=f"{host}:{port}", + auto_login=auto_login, + reconnection=QuicReconnectionConfig(reestablish_after=timedelta(0)), + ) + host, port = get_websocket_server_config() + return WebSocketConfig(server_address=f"{host}:{port}", auto_login=auto_login) + + +async def reconnect(client: IggyClient) -> None: + """Drop the connection, sign in again and prove the session is privileged.""" + await client.disconnect() + await client.connect() + await wait_for_ping(client) + await client.get_users()
diff --git a/foreign/python/tests/test_segment.py b/foreign/python/tests/test_segment.py index f9b3def..7e7e1c0 100644 --- a/foreign/python/tests/test_segment.py +++ b/foreign/python/tests/test_segment.py
@@ -27,6 +27,7 @@ PollingStrategy, SendMessage, StreamPermissions, + TopicOptions, TopicPermissions, ) @@ -55,10 +56,11 @@ await iggy_client.create_topic( stream=stream_name, name=topic_name, - partitions_count=1, - segment_size=SEGMENT_SIZE, - durability=Durability.PERSISTED, - messages_required_to_save=1, + options=TopicOptions( + segment_size=SEGMENT_SIZE, + durability=Durability.PERSISTED, + messages_required_to_save=1, + ), ) stream = await iggy_client.get_stream(stream_name) assert stream is not None @@ -102,10 +104,11 @@ await iggy_client.create_topic( stream=stream_name, name=topic_name, - partitions_count=1, - segment_size=SEGMENT_SIZE, - durability=Durability.PERSISTED, - messages_required_to_save=1, + options=TopicOptions( + segment_size=SEGMENT_SIZE, + durability=Durability.PERSISTED, + messages_required_to_save=1, + ), ) for _ in range(2 * MESSAGES_PER_SEALED_SEGMENT + 2): await iggy_client.send_messages( @@ -145,10 +148,11 @@ await iggy_client.create_topic( stream=stream_name, name=topic_name, - partitions_count=1, - segment_size=SEGMENT_SIZE, - durability=Durability.PERSISTED, - messages_required_to_save=1, + options=TopicOptions( + segment_size=SEGMENT_SIZE, + durability=Durability.PERSISTED, + messages_required_to_save=1, + ), ) for _ in range(2 * MESSAGES_PER_SEALED_SEGMENT + 2): await iggy_client.send_messages( @@ -198,10 +202,11 @@ await iggy_client.create_topic( stream=stream_name, name=topic_name, - partitions_count=1, - segment_size=SEGMENT_SIZE, - durability=Durability.PERSISTED, - messages_required_to_save=1, + options=TopicOptions( + segment_size=SEGMENT_SIZE, + durability=Durability.PERSISTED, + messages_required_to_save=1, + ), ) for _ in range(MESSAGES_PER_SEALED_SEGMENT + 1): await iggy_client.send_messages( @@ -235,7 +240,7 @@ stream_name = unique_name() topic_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic(stream_name, topic_name, partitions_count=1) + await iggy_client.create_topic(stream_name, topic_name) missing_name = unique_name() stream_id = missing_name if missing == "stream" else stream_name topic_id = missing_name if missing == "topic" else topic_name @@ -287,10 +292,8 @@ topic_name = unique_name() other_topic_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic(stream_name, topic_name, partitions_count=1) - await iggy_client.create_topic( - stream_name, other_topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream_name, topic_name) + await iggy_client.create_topic(stream_name, other_topic_name) stream = await iggy_client.get_stream(stream_name) assert stream is not None topic = await iggy_client.get_topic(stream.id, topic_name)
diff --git a/foreign/python/tests/test_stats.py b/foreign/python/tests/test_stats.py index 240ab5e..91c0cfb 100644 --- a/foreign/python/tests/test_stats.py +++ b/foreign/python/tests/test_stats.py
@@ -19,7 +19,13 @@ import pytest -from apache_iggy import CacheMetricsKey, GlobalPermissions, IggyClient, Permissions +from apache_iggy import ( + CacheMetricsKey, + GlobalPermissions, + IggyClient, + Permissions, + TopicOptions, +) from apache_iggy import SendMessage as Message from .utils import ( @@ -54,9 +60,7 @@ stream_name = unique_name() topic_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.send_messages( stream=stream_name, topic=topic_name, @@ -122,7 +126,7 @@ await iggy_client.create_topic( stream=stream_name, name=topic_name, - partitions_count=partitions_per_topic, + options=TopicOptions(partitions_count=partitions_per_topic), ) await iggy_client.create_consumer_group( stream_name, topic_name, unique_name()
diff --git a/foreign/python/tests/test_stream.py b/foreign/python/tests/test_stream.py index 0b2a0ef..9148dc7 100644 --- a/foreign/python/tests/test_stream.py +++ b/foreign/python/tests/test_stream.py
@@ -243,9 +243,7 @@ topic_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) streams = await iggy_client.get_streams() listed = next(
diff --git a/foreign/python/tests/test_tls.py b/foreign/python/tests/test_tls.py index f1daf88..acbdb19 100644 --- a/foreign/python/tests/test_tls.py +++ b/foreign/python/tests/test_tls.py
@@ -128,7 +128,7 @@ # Create stream and topic await tls_client.create_stream(stream_name) - await tls_client.create_topic(stream_name, topic_name, partitions_count=1) + await tls_client.create_topic(stream_name, topic_name) # Produce messages test_messages = [f"tls-message-{i}" for i in range(3)]
diff --git a/foreign/python/tests/test_topic.py b/foreign/python/tests/test_topic.py index f25105a..f590da97 100644 --- a/foreign/python/tests/test_topic.py +++ b/foreign/python/tests/test_topic.py
@@ -25,6 +25,7 @@ IggyClient, IggyExpiry, MaxTopicSize, + TopicOptions, ) from .utils import ( @@ -75,7 +76,9 @@ await iggy_client.create_stream(stream_name) await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=2 + stream=stream_name, + name=topic_name, + options=TopicOptions(partitions_count=2), ) topic = await iggy_client.get_topic(stream_name, topic_name) @@ -121,9 +124,7 @@ await iggy_client.create_stream(stream_name) with pytest.raises(RuntimeError): - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) @pytest.mark.asyncio async def test_create_and_get_topic_with_numeric_stream_id( @@ -138,7 +139,7 @@ assert stream is not None await iggy_client.create_topic( - stream=stream.id, name=topic_name, partitions_count=2 + stream=stream.id, name=topic_name, options=TopicOptions(partitions_count=2) ) topic_by_name = await iggy_client.get_topic(stream.id, topic_name) @@ -158,14 +159,10 @@ topic_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) with pytest.raises(RuntimeError) as exc_info: - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) assert "already exists" in str(exc_info.value) @@ -181,12 +178,8 @@ await iggy_client.create_stream(first_stream_name) await iggy_client.create_stream(second_stream_name) - await iggy_client.create_topic( - stream=first_stream_name, name=topic_name, partitions_count=1 - ) - await iggy_client.create_topic( - stream=second_stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=first_stream_name, name=topic_name) + await iggy_client.create_topic(stream=second_stream_name, name=topic_name) first_topic = await iggy_client.get_topic(first_stream_name, topic_name) second_topic = await iggy_client.get_topic(second_stream_name, topic_name) @@ -211,8 +204,7 @@ await iggy_client.create_topic( stream=stream_name, name=topic_name, - partitions_count=1, - compression_algorithm=compression_algorithm, + options=TopicOptions(compression_algorithm=compression_algorithm), ) topic = await iggy_client.get_topic(stream_name, topic_name) @@ -237,8 +229,7 @@ await iggy_client.create_topic( stream=stream_name, name=topic_name, - partitions_count=1, - compression_algorithm=compression_algorithm, + options=TopicOptions(compression_algorithm=compression_algorithm), ) @pytest.mark.asyncio @@ -268,8 +259,7 @@ await iggy_client.create_topic( stream=stream_name, name=topic_name, - partitions_count=1, - message_expiry=message_expiry, + options=TopicOptions(message_expiry=message_expiry), ) topic = await iggy_client.get_topic(stream_name, topic_name) @@ -304,8 +294,9 @@ await iggy_client.create_topic( stream=stream_name, name=topic_name, - partitions_count=1, - message_expiry=IggyExpiry.ExpireDuration(invalid_duration), + options=TopicOptions( + message_expiry=IggyExpiry.ExpireDuration(invalid_duration) + ), ) @pytest.mark.asyncio @@ -325,8 +316,7 @@ await iggy_client.create_topic( stream=stream_name, name=topic_name, - partitions_count=1, - message_expiry=invalid_message_expiry, + options=TopicOptions(message_expiry=invalid_message_expiry), ) @pytest.mark.asyncio @@ -353,8 +343,7 @@ await iggy_client.create_topic( stream=stream_name, name=topic_name, - partitions_count=1, - max_topic_size=max_topic_size, + options=TopicOptions(max_topic_size=max_topic_size), ) topic = await iggy_client.get_topic(stream_name, topic_name) @@ -395,8 +384,9 @@ await iggy_client.create_topic( stream=stream_name, name=topic_name, - partitions_count=1, - max_topic_size=MaxTopicSize.Custom(max_topic_size_bytes), + options=TopicOptions( + max_topic_size=MaxTopicSize.Custom(max_topic_size_bytes) + ), ) @pytest.mark.asyncio @@ -412,7 +402,9 @@ with pytest.raises(RuntimeError) as exc_info: await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=partitions_count + stream=stream_name, + name=topic_name, + options=TopicOptions(partitions_count=partitions_count), ) assert "Too many partitions" in str(exc_info.value) @@ -437,7 +429,9 @@ await iggy_client.create_stream(stream_name) await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=partitions_count + stream=stream_name, + name=topic_name, + options=TopicOptions(partitions_count=partitions_count), ) topic = await iggy_client.get_topic(stream_name, topic_name) @@ -460,12 +454,8 @@ assert stream_before is not None assert stream_before.topics_count == 0 - await iggy_client.create_topic( - stream=stream_name, name=first_topic_name, partitions_count=1 - ) - await iggy_client.create_topic( - stream=stream_name, name=second_topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=first_topic_name) + await iggy_client.create_topic(stream=stream_name, name=second_topic_name) stream_after = await iggy_client.get_stream(stream_name) assert stream_after is not None @@ -480,9 +470,7 @@ topic_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) host, port = get_server_config() wait_for_server(host, port) @@ -506,9 +494,7 @@ topic_name = unique_name() with pytest.raises(RuntimeError): - await iggy_client.create_topic( - stream=nonexistent_stream, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=nonexistent_stream, name=topic_name) @pytest.mark.asyncio async def test_create_topic_requires_connection_and_auth(self, unique_name): @@ -518,15 +504,11 @@ client = IggyClient(f"{host}:{port}") with pytest.raises(RuntimeError): - await client.create_topic( - stream=unique_name(), name=unique_name(), partitions_count=1 - ) + await client.create_topic(stream=unique_name(), name=unique_name()) await client.connect() with pytest.raises(RuntimeError): - await client.create_topic( - stream=unique_name(), name=unique_name(), partitions_count=1 - ) + await client.create_topic(stream=unique_name(), name=unique_name()) class TestGetTopic: @@ -539,9 +521,7 @@ topic_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) topic_by_name = await iggy_client.get_topic(stream_name, topic_name) assert topic_by_name is not None @@ -564,7 +544,9 @@ await iggy_client.create_stream(stream_name) await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=3 + stream=stream_name, + name=topic_name, + options=TopicOptions(partitions_count=3), ) topic = await iggy_client.get_topic(stream_name, topic_name) @@ -647,9 +629,7 @@ await iggy_client.create_stream(stream_name) for name in topic_names: - await iggy_client.create_topic( - stream=stream_name, name=name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=name) topics = await iggy_client.get_topics(stream_name) assert len(topics) == len(topic_names) @@ -671,9 +651,7 @@ await iggy_client.create_stream(stream_name) for name in topic_names: - await iggy_client.create_topic( - stream=stream_name, name=name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=name) first = await iggy_client.get_topics(stream_name) second = await iggy_client.get_topics(stream_name) @@ -690,9 +668,7 @@ topic_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) stream = await iggy_client.get_stream(stream_name) assert stream is not None @@ -757,9 +733,7 @@ invalid_name = unique_name(prefix, min_bytes=min_bytes, max_bytes=max_bytes) await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) with pytest.raises(RuntimeError): await iggy_client.update_topic( @@ -776,9 +750,7 @@ new_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.update_topic( stream_id=stream_name, topic_id=topic_name, name=new_name @@ -801,9 +773,7 @@ new_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) before = await iggy_client.get_topic(stream_name, topic_name) assert before is not None @@ -825,9 +795,7 @@ new_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) topic = await iggy_client.get_topic(stream_name, topic_name) assert topic is not None @@ -841,53 +809,43 @@ @pytest.mark.asyncio @pytest.mark.parametrize("compression_algorithm", ["gzip", "none"]) - async def test_update_topic_with_valid_compression_algorithm( + async def test_update_topic_keeps_the_compression_algorithm_set_at_creation( self, iggy_client: IggyClient, unique_name, compression_algorithm: str ): - """Test update_topic accepts a supported compression algorithm.""" + """Test compression is fixed at creation and no update can change it.""" stream_name = unique_name() topic_name = unique_name() + other_algorithm = "none" if compression_algorithm == "gzip" else "gzip" await iggy_client.create_stream(stream_name) await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 + stream=stream_name, + name=topic_name, + options=TopicOptions(compression_algorithm=compression_algorithm), ) + with pytest.raises(TypeError): + await iggy_client.update_topic( + stream_id=stream_name, + topic_id=topic_name, + # pyrefly: ignore # unexpected-keyword + compression_algorithm=other_algorithm, + ) + with pytest.raises(RuntimeError, match="Option cannot be updated"): + await iggy_client.update_topic( + stream_id=stream_name, + topic_id=topic_name, + options={"compression_algorithm": other_algorithm}, + ) await iggy_client.update_topic( - stream_id=stream_name, - topic_id=topic_name, - name=topic_name, - compression_algorithm=compression_algorithm, + stream_id=stream_name, topic_id=topic_name, name=topic_name ) topic = await iggy_client.get_topic(stream_name, topic_name) assert topic is not None - assert topic.name == topic_name assert topic.compression_algorithm == compression_algorithm @pytest.mark.asyncio - @pytest.mark.parametrize("compression_algorithm", ["brotli", "gzipp", ""]) - async def test_update_topic_invalid_compression_algorithm( - self, iggy_client: IggyClient, unique_name, compression_algorithm: str - ): - """Test update_topic rejects unsupported compression algorithm values.""" - stream_name = unique_name() - topic_name = unique_name() - - await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) - - with pytest.raises(RuntimeError, match="Unknown compression type"): - await iggy_client.update_topic( - stream_id=stream_name, - topic_id=topic_name, - name=topic_name, - compression_algorithm=compression_algorithm, - ) - - @pytest.mark.asyncio @pytest.mark.parametrize( "invalid_message_expiry", [1, "1s", object(), timedelta(seconds=1)] ) @@ -899,9 +857,7 @@ topic_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) with pytest.raises(TypeError): await iggy_client.update_topic( @@ -933,9 +889,7 @@ topic_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) with pytest.raises(expected_exception): await iggy_client.update_topic( @@ -954,9 +908,7 @@ topic_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.update_topic( stream_id=stream_name, @@ -992,9 +944,7 @@ topic_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) with pytest.raises(ValueError): await iggy_client.update_topic( @@ -1025,9 +975,7 @@ topic_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) created = await iggy_client.get_topic(stream_name, topic_name) assert created is not None @@ -1044,11 +992,9 @@ assert topic is not None assert topic.name == topic_name if expected_kind == "server_default": - # Every setting rides the options block and 0 is its "resolve the - # default" sentinel, so a ServerDefault update carries no key at all - # and the topic keeps the value admission resolved when it was - # created. Resetting a setting back to the node default is - # deliberately not expressible. + # A ServerDefault update carries no key at all, so the topic keeps + # the value admission resolved when it was created. Returning to + # the default is a reset of `OptionKey.MaxTopicSize` instead. assert isinstance(topic.max_topic_size, type(resolved_at_creation)) assert not isinstance(topic.max_topic_size, MaxTopicSize.ServerDefault) elif expected_kind == "unlimited": @@ -1069,9 +1015,7 @@ second_rename = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.update_topic( stream_id=stream_name, topic_id=topic_name, name=first_rename @@ -1123,12 +1067,8 @@ second_topic = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=first_topic, partitions_count=1 - ) - await iggy_client.create_topic( - stream=stream_name, name=second_topic, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=first_topic) + await iggy_client.create_topic(stream=stream_name, name=second_topic) with pytest.raises(RuntimeError): await iggy_client.update_topic( @@ -1166,9 +1106,7 @@ topic_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.delete_topic(stream_name, topic_name) @@ -1188,9 +1126,7 @@ topic_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) topic = await iggy_client.get_topic(stream_name, topic_name) assert topic is not None @@ -1208,12 +1144,8 @@ topic_to_keep = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_to_delete, partitions_count=1 - ) - await iggy_client.create_topic( - stream=stream_name, name=topic_to_keep, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_to_delete) + await iggy_client.create_topic(stream=stream_name, name=topic_to_keep) await iggy_client.delete_topic(stream_name, topic_to_delete) @@ -1242,9 +1174,7 @@ topic_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.delete_topic(stream_name, topic_name) with pytest.raises(RuntimeError): @@ -1278,9 +1208,9 @@ await iggy_client.create_topic( stream=stream_name, name=topic_name, - partitions_count=1, - durability=Durability.PERSISTED, - options={"segment_size": "128 MiB"}, + options=TopicOptions( + durability=Durability.PERSISTED, raw={"segment_size": "128 MiB"} + ), ) topic = await iggy_client.get_topic(stream_name, topic_name) @@ -1309,30 +1239,32 @@ topic_name = unique_name() await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) + await iggy_client.create_topic(stream=stream_name, name=topic_name) await iggy_client.update_topic( stream_id=stream_name, topic_id=topic_name, - name=topic_name, - options={"compression_algorithm": "gzip"}, + options={"message_expiry": "3600000000"}, ) topic = await iggy_client.get_topic(stream_name, topic_name) assert topic is not None - assert topic.compression_algorithm == "gzip" + assert topic.name == topic_name + assert isinstance(topic.message_expiry, IggyExpiry.ExpireDuration) + assert topic.message_expiry.duration == timedelta(hours=1) # A create-time key is refused by name, so nothing re-pushes it to the # partitions of a live topic. - with pytest.raises(RuntimeError): - await iggy_client.update_topic( - stream_id=stream_name, - topic_id=topic_name, - name=topic_name, - options={"segment_size": "2 MiB"}, - ) + for create_only in ( + {"segment_size": "2 MiB"}, + {"compression_algorithm": "gzip"}, + ): + with pytest.raises(RuntimeError, match="Option cannot be updated"): + await iggy_client.update_topic( + stream_id=stream_name, + topic_id=topic_name, + options=create_only, + ) @pytest.mark.asyncio async def test_describe_options_lists_the_topic_catalog( @@ -1354,9 +1286,29 @@ assert default.value == 1024 * 1024 * 1024 assert segment_size.description - # Streams and users have no catalog keys yet. - assert await iggy_client.describe_options("stream") == [] - assert await iggy_client.describe_options("user") == [] + flags = { + spec.key: (spec.updatable, spec.resettable, spec.secret, spec.read_only) + for spec in specs + } + assert flags["message_expiry"] == (True, True, False, False) + assert flags["compression_algorithm"] == (False, False, False, False) + + streams = await iggy_client.describe_options("stream") + assert { + spec.key: (spec.updatable, spec.secret, spec.read_only, spec.critical) + for spec in streams + } == { + "name": (True, False, False, False), + "encryption_key": (False, True, False, True), + "encryption_key_source": (False, False, True, False), + "encryption_key_fingerprint": (False, False, True, False), + "encryption_kek_fingerprint": (False, False, True, False), + } + users = await iggy_client.describe_options("user") + assert [(spec.key, spec.updatable) for spec in users] == [ + ("username", True), + ("status", True), + ] @pytest.mark.asyncio async def test_describe_options_rejects_an_unknown_scope(
diff --git a/foreign/python/tests/test_user.py b/foreign/python/tests/test_user.py index 6ae220c..bcb2bc3 100644 --- a/foreign/python/tests/test_user.py +++ b/foreign/python/tests/test_user.py
@@ -654,14 +654,15 @@ await iggy_client.delete_user(created.id) @pytest.mark.asyncio - async def test_update_user_with_no_fields_is_a_noop( + async def test_update_user_with_no_fields_is_refused_before_sending( self, iggy_client: IggyClient, unique_name ): - """Test the server permits an update with neither username nor status.""" + """Test an update that changes nothing is refused and leaves the user.""" username, password = unique_credentials(unique_name) created = await iggy_client.create_user(username, password) - await iggy_client.update_user(created.id) + with pytest.raises(RuntimeError, match="Invalid command"): + await iggy_client.update_user(created.id) user = await iggy_client.get_user(created.id) assert user is not None @@ -920,7 +921,7 @@ "get_user": (username,), "get_users": (), "create_user": (username, "secret"), - "update_user": (username,), + "update_user": (username, username), "delete_user": (username,), "update_permissions": (username, None), "change_password": (username, "secret", "secret2"),
diff --git a/foreign/python/uv.lock b/foreign/python/uv.lock index bad59a7..d05933e 100644 --- a/foreign/python/uv.lock +++ b/foreign/python/uv.lock
@@ -6,9 +6,12 @@ exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values. exclude-newer-span = "P7D" +[options.exclude-newer-package] +certifi = { timestamp = "0001-01-01T00:00:00Z", span = "P2D" } + [[package]] name = "apache-iggy" -version = "0.9.1.dev1" +version = "0.9.1.dev2" source = { editable = "." } [package.optional-dependencies] @@ -60,11 +63,11 @@ [[package]] name = "certifi" -version = "2026.7.22" +version = "2026.10.19" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/1436c50cc46699de0230fbaacc1d1de7bfec413176099949e9d5112e88f2/certifi-2026.10.19.tar.gz", hash = "sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae", size = 137598, upload-time = "2026-10-20T02:24:16.604Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, + { url = "https://files.pythonhosted.org/packages/c4/51/29046e48f693c54e5b7412216d447a1d8362c9653148d98c02cdf65fa733/certifi-2026.10.19-py3-none-any.whl", hash = "sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b", size = 136464, upload-time = "2026-10-20T02:24:15.072Z" }, ] [[package]]
diff --git a/foreign/swift/Package.resolved b/foreign/swift/Package.resolved index 762d62b..12cd9c4 100644 --- a/foreign/swift/Package.resolved +++ b/foreign/swift/Package.resolved
@@ -1,7 +1,16 @@ { - "originHash" : "62946d564ad9c5d0b9eff5c2610c835de3d4e4e178e822878bdce27e328763ee", + "originHash" : "5779d5213e3a88ea6669d73995c16d99f4400dc8c2a7362223e4d44682e2dafb", "pins" : [ { + "identity" : "swift-asn1", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-asn1.git", + "state" : { + "revision" : "3b6410f7dee09eb33cdd26260c5fd47fda19b0e2", + "version" : "1.7.3" + } + }, + { "identity" : "swift-atomics", "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-atomics.git", @@ -20,6 +29,15 @@ } }, { + "identity" : "swift-crypto", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-crypto.git", + "state" : { + "revision" : "95ba0316a9b733e92bb6b071255ff46263bbe7dc", + "version" : "3.15.1" + } + }, + { "identity" : "swift-log", "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-log.git",
diff --git a/foreign/swift/Package.swift b/foreign/swift/Package.swift index 839cdd5..de70d68 100644 --- a/foreign/swift/Package.swift +++ b/foreign/swift/Package.swift
@@ -34,6 +34,7 @@ .package(url: "https://github.com/apple/swift-nio.git", from: "2.80.0"), .package(url: "https://github.com/apple/swift-nio-ssl.git", from: "2.29.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.6.0"), + .package(url: "https://github.com/apple/swift-crypto.git", "3.0.0"..<"4.0.0"), ], targets: [ .target( @@ -45,6 +46,7 @@ .product(name: "NIOTLS", package: "swift-nio"), .product(name: "NIOSSL", package: "swift-nio-ssl"), .product(name: "Logging", package: "swift-log"), + .product(name: "Crypto", package: "swift-crypto"), ] ), .testTarget(
diff --git a/foreign/swift/README.md b/foreign/swift/README.md index 79350b0..961c339 100644 --- a/foreign/swift/README.md +++ b/foreign/swift/README.md
@@ -29,8 +29,9 @@ - An Iggy server that speaks the consensus protocol (`iggy-server` from this repository) The SDK depends on [SwiftNIO](https://github.com/apple/swift-nio), -[swift-nio-ssl](https://github.com/apple/swift-nio-ssl), and -[swift-log](https://github.com/apple/swift-log). +[swift-nio-ssl](https://github.com/apple/swift-nio-ssl), +[swift-log](https://github.com/apple/swift-log), and +[swift-crypto](https://github.com/apple/swift-crypto) for the SHA-256 behind stream key fingerprints. ## Installation @@ -41,7 +42,7 @@ ```swift // Package.swift dependencies: [ - .package(url: "https://github.com/apache/iggy-swift.git", from: "0.1.0") + .package(url: "https://github.com/apache/iggy-swift.git", from: "0.1.1") ], targets: [ .target(name: "MyApp", dependencies: [.product(name: "Iggy", package: "iggy-swift")]) @@ -94,6 +95,36 @@ The server certificate is verified against the system roots, or against `caFile` when set, and against the host of the address unless `TLSOptions.domain` names another one. +### Resource options + +Streams, topics, and users carry options. A create call takes a typed options value, and an update names only the keys it changes, plus the keys to return to their defaults. Every other key keeps its stored value: + +```swift +let key = try EncryptionKey.decode(base64: base64Key) +let stream = try await client.createStream(name: "payments", options: StreamCreateOptions(encryptionKey: key)) +let topic = try await client.createTopic( + streamID: "payments", name: "orders", + options: TopicCreateOptions(partitionsCount: 3, messageExpiry: .after(.seconds(604_800)), partitionResizePolicy: .fixed)) +try await client.updateStream("payments", options: StreamUpdateOptions(name: "payments-v2")) +try await client.updateTopic( + streamID: "payments-v2", topicID: "orders", options: TopicUpdateOptions(messageExpiry: .after(.seconds(86_400)), reset: [.maxTopicSize])) +try await client.updateUser("alice", options: UserUpdateOptions(status: .inactive)) +``` + +Only `message_expiry` and `max_topic_size` can be reset. Compression and the other topic settings are fixed at create, and no stream or user key is resettable yet. A typed update that breaks these rules fails before anything is sent, with the error the server would return. + +Everything else, such as a taken name or a missing resource, is refused by the server. `describeOptions(scope:)` lists every key of a scope with flags saying what an update may do with it. Renaming the signed-in user, through `username` or a raw `username` entry, moves the credentials later reconnects use. + +Keys this SDK has no typed member for go in `raw`, values as strings the server parses, and reset names in `rawReset`. The server alone decides on a patch that carries either, since it may know keys this SDK does not: + +```swift +try await client.updateTopic(streamID: "payments-v2", topicID: "orders", options: TopicUpdateOptions(rawReset: ["future_retention"])) +``` + +`StreamCreateOptions.encryptionKey` gives the stream its own data key. It must be 32 random bytes, never derived from a password. It travels inside the create request over whatever transport the client uses, so protecting it in transit with TLS is the caller's choice. + +The server stores it wrapped and never returns it: reads report `encryptionKeySource`, `encryptionKeyFingerprint`, and `encryptionKekFingerprints`, and `EncryptionKey.fingerprint(_:)` computes the fingerprint of a key to compare. Without a key, a server with encryption enabled generates one for the stream. + ### Reconnection If a request hits a dropped connection, the client reconnects and signs in again within @@ -240,9 +271,13 @@ ```bash cargo build --bin iggy-server -IGGY_PATH=/tmp/iggy-swift IGGY_ROOT_USERNAME=iggy IGGY_ROOT_PASSWORD=iggy target/debug/iggy-server +IGGY_PATH=/tmp/iggy-swift IGGY_ROOT_USERNAME=iggy IGGY_ROOT_PASSWORD=iggy \ +IGGY_ENCRYPTION_ENABLED=true IGGY_ENCRYPTION_KEY=/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE= \ +target/debug/iggy-server ``` +The resource options scenarios need encryption enabled, so a stream created without a key gets one the server generates. + Then, from `foreign/swift`: ```bash @@ -260,6 +295,8 @@ cargo run --manifest-path Tools/golden-vectors/Cargo.toml -- Tests/IggyTests/Fixtures/golden.json ``` +The resource options tests compare against `Tests/IggyTests/Fixtures/resource_options.json`, the fixture every SDK shares, generated by `core/common/examples/resource_options_fixtures.rs`. Never edit the copy. + ## Contributing Format the sources with the toolchain's formatter before opening a pull request; CI runs it
diff --git a/foreign/swift/Sources/Iggy/Client/ClientCore.swift b/foreign/swift/Sources/Iggy/Client/ClientCore.swift index 65503a7..a225fe9 100644 --- a/foreign/swift/Sources/Iggy/Client/ClientCore.swift +++ b/foreign/swift/Sources/Iggy/Client/ClientCore.swift
@@ -1386,10 +1386,9 @@ case .changePassword: let change = try Requests.changedPassword(payload) refreshSessionPassword(user: change.userID, newPassword: change.newPassword) - case .updateUser: - let update = try Requests.renamedUser(payload) - if let username = update.username { - refreshSessionUsername(user: update.userID, newUsername: username) + case .updateResource: + if let rename = try Requests.renamedUser(payload) { + refreshSessionUsername(user: rename.userID, newUsername: rename.username) } default: break
diff --git a/foreign/swift/Sources/Iggy/Client/IggyClient+Resources.swift b/foreign/swift/Sources/Iggy/Client/IggyClient+Resources.swift index 040ef84..26632c2 100644 --- a/foreign/swift/Sources/Iggy/Client/IggyClient+Resources.swift +++ b/foreign/swift/Sources/Iggy/Client/IggyClient+Resources.swift
@@ -32,16 +32,19 @@ return body.isEmpty ? [] : try Responses.streams(body[...]) } - /// Creates a stream and returns it. - public func createStream(name: String) async throws -> StreamDetails { + /// Creates a stream and returns it. A key in ``StreamCreateOptions/encryptionKey`` + /// is refused before anything is sent unless it is exactly 32 bytes. + public func createStream(name: String, options: StreamCreateOptions = StreamCreateOptions()) async throws -> StreamDetails { + let payload = try Requests.createStream(name: name, options: try options.encode()) let lineage = try await requireAuthenticated() - return try Responses.streamDetails(try await send(.createStream, try Requests.createStream(name: name), lineage: lineage)[...]) + return try Responses.streamDetails(try await send(.createStream, payload, lineage: lineage)[...]) } - /// Renames a stream. - public func updateStream(_ streamID: Identifier, name: String, options: StreamUpdateOptions = StreamUpdateOptions()) async throws { + /// Updates a stream: renames it today. Unmentioned keys keep their values. + public func updateStream(_ streamID: Identifier, options: StreamUpdateOptions) async throws { + let payload = try Requests.updateResource(.stream(streamID), try options.toPatch()) let lineage = try await requireAuthenticated() - _ = try await send(.updateStream, try Requests.updateStream(streamID: streamID, name: name, options: try options.encode()), lineage: lineage) + _ = try await send(.updateResource, payload, lineage: lineage) } /// Deletes a stream with every topic and message in it. @@ -92,11 +95,12 @@ return try Responses.topicDetails(try await send(.createTopic, payload, lineage: lineage)[...]) } - /// Renames a topic and updates its updatable options. - public func updateTopic(streamID: Identifier, topicID: Identifier, name: String, options: TopicUpdateOptions = TopicUpdateOptions()) async throws { + /// Renames a topic, sets its updatable options, or resets them to their + /// defaults. Unmentioned keys keep their values. + public func updateTopic(streamID: Identifier, topicID: Identifier, options: TopicUpdateOptions) async throws { + let payload = try Requests.updateResource(.topic(stream: streamID, topic: topicID), try options.toPatch()) let lineage = try await requireAuthenticated() - _ = try await send( - .updateTopic, try Requests.updateTopic(streamID: streamID, topicID: topicID, name: name, options: try options.encode()), lineage: lineage) + _ = try await send(.updateResource, payload, lineage: lineage) } /// Deletes a topic with every message in it. @@ -198,9 +202,11 @@ } /// Creates a user and returns it. - public func createUser(username: String, password: String, status: UserStatus = .active, permissions: Permissions? = nil) async throws -> UserInfoDetails { + public func createUser( + username: String, password: String, status: UserStatus = .active, permissions: Permissions? = nil, options: UserCreateOptions = UserCreateOptions() + ) async throws -> UserInfoDetails { + let payload = try Requests.createUser(username: username, password: password, status: status, permissions: permissions, options: try options.encode()) let lineage = try await requireAuthenticated() - let payload = try Requests.createUser(username: username, password: password, status: status, permissions: permissions) return try Responses.userDetails(try await send(.createUser, payload, lineage: lineage)[...]) } @@ -210,13 +216,12 @@ _ = try await send(.deleteUser, Requests.identifierOnly(userID), lineage: lineage) } - /// Changes a user's name and/or status; nil leaves a field as is. - public func updateUser( - _ userID: Identifier, username: String? = nil, status: UserStatus? = nil, options: UserUpdateOptions = UserUpdateOptions() - ) async throws { + /// Changes a user's name and/or status. Renaming the signed-in or + /// auto-login user moves the credentials later reconnects sign in with. + public func updateUser(_ userID: Identifier, options: UserUpdateOptions) async throws { + let payload = try Requests.updateResource(.user(userID), try options.toPatch()) let lineage = try await requireAuthenticated() - _ = try await send( - .updateUser, try Requests.updateUser(userID: userID, username: username, status: status, options: try options.encode()), lineage: lineage) + _ = try await send(.updateResource, payload, lineage: lineage) } /// Replaces a user's permissions; nil removes them all.
diff --git a/foreign/swift/Sources/Iggy/Client/IggyClient.swift b/foreign/swift/Sources/Iggy/Client/IggyClient.swift index 8516b49..ba22794 100644 --- a/foreign/swift/Sources/Iggy/Client/IggyClient.swift +++ b/foreign/swift/Sources/Iggy/Client/IggyClient.swift
@@ -259,8 +259,8 @@ return body.isEmpty ? [] : try Responses.clients(body[...]) } - /// The option catalog of a resource scope: the keys create accepts, their - /// kinds, and this server's defaults. + /// The option catalog of a resource scope: every key a client can name, + /// its kind, what an update may do with it, and this server's default. public func describeOptions(scope: OptionsScope) async throws -> [OptionSpec] { let lineage = try await requireAuthenticated() return try Responses.optionSpecs(try await send(.describeOptions, Requests.describeOptions(scope: scope), lineage: lineage)[...])
diff --git a/foreign/swift/Sources/Iggy/Errors/IggyErrorCode.swift b/foreign/swift/Sources/Iggy/Errors/IggyErrorCode.swift index 98b80a7..8747bde 100644 --- a/foreign/swift/Sources/Iggy/Errors/IggyErrorCode.swift +++ b/foreign/swift/Sources/Iggy/Errors/IggyErrorCode.swift
@@ -228,6 +228,12 @@ case invalidOffset = 4100 case invalidReservedField = 4101 case segmentSizeMismatchAtOpen = 4102 + case optionNotUpdatable = 4103 + case optionNotResettable = 4104 + case readOnlyOption = 4105 + case streamEncryptionUnavailable = 4106 + case unsupportedCriticalOption = 4107 + case cannotModifyRootUser = 4108 case consumerGroupIdNotFound = 5000 case invalidConsumerGroupId = 5002 case consumerGroupNameNotFound = 5003 @@ -487,6 +493,12 @@ case .invalidOffset: "invalid_offset" case .invalidReservedField: "invalid_reserved_field" case .segmentSizeMismatchAtOpen: "segment_size_mismatch_at_open" + case .optionNotUpdatable: "option_not_updatable" + case .optionNotResettable: "option_not_resettable" + case .readOnlyOption: "read_only_option" + case .streamEncryptionUnavailable: "stream_encryption_unavailable" + case .unsupportedCriticalOption: "unsupported_critical_option" + case .cannotModifyRootUser: "cannot_modify_root_user" case .consumerGroupIdNotFound: "consumer_group_id_not_found" case .invalidConsumerGroupId: "invalid_consumer_group_id" case .consumerGroupNameNotFound: "consumer_group_name_not_found"
diff --git a/foreign/swift/Sources/Iggy/Iggy.swift b/foreign/swift/Sources/Iggy/Iggy.swift index bddbbbe..9321385 100644 --- a/foreign/swift/Sources/Iggy/Iggy.swift +++ b/foreign/swift/Sources/Iggy/Iggy.swift
@@ -21,5 +21,5 @@ public static let sdkName = "swift-sdk" /// Version of this SDK as reported to the server. Provisional until the /// release wiring lands; the release tooling reads this constant. - public static let sdkVersion = "0.1.0" + public static let sdkVersion = "0.1.1" }
diff --git a/foreign/swift/Sources/Iggy/Models/EncryptionKey.swift b/foreign/swift/Sources/Iggy/Models/EncryptionKey.swift new file mode 100644 index 0000000..99575f5 --- /dev/null +++ b/foreign/swift/Sources/Iggy/Models/EncryptionKey.swift
@@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Crypto +import Foundation + +/// A stream's own data encryption key: exactly 32 random bytes, sent with +/// ``StreamCreateOptions/encryptionKey``. Never derive one from a password, +/// since its fingerprint is public to anyone who can read the stream. +public enum EncryptionKey { + /// Byte length of a stream encryption key. + public static let length = 32 + + private static let fingerprintLabel = Array("iggy/stream-key-fingerprint/v1".utf8) + private static let fingerprintLength = 8 + + /// The fingerprint reads report in ``StreamInfo/encryptionKeyFingerprint`` + /// for a stream encrypted under `key`: the first 8 bytes of a labelled + /// SHA-256 of the key, as 16 lowercase hex characters. It names the key + /// without revealing it. + public static func fingerprint(_ key: Data) throws -> String { + try validate(key) + var hasher = SHA256() + hasher.update(data: fingerprintLabel) + hasher.update(data: key) + return hasher.finalize().prefix(fingerprintLength).map { String($0, radix: 16).leftPadded(to: 2) }.joined() + } + + /// ``fingerprint(_:)`` of a key given as base64, decoded by + /// ``decode(base64:)``. + public static func fingerprint(base64 key: String) throws -> String { + try fingerprint(decode(base64: key)) + } + + /// Decodes a key from canonical standard padded base64 of exactly 32 + /// bytes. Whitespace, the URL-safe alphabet, missing padding and + /// noncanonical trailing bits are refused, as the server refuses them. + public static func decode(base64 text: String) throws -> Data { + guard let key = Data(base64Encoded: text), key.base64EncodedString() == text else { + throw IggyError(.invalidEncryptionKey, context: "stream encryption key must be canonical standard padded base64") + } + try validate(key) + return key + } + + static func validate(_ key: Data) throws { + guard key.count == length else { + throw IggyError(.invalidEncryptionKey, context: "stream encryption key must be \(length) bytes, got \(key.count)") + } + } +}
diff --git a/foreign/swift/Sources/Iggy/Models/ResourceOptions.swift b/foreign/swift/Sources/Iggy/Models/ResourceOptions.swift index 7bd271c..43d935a 100644 --- a/foreign/swift/Sources/Iggy/Models/ResourceOptions.swift +++ b/foreign/swift/Sources/Iggy/Models/ResourceOptions.swift
@@ -48,22 +48,103 @@ case user = 3 } -/// One entry of the option catalog: a key the create command accepts, its -/// canonical kind, and the server's current default. +/// One entry of a scope's option catalog: a key, its canonical kind, what an +/// update may do with it, and the server's default. public struct OptionSpec: Sendable, Hashable { public var key: String public var kind: HeaderKind + public var flags: OptionFlags public var defaultValue: [UInt8] public var description: String - public init(key: String, kind: HeaderKind, defaultValue: [UInt8], description: String) { + public init(key: String, kind: HeaderKind, flags: OptionFlags = [], defaultValue: [UInt8], description: String) { self.key = key self.kind = kind + self.flags = flags self.defaultValue = defaultValue self.description = description } } +/// What a catalog key allows, as ``IggyClient/describeOptions(scope:)`` +/// reports it. A key with none of the first two flags is create-only. +public struct OptionFlags: OptionSet, Sendable, Hashable { + public let rawValue: UInt8 + + public init(rawValue: UInt8) { + self.rawValue = rawValue + } + + /// An update may set the key. + public static let updatable = OptionFlags(rawValue: 1 << 0) + /// An update may reset the key to its default. + public static let resettable = OptionFlags(rawValue: 1 << 1) + /// Write-only: accepted at create and never returned. + public static let secret = OptionFlags(rawValue: 1 << 2) + /// Computed by the server and returned on reads, never accepted. + public static let readOnly = OptionFlags(rawValue: 1 << 3) + /// A server that does not understand the key must not serve the resource. + public static let critical = OptionFlags(rawValue: 1 << 4) +} + +/// A key of one scope's option catalog, with the flags the server's catalog +/// gives it. Cases are declared in catalog order. +protocol CatalogOptionKey: RawRepresentable, CaseIterable, Hashable where RawValue == String { + var flags: OptionFlags { get } +} + +/// An update's options: the keys to set and the keys to return to their +/// defaults. Every other key keeps its stored value. +struct OptionsPatch { + /// Reset names the `u8` count of the request carries at most. + static let maxResetKeys = Int(UInt8.max) + + let set: ResourceOptions + /// In byte order, as the Rust encoder writes them. + let reset: [String] + + /// Refuses the patch as server admission would, in the server's order: + /// the request shape, then the key categories, then the values `values` + /// encodes. A patch with a raw key or raw reset name gets the shape + /// checks only, since the server's catalog may know keys or flags this + /// build does not. + init<Key: CatalogOptionKey>( + typed: [Key], raw: [String: String], reset: Set<Key>, rawReset: Set<String>, values: () throws -> ResourceOptions + ) throws { + let setKeys = Set(typed.map(\.rawValue)).union(raw.keys) + let names = Set(reset.map(\.rawValue)).union(rawReset).sorted { $0.utf8.lexicographicallyPrecedes($1.utf8) } + for name in names where name.isEmpty || name.utf8.count > HeaderTLV.maxFieldLength { + throw IggyError(.unsupportedOptionKey, context: String(name.prefix(errorKeyPreviewLength))) + } + if names.count > Self.maxResetKeys { + let unknown = names.first { Key(rawValue: $0) == nil } + throw unknown.map { IggyError(.unsupportedOptionKey, context: String($0.prefix(errorKeyPreviewLength))) } + ?? IggyError(.invalidCommand, context: "more than \(Self.maxResetKeys) reset keys") + } + if setKeys.isEmpty && names.isEmpty { + throw IggyError(.invalidCommand, context: "the update sets and resets nothing") + } + if let name = names.first(where: setKeys.contains) { + throw IggyError(.invalidCommand, context: "\(name) is both set and reset") + } + if raw.isEmpty && rawReset.isEmpty { + let resetKeys = names.compactMap(Key.init(rawValue:)) + let allKeys = typed.sorted { $0.rawValue.utf8.lexicographicallyPrecedes($1.rawValue.utf8) } + resetKeys + if let key = allKeys.first(where: { $0.flags.contains(.readOnly) }) { + throw IggyError(.readOnlyOption, context: key.rawValue) + } + if let key = allKeys.first(where: { !$0.flags.contains(.updatable) }) { + throw IggyError(.optionNotUpdatable, context: key.rawValue) + } + if let key = resetKeys.first(where: { !$0.flags.contains(.resettable) }) { + throw IggyError(.optionNotResettable, context: key.rawValue) + } + } + self.set = try values() + self.reset = names + } +} + /// Key-value options block for create and update requests: the user-headers /// TLV with the stricter contract options need. Mirrors /// `core/binary_protocol/src/primitives/options.rs`. @@ -154,3 +235,32 @@ return options } } + +/// Characters of an offending key an error message shows, +/// `ERROR_KEY_PREVIEW_LEN` in the Rust crate. +private let errorKeyPreviewLength = 64 + +/// Keys sent as `String` values, all explicit. Typed fields are inserted +/// afterwards, so they win on collision. +func rawOptions(_ raw: [String: String]) throws -> ResourceOptions { + var options: ResourceOptions = [:] + for (key, value) in raw { + guard !key.isEmpty, key.utf8.count <= HeaderTLV.maxFieldLength else { + throw IggyError(.unsupportedOptionKey, context: String(key.prefix(errorKeyPreviewLength))) + } + guard let headerValue = try? HeaderValue.string(value) else { + throw IggyError(.invalidOptionValue, context: String(key.prefix(errorKeyPreviewLength))) + } + options[key] = .explicit(headerValue) + } + return options +} + +/// The value of a column key such as a name, refused with the scope's own +/// error when it does not fit an option value. +func columnValue(_ text: String, refusal: IggyErrorCode) throws -> HeaderValue { + guard let value = try? HeaderValue.string(text) else { + throw IggyError(refusal, context: "must be 1-\(HeaderTLV.maxFieldLength) bytes, got \(text.utf8.count)") + } + return value +}
diff --git a/foreign/swift/Sources/Iggy/Models/Resources.swift b/foreign/swift/Sources/Iggy/Models/Resources.swift index 82765ca..4c0accb 100644 --- a/foreign/swift/Sources/Iggy/Models/Resources.swift +++ b/foreign/swift/Sources/Iggy/Models/Resources.swift
@@ -98,6 +98,8 @@ /// A stream as listed. Named `StreamInfo` so that a file importing /// Foundation, which has its own `Stream`, never sees an ambiguous type. +/// ``options`` holds the keys the client set and the projections the server +/// computes, such as ``encryptionKeySource``. public struct StreamInfo: Sendable, Hashable { public var id: UInt32 public var createdAt: IggyTimestamp @@ -146,30 +148,26 @@ } } -/// A topic as listed. +/// A topic as listed. Message expiry, compression and the size cap live in +/// ``options``, read typed through ``messageExpiry``, +/// ``compressionAlgorithm`` and ``maxTopicSize``. public struct Topic: Sendable, Hashable { public var id: UInt32 public var createdAt: IggyTimestamp public var name: String public var sizeBytes: UInt64 - public var messageExpiry: IggyExpiry - public var compressionAlgorithm: CompressionAlgorithm - public var maxTopicSize: MaxTopicSize public var messagesCount: UInt64 public var partitionsCount: UInt32 public var options: ResourceOptions public init( - id: UInt32, createdAt: IggyTimestamp, name: String, sizeBytes: UInt64, messageExpiry: IggyExpiry, compressionAlgorithm: CompressionAlgorithm, - maxTopicSize: MaxTopicSize, messagesCount: UInt64, partitionsCount: UInt32, options: ResourceOptions = [:] + id: UInt32, createdAt: IggyTimestamp, name: String, sizeBytes: UInt64, messagesCount: UInt64, partitionsCount: UInt32, + options: ResourceOptions = [:] ) { self.id = id self.createdAt = createdAt self.name = name self.sizeBytes = sizeBytes - self.messageExpiry = messageExpiry - self.compressionAlgorithm = compressionAlgorithm - self.maxTopicSize = maxTopicSize self.messagesCount = messagesCount self.partitionsCount = partitionsCount self.options = options @@ -182,9 +180,6 @@ public var createdAt: IggyTimestamp public var name: String public var sizeBytes: UInt64 - public var messageExpiry: IggyExpiry - public var compressionAlgorithm: CompressionAlgorithm - public var maxTopicSize: MaxTopicSize public var messagesCount: UInt64 public var partitionsCount: UInt32 public var partitions: [Partition] @@ -195,9 +190,6 @@ createdAt = topic.createdAt name = topic.name sizeBytes = topic.sizeBytes - messageExpiry = topic.messageExpiry - compressionAlgorithm = topic.compressionAlgorithm - maxTopicSize = topic.maxTopicSize messagesCount = topic.messagesCount partitionsCount = topic.partitionsCount self.partitions = partitions
diff --git a/foreign/swift/Sources/Iggy/Models/StreamOptions.swift b/foreign/swift/Sources/Iggy/Models/StreamOptions.swift new file mode 100644 index 0000000..320e02c --- /dev/null +++ b/foreign/swift/Sources/Iggy/Models/StreamOptions.swift
@@ -0,0 +1,191 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +/// Keys of the stream option catalog a client can name, in catalog order. +public enum StreamOptionKey: String, Sendable, Hashable, CaseIterable { + /// The stream's name. Updatable, never resettable. + case name + /// The stream's own data key. Create-only and write-only. + case encryptionKey = "encryption_key" + /// Where the data key came from. Read-only. + case encryptionKeySource = "encryption_key_source" + /// Fingerprint of the data key. Read-only. + case encryptionKeyFingerprint = "encryption_key_fingerprint" + /// Fingerprints of the server keys that wrap the data key. Read-only. + case encryptionKekFingerprint = "encryption_kek_fingerprint" +} + +/// Where a stream's data key came from, as reads report it. +public enum StreamEncryptionSource: String, Sendable, Hashable, CaseIterable { + /// The client sent the key with the create. + case stream + /// The server generated the key, because encryption was enabled when the + /// stream was created. + case server + /// The stream has no key and stays plaintext for life. `none` on the wire. + case plaintext = "none" +} + +/// Options of a stream being created. +/// +/// ``encryptionKey`` gives the stream its own data key: exactly 32 random +/// bytes, ``EncryptionKey/decode(base64:)`` reads one from base64. The key +/// travels inside the create request over whatever transport the client +/// uses, so protecting it in transit with TLS is the caller's choice. The +/// server stores it wrapped and never returns it: reads report only +/// ``StreamInfo/encryptionKeySource`` and the fingerprints. Without a key, a +/// server with encryption enabled generates one for the stream. +/// +/// `raw` carries keys this SDK has no typed field for, as strings the server +/// parses. The SDK's own descriptions of the value never show the key. +public struct StreamCreateOptions: Sendable, Hashable { + public var encryptionKey: Data? + public var raw: [String: String] + + public init(encryptionKey: Data? = nil, raw: [String: String] = [:]) { + self.encryptionKey = encryptionKey + self.raw = raw + } + + func toResourceOptions() throws -> ResourceOptions { + var options = try rawOptions(raw) + if let encryptionKey { + try EncryptionKey.validate(encryptionKey) + options[StreamOptionKey.encryptionKey.rawValue] = .explicit(try .raw(Array(encryptionKey))) + } + return options + } + + func encode() throws -> [UInt8] { + try OptionsBlock.encode(try toResourceOptions()) + } +} + +/// Options of a stream being updated. Absent keys keep their stored values. +public struct StreamUpdateOptions: Sendable, Hashable { + /// Nil keeps the stream's name. + public var name: String? + /// Keys to return to their defaults. No stream key is resettable yet, so + /// a non-empty set is refused before anything is sent. + public var reset: Set<StreamOptionKey> + /// Updatable keys newer than this SDK, as strings the server parses. + public var raw: [String: String] + /// Reset names newer than this SDK. The server alone decides whether a + /// patch carrying raw keys or names is admitted. + public var rawReset: Set<String> + + public init(name: String? = nil, reset: Set<StreamOptionKey> = [], raw: [String: String] = [:], rawReset: Set<String> = []) { + self.name = name + self.reset = reset + self.raw = raw + self.rawReset = rawReset + } + + func toResourceOptions() throws -> ResourceOptions { + var options = try rawOptions(raw) + if let name { + options[StreamOptionKey.name.rawValue] = .explicit(try columnValue(name, refusal: .invalidStreamName)) + } + return options + } + + func toPatch() throws -> OptionsPatch { + try OptionsPatch(typed: name == nil ? [] : [.name], raw: raw, reset: reset, rawReset: rawReset, values: toResourceOptions) + } +} + +extension StreamOptionKey: CatalogOptionKey { + var flags: OptionFlags { + switch self { + case .name: .updatable + case .encryptionKey: [.secret, .critical] + case .encryptionKeySource, .encryptionKeyFingerprint, .encryptionKekFingerprint: .readOnly + } + } +} + +extension StreamCreateOptions: CustomStringConvertible, CustomDebugStringConvertible, CustomReflectable { + public var description: String { + "StreamCreateOptions(encryptionKey: \(encryptionKey == nil ? "nil" : redacted), raw: \(redactedRaw))" + } + + public var debugDescription: String { + description + } + + public var customMirror: Mirror { + Mirror(self, children: ["encryptionKey": encryptionKey.map { _ in redacted } as Any, "raw": redactedRaw], displayStyle: .struct) + } + + private var redacted: String { "<redacted>" } + + private var redactedRaw: [String: String] { + raw.reduce(into: [:]) { shown, entry in + shown[entry.key] = entry.key == StreamOptionKey.encryptionKey.rawValue ? redacted : entry.value + } + } +} + +extension StreamInfo { + /// Where the stream's data key came from. Nil when the server reported + /// a source this SDK does not know. + public var encryptionKeySource: StreamEncryptionSource? { + streamEncryptionSource(options) + } + + /// Fingerprint of the stream's data key, nil for a plaintext stream. + /// Compare it with ``EncryptionKey/fingerprint(_:)`` of a key. + public var encryptionKeyFingerprint: String? { + options[StreamOptionKey.encryptionKeyFingerprint.rawValue]?.value.stringValue + } + + /// Fingerprints of the server keys wrapping the data key. More than one + /// while the server's key is being rotated, none for a plaintext stream. + public var encryptionKekFingerprints: [String] { + kekFingerprints(options) + } +} + +extension StreamDetails { + /// Where the stream's data key came from. Nil when the server reported + /// a source this SDK does not know. + public var encryptionKeySource: StreamEncryptionSource? { + streamEncryptionSource(options) + } + + /// Fingerprint of the stream's data key, nil for a plaintext stream. + /// Compare it with ``EncryptionKey/fingerprint(_:)`` of a key. + public var encryptionKeyFingerprint: String? { + options[StreamOptionKey.encryptionKeyFingerprint.rawValue]?.value.stringValue + } + + /// Fingerprints of the server keys wrapping the data key. More than one + /// while the server's key is being rotated, none for a plaintext stream. + public var encryptionKekFingerprints: [String] { + kekFingerprints(options) + } +} + +private func streamEncryptionSource(_ options: ResourceOptions) -> StreamEncryptionSource? { + options[StreamOptionKey.encryptionKeySource.rawValue]?.value.stringValue.flatMap(StreamEncryptionSource.init(rawValue:)) +} + +private func kekFingerprints(_ options: ResourceOptions) -> [String] { + options[StreamOptionKey.encryptionKekFingerprint.rawValue]?.value.stringValue?.split(separator: ",").map(String.init) ?? [] +}
diff --git a/foreign/swift/Sources/Iggy/Models/TopicOptions.swift b/foreign/swift/Sources/Iggy/Models/TopicOptions.swift index 33b6f66..e6699c0 100644 --- a/foreign/swift/Sources/Iggy/Models/TopicOptions.swift +++ b/foreign/swift/Sources/Iggy/Models/TopicOptions.swift
@@ -15,17 +15,23 @@ // specific language governing permissions and limitations // under the License. -/// Keys of the topic option catalog. -public enum TopicOptionKey { - public static let compressionAlgorithm = "compression_algorithm" - public static let messageExpiry = "message_expiry" - public static let maxTopicSize = "max_topic_size" - public static let segmentSize = "segment_size" - public static let durability = "durability" - public static let consumerOffsetDurability = "consumer_offset_durability" - public static let messagesRequiredToSave = "messages_required_to_save" - public static let sizeOfMessagesRequiredToSave = "size_of_messages_required_to_save" - public static let preallocateSegments = "preallocate_segments" +/// Keys of the topic option catalog, in catalog order. +public enum TopicOptionKey: String, Sendable, Hashable, CaseIterable { + /// The topic's name. Updatable, never resettable. + case name + /// Create-only: the server stores it and does not apply it. + case compressionAlgorithm = "compression_algorithm" + /// Updatable and resettable to never expire. + case messageExpiry = "message_expiry" + /// Updatable and resettable to unlimited. + case maxTopicSize = "max_topic_size" + case segmentSize = "segment_size" + case durability + case consumerOffsetDurability = "consumer_offset_durability" + case partitionResizePolicy = "partition_resize_policy" + case messagesRequiredToSave = "messages_required_to_save" + case sizeOfMessagesRequiredToSave = "size_of_messages_required_to_save" + case preallocateSegments = "preallocate_segments" } /// How far a write has to get before the server completes it. @@ -41,13 +47,20 @@ } } +/// Whether a topic's partition count may change after create. +public enum PartitionResizePolicy: String, Sendable, Hashable, CaseIterable { + /// Partitions may be added and deleted. + case mutable + /// The partition set is fixed at create: the server refuses later + /// partition creation and deletion on the topic. + case fixed +} + /// Options of a topic being created. Every knob is optional; an absent key /// resolves against the server's defaults at admission. /// /// `raw` carries keys this SDK has no typed field for, as strings the server -/// parses, so a newer server's option costs no SDK change. For example, -/// `["partition_resize_policy": "fixed"]` makes the server reject later -/// partition creation and deletion on this topic. +/// parses, so a newer server's option costs no SDK change. public struct TopicCreateOptions: Sendable, Hashable { /// Number of partitions the topic starts with. A field of the create /// command rather than an option key; nil means 1, the server's @@ -62,6 +75,8 @@ public var durability: Durability /// Completion policy for stored consumer offsets, independent of `durability`. public var consumerOffsetDurability: Durability + /// Nil leaves the server's default, ``PartitionResizePolicy/mutable``. + public var partitionResizePolicy: PartitionResizePolicy? public var messagesRequiredToSave: UInt32? public var sizeOfMessagesRequiredToSaveBytes: UInt64? public var preallocateSegments: Bool? @@ -70,7 +85,7 @@ public init( partitionsCount: UInt32? = nil, compressionAlgorithm: CompressionAlgorithm? = nil, messageExpiry: IggyExpiry? = nil, maxTopicSize: MaxTopicSize? = nil, segmentSizeBytes: UInt64? = nil, durability: Durability = .replicated, - consumerOffsetDurability: Durability = .replicated, messagesRequiredToSave: UInt32? = nil, + consumerOffsetDurability: Durability = .replicated, partitionResizePolicy: PartitionResizePolicy? = nil, messagesRequiredToSave: UInt32? = nil, sizeOfMessagesRequiredToSaveBytes: UInt64? = nil, preallocateSegments: Bool? = nil, raw: [String: String] = [:] ) { self.partitionsCount = partitionsCount @@ -80,6 +95,7 @@ self.segmentSizeBytes = segmentSizeBytes self.durability = durability self.consumerOffsetDurability = consumerOffsetDurability + self.partitionResizePolicy = partitionResizePolicy self.messagesRequiredToSave = messagesRequiredToSave self.sizeOfMessagesRequiredToSaveBytes = sizeOfMessagesRequiredToSaveBytes self.preallocateSegments = preallocateSegments @@ -90,13 +106,20 @@ public static let defaultPartitionsCount: UInt32 = 1 func toResourceOptions() throws -> ResourceOptions { - var options = try TopicUpdateOptions( - compressionAlgorithm: compressionAlgorithm, messageExpiry: messageExpiry, maxTopicSize: maxTopicSize, raw: raw - ).toResourceOptions() - if let segmentSizeBytes { - options[TopicOptionKey.segmentSize] = .explicit(.uint64(segmentSizeBytes)) + var options = try rawOptions(raw) + if let compressionAlgorithm { + options[TopicOptionKey.compressionAlgorithm.rawValue] = .explicit(try .string(compressionAlgorithm.description)) } - for (key, policy) in [(TopicOptionKey.durability, durability), (TopicOptionKey.consumerOffsetDurability, consumerOffsetDurability)] { + if let messageExpiry, let microseconds = try messageExpiry.wireMicroseconds() { + options[TopicOptionKey.messageExpiry.rawValue] = .explicit(.uint64(microseconds)) + } + if let maxTopicSize { + options[TopicOptionKey.maxTopicSize.rawValue] = .explicit(.uint64(maxTopicSize.wireValue)) + } + if let segmentSizeBytes { + options[TopicOptionKey.segmentSize.rawValue] = .explicit(.uint64(segmentSizeBytes)) + } + for (key, policy) in [(TopicOptionKey.durability.rawValue, durability), (TopicOptionKey.consumerOffsetDurability.rawValue, consumerOffsetDurability)] { // A raw entry may strengthen the default; it may not weaken a // typed `persisted`, which would make the two fields disagree. // Whatever spelling it used, the canonical one is sent. @@ -112,14 +135,17 @@ } options[key] = .explicit(try .string(effective.rawValue)) } + if let partitionResizePolicy { + options[TopicOptionKey.partitionResizePolicy.rawValue] = .explicit(try .string(partitionResizePolicy.rawValue)) + } if let messagesRequiredToSave { - options[TopicOptionKey.messagesRequiredToSave] = .explicit(.uint32(messagesRequiredToSave)) + options[TopicOptionKey.messagesRequiredToSave.rawValue] = .explicit(.uint32(messagesRequiredToSave)) } if let sizeOfMessagesRequiredToSaveBytes { - options[TopicOptionKey.sizeOfMessagesRequiredToSave] = .explicit(.uint64(sizeOfMessagesRequiredToSaveBytes)) + options[TopicOptionKey.sizeOfMessagesRequiredToSave.rawValue] = .explicit(.uint64(sizeOfMessagesRequiredToSaveBytes)) } if let preallocateSegments { - options[TopicOptionKey.preallocateSegments] = .explicit(.bool(preallocateSegments)) + options[TopicOptionKey.preallocateSegments.rawValue] = .explicit(.bool(preallocateSegments)) } return options } @@ -129,83 +155,124 @@ } } -/// Options of a topic being updated. +/// Options of a topic being updated. Absent keys keep their stored values. +/// Create-only settings, compression among them, have no field here, so they +/// cannot be sent only to be refused by the server. public struct TopicUpdateOptions: Sendable, Hashable { - public var compressionAlgorithm: CompressionAlgorithm? + /// Nil keeps the topic's name. + public var name: String? + /// Nil keeps the current expiry. ``IggyExpiry/serverDefault`` changes + /// nothing: reset ``TopicOptionKey/messageExpiry`` to return to the + /// default. public var messageExpiry: IggyExpiry? + /// Nil keeps the current cap. ``MaxTopicSize/serverDefault`` changes + /// nothing: reset ``TopicOptionKey/maxTopicSize`` to return to the + /// default. public var maxTopicSize: MaxTopicSize? + /// Keys to return to their defaults: ``TopicOptionKey/messageExpiry`` or + /// ``TopicOptionKey/maxTopicSize``. Any other key is refused before + /// anything is sent. + public var reset: Set<TopicOptionKey> + /// Updatable keys newer than this SDK, as strings the server parses. public var raw: [String: String] + /// Reset names newer than this SDK. The server alone decides whether a + /// patch carrying raw keys or names is admitted. + public var rawReset: Set<String> public init( - compressionAlgorithm: CompressionAlgorithm? = nil, messageExpiry: IggyExpiry? = nil, maxTopicSize: MaxTopicSize? = nil, raw: [String: String] = [:] + name: String? = nil, messageExpiry: IggyExpiry? = nil, maxTopicSize: MaxTopicSize? = nil, reset: Set<TopicOptionKey> = [], + raw: [String: String] = [:], rawReset: Set<String> = [] ) { - self.compressionAlgorithm = compressionAlgorithm + self.name = name self.messageExpiry = messageExpiry self.maxTopicSize = maxTopicSize + self.reset = reset self.raw = raw + self.rawReset = rawReset } func toResourceOptions() throws -> ResourceOptions { var options = try rawOptions(raw) - if let compressionAlgorithm { - options[TopicOptionKey.compressionAlgorithm] = .explicit(try .string(compressionAlgorithm.description)) + if let name { + options[TopicOptionKey.name.rawValue] = .explicit(try columnValue(name, refusal: .invalidTopicName)) } if let messageExpiry, let microseconds = try messageExpiry.wireMicroseconds() { - options[TopicOptionKey.messageExpiry] = .explicit(.uint64(microseconds)) + options[TopicOptionKey.messageExpiry.rawValue] = .explicit(.uint64(microseconds)) } - if let maxTopicSize { - options[TopicOptionKey.maxTopicSize] = .explicit(.uint64(maxTopicSize.wireValue)) + if let maxTopicSize, maxTopicSize != .serverDefault { + options[TopicOptionKey.maxTopicSize.rawValue] = .explicit(.uint64(maxTopicSize.wireValue)) } return options } - func encode() throws -> [UInt8] { - try OptionsBlock.encode(try toResourceOptions()) + func toPatch() throws -> OptionsPatch { + // A server default sets nothing, so it is no set key either. + let typed: [TopicOptionKey] = [ + name.map { _ in .name }, + messageExpiry.flatMap { $0 == .serverDefault ? nil : .messageExpiry }, + maxTopicSize.flatMap { $0 == .serverDefault ? nil : .maxTopicSize }, + ].compactMap { $0 } + return try OptionsPatch(typed: typed, raw: raw, reset: reset, rawReset: rawReset, values: toResourceOptions) } } -/// Options of a stream being updated. The catalog has no stream keys today; -/// `raw` is forwarded as-is for forward compatibility. -public struct StreamUpdateOptions: Sendable, Hashable { - public var raw: [String: String] - - public init(raw: [String: String] = [:]) { - self.raw = raw - } - - func encode() throws -> [UInt8] { - try OptionsBlock.encode(try rawOptions(raw)) - } -} - -/// Options of a user being updated. The catalog has no user keys today; -/// `raw` is forwarded as-is for forward compatibility. -public struct UserUpdateOptions: Sendable, Hashable { - public var raw: [String: String] - - public init(raw: [String: String] = [:]) { - self.raw = raw - } - - func encode() throws -> [UInt8] { - try OptionsBlock.encode(try rawOptions(raw)) - } -} - -/// Characters of an offending key an error message shows, -/// `ERROR_KEY_PREVIEW_LEN` in the Rust crate. -private let errorKeyPreviewLength = 64 - -private func rawOptions(_ raw: [String: String]) throws -> ResourceOptions { - var options: ResourceOptions = [:] - for (key, value) in raw { - guard !key.isEmpty, key.utf8.count <= HeaderTLV.maxFieldLength else { - throw IggyError(.unsupportedOptionKey, context: String(key.prefix(errorKeyPreviewLength))) +extension TopicOptionKey: CatalogOptionKey { + var flags: OptionFlags { + switch self { + case .name: .updatable + case .messageExpiry, .maxTopicSize: [.updatable, .resettable] + default: [] } - guard let headerValue = try? HeaderValue.string(value) else { - throw IggyError(.invalidOptionValue, context: String(key.prefix(errorKeyPreviewLength))) - } - options[key] = .explicit(headerValue) } - return options +} + +extension Topic { + /// How long messages are kept, from ``options``. + public var messageExpiry: IggyExpiry { + resolvedMessageExpiry(options) + } + + /// The compression label the topic was created with, from ``options``. + public var compressionAlgorithm: CompressionAlgorithm { + resolvedCompressionAlgorithm(options) + } + + /// The topic's size cap, from ``options``. + public var maxTopicSize: MaxTopicSize { + resolvedMaxTopicSize(options) + } +} + +extension TopicDetails { + /// How long messages are kept, from ``options``. + public var messageExpiry: IggyExpiry { + resolvedMessageExpiry(options) + } + + /// The compression label the topic was created with, from ``options``. + public var compressionAlgorithm: CompressionAlgorithm { + resolvedCompressionAlgorithm(options) + } + + /// The topic's size cap, from ``options``. + public var maxTopicSize: MaxTopicSize { + resolvedMaxTopicSize(options) + } +} + +// An unreported or unreadable key resolves to its catalog default, as in the Rust crate. + +private func resolvedMessageExpiry(_ options: ResourceOptions) -> IggyExpiry { + guard let microseconds = options[TopicOptionKey.messageExpiry.rawValue]?.value.uint64Value, microseconds != 0 else { + return .never + } + return IggyExpiry(wireMicroseconds: microseconds) +} + +private func resolvedCompressionAlgorithm(_ options: ResourceOptions) -> CompressionAlgorithm { + options[TopicOptionKey.compressionAlgorithm.rawValue]?.value.stringValue.flatMap(CompressionAlgorithm.init(name:)) ?? .none +} + +private func resolvedMaxTopicSize(_ options: ResourceOptions) -> MaxTopicSize { + options[TopicOptionKey.maxTopicSize.rawValue]?.value.uint64Value.map(MaxTopicSize.init(wireValue:)) ?? .unlimited }
diff --git a/foreign/swift/Sources/Iggy/Models/UserHeaders.swift b/foreign/swift/Sources/Iggy/Models/UserHeaders.swift index 2c46c59..ed698f8 100644 --- a/foreign/swift/Sources/Iggy/Models/UserHeaders.swift +++ b/foreign/swift/Sources/Iggy/Models/UserHeaders.swift
@@ -331,7 +331,7 @@ public typealias UserHeaders = [HeaderKey: HeaderValue] extension String { - fileprivate func leftPadded(to width: Int) -> String { + func leftPadded(to width: Int) -> String { count >= width ? self : String(repeating: "0", count: width - count) + self } }
diff --git a/foreign/swift/Sources/Iggy/Models/UserOptions.swift b/foreign/swift/Sources/Iggy/Models/UserOptions.swift new file mode 100644 index 0000000..b5a5795 --- /dev/null +++ b/foreign/swift/Sources/Iggy/Models/UserOptions.swift
@@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/// Keys of the user option catalog a client can name, in catalog order. +public enum UserOptionKey: String, Sendable, Hashable, CaseIterable { + /// The user's name. Updatable, never resettable. + case username + /// Whether the user can sign in. Updatable, never resettable. + case status +} + +/// Options of a user being created. The catalog has no user option yet, so +/// `raw`, keys newer than this SDK as strings the server parses, is the only +/// field. +public struct UserCreateOptions: Sendable, Hashable { + public var raw: [String: String] + + public init(raw: [String: String] = [:]) { + self.raw = raw + } + + func encode() throws -> [UInt8] { + try OptionsBlock.encode(try rawOptions(raw)) + } +} + +/// Options of a user being updated. Absent keys keep their stored values. +public struct UserUpdateOptions: Sendable, Hashable { + /// Nil keeps the username. + public var username: String? + /// Nil keeps the status. Deactivating a user ends its sessions. + public var status: UserStatus? + /// Keys to return to their defaults. No user key is resettable yet, so a + /// non-empty set is refused before anything is sent. + public var reset: Set<UserOptionKey> + /// Updatable keys newer than this SDK, as strings the server parses. + public var raw: [String: String] + /// Reset names newer than this SDK. The server alone decides whether a + /// patch carrying raw keys or names is admitted. + public var rawReset: Set<String> + + public init( + username: String? = nil, status: UserStatus? = nil, reset: Set<UserOptionKey> = [], raw: [String: String] = [:], rawReset: Set<String> = [] + ) { + self.username = username + self.status = status + self.reset = reset + self.raw = raw + self.rawReset = rawReset + } + + func toResourceOptions() throws -> ResourceOptions { + var options = try rawOptions(raw) + if let username { + try Requests.validateUsername(username) + options[UserOptionKey.username.rawValue] = .explicit(try columnValue(username, refusal: .invalidUsername)) + } + if let status { + options[UserOptionKey.status.rawValue] = .explicit(try .string(status.description)) + } + return options + } + + func toPatch() throws -> OptionsPatch { + let typed: [UserOptionKey] = [username.map { _ in .username }, status.map { _ in .status }].compactMap { $0 } + return try OptionsPatch(typed: typed, raw: raw, reset: reset, rawReset: rawReset, values: toResourceOptions) + } +} + +extension UserOptionKey: CatalogOptionKey { + var flags: OptionFlags { + .updatable + } +}
diff --git a/foreign/swift/Sources/Iggy/Protocol/CommandCode.swift b/foreign/swift/Sources/Iggy/Protocol/CommandCode.swift index d1dbf68..b292867 100644 --- a/foreign/swift/Sources/Iggy/Protocol/CommandCode.swift +++ b/foreign/swift/Sources/Iggy/Protocol/CommandCode.swift
@@ -25,6 +25,7 @@ case describeOptions = 13 // 14 was attach_consumer_session; the server keeps the code retired. case bindSession = 15 + case updateResource = 16 case getMe = 20 case getClient = 21 case getClients = 22 @@ -32,7 +33,7 @@ case getUsers = 32 case createUser = 33 case deleteUser = 34 - case updateUser = 35 + // 35 was update_user. The server keeps the code retired. case updatePermissions = 36 case changePassword = 37 case loginUser = 38 @@ -56,14 +57,12 @@ case getStreams = 201 case createStream = 202 case deleteStream = 203 - case updateStream = 204 - // 205 was purge_stream; the server keeps the code retired. + // 204 was update_stream and 205 purge_stream. The server keeps both retired. case getTopic = 300 case getTopics = 301 case createTopic = 302 case deleteTopic = 303 - case updateTopic = 304 - // 305 was purge_topic; the server keeps the code retired. + // 304 was update_topic and 305 purge_topic. The server keeps both retired. case createPartitions = 402 case deletePartitions = 403 case deleteSegments = 503 @@ -84,6 +83,7 @@ case .getClusterMetadata: "cluster.metadata" case .describeOptions: "options.describe" case .bindSession: "session.bind" + case .updateResource: "resource.update" case .getMe: "me" case .getClient: "client.get" case .getClients: "client.list" @@ -91,7 +91,6 @@ case .getUsers: "user.list" case .createUser: "user.create" case .deleteUser: "user.delete" - case .updateUser: "user.update" case .updatePermissions: "user.permissions" case .changePassword: "user.password" case .loginUser: "user.login" @@ -112,12 +111,10 @@ case .getStreams: "stream.list" case .createStream: "stream.create" case .deleteStream: "stream.delete" - case .updateStream: "stream.update" case .getTopic: "topic.get" case .getTopics: "topic.list" case .createTopic: "topic.create" case .deleteTopic: "topic.delete" - case .updateTopic: "topic.update" case .createPartitions: "partition.create" case .deletePartitions: "partition.delete" case .deleteSegments: "segment.delete"
diff --git a/foreign/swift/Sources/Iggy/Protocol/ConsensusOperation.swift b/foreign/swift/Sources/Iggy/Protocol/ConsensusOperation.swift index ae9bbfb..5aee612 100644 --- a/foreign/swift/Sources/Iggy/Protocol/ConsensusOperation.swift +++ b/foreign/swift/Sources/Iggy/Protocol/ConsensusOperation.swift
@@ -32,13 +32,20 @@ case truncatePartition = 68 case finalizeSession = 70 case completeLifecycle = 71 + /// The resolved forms of `updateResource`, one per scope. The primary + /// commits the generic update under one of them, so its reply carries it. + case updateStreamResolved = 72 + case updateTopicResolved = 73 + case updateUserResolved = 74 + /// A replicated rewrap of stream keys the server proposes itself. + case rewrapStreamKeys = 75 case createStream = 128 - case updateStream = 129 + // 129 was updateStream. The server keeps the code retired. case deleteStream = 130 // 131 was purgeStream; the server keeps the code retired. case createTopic = 132 - case updateTopic = 133 + // 133 was updateTopic. The server keeps the code retired. case deleteTopic = 134 // 135 was purgeTopic; the server keeps the code retired. case createPartitions = 136 @@ -47,7 +54,7 @@ case createConsumerGroup = 139 case deleteConsumerGroup = 140 case createUser = 141 - case updateUser = 142 + // 142 was updateUser. The server keeps the code retired. case deleteUser = 143 case changePassword = 144 case updatePermissions = 145 @@ -55,6 +62,7 @@ case deletePersonalAccessToken = 147 case joinConsumerGroup = 148 case leaveConsumerGroup = 149 + case updateResource = 150 case sendMessages = 160 case storeConsumerOffset = 161 @@ -86,11 +94,10 @@ return true } switch self { - case .createStream, .updateStream, .deleteStream, .createTopic, .updateTopic, - .deleteTopic, .createPartitions, .deletePartitions, .createConsumerGroup, - .deleteConsumerGroup, .createUser, .updateUser, .deleteUser, .changePassword, - .updatePermissions, .createPersonalAccessToken, .deletePersonalAccessToken, - .joinConsumerGroup, .leaveConsumerGroup: + case .createStream, .deleteStream, .createTopic, .deleteTopic, .createPartitions, + .deletePartitions, .createConsumerGroup, .deleteConsumerGroup, .createUser, + .deleteUser, .changePassword, .updatePermissions, .createPersonalAccessToken, + .deletePersonalAccessToken, .joinConsumerGroup, .leaveConsumerGroup, .updateResource: return true default: return false @@ -112,7 +119,6 @@ switch CommandCode(rawValue: code) { case .createUser: .createUser case .deleteUser: .deleteUser - case .updateUser: .updateUser case .updatePermissions: .updatePermissions case .changePassword: .changePassword case .createPersonalAccessToken: .createPersonalAccessToken @@ -122,10 +128,8 @@ case .deleteConsumerOffset: .deleteConsumerOffset case .createStream: .createStream case .deleteStream: .deleteStream - case .updateStream: .updateStream case .createTopic: .createTopic case .deleteTopic: .deleteTopic - case .updateTopic: .updateTopic case .createPartitions: .createPartitions case .deletePartitions: .deletePartitions case .deleteSegments: .deleteSegments @@ -133,6 +137,7 @@ case .deleteConsumerGroup: .deleteConsumerGroup case .joinConsumerGroup: .joinConsumerGroup case .leaveConsumerGroup: .leaveConsumerGroup + case .updateResource: .updateResource default: nil } }
diff --git a/foreign/swift/Sources/Iggy/Protocol/ConsensusSession.swift b/foreign/swift/Sources/Iggy/Protocol/ConsensusSession.swift index 3fd0025..0eeee59 100644 --- a/foreign/swift/Sources/Iggy/Protocol/ConsensusSession.swift +++ b/foreign/swift/Sources/Iggy/Protocol/ConsensusSession.swift
@@ -37,12 +37,14 @@ /// Bearer proof that binds connections to this logical session, sent with /// every login. Generated per session and never logged. let bindSecret: [UInt8] + let registerRequestIdentifier: UInt128Value private var requestCounter: UInt64 init(clientID: UInt128Value = .random()) { var generator = SystemRandomNumberGenerator() self.clientID = clientID self.bindSecret = (0..<Requests.bindSecretSize).map { _ in UInt8.random(in: UInt8.min...UInt8.max, using: &generator) } + self.registerRequestIdentifier = .random() self.session = nil self.requestCounter = 1 }
diff --git a/foreign/swift/Sources/Iggy/Protocol/ProtocolVersion.swift b/foreign/swift/Sources/Iggy/Protocol/ProtocolVersion.swift index 03a65a2..70440e4 100644 --- a/foreign/swift/Sources/Iggy/Protocol/ProtocolVersion.swift +++ b/foreign/swift/Sources/Iggy/Protocol/ProtocolVersion.swift
@@ -32,7 +32,7 @@ /// The protocol version this SDK implements: the version of the /// `iggy_binary_protocol` crate it was ported from. The golden fixture /// carries the crate's value and a test fails when the two diverge. - public static let current = ProtocolVersion(major: 0, minor: 11, patch: 1) + public static let current = ProtocolVersion(major: 0, minor: 12, patch: 0) public var major: UInt32 { packed >> 20 } public var minor: UInt32 { (packed >> 10) & 0x3FF }
diff --git a/foreign/swift/Sources/Iggy/Protocol/Requests.swift b/foreign/swift/Sources/Iggy/Protocol/Requests.swift index 86a5ba0..d734be9 100644 --- a/foreign/swift/Sources/Iggy/Protocol/Requests.swift +++ b/foreign/swift/Sources/Iggy/Protocol/Requests.swift
@@ -34,14 +34,6 @@ return writer.bytes } - static func updateStream(streamID: Identifier, name: String, options: [UInt8]) throws -> [UInt8] { - var writer = ByteWriter() - try streamID.encode(into: &writer) - try writer.writeName(validatedWireName(name)) - writer.write(options) - return writer.bytes - } - // MARK: Topics static func streamAndTopic(_ streamID: Identifier, _ topicID: Identifier) throws -> [UInt8] { @@ -60,12 +52,36 @@ return writer.bytes } - static func updateTopic(streamID: Identifier, topicID: Identifier, name: String, options: [UInt8]) throws -> [UInt8] { + // MARK: Resource updates + + /// What an `UpdateResource` patches, tagged on the wire with its options + /// scope. + enum ResourceTarget { + case topic(stream: Identifier, topic: Identifier) + case stream(Identifier) + case user(Identifier) + } + + /// `[scope:u8][target ids][reset_count:u8][reset_key:name]*[options block]` + static func updateResource(_ target: ResourceTarget, _ patch: OptionsPatch) throws -> [UInt8] { var writer = ByteWriter() - try streamID.encode(into: &writer) - try topicID.encode(into: &writer) - try writer.writeName(validatedWireName(name)) - writer.write(options) + switch target { + case .topic(let stream, let topic): + writer.write(OptionsScope.topic.rawValue) + try stream.encode(into: &writer) + try topic.encode(into: &writer) + case .stream(let stream): + writer.write(OptionsScope.stream.rawValue) + try stream.encode(into: &writer) + case .user(let user): + writer.write(OptionsScope.user.rawValue) + try user.encode(into: &writer) + } + writer.write(UInt8(patch.reset.count)) + for key in patch.reset { + try writer.writeName(key) + } + writer.write(try OptionsBlock.encode(patch.set)) return writer.bytes } @@ -216,33 +232,19 @@ } } - static func updateUser(userID: Identifier, username: String?, status: UserStatus?, options: [UInt8]) throws -> [UInt8] { - var writer = ByteWriter() - try userID.encode(into: &writer) - if let username { - try validateUsername(username) - writer.write(UInt8(1)) - try writer.writeName(username) - } else { - writer.write(UInt8(0)) + /// The user a user-scope ``updateResource(_:_:)`` payload names and the + /// name it sets, nil when it patches another scope or keeps the name. + static func renamedUser(_ payload: [UInt8]) throws -> (userID: Identifier, username: String)? { + guard payload.first == OptionsScope.user.rawValue else { + return nil } - if let status { - writer.write(UInt8(1)) - writer.write(status.rawValue) - } else { - writer.write(UInt8(0)) - } - writer.write(options) - return writer.bytes - } - - /// The user an ``updateUser(userID:username:status:options:)`` payload - /// names and its new name, nil when the name stays. - static func renamedUser(_ payload: [UInt8]) throws -> (userID: Identifier, username: String?) { - try Responses.decode(payload[...]) { reader in + return try Responses.decode(payload.dropFirst()) { reader in let userID = try Identifier.decode(from: &reader) - let renames = try reader.readUInt8() != 0 - return (userID, renames ? try reader.readName() : nil) + for _ in 0..<(try reader.readUInt8()) { + _ = try reader.readName() + } + let options = try OptionsBlock.decode(try reader.readBytes(reader.remaining), explicit: true) + return options[UserOptionKey.username.rawValue]?.value.stringValue.map { (userID, $0) } } }
diff --git a/foreign/swift/Sources/Iggy/Protocol/Responses.swift b/foreign/swift/Sources/Iggy/Protocol/Responses.swift index 83772cc..ba710b7 100644 --- a/foreign/swift/Sources/Iggy/Protocol/Responses.swift +++ b/foreign/swift/Sources/Iggy/Protocol/Responses.swift
@@ -43,7 +43,9 @@ let sizeBytes = try reader.readUInt64() let messagesCount = try reader.readUInt64() let name = try reader.readName() - let options = try OptionsBlock.decode(try reader.readPrefixedBytes(), explicit: true) + let explicit = try reader.readPrefixedBytes() + let derived = try reader.readPrefixedBytes() + let options = try OptionsBlock.decodeSplit(explicit: explicit, derived: derived) return StreamInfo( id: id, createdAt: IggyTimestamp(microseconds: createdAt), name: name, sizeBytes: sizeBytes, messagesCount: messagesCount, topicsCount: topicsCount, options: options) @@ -78,25 +80,14 @@ let id = try reader.readUInt32() let createdAt = try reader.readUInt64() let partitionsCount = try reader.readUInt32() - let messageExpiry = try reader.readUInt64() - let compression = try reader.readUInt8() - let maxTopicSize = try reader.readUInt64() let sizeBytes = try reader.readUInt64() let messagesCount = try reader.readUInt64() let name = try reader.readName() let explicit = try reader.readPrefixedBytes() let derived = try reader.readPrefixedBytes() - guard let compressionAlgorithm = CompressionAlgorithm(rawValue: compression) else { - throw IggyError(.invalidCommand, context: "unknown compression algorithm \(compression)") - } - // A stored expiry of zero means the topic never expires, unlike the - // request side where zero is the server default. - let expiry: IggyExpiry = messageExpiry == 0 ? .never : IggyExpiry(wireMicroseconds: messageExpiry) return Topic( - id: id, createdAt: IggyTimestamp(microseconds: createdAt), name: name, sizeBytes: sizeBytes, messageExpiry: expiry, - compressionAlgorithm: compressionAlgorithm, maxTopicSize: MaxTopicSize(wireValue: maxTopicSize), - messagesCount: messagesCount, partitionsCount: partitionsCount, - options: try OptionsBlock.decodeSplit(explicit: explicit, derived: derived)) + id: id, createdAt: IggyTimestamp(microseconds: createdAt), name: name, sizeBytes: sizeBytes, messagesCount: messagesCount, + partitionsCount: partitionsCount, options: try OptionsBlock.decodeSplit(explicit: explicit, derived: derived)) } static func partition(from reader: inout ByteReader) throws -> Partition { @@ -248,7 +239,9 @@ let createdAt = try reader.readUInt64() let statusCode = try reader.readUInt8() let username = try reader.readName() - let options = try OptionsBlock.decode(try reader.readPrefixedBytes(), explicit: true) + let explicit = try reader.readPrefixedBytes() + let derived = try reader.readPrefixedBytes() + let options = try OptionsBlock.decodeSplit(explicit: explicit, derived: derived) guard let status = UserStatus(rawValue: statusCode) else { throw IggyError(.invalidUserStatus, context: "unknown user status \(statusCode)") } @@ -415,12 +408,13 @@ for _ in 0..<count { let key = try reader.readName() let kindCode = try reader.readUInt8() + let flags = OptionFlags(rawValue: try reader.readUInt8()) let defaultValue = Array(try reader.readPrefixedBytes()) let description = String(decoding: try reader.readPrefixedBytes(), as: UTF8.self) guard let kind = HeaderKind(rawValue: kindCode) else { throw IggyError(.invalidHeaderKind, context: "unknown option kind \(kindCode)") } - specs.append(OptionSpec(key: key, kind: kind, defaultValue: defaultValue, description: description)) + specs.append(OptionSpec(key: key, kind: kind, flags: flags, defaultValue: defaultValue, description: description)) } return specs }
diff --git a/foreign/swift/Sources/Iggy/Protocol/VSRFrame.swift b/foreign/swift/Sources/Iggy/Protocol/VSRFrame.swift index 65b69eb..9a1d95d 100644 --- a/foreign/swift/Sources/Iggy/Protocol/VSRFrame.swift +++ b/foreign/swift/Sources/Iggy/Protocol/VSRFrame.swift
@@ -106,10 +106,9 @@ /// Builds the complete request frame for a command code and its encoded /// payload, deriving sequencing from the command code and the session. /// - /// The request checksum is stamped for the operations the server's client - /// table deduplicates, like the Rust SDK does; partition and - /// non-replicated operations leave it zero, which the server reads as - /// unstamped. + /// Metadata and session requests carry a random identifier retained in + /// the encoded frame across retries. Partition and non-replicated + /// operations leave it zero. /// /// `partitionContext` is the context captured with the partition the /// request targets. The server fences a send or an offset change on it, so @@ -138,8 +137,14 @@ sessionID = bound requestID = try session.nextRequestID() } - let requestChecksum: UInt64 = - (operation.isPartition || operation == .nonReplicated) ? 0 : XXH3.hash64(payload) + let requestIdentifier: UInt128Value = + if operation.isPartition || operation == .nonReplicated { + .zero + } else if operation == .register { + session.registerRequestIdentifier + } else { + .random() + } var frame = [UInt8]() frame.reserveCapacity(totalSize) @@ -149,7 +154,8 @@ buffer[RequestOffset.command] = Command.request.rawValue buffer.storeBytes(of: session.clientID.low.littleEndian, toByteOffset: RequestOffset.client, as: UInt64.self) buffer.storeBytes(of: session.clientID.high.littleEndian, toByteOffset: RequestOffset.client + 8, as: UInt64.self) - buffer.storeBytes(of: requestChecksum.littleEndian, toByteOffset: RequestOffset.requestChecksum, as: UInt64.self) + buffer.storeBytes(of: requestIdentifier.low.littleEndian, toByteOffset: RequestOffset.requestChecksum, as: UInt64.self) + buffer.storeBytes(of: requestIdentifier.high.littleEndian, toByteOffset: RequestOffset.requestChecksum + 8, as: UInt64.self) buffer.storeBytes(of: requestID.littleEndian, toByteOffset: RequestOffset.request, as: UInt64.self) buffer[RequestOffset.operation] = operation.rawValue buffer.storeBytes(of: sessionID.littleEndian, toByteOffset: RequestOffset.session, as: UInt64.self)
diff --git a/foreign/swift/Tests/IggyE2ETests/ClientE2ETests.swift b/foreign/swift/Tests/IggyE2ETests/ClientE2ETests.swift index 8d70c8d..cc155fc 100644 --- a/foreign/swift/Tests/IggyE2ETests/ClientE2ETests.swift +++ b/foreign/swift/Tests/IggyE2ETests/ClientE2ETests.swift
@@ -166,7 +166,7 @@ #expect(try await client.getStreams().contains { $0.id == created.id }) let renamed = E2EEnvironment.uniqueName("swift-renamed") - try await client.updateStream(id, name: renamed) + try await client.updateStream(id, options: StreamUpdateOptions(name: renamed)) #expect(try await client.getStream(id)?.name == renamed) try await client.deleteStream(id) @@ -203,11 +203,15 @@ await session.track(stream: try Identifier(named: name)) let specs = try await client.describeOptions(scope: .topic) - let keys = Set(specs.map(\.key)) - #expect(keys.contains(TopicOptionKey.durability)) - #expect(keys.contains(TopicOptionKey.segmentSize)) - #expect(try #require(specs.first { $0.key == TopicOptionKey.segmentSize }).kind == .uint64) - #expect(try await client.describeOptions(scope: .stream).isEmpty) + #expect(specs.map(\.key) == TopicOptionKey.allCases.map(\.rawValue)) + #expect(try #require(specs.first { $0.key == TopicOptionKey.segmentSize.rawValue }).kind == .uint64) + #expect(try #require(specs.first { $0.key == TopicOptionKey.messageExpiry.rawValue }).flags == [.updatable, .resettable]) + #expect(try #require(specs.first { $0.key == TopicOptionKey.compressionAlgorithm.rawValue }).flags.isEmpty) + let streamSpecs = try await client.describeOptions(scope: .stream) + #expect(streamSpecs.map(\.key) == StreamOptionKey.allCases.map(\.rawValue)) + #expect(try #require(streamSpecs.first { $0.key == StreamOptionKey.encryptionKey.rawValue }).flags == [.secret, .critical]) + #expect(try #require(streamSpecs.first { $0.key == StreamOptionKey.encryptionKeySource.rawValue }).flags == .readOnly) + #expect(try await client.describeOptions(scope: .user).map(\.key) == UserOptionKey.allCases.map(\.rawValue)) let options = TopicCreateOptions( partitionsCount: 3, compressionAlgorithm: .gzip, messageExpiry: .after(.seconds(3600)), @@ -222,20 +226,18 @@ let topicID = Identifier(numeric: created.id) let fetched = try #require(try await client.getTopic(streamID: streamID, topicID: topicID)) - #expect(fetched.options[TopicOptionKey.durability] == .explicit("persisted")) + #expect(fetched.options[TopicOptionKey.durability.rawValue] == .explicit("persisted")) // Keys the client left alone are resolved by admission and reported // as derived. - #expect(fetched.options[TopicOptionKey.segmentSize]?.explicit == false) + #expect(fetched.options[TopicOptionKey.segmentSize.rawValue]?.explicit == false) #expect(try await client.getTopics(streamID: streamID).map(\.id) == [created.id]) #expect(try await client.getStream(streamID)?.topics.first?.name == name) let renamed = E2EEnvironment.uniqueName("swift-topic-renamed") - try await client.updateTopic( - streamID: streamID, topicID: topicID, name: renamed, - options: TopicUpdateOptions(compressionAlgorithm: CompressionAlgorithm.none, messageExpiry: .never)) + try await client.updateTopic(streamID: streamID, topicID: topicID, options: TopicUpdateOptions(name: renamed, messageExpiry: .never)) let updated = try #require(try await client.getTopic(streamID: streamID, topicID: topicID)) #expect(updated.name == renamed) - #expect(updated.compressionAlgorithm == CompressionAlgorithm.none) + #expect(updated.compressionAlgorithm == .gzip) #expect(updated.messageExpiry == .never) try await client.createPartitions(streamID: streamID, topicID: topicID, count: 2) @@ -261,9 +263,9 @@ // A string value for a typed key is parsed by admission and reported // with its canonical kind. let created = try await client.createTopic( - streamID: streamID, name: name, options: TopicCreateOptions(raw: [TopicOptionKey.durability: "persisted"])) + streamID: streamID, name: name, options: TopicCreateOptions(raw: [TopicOptionKey.durability.rawValue: "persisted"])) let topic = try #require(try await client.getTopic(streamID: streamID, topicID: Identifier(numeric: created.id))) - #expect(topic.options[TopicOptionKey.durability] == .explicit("persisted")) + #expect(topic.options[TopicOptionKey.durability.rawValue] == .explicit("persisted")) } } @@ -327,7 +329,7 @@ } try await client.changePassword(userID, currentPassword: "secret-1", newPassword: "secret-2") - try await client.updateUser(userID, username: username + "-renamed", status: .inactive) + try await client.updateUser(userID, options: UserUpdateOptions(username: username + "-renamed", status: .inactive)) let updated = try #require(try await client.getUser(userID)) #expect(updated.username == username + "-renamed") #expect(updated.status == .inactive) @@ -541,7 +543,8 @@ streamID: scratch.stream, topicID: scratch.topic, partitioning: .partition(0), messages: try E2EEnvironment.messages(2)) let target = Self.rawConsumerTarget(consumer: 7, stream: scratch.streamID, topic: scratch.topicID) // Strategy first, count 10, no auto-commit. - let poll = target + [3] + Self.littleEndian(UInt64(0)) + Self.littleEndian(UInt32(10)) + [0] + let strategy: [UInt8] = [3] + Self.littleEndian(UInt64(0)) + let poll: [UInt8] = target + strategy + Self.littleEndian(UInt32(10)) + [0] #expect(!(try await client.sendRawRequest(code: CommandCode.pollMessages.rawValue, payload: poll)).isEmpty) // Offset 1, quorum acknowledgement. let store = target + Self.littleEndian(UInt64(1)) + [1] @@ -554,8 +557,10 @@ /// `[consumer kind:1][consumer][stream][topic][has partition:1][partition 0:4]` /// with numeric identifiers `[kind 1][length 4][id]`. private static func rawConsumerTarget(consumer: UInt32, stream: UInt32, topic: UInt32) -> [UInt8] { - let numeric = { (id: UInt32) in [1, 4] + littleEndian(id) } - return [1] + numeric(consumer) + numeric(stream) + numeric(topic) + [1] + littleEndian(UInt32(0)) + let numeric = { (id: UInt32) -> [UInt8] in [1, 4] + littleEndian(id) } + let identifiers: [UInt8] = numeric(consumer) + numeric(stream) + numeric(topic) + let partition: [UInt8] = [1] + littleEndian(UInt32(0)) + return [1] + identifiers + partition } private static func littleEndian<T: FixedWidthInteger>(_ value: T) -> [UInt8] {
diff --git a/foreign/swift/Tests/IggyE2ETests/ResourceOptionsE2ETests.swift b/foreign/swift/Tests/IggyE2ETests/ResourceOptionsE2ETests.swift new file mode 100644 index 0000000..aad50e5 --- /dev/null +++ b/foreign/swift/Tests/IggyE2ETests/ResourceOptionsE2ETests.swift
@@ -0,0 +1,309 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation +import Iggy +import Testing + +/// The shared resource options scenarios. The server must run with +/// encryption enabled (`IGGY_ENCRYPTION_ENABLED=true` and a 32-byte +/// `IGGY_ENCRYPTION_KEY`), so a stream created without a key gets one the +/// server generates. +@Suite("Resource options", requiresServer) +struct ResourceOptionsE2ETests { + /// Keys no read may ever carry. + static let neverReturnedKeys: Set<String> = ["encryption_key", "encryption_key_wrapped", "critical_keys"] + + @Test func givenStreamKeyWhenCreatingShouldReadMatchingProjection() async throws { + try await E2EEnvironment.withSession() { session in + let client = session.client + let key = Self.randomKey() + let name = E2EEnvironment.uniqueName("swift-keyed") + let created = try await client.createStream(name: name, options: StreamCreateOptions(encryptionKey: key)) + await session.track(stream: try Identifier(named: name)) + let fetched = try #require(try await client.getStream(Identifier(numeric: created.id))) + let listed = try #require(try await client.getStreams().first { $0.id == created.id }) + + let fingerprint = try EncryptionKey.fingerprint(key) + for stream in [created.projection, fetched.projection, listed.projection] { + #expect(stream.source == .stream) + #expect(stream.keyFingerprint == fingerprint) + #expect(stream.kekFingerprints.count == 1) + #expect(stream.kekFingerprints.allSatisfy(Self.isFingerprint)) + #expect(stream.kekFingerprints == created.encryptionKekFingerprints) + } + for options in [created.options, fetched.options, listed.options] { + Self.expectNoKeyMaterial(options, key: key) + } + } + } + + @Test func givenKeylessStreamWhenReadingShouldReportServerGeneratedKey() async throws { + try await E2EEnvironment.withSession() { session in + let client = session.client + var fingerprints: [String] = [] + for _ in 0..<2 { + let name = E2EEnvironment.uniqueName("swift-keyless") + let created = try await client.createStream(name: name) + await session.track(stream: try Identifier(named: name)) + let fetched = try #require(try await client.getStream(Identifier(numeric: created.id))) + let listed = try #require(try await client.getStreams().first { $0.id == created.id }) + for stream in [created.projection, fetched.projection, listed.projection] { + #expect(stream.source == .server) + #expect(stream.keyFingerprint.map(Self.isFingerprint) == true) + #expect(stream.keyFingerprint == created.encryptionKeyFingerprint) + #expect(stream.kekFingerprints.count == 1) + #expect(stream.kekFingerprints.allSatisfy(Self.isFingerprint)) + } + #expect(Self.neverReturnedKeys.isDisjoint(with: fetched.options.keys)) + fingerprints.append(try #require(created.encryptionKeyFingerprint)) + } + // Every keyless stream gets its own key. + #expect(Set(fingerprints).count == 2) + } + } + + @Test func givenStreamRenameWhenUpdatingShouldPreserveEncryptionProjection() async throws { + try await E2EEnvironment.withSession() { session in + let client = session.client + let key = Self.randomKey() + let name = E2EEnvironment.uniqueName("swift-rename") + let created = try await client.createStream(name: name, options: StreamCreateOptions(encryptionKey: key)) + let streamID = Identifier(numeric: created.id) + let topic = try await client.createTopic(streamID: streamID, name: name, options: TopicCreateOptions(partitionsCount: 1)) + let topicID = Identifier(numeric: topic.id) + try await client.sendBatch(streamID: streamID, topicID: topicID, partitioning: .partition(0), messages: [IggyMessage("sealed payload")]) + + let renamed = E2EEnvironment.uniqueName("swift-renamed") + await session.track(stream: try Identifier(named: renamed)) + try await client.updateStream(streamID, options: StreamUpdateOptions(name: renamed)) + let fetched = try #require(try await client.getStream(streamID)) + #expect(fetched.id == created.id) + #expect(fetched.name == renamed) + #expect(fetched.projection == created.projection) + #expect(fetched.encryptionKeyFingerprint == (try EncryptionKey.fingerprint(key))) + let polled = try await client.pollMessages( + streamID: Identifier(named: renamed), topicID: topicID, partitionID: 0, strategy: .first, count: 1, autoCommit: false) + #expect(polled.messages.map(\.payloadString) == ["sealed payload"]) + + // No stream key is resettable and the projections are read-only, + // so these never leave the client. + await expectIggyError(.optionNotResettable) { try await client.updateStream(streamID, options: StreamUpdateOptions(reset: [.name])) } + await expectIggyError(.optionNotUpdatable) { try await client.updateStream(streamID, options: StreamUpdateOptions(reset: [.encryptionKey])) } + await expectIggyError(.readOnlyOption) { + try await client.updateStream(streamID, options: StreamUpdateOptions(reset: [.encryptionKeySource])) + } + await expectIggyError(.invalidCommand) { try await client.updateStream(streamID, options: StreamUpdateOptions()) } + // The raw escape hatch still goes through server admission. + let unknown = E2EEnvironment.uniqueName("swift-unknown-key") + await expectReplicatedIggyError(.unsupportedOptionKey) { + _ = try await client.createStream(name: unknown, options: StreamCreateOptions(raw: ["future_option": "enabled"])) + } + #expect(try await client.getStream(Identifier(named: unknown)) == nil) + #expect(try await client.getStream(streamID)?.projection == created.projection) + } + } + + @Test func givenTopicResetWhenUpdatingShouldPreserveUnmentionedValues() async throws { + try await E2EEnvironment.withSession() { session in + let client = session.client + let name = E2EEnvironment.uniqueName("swift-reset") + let stream = try await client.createStream(name: name) + let streamID = Identifier(numeric: stream.id) + await session.track(stream: try Identifier(named: name)) + let created = try await client.createTopic( + streamID: streamID, name: name, + options: TopicCreateOptions( + compressionAlgorithm: .gzip, messageExpiry: .after(.seconds(3600)), maxTopicSize: .bytes(10_737_418_240), durability: .persisted)) + let topicID = Identifier(numeric: created.id) + #expect(created.options[TopicOptionKey.maxTopicSize.rawValue] == .explicit(.uint64(10_737_418_240))) + + try await client.updateTopic( + streamID: streamID, topicID: topicID, options: TopicUpdateOptions(messageExpiry: .after(.seconds(7200)), reset: [.maxTopicSize])) + let updated = try #require(try await client.getTopic(streamID: streamID, topicID: topicID)) + #expect(updated.name == name) + #expect(updated.messageExpiry == .after(.seconds(7200))) + #expect(updated.options[TopicOptionKey.messageExpiry.rawValue] == .explicit(.uint64(7_200_000_000))) + #expect(updated.maxTopicSize == .unlimited) + #expect(updated.options[TopicOptionKey.maxTopicSize.rawValue] == .derived(.uint64(UInt64.max))) + #expect(updated.compressionAlgorithm == .gzip) + #expect(updated.options[TopicOptionKey.compressionAlgorithm.rawValue] == .explicit("gzip")) + #expect(updated.options[TopicOptionKey.durability.rawValue] == .explicit("persisted")) + #expect(updated.options[TopicOptionKey.segmentSize.rawValue] == created.options[TopicOptionKey.segmentSize.rawValue]) + + // A rename alone keeps every option and its provenance. + let renamed = E2EEnvironment.uniqueName("swift-reset-renamed") + try await client.updateTopic(streamID: streamID, topicID: topicID, options: TopicUpdateOptions(name: renamed)) + let after = try #require(try await client.getTopic(streamID: streamID, topicID: topicID)) + #expect(after.name == renamed) + #expect(after.options == updated.options) + + try await client.updateTopic(streamID: streamID, topicID: topicID, options: TopicUpdateOptions(reset: [.messageExpiry])) + let reset = try #require(try await client.getTopic(streamID: streamID, topicID: topicID)) + #expect(reset.messageExpiry == .never) + #expect(reset.options[TopicOptionKey.messageExpiry.rawValue]?.explicit == false) + + // Create-only keys cannot be reset, and a key cannot be both set and reset. + await expectIggyError(.optionNotUpdatable) { + try await client.updateTopic(streamID: streamID, topicID: topicID, options: TopicUpdateOptions(reset: [.compressionAlgorithm])) + } + await expectIggyError(.invalidCommand) { + try await client.updateTopic( + streamID: streamID, topicID: topicID, options: TopicUpdateOptions(messageExpiry: .never, reset: [.messageExpiry])) + } + #expect(try await client.getTopic(streamID: streamID, topicID: topicID)?.options == reset.options) + + // Raw keys and reset names are the server's to judge. + await expectReplicatedIggyError(.optionNotUpdatable) { + try await client.updateTopic( + streamID: streamID, topicID: topicID, options: TopicUpdateOptions(reset: [.name], raw: [TopicOptionKey.segmentSize.rawValue: "1 GiB"])) + } + await expectReplicatedIggyError(.unsupportedOptionKey) { + try await client.updateTopic(streamID: streamID, topicID: topicID, options: TopicUpdateOptions(rawReset: ["future_retention"])) + } + await expectReplicatedIggyError(.optionNotUpdatable) { + try await client.updateTopic( + streamID: streamID, topicID: topicID, options: TopicUpdateOptions(rawReset: [TopicOptionKey.segmentSize.rawValue])) + } + #expect(try await client.getTopic(streamID: streamID, topicID: topicID)?.options == reset.options) + + let fixedName = E2EEnvironment.uniqueName("swift-fixed") + let fixed = try await client.createTopic(streamID: streamID, name: fixedName, options: TopicCreateOptions(partitionResizePolicy: .fixed)) + #expect(fixed.options[TopicOptionKey.partitionResizePolicy.rawValue] == .explicit("fixed")) + } + } + + @Test func givenUserOptionsWhenCreatingAndUpdatingShouldPreserveUserBehavior() async throws { + try await E2EEnvironment.withSession() { session in + let client = session.client + let first = String(E2EEnvironment.uniqueName("swift-opt-a").prefix(50)) + let second = String(E2EEnvironment.uniqueName("swift-opt-b").prefix(50)) + let created = try await client.createUser(username: first, password: "secret-1", options: UserCreateOptions()) + let other = try await client.createUser(username: second, password: "secret-2", options: UserCreateOptions()) + let userID = Identifier(numeric: created.id) + let otherID = Identifier(numeric: other.id) + #expect(created.options.isEmpty) + + let renamed = String((first + "-x").suffix(50)) + try await client.updateUser(userID, options: UserUpdateOptions(username: renamed, status: .inactive)) + let updated = try #require(try await client.getUser(userID)) + #expect(updated.username == renamed) + #expect(updated.status == .inactive) + #expect(updated.options.isEmpty) + #expect(try await client.getUsers().first { $0.id == created.id }?.username == renamed) + + // A taken username refuses the whole patch: the status stays. + await expectReplicatedIggyError(.userAlreadyExists) { + try await client.updateUser(otherID, options: UserUpdateOptions(username: renamed, status: .inactive)) + } + let untouched = try #require(try await client.getUser(otherID)) + #expect(untouched.username == second) + #expect(untouched.status == .active) + + await expectIggyError(.optionNotResettable) { try await client.updateUser(userID, options: UserUpdateOptions(reset: [.status])) } + await expectIggyError(.invalidCommand) { try await client.updateUser(userID, options: UserUpdateOptions()) } + await expectIggyError(.invalidUsername) { try await client.updateUser(userID, options: UserUpdateOptions(username: "ab")) } + await expectReplicatedIggyError(.readOnlyOption) { + try await client.updateUser(userID, options: UserUpdateOptions(rawReset: ["critical_keys"])) + } + try await client.deleteUser(userID) + try await client.deleteUser(otherID) + } + } + + @Test func givenLoggedInUserRenameWhenReconnectingShouldKeepNewUsername() async throws { + try await E2EEnvironment.withSession() { session in + let root = session.client + let username = String(E2EEnvironment.uniqueName("swift-self").prefix(50)) + let permissions = Permissions(global: GlobalPermissions(manageUsers: true, readUsers: true, manageStreams: true, readStreams: true)) + let user = try await root.createUser(username: username, password: "secret-1", permissions: permissions) + let userID = Identifier(numeric: user.id) + + let client = IggyClient( + configuration: E2EEnvironment.configuration(autoLogin: .usernamePassword(username: username, password: "secret-1"))) + do { + try await client.connect() + let renamed = String((username + "-r").suffix(50)) + try await client.updateUser(userID, options: UserUpdateOptions(username: renamed)) + await client.disconnect() + try await client.connect() + #expect(try await client.getMe().userID == user.id) + + // A refused rename keeps the credentials that work. + await expectReplicatedIggyError(.userAlreadyExists) { + try await client.updateUser(userID, options: UserUpdateOptions(username: E2EEnvironment.rootUsername)) + } + await client.disconnect() + try await client.connect() + #expect(try await client.getMe().userID == user.id) + + // Renaming a stream does not touch the credentials. + let name = E2EEnvironment.uniqueName("swift-self-stream") + let stream = try await client.createStream(name: name) + let streamName = E2EEnvironment.uniqueName("swift-self-renamed") + await session.track(stream: try Identifier(named: streamName)) + try await client.updateStream(Identifier(numeric: stream.id), options: StreamUpdateOptions(name: streamName)) + await client.disconnect() + try await client.connect() + #expect(try await client.getMe().userID == user.id) + #expect(try await root.getUser(userID)?.username == renamed) + } catch { + await client.shutdown() + try? await root.deleteUser(userID) + throw error + } + await client.shutdown() + try await root.deleteUser(userID) + } + } + + static func randomKey() -> Data { + Data((0..<EncryptionKey.length).map { _ in UInt8.random(in: .min ... .max) }) + } + + static func isFingerprint(_ text: String) -> Bool { + text.count == 16 && text.allSatisfy { $0.isHexDigit && !$0.isUppercase } + } + + static func expectNoKeyMaterial(_ options: ResourceOptions, key: Data, sourceLocation: SourceLocation = #_sourceLocation) { + #expect(neverReturnedKeys.isDisjoint(with: options.keys), sourceLocation: sourceLocation) + let encoded = [key.base64EncodedString(), key.map { String(format: "%02x", $0) }.joined()] + for (name, option) in options { + #expect(option.value.bytes != Array(key), "\(name)", sourceLocation: sourceLocation) + #expect(!encoded.contains { option.value.description.contains($0) }, "\(name)", sourceLocation: sourceLocation) + } + } +} + +/// The encryption projection a stream read reports. +struct StreamProjection: Equatable { + let source: StreamEncryptionSource? + let keyFingerprint: String? + let kekFingerprints: [String] +} + +extension StreamInfo { + var projection: StreamProjection { + StreamProjection(source: encryptionKeySource, keyFingerprint: encryptionKeyFingerprint, kekFingerprints: encryptionKekFingerprints) + } +} + +extension StreamDetails { + var projection: StreamProjection { + StreamProjection(source: encryptionKeySource, keyFingerprint: encryptionKeyFingerprint, kekFingerprints: encryptionKekFingerprints) + } +}
diff --git a/foreign/swift/Tests/IggyE2ETests/Support/E2EEnvironment.swift b/foreign/swift/Tests/IggyE2ETests/Support/E2EEnvironment.swift index 05a809e..9fb3880 100644 --- a/foreign/swift/Tests/IggyE2ETests/Support/E2EEnvironment.swift +++ b/foreign/swift/Tests/IggyE2ETests/Support/E2EEnvironment.swift
@@ -27,6 +27,7 @@ /// cargo build --bin iggy-server /// IGGY_PATH=/tmp/iggy-swift IGGY_TCP_ADDRESS=127.0.0.1:8090 \ /// IGGY_HTTP_ENABLED=false IGGY_QUIC_ENABLED=false IGGY_WEBSOCKET_ENABLED=false \ +/// IGGY_ENCRYPTION_ENABLED=true IGGY_ENCRYPTION_KEY=/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE= \ /// IGGY_ROOT_USERNAME=iggy IGGY_ROOT_PASSWORD=iggy target/debug/iggy-server /// /// IGGY_TCP_ADDRESS=127.0.0.1:8090 swift test --filter IggyE2ETests @@ -34,6 +35,8 @@ /// /// Set `IGGY_TCP_TLS_ENABLED=true` to run against a server started with /// `IGGY_TCP_TLS_ENABLED=true` and the certificate pair in `core/certs`. +/// The resource options suite needs server encryption enabled, so a stream +/// created without a key gets one the server generates. enum E2EEnvironment { static let rootUsername = ProcessInfo.processInfo.environment["IGGY_ROOT_USERNAME"] ?? "iggy" static let rootPassword = ProcessInfo.processInfo.environment["IGGY_ROOT_PASSWORD"] ?? "iggy"
diff --git a/foreign/swift/Tests/IggyTests/ClientTransportTests.swift b/foreign/swift/Tests/IggyTests/ClientTransportTests.swift index 060c67b..1abf3d8 100644 --- a/foreign/swift/Tests/IggyTests/ClientTransportTests.swift +++ b/foreign/swift/Tests/IggyTests/ClientTransportTests.swift
@@ -199,7 +199,8 @@ #expect(requests[0].session == 0 && requests[0].requestID == 0 && requests[0].checksum != 0) #expect(requests[1].code == CommandCode.getClusterMetadata.rawValue) #expect(requests[2].code == CommandCode.ping.rawValue && requests[2].session == 1) - #expect(requests[3].session == 1 && requests[3].checksum == XXH3.hash64(requests[3].payload)) + #expect(requests[3].session == 1 && requests[3].checksum != XXH3.hash64(requests[3].payload)) + #expect(requests[3].checksum != requests[4].checksum) #expect(requests[5].checksum == 0) #expect(requests.allSatisfy { $0.clientID == requests[0].clientID }) #expect(await client.serverAddress == server.address) @@ -2102,7 +2103,7 @@ await Self.forgetSessions(on: server) let client = makeClient(server) try await client.connect() - try await client.updateUser("iggy", username: "renamed") + try await client.updateUser("iggy", options: UserUpdateOptions(username: "renamed")) await server.closeAllConnections() try await Task.sleep(for: .milliseconds(20)) try await client.ping() @@ -2122,6 +2123,22 @@ await client.shutdown() } + @Test func rawUsernameRenameUpdatesTheReconnectCredentials() async throws { + let server = try await FakeVSRServer.started() + defer { Task { await server.stop() } } + await Self.forgetSessions(on: server) + let client = makeClient(server) + try await client.connect() + try await client.updateUser("iggy", options: UserUpdateOptions(raw: [UserOptionKey.username.rawValue: "renamed"])) + await server.closeAllConnections() + try await Task.sleep(for: .milliseconds(20)) + try await client.ping() + let registers = await server.requests.filter { $0.operation == .register } + #expect(registers.count == 2) + #expect(Self.username(in: registers[1].payload) == "renamed") + await client.shutdown() + } + @Test func rawPasswordChangeUpdatesTheReconnectCredentials() async throws { let server = try await FakeVSRServer.started() defer { Task { await server.stop() } } @@ -2145,8 +2162,8 @@ await Self.forgetSessions(on: server) let client = makeClient(server) try await client.connect() - let rename = try Requests.updateUser(userID: "iggy", username: "renamed", status: nil, options: []) - _ = try await client.sendRawRequest(code: CommandCode.updateUser.rawValue, payload: rename) + let rename = try Requests.updateResource(.user("iggy"), try UserUpdateOptions(username: "renamed").toPatch()) + _ = try await client.sendRawRequest(code: CommandCode.updateResource.rawValue, payload: rename) await server.closeAllConnections() try await Task.sleep(for: .milliseconds(20)) try await client.ping()
diff --git a/foreign/swift/Tests/IggyTests/Fixtures/golden.json b/foreign/swift/Tests/IggyTests/Fixtures/golden.json index a5f24e7..83235eb 100644 --- a/foreign/swift/Tests/IggyTests/Fixtures/golden.json +++ b/foreign/swift/Tests/IggyTests/Fixtures/golden.json
@@ -1,6 +1,6 @@ { - "protocol_version": 11265, - "protocol_version_min": 11265, + "protocol_version": 12288, + "protocol_version_min": 12288, "no_assigned_partition": 4294967294, "resync_required_partition": 4294967295, "xxh3_64": { @@ -848,6 +848,12 @@ "4100": "invalid_offset", "4101": "invalid_reserved_field", "4102": "segment_size_mismatch_at_open", + "4103": "option_not_updatable", + "4104": "option_not_resettable", + "4105": "read_only_option", + "4106": "stream_encryption_unavailable", + "4107": "unsupported_critical_option", + "4108": "cannot_modify_root_user", "5000": "consumer_group_id_not_found", "5002": "invalid_consumer_group_id", "5003": "consumer_group_name_not_found", @@ -904,26 +910,30 @@ "identifier.named.my-stream": "02096d792d73747265616d", "identifier.numeric.1": "010401000000", "options.golden": "020a0000006475726162696c6974790209000000706572736973746564020c0000007365676d656e745f73697a650c080000000000004000000000", + "options.golden_block": "0214000000707265616c6c6f636174655f7365676d656e7473030100000001020c0000007365676d656e745f73697a650c080000000000004000000000020a0000006475726162696c6974790209000000706572736973746564", + "options.stream_create.key": "020e000000656e6372797074696f6e5f6b65790120000000000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", "options.topic_create.full": "0215000000636f6d7072657373696f6e5f616c676f726974686d0204000000677a6970021a000000636f6e73756d65725f6f66667365745f6475726162696c697479020a0000007265706c696361746564020a0000006475726162696c6974790209000000706572736973746564020e0000006d61785f746f7069635f73697a650c080000000000004000000000020e0000006d6573736167655f6578706972790c0800000000a0e3d08c00000002190000006d657373616765735f72657175697265645f746f5f736176650b04000000070000000214000000707265616c6c6f636174655f7365676d656e7473030100000000020c0000007365676d656e745f73697a650c080000000000100000000000022100000073697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c080000000010000000000000", "options.topic_create.raw": "021a000000636f6e73756d65725f6f66667365745f6475726162696c697479020a0000007265706c696361746564020a0000006475726162696c6974790209000000706572736973746564", - "options.topic_update.full": "0215000000636f6d7072657373696f6e5f616c676f726974686d02040000006e6f6e65020e0000006d61785f746f7069635f73697a650c08000000ffffffffffffffff020e0000006d6573736167655f6578706972790c08000000ffffffffffffffff", + "options.topic_update.full": "020e0000006d61785f746f7069635f73697a650c08000000ffffffffffffffff020e0000006d6573736167655f6578706972790c08000000ffffffffffffffff", "partition.context": "110000000000000009000000000000003400000000000000", "partitioning.balanced": "0100", "partitioning.messages_key.user-123": "0308757365722d313233", "partitioning.partition_id.7": "020407000000", "payload.create_stream": "0673747265616d", + "payload.update_resource": "01010401000000010402000000010e6d61785f746f7069635f73697a65020e0000006d6573736167655f6578706972790c0800000000a0e3d08c00000002040000006e616d65020d000000757064617465642d746f706963", "permissions.sample": "010100010001000101000101000000010100010100010a000000010100010114000000000101010001020000000001010000010000", "polling.first": "030000000000000000", "polling.last": "040000000000000000", "polling.next": "050000000000000000", "polling.offset.100": "016400000000000000", "polling.timestamp.1700000000000": "020068e5cf8b010000", - "request.bind_session": "012c00000973776966742d73646b05302e312e3001000000000000000000000000000000640000000000000078000000000000005a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a", + "request.bind_session": "003000000973776966742d73646b05302e312e3101000000000000000000000000000000640000000000000078000000000000005a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a", "request.change_password": "010401000000086f6c642d706173730c6e65772d706173732d313233", "request.create_consumer_group": "01040100000001040200000003677270", "request.create_partitions": "01040100000001040200000005000000", "request.create_personal_access_token": "086d792d746f6b656e100e000000000000", "request.create_stream": "0b746573742d73747265616d", + "request.create_stream.key": "0b746573742d73747265616d020e000000656e6372797074696f6e5f6b65790120000000000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", "request.create_stream.options": "0b746573742d73747265616d020a0000006475726162696c6974790209000000706572736973746564020c0000007365676d656e745f73697a650c080000000000004000000000", "request.create_topic": "01040100000003000000066f7264657273020a0000006475726162696c6974790209000000706572736973746564020c0000007365676d656e745f73697a650c080000000000004000000000", "request.create_user": "0561646d696e087040737377307264010135000000010100010001000101000101000000010100010100010a000000010100010114000000000101010001020000000001010000010000", @@ -956,8 +966,8 @@ "request.get_users": "", "request.join_consumer_group": "010401000000010402000000010403000000", "request.leave_consumer_group": "010401000000010402000000020167", - "request.login_register": "012c00000973776966742d73646b05302e312e305a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a0469676779046967677900000000", - "request.login_register_with_pat": "012c00000973776966742d73646b05302e312e305a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a107061742d61626331323364656634353600000000", + "request.login_register": "003000000973776966742d73646b05302e312e315a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a0469676779046967677900000000", + "request.login_register_with_pat": "003000000973776966742d73646b05302e312e315a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a107061742d61626331323364656634353600000000", "request.logout_user": "", "request.ping": "", "request.poll_messages": "0101040100000001040a00000001041400000001050000000164000000000000003200000001", @@ -968,10 +978,11 @@ "request.sync_consumer_group": "020873747265616d2d310207746f7069632d31020767726f75702d31", "request.update_permissions": "0104010000000135000000010100010001000101000101000000010100010100010a000000010100010114000000000101010001020000000001010000010000", "request.update_permissions.none": "020561646d696e00", - "request.update_stream": "02086f6c642d6e616d65086e65772d6e616d65", - "request.update_topic": "0104010000000104020000000d757064617465642d746f706963020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000", - "request.update_user": "01040100000001086e65772d6e616d650102", - "request.update_user.none": "0104050000000000", + "request.update_resource.stream": "0202086f6c642d6e616d650002040000006e616d6502080000006e65772d6e616d65", + "request.update_resource.topic": "01010401000000010402000000010e6d61785f746f7069635f73697a65020e0000006d6573736167655f6578706972790c0800000000a0e3d08c00000002040000006e616d65020d000000757064617465642d746f706963", + "request.update_resource.topic.reset_only": "01020673747265616d0205746f706963020e6d61785f746f7069635f73697a650e6d6573736167655f657870697279", + "request.update_resource.user": "030104010000000002060000007374617475730208000000696e6163746976650208000000757365726e616d6502080000006e65772d6e616d65", + "request.update_resource.user.status": "03020561646d696e0002060000007374617475730206000000616374697665", "response.client": "010000000a000000010e0000003132372e302e302e313a3830383002000000", "response.client_details": "010000000a000000010e0000003132372e302e302e313a3830383002000000010000000200000003000000040000000500000006000000", "response.client_details.no_user": "02000000ffffffff010d00000031302e302e302e313a3630303000000000", @@ -979,18 +990,18 @@ "response.consumer_group": "010000000400000002000000086d792d67726f7570", "response.consumer_group_details": "010000000600000002000000086d792d67726f757001000000030000000000000001000000020000000200000003000000030000000400000005000000", "response.consumer_offset": "01000000e803000000000000f401000000000000", - "response.describe_options": "020000000c7365676d656e745f73697a650c0800000000000040000000000c0000005365676d656e742073697a650a6475726162696c697479020a0000007265706c69636174656400000000", + "response.describe_options": "040000000c7365676d656e745f73697a650c000800000000000040000000000c0000005365676d656e742073697a650a6475726162696c69747902000a0000007265706c696361746564000000000e6d6573736167655f6578706972790c0308000000ffffffffffffffff000000000e656e6372797074696f6e5f6b657901140000000000000000", "response.get_clients": "010000000a000000010e0000003132372e302e302e313a38303830020000000200000014000000020d00000031302e302e302e313a3630303003000000", "response.get_consumer_groups": "0100000004000000020000000767726f75702d610200000004000000000000000767726f75702d62", "response.get_personal_access_tokens": "07746f6b656e2d6100e090663c13060007746f6b656e2d620000000000000000", - "response.get_stream": "0100000000e090663c130600020000000008000000000000c800000000000000096d792d73747265616d000000000100000000e090663c1306000300000000a0e3d08c0000000100000040000000000010000000000000640000000000000007746f7069632d613b000000020a0000006475726162696c6974790209000000706572736973746564020c0000007365676d656e745f73697a650c08000000000000400000000020000000020e0000006d61785f746f7069635f73697a650c08000000ffffffffffffffff0200000000e090663c1306000300000000a0e3d08c0000000100000040000000000010000000000000640000000000000007746f7069632d623b000000020a0000006475726162696c6974790209000000706572736973746564020c0000007365676d656e745f73697a650c08000000000000400000000020000000020e0000006d61785f746f7069635f73697a650c08000000ffffffffffffffff", - "response.get_streams": "0100000000e090663c130600020000000008000000000000c800000000000000096d792d73747265616d000000000200000000e090663c130600000000000008000000000000c800000000000000096d792d73747265616d00000000", - "response.get_topic": "0100000000e090663c1306000300000000a0e3d08c00000001000000400000000000100000000000006400000000000000086d792d746f7069633b000000020a0000006475726162696c6974790209000000706572736973746564020c0000007365676d656e745f73697a650c08000000000000400000000020000000020e0000006d61785f746f7069635f73697a650c08000000ffffffffffffffff0000000000e090663c130600020000006300000000000000000400000000000064000000000000000000000000000000000000000000000000000000000000000100000001e090663c130600010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200000002e090663c13060003000000070000000000000000020000000000000800000000000000000000000000000000000000000000000000000000000000", - "response.get_topics": "020000000100000000e090663c1306000300000000a0e3d08c00000001000000400000000000100000000000006400000000000000066576656e74733b000000020a0000006475726162696c6974790209000000706572736973746564020c0000007365676d656e745f73697a650c08000000000000400000000020000000020e0000006d61785f746f7069635f73697a650c08000000ffffffffffffffff0200000000e090663c1306000300000000a0e3d08c00000001000000400000000000100000000000006400000000000000046c6f67733b000000020a0000006475726162696c6974790209000000706572736973746564020c0000007365676d656e745f73697a650c08000000000000400000000020000000020e0000006d61785f746f7069635f73697a650c08000000ffffffffffffffff", + "response.get_stream": "0100000000e090663c130600020000000008000000000000c800000000000000096d792d73747265616d000000009e000000021a000000656e6372797074696f6e5f6b656b5f66696e6765727072696e740221000000333965346630663935633336393363372c64353564343739383239626332393336021a000000656e6372797074696f6e5f6b65795f66696e6765727072696e740210000000383434316533383533386362666135370215000000656e6372797074696f6e5f6b65795f736f75726365020600000073747265616d0100000000e090663c130600030000000010000000000000640000000000000007746f7069632d615b000000020a0000006475726162696c6974790209000000706572736973746564020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000020c0000007365676d656e745f73697a650c080000000000004000000000430000000215000000636f6d7072657373696f6e5f616c676f726974686d02040000006e6f6e65020e0000006d61785f746f7069635f73697a650c08000000ffffffffffffffff0200000000e090663c130600030000000010000000000000640000000000000007746f7069632d625b000000020a0000006475726162696c6974790209000000706572736973746564020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000020c0000007365676d656e745f73697a650c080000000000004000000000430000000215000000636f6d7072657373696f6e5f616c676f726974686d02040000006e6f6e65020e0000006d61785f746f7069635f73697a650c08000000ffffffffffffffff", + "response.get_streams": "0100000000e090663c130600020000000008000000000000c800000000000000096d792d73747265616d000000009e000000021a000000656e6372797074696f6e5f6b656b5f66696e6765727072696e740221000000333965346630663935633336393363372c64353564343739383239626332393336021a000000656e6372797074696f6e5f6b65795f66696e6765727072696e740210000000383434316533383533386362666135370215000000656e6372797074696f6e5f6b65795f736f75726365020600000073747265616d0200000000e090663c130600000000000008000000000000c800000000000000096d792d73747265616d000000009e000000021a000000656e6372797074696f6e5f6b656b5f66696e6765727072696e740221000000333965346630663935633336393363372c64353564343739383239626332393336021a000000656e6372797074696f6e5f6b65795f66696e6765727072696e740210000000383434316533383533386362666135370215000000656e6372797074696f6e5f6b65795f736f75726365020600000073747265616d", + "response.get_topic": "0100000000e090663c1306000300000000100000000000006400000000000000086d792d746f7069635b000000020a0000006475726162696c6974790209000000706572736973746564020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000020c0000007365676d656e745f73697a650c080000000000004000000000430000000215000000636f6d7072657373696f6e5f616c676f726974686d02040000006e6f6e65020e0000006d61785f746f7069635f73697a650c08000000ffffffffffffffff0000000000e090663c130600020000006300000000000000000400000000000064000000000000000000000000000000000000000000000000000000000000000100000001e090663c130600010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200000002e090663c13060003000000070000000000000000020000000000000800000000000000000000000000000000000000000000000000000000000000", + "response.get_topics": "020000000100000000e090663c1306000300000000100000000000006400000000000000066576656e74735b000000020a0000006475726162696c6974790209000000706572736973746564020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000020c0000007365676d656e745f73697a650c080000000000004000000000430000000215000000636f6d7072657373696f6e5f616c676f726974686d02040000006e6f6e65020e0000006d61785f746f7069635f73697a650c08000000ffffffffffffffff0200000000e090663c1306000300000000100000000000006400000000000000046c6f67735b000000020a0000006475726162696c6974790209000000706572736973746564020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000020c0000007365676d656e745f73697a650c080000000000004000000000430000000215000000636f6d7072657373696f6e5f616c676f726974686d02040000006e6f6e65020e0000006d61785f746f7069635f73697a650c08000000ffffffffffffffff", "response.get_topics.empty": "00000000", - "response.get_users": "0100000000e090663c130600010561646d696e0000000002000000c8000000000000000205616c69636500000000", + "response.get_users": "0100000000e090663c130600010561646d696e000000000000000002000000c8000000000000000205616c6963650000000000000000", "response.identity": "2a000000", - "response.login_register": "2a0000006400000000000000012c000006302e31312e30", + "response.login_register": "2a00000064000000000000000030000006302e31312e30", "response.partition_route": "07000000000000000a00000000000000030000000400000002", "response.poll_messages": "07000000660000000000000003000000000000000000000000000000000000000000000000000000070000000000000064000000000000008813000000000000e8030000000000006e01000000000000f576a2492e1328b302000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000fa7fe16d19356ec20b00000000000000000000000000000000000000000000000000000001000000000000000000000061b7f159184e9d8ac20c000000000000000000000000000000010000000a0000000c0000000100000000000000000000006202010000006b020100000076070000000000000066000000000000007017000000000000d00700000000000031010000000000006739773bc7d38608010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000009377fb7d5180f51e0d00000000000000000000000000000000000000000000000000000001000000000000000000000063", "response.poll_messages.empty": "00000000000000000000000000000000000000000000000000000000000000000000000000000000", @@ -998,26 +1009,29 @@ "response.send_messages": "0200000001000000020000000300000004000000000000000100000002000000000000002a00000000000000", "response.send_messages.empty": "00000000", "response.stats": "d20400000000cc4100004842000000400000000000000000020000000000000001000000100e00000000000000e090663c13060040420f000000000020a107000000000080841e0000000000030000000a0000001e0000005a00000050c30000000000000500000002000000060000006e6f64652d31050000004c696e757803000000362e3105000000362e312e3006000000302e31312e30002c000001000000010000000100000000000000e80300000000000032000000000000003dcf733f1000000000000000190000000060253c7700000080000000000000000028000000000000", - "response.stream": "0100000000e090663c130600020000000008000000000000c800000000000000096d792d73747265616d00000000", + "response.stream": "0100000000e090663c130600020000000008000000000000c800000000000000096d792d73747265616d000000009e000000021a000000656e6372797074696f6e5f6b656b5f66696e6765727072696e740221000000333965346630663935633336393363372c64353564343739383239626332393336021a000000656e6372797074696f6e5f6b65795f66696e6765727072696e740210000000383434316533383533386362666135370215000000656e6372797074696f6e5f6b65795f736f75726365020600000073747265616d", "response.sync_consumer_group": "070000000000000003000000000000000200000004000000", - "response.topic_header": "0500000000e090663c1306000300000000a0e3d08c00000001000000400000000000100000000000006400000000000000066576656e74733b000000020a0000006475726162696c6974790209000000706572736973746564020c0000007365676d656e745f73697a650c08000000000000400000000020000000020e0000006d61785f746f7069635f73697a650c08000000ffffffffffffffff", - "response.user": "0100000000e090663c130600010561646d696e00000000", - "response.user_details": "0100000000e090663c130600010561646d696e000000000135000000010100010001000101000101000000010100010100010a000000010100010114000000000101010001020000000001010000010000", - "response.user_details.no_permissions": "0100000000e090663c130600010561646d696e0000000000000000", + "response.topic_header": "0500000000e090663c1306000300000000100000000000006400000000000000066576656e74735b000000020a0000006475726162696c6974790209000000706572736973746564020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000020c0000007365676d656e745f73697a650c080000000000004000000000430000000215000000636f6d7072657373696f6e5f616c676f726974686d02040000006e6f6e65020e0000006d61785f746f7069635f73697a650c08000000ffffffffffffffff", + "response.user": "0100000000e090663c130600010561646d696e0000000000000000", + "response.user_details": "0100000000e090663c130600010561646d696e00000000000000000135000000010100010001000101000101000000010100010100010a000000010100010114000000000101010001020000000001010000010000", + "response.user_details.no_permissions": "0100000000e090663c130600010561646d696e000000000000000000000000", + "stream_key.fingerprint": "8441e38538cbfa57", "user_headers.sample": "0207000000617474656d70740b0400000003000000020800000074726163652d69640203000000616263", - "version_info": "012c00000973776966742d73646b05302e312e30", - "vsr.eviction.incompatible_protocol": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000d000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001032547698badcfeefcdab8967452301012c0000012c0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e", + "version_info": "003000000973776966742d73646b05302e312e31", + "vsr.eviction.incompatible_protocol": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000d000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001032547698badcfeefcdab89674523010030000000300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e", "vsr.eviction.stale_client": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000d000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001032547698badcfeefcdab89674523010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000d", "vsr.reply.committed": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c0100000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001032547698badcfeefcdab896745230100000000000000000c0000000000000000000000000000000700000000000000800000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000", "vsr.reply.denied": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000008000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000000000000020000000000000029000000000000000000000000000000000000000000000000000000000000000000000000000000", "vsr.reply.ok": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c0100000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001032547698badcfeefcdab896745230100000000000000000c0000000000000000000000000000000700000000000000800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "vsr.request.create_stream": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007010000000000000000000005000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001032547698badcfeefcdab89674523013a9e6b43adf27bf20000000000000000000000000000000001000000000000008000000000000000630000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "vsr.request.logout": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000005000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001032547698badcfeefcdab8967452301c294d3380580062d0000000000000000000000000000000004000000000000000300000000000000630000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "vsr.reply.update_topic_resolved": "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000040100000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001032547698badcfeefcdab896745230100000000000000000c0000000000000000000000000000000700000000000000490000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000", + "vsr.request.create_stream": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007010000000000000000000005000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001032547698badcfeefcdab896745230100000000000000000000000000000000000000000000000001000000000000008000000000000000630000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "vsr.request.logout": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000005000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001032547698badcfeefcdab896745230100000000000000000000000000000000000000000000000004000000000000000300000000000000630000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "vsr.request.ping": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000005000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001032547698badcfeefcdab896745230100000000000000000000000000000000000000000000000001000000000000000200000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "vsr.request.ping.bound": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000005000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001032547698badcfeefcdab896745230100000000000000000000000000000000000000000000000003000000000000000200000000000000630000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "vsr.request.register": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003010000000000000000000005000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001032547698badcfeefcdab89674523013b73ae32769bceeb0000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "vsr.request.register": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003010000000000000000000005000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001032547698badcfeefcdab896745230100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "vsr.request.send_messages": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004010000000000000000000005000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001032547698badcfeefcdab89674523010000000000000000000000000000000000000000000000000200000000000000a000000000000000630000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "vsr.request.send_messages.fenced": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004010000000000000000000005000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001032547698badcfeefcdab89674523010000000000000000000000000000000000000000000000000200000000000000a000000000000000630000000000000000000000000000001100000000000000090000000000000034000000000000000000000000000000000000000000000000000000000000000000000000000000", - "vsr.request.unknown_code": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000005000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001032547698badcfeefcdab89674523010000000000000000000000000000000000000000000000000500000000000000020000000000000063000000000000000000000060ea00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "vsr.request.unknown_code": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000005000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001032547698badcfeefcdab89674523010000000000000000000000000000000000000000000000000500000000000000020000000000000063000000000000000000000060ea00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "vsr.request.update_resource": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000058010000000000000000000005000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001032547698badcfeefcdab896745230100000000000000000000000000000000000000000000000001000000000000009600000000000000630000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" } }
diff --git a/foreign/swift/Tests/IggyTests/Fixtures/resource_options.json b/foreign/swift/Tests/IggyTests/Fixtures/resource_options.json new file mode 100644 index 0000000..c5cc2ac --- /dev/null +++ b/foreign/swift/Tests/IggyTests/Fixtures/resource_options.json
@@ -0,0 +1,1619 @@ +{ + "schema_version": 1, + "generated_by": "core/common/examples/resource_options_fixtures.rs", + "conventions": { + "byte_strings": "Every *_hex field is lowercase hex without separators.", + "errors": "name is the snake_case IggyError name and code its numeric code.", + "max_topic_size": "Bytes. 18446744073709551615 means unlimited.", + "message_expiry": "Microseconds. 18446744073709551615 means never expire.", + "never_returned_keys": "No response, decoded projection or accessor may contain these keys or their values.", + "option_maps": "Option maps use the HTTP JSON form: value as reversible text, kind, and explicit provenance. Request blocks mark every entry explicit.", + "options_block_order": "Options block entries are encoded in ascending key byte order.", + "request_bodies": "body_hex and update_resource_hex are command payloads without framing. constants.command_codes maps each command to its code and VSR operation.", + "reset_order": "The typed encoders and the HTTP handler write reset keys in ascending key byte order.", + "stream_state": "state is the stored server state a stream response was projected from. SDKs need only response_hex and expected.", + "u64_values": "u64 option values and typed u64 inputs are decimal strings, so values above 2^53 keep their precision." + }, + "constants": { + "protocol_version": { + "packed": 12288, + "text": "0.12.0", + "major": 0, + "minor": 12, + "patch": 0 + }, + "protocol_version_min": { + "packed": 12288, + "text": "0.12.0", + "major": 0, + "minor": 12, + "patch": 0 + }, + "command_codes": { + "create_stream": { + "code": 202, + "operation": 128 + }, + "create_topic": { + "code": 302, + "operation": 132 + }, + "create_user": { + "code": 33, + "operation": 141 + }, + "describe_options": { + "code": 13, + "operation": null + }, + "get_stream": { + "code": 200, + "operation": null + }, + "get_streams": { + "code": 201, + "operation": null + }, + "get_topic": { + "code": 300, + "operation": null + }, + "get_topics": { + "code": 301, + "operation": null + }, + "get_user": { + "code": 31, + "operation": null + }, + "get_users": { + "code": 32, + "operation": null + }, + "update_resource": { + "code": 16, + "operation": 150 + } + }, + "error_codes": { + "cannot_modify_root_user": 4108 + }, + "retired_command_codes": { + "update_stream": 204, + "update_topic": 304, + "update_user": 35 + }, + "operations": { + "rewrap_stream_keys": { + "value": 75, + "internal": true, + "request_operation": 75 + }, + "update_resource": { + "value": 150, + "internal": false, + "request_operation": 150 + }, + "update_stream_resolved": { + "value": 72, + "internal": true, + "request_operation": 150 + }, + "update_topic_resolved": { + "value": 73, + "internal": true, + "request_operation": 150 + }, + "update_user_resolved": { + "value": 74, + "internal": true, + "request_operation": 150 + } + }, + "retired_operations": { + "update_stream": 129, + "update_topic": 133, + "update_user": 142 + }, + "scope_codes": { + "stream": 2, + "topic": 1, + "user": 3 + }, + "header_kind_codes": { + "raw": 1, + "string": 2, + "bool": 3, + "uint32": 11, + "uint64": 12 + }, + "option_flags": { + "critical": 16, + "read_only": 8, + "resettable": 2, + "secret": 4, + "updatable": 1 + }, + "user_status_codes": { + "active": 1, + "inactive": 2 + }, + "stream_key_length": 32, + "never_returned_keys": [ + "encryption_key", + "encryption_key_wrapped", + "critical_keys" + ] + }, + "catalog": [ + { + "key": "name", + "scopes": [ + "stream", + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "column", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 1, + "described": true, + "default": null + }, + { + "key": "username", + "scopes": [ + "user" + ], + "kind": "string", + "kind_code": 2, + "storage": "column", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 1, + "described": true, + "default": null + }, + { + "key": "status", + "scopes": [ + "user" + ], + "kind": "string", + "kind_code": 2, + "storage": "column", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 1, + "described": true, + "default": null + }, + { + "key": "compression_algorithm", + "scopes": [ + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "none" + }, + { + "key": "message_expiry", + "scopes": [ + "topic" + ], + "kind": "uint64", + "kind_code": 12, + "storage": "map", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 3, + "described": true, + "default": "18446744073709551615" + }, + { + "key": "max_topic_size", + "scopes": [ + "topic" + ], + "kind": "uint64", + "kind_code": 12, + "storage": "map", + "mutability": "updatable", + "visibility": "public", + "critical": false, + "flags": 3, + "described": true, + "default": "18446744073709551615" + }, + { + "key": "segment_size", + "scopes": [ + "topic" + ], + "kind": "uint64", + "kind_code": 12, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "1073741824" + }, + { + "key": "durability", + "scopes": [ + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "replicated" + }, + { + "key": "consumer_offset_durability", + "scopes": [ + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "replicated" + }, + { + "key": "partition_resize_policy", + "scopes": [ + "topic" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "mutable" + }, + { + "key": "messages_required_to_save", + "scopes": [ + "topic" + ], + "kind": "uint32", + "kind_code": 11, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "1024" + }, + { + "key": "size_of_messages_required_to_save", + "scopes": [ + "topic" + ], + "kind": "uint64", + "kind_code": 12, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "1048576" + }, + { + "key": "preallocate_segments", + "scopes": [ + "topic" + ], + "kind": "bool", + "kind_code": 3, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": "false" + }, + { + "key": "encryption_key", + "scopes": [ + "stream" + ], + "kind": "raw", + "kind_code": 1, + "storage": "map", + "mutability": "create_only", + "visibility": "secret", + "critical": true, + "flags": 20, + "described": true, + "default": null + }, + { + "key": "encryption_key_wrapped", + "scopes": [ + "stream" + ], + "kind": "raw", + "kind_code": 1, + "storage": "map", + "mutability": "create_only", + "visibility": "internal", + "critical": true, + "flags": 16, + "described": false, + "default": null + }, + { + "key": "encryption_key_source", + "scopes": [ + "stream" + ], + "kind": "string", + "kind_code": 2, + "storage": "computed", + "mutability": "read_only", + "visibility": "read_only", + "critical": false, + "flags": 8, + "described": true, + "default": null + }, + { + "key": "encryption_key_fingerprint", + "scopes": [ + "stream" + ], + "kind": "string", + "kind_code": 2, + "storage": "computed", + "mutability": "read_only", + "visibility": "read_only", + "critical": false, + "flags": 8, + "described": true, + "default": null + }, + { + "key": "encryption_kek_fingerprint", + "scopes": [ + "stream" + ], + "kind": "string", + "kind_code": 2, + "storage": "computed", + "mutability": "read_only", + "visibility": "read_only", + "critical": false, + "flags": 8, + "described": true, + "default": null + }, + { + "key": "critical_keys", + "scopes": [ + "stream", + "topic", + "user" + ], + "kind": "string", + "kind_code": 2, + "storage": "map", + "mutability": "create_only", + "visibility": "internal", + "critical": false, + "flags": 0, + "described": false, + "default": null + } + ], + "describe_options": [ + { + "scope": "topic", + "request_hex": "01", + "response_hex": "0b000000046e616d650201000000007e0000005265736f75726365206e616d652c203120746f203235352062797465732c20756e697175652077697468696e2069747320706172656e742e20536574206279207468652063726561746520636f6d6d616e642773206e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e15636f6d7072657373696f6e5f616c676f726974686d0200040000006e6f6e6550000000436f6d7072657373696f6e20616c676f726974686d20286e6f6e652c20677a6970292e2053746f72656420616e64207265706f727465642c206e6f74206170706c69656420746f206d657373616765730e6d6573736167655f6578706972790c0308000000ffffffffffffffff430000004d6573736167652065787069727920696e206d6963726f7365636f6e64732c206f7220612068756d616e74696d6520737472696e672028652e672e20372064617973290e6d61785f746f7069635f73697a650c0308000000ffffffffffffffff9d000000546f7069632d77696465207365616c65642d7365676d656e742073697a65206361702c2073706c6974206163726f737320616c6c20706172746974696f6e732c20696e206279746573206f72206120627974652d73697a6520737472696e672028652e672e203120476942293b2066696e6974652076616c756573206d757374206265206174206c6561737420746865207365676d656e742073697a650c7365676d656e745f73697a650c000800000000000040000000006c0000005365676d656e742073697a6520696e2062797465732c206f72206120627974652d73697a6520737472696e672028652e672e20313238204d6942293b2061203531322d62797465206d756c7469706c652077697468696e20313034383537362e2e3d313037333734313832340a6475726162696c69747902000a0000007265706c6963617465643e0100004d65737361676520636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20412073696e676c65746f6e2071756f72756d20686173206f6e6520636f70792e20496e207265706c6963617465642067726f7570732c20706572736973746564206d65737361676573207573652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c2057414c207265636c616d6174696f6e2e205468652066756c6c20626f64792073697a65207374696c6c20636f756e747320616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e1a636f6e73756d65725f6f66667365745f6475726162696c69747902000a0000007265706c6963617465646e0100004578706c69636974206f666673657420636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20506f6c6c206175746f2d636f6d6d69742072656d61696e73206173796e6368726f6e6f75732e20496e207265706c6963617465642067726f7570732c20706572736973746564206f66667365747320616c736f20656e61626c652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c207265636c616d6174696f6e2c206576656e2077697468207265706c696361746564206d657373616765206475726162696c6974792e2046756c6c20626f64792073697a657320636f756e7420616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e17706172746974696f6e5f726573697a655f706f6c6963790200070000006d757461626c65c8000000496d6d757461626c65206166746572206372656174696f6e3a206d757461626c65207065726d69747320706172746974696f6e206164646974696f6e2f64656c6574696f6e20776974686f7574206120626f756e646564206375746f766572206f72207065722d6b6579206f72646572696e67206163726f7373206c61796f7574733b2066697865642072656a6563747320626f74682e20526574656e74696f6e20616e6420746f7069632064656c6574696f6e2072656d61696e2064657374727563746976652e196d657373616765735f72657175697265645f746f5f736176650b000400000000040000870000004f7264696e617279206d6573736167652d636f756e7420666c75736820747269676765723b20312e2e3d31363737373231362e2052657175697265642070657273697374656e63652c2063617061636974792070726573737572652c20616e64206c6966656379636c65206f7065726174696f6e732063616e20666c757368206561726c6965722173697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c0008000000000010000000000083000000466c75736820746865206a6f75726e616c206f6e636520697420686f6c64732074686973206d616e792062797465732c206f72206120627974652d73697a6520737472696e673b20776869636865766572207468726573686f6c6420747269707320666972737420666c75736865732e204174206d6f7374203130373337343138323414707265616c6c6f636174655f7365676d656e7473030001000000001201000052657175657374207365676d656e745f73697a65206279746573206f66206469736b207265736572766174696f6e207768656e2065616368207365676d656e74206f70656e732e20556e737570706f72746564206f72206661696c6564207265736572766174696f6e732066616c6c206261636b20746f206f7264696e61727920616c6c6f636174696f6e20776974682061207761726e696e672e205265736572766174696f6e2072756e7320696e6c696e65206f6e20746865206f776e696e672073686172642e207365676d656e745f73697a65202a20706172746974696f6e735f636f756e74206973206361707065642061742036383731393437363733362062797465732070657220637265617465", + "entries": [ + { + "key": "name", + "kind": "string", + "flags": 1, + "default": null + }, + { + "key": "compression_algorithm", + "kind": "string", + "flags": 0, + "default": "none" + }, + { + "key": "message_expiry", + "kind": "uint64", + "flags": 3, + "default": "18446744073709551615" + }, + { + "key": "max_topic_size", + "kind": "uint64", + "flags": 3, + "default": "18446744073709551615" + }, + { + "key": "segment_size", + "kind": "uint64", + "flags": 0, + "default": "1073741824" + }, + { + "key": "durability", + "kind": "string", + "flags": 0, + "default": "replicated" + }, + { + "key": "consumer_offset_durability", + "kind": "string", + "flags": 0, + "default": "replicated" + }, + { + "key": "partition_resize_policy", + "kind": "string", + "flags": 0, + "default": "mutable" + }, + { + "key": "messages_required_to_save", + "kind": "uint32", + "flags": 0, + "default": "1024" + }, + { + "key": "size_of_messages_required_to_save", + "kind": "uint64", + "flags": 0, + "default": "1048576" + }, + { + "key": "preallocate_segments", + "kind": "bool", + "flags": 0, + "default": "false" + } + ] + }, + { + "scope": "stream", + "request_hex": "02", + "response_hex": "05000000046e616d650201000000007e0000005265736f75726365206e616d652c203120746f203235352062797465732c20756e697175652077697468696e2069747320706172656e742e20536574206279207468652063726561746520636f6d6d616e642773206e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e0e656e6372797074696f6e5f6b6579011400000000a30100005468652073747265616d2773206f776e206461746120656e6372797074696f6e206b65793a2033322072616e646f6d2062797465732c20726177206f72207374616e6461726420706164646564206261736536342e204372656174652d6f6e6c7920616e642077726974652d6f6e6c793a20746865207365727665722073746f726573206974207772617070656420756e6465722061206b657920646572697665642066726f6d207468652073657276657220656e6372797074696f6e206b657920616e64206e657665722072657475726e732069742e205265717569726573207468652073657276657220656e6372797074696f6e206b657920746f20626520636f6e666967757265642e20576974686f75742069742c206120736572766572207769746820656e6372797074696f6e20656e61626c65642067656e6572617465732061206b657920666f72207468652073747265616d2e20546865206b65792074726176656c7320696e20746865206372656174652072657175657374206f76657220746865207472616e73706f72742074686520636c69656e7420757365732e15656e6372797074696f6e5f6b65795f736f757263650208000000009f00000057686f2063686f7365207468652073747265616d27732061742d72657374206b65793a2073747265616d207768656e2074686520636c69656e742073656e742069742c20736572766572207768656e20746865207365727665722067656e6572617465642069742c206e6f6e65207768656e207468652073747265616d2773206d65737361676573206172652073746f72656420756e656e637279707465641a656e6372797074696f6e5f6b65795f66696e6765727072696e740208000000009f0000005468652066697273742038206279746573206f66205348412d323536206f76657220746865206c6162656c20696767792f73747265616d2d6b65792d66696e6765727072696e742f763120616e64207468652073747265616d2773206b65792c206173203136206c6f776572636173652068657820636861726163746572732e20416273656e74207768656e2074686520736f75726365206973206e6f6e651a656e6372797074696f6e5f6b656b5f66696e6765727072696e74020800000000b800000046696e6765727072696e7473206f6620746865206b65792d656e6372797074696f6e206b65797320746861742077726170207468652073747265616d2773206b65792c20636f6d6d612d7365706172617465643a206f6e65206f7574736964652061206b657920726f746174696f6e2c206f6e652070657220636f6e6669677572656420736572766572206b657920647572696e672069742e20416273656e74207768656e2074686520736f75726365206973206e6f6e65", + "entries": [ + { + "key": "name", + "kind": "string", + "flags": 1, + "default": null + }, + { + "key": "encryption_key", + "kind": "raw", + "flags": 20, + "default": null + }, + { + "key": "encryption_key_source", + "kind": "string", + "flags": 8, + "default": null + }, + { + "key": "encryption_key_fingerprint", + "kind": "string", + "flags": 8, + "default": null + }, + { + "key": "encryption_kek_fingerprint", + "kind": "string", + "flags": 8, + "default": null + } + ] + }, + { + "scope": "user", + "request_hex": "03", + "response_hex": "0200000008757365726e616d650201000000006b00000055736572206e616d652c203320746f2035302062797465732c20756e697175652e20536574206279207468652063726561746520636f6d6d616e64277320757365726e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e067374617475730201000000008f00000055736572207374617475732c20616374697665206f7220696e6163746976652e20536574206279207468652063726561746520636f6d6d616e64277320737461747573206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e20446561637469766174696e672061207573657220656e6473206974732073657373696f6e732e", + "entries": [ + { + "key": "username", + "kind": "string", + "flags": 1, + "default": null + }, + { + "key": "status", + "kind": "string", + "flags": 1, + "default": null + } + ] + } + ], + "stream_key": { + "key_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "key_base64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", + "fingerprint_label": "iggy/stream-key-fingerprint/v1", + "fingerprint": "8441e38538cbfa57", + "invalid_keys": [ + { + "case": "raw_31_bytes", + "input": { + "raw_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + }, + { + "case": "raw_33_bytes", + "input": { + "raw_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + }, + { + "case": "base64_with_whitespace", + "input": { + "base64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=\n" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + }, + { + "case": "base64_missing_padding", + "input": { + "base64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + }, + { + "case": "base64_noncanonical_trailing_bits", + "input": { + "base64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh9=" + }, + "error": { + "name": "invalid_encryption_key", + "code": 70 + } + } + ] + }, + "update_resource": [ + { + "case": "topic_set_expiry_without_name_reset_max_topic_size", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "message_expiry": "604800000000", + "reset": [ + "max_topic_size" + ] + }, + "wire_set": { + "message_expiry": { + "value": "604800000000", + "kind": "uint64", + "explicit": true + } + }, + "wire_reset": [ + "max_topic_size" + ], + "body_hex": "01010401000000010402000000010e6d61785f746f7069635f73697a65020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000" + }, + { + "case": "topic_rename_only_named_identifiers", + "scope": "topic", + "target": { + "stream_id": "orders", + "topic_id": "events" + }, + "input": { + "name": "events-v2" + }, + "wire_set": { + "name": { + "value": "events-v2", + "kind": "string", + "explicit": true + } + }, + "wire_reset": [], + "body_hex": "0102066f726465727302066576656e74730002040000006e616d6502090000006576656e74732d7632" + }, + { + "case": "topic_rename_set_expiry_reset_max_topic_size", + "scope": "topic", + "target": { + "stream_id": 7, + "topic_id": "events" + }, + "input": { + "name": "events-v3", + "message_expiry": "86400000000", + "reset": [ + "max_topic_size" + ] + }, + "wire_set": { + "message_expiry": { + "value": "86400000000", + "kind": "uint64", + "explicit": true + }, + "name": { + "value": "events-v3", + "kind": "string", + "explicit": true + } + }, + "wire_reset": [ + "max_topic_size" + ], + "body_hex": "0101040700000002066576656e7473010e6d61785f746f7069635f73697a65020e0000006d6573736167655f6578706972790c080000000060d71d1400000002040000006e616d6502090000006576656e74732d7633" + }, + { + "case": "topic_reset_both_defaults", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "reset": [ + "max_topic_size", + "message_expiry" + ] + }, + "wire_set": {}, + "wire_reset": [ + "max_topic_size", + "message_expiry" + ], + "body_hex": "01010401000000010402000000020e6d61785f746f7069635f73697a650e6d6573736167655f657870697279" + }, + { + "case": "topic_max_topic_size_above_2_pow_53", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "max_topic_size": "9007199254740993" + }, + "wire_set": { + "max_topic_size": { + "value": "9007199254740993", + "kind": "uint64", + "explicit": true + } + }, + "wire_reset": [], + "body_hex": "0101040100000001040200000000020e0000006d61785f746f7069635f73697a650c080000000100000000002000" + }, + { + "case": "stream_rename", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "input": { + "name": "payments-v2" + }, + "wire_set": { + "name": { + "value": "payments-v2", + "kind": "string", + "explicit": true + } + }, + "wire_reset": [], + "body_hex": "020104030000000002040000006e616d65020b0000007061796d656e74732d7632" + }, + { + "case": "user_username_and_status", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "username": "alice2", + "status": "inactive" + }, + "wire_set": { + "status": { + "value": "inactive", + "kind": "string", + "explicit": true + }, + "username": { + "value": "alice2", + "kind": "string", + "explicit": true + } + }, + "wire_reset": [], + "body_hex": "030104050000000002060000007374617475730208000000696e6163746976650208000000757365726e616d650206000000616c69636532" + } + ], + "update_resource_errors": [ + { + "case": "topic_empty_update", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": {}, + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_set_and_reset_same_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "message_expiry": "604800000000", + "reset": [ + "message_expiry" + ] + }, + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_reset_create_only_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "reset": [ + "compression_algorithm" + ] + }, + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "stream_reset_name", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "input": { + "reset": [ + "name" + ] + }, + "error": { + "name": "option_not_resettable", + "code": 4104 + } + }, + { + "case": "user_reset_status", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "reset": [ + "status" + ] + }, + "error": { + "name": "option_not_resettable", + "code": 4104 + } + }, + { + "case": "stream_reset_read_only_key", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "input": { + "reset": [ + "encryption_key_source" + ] + }, + "error": { + "name": "read_only_option", + "code": 4105 + } + }, + { + "case": "user_reset_internal_key", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "reset": [ + "critical_keys" + ] + }, + "error": { + "name": "read_only_option", + "code": 4105 + } + }, + { + "case": "topic_empty_name", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "name": "" + }, + "error": { + "name": "invalid_topic_name", + "code": 2014 + } + }, + { + "case": "topic_empty_name_and_create_only_reset", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "input": { + "name": "", + "reset": [ + "segment_size" + ] + }, + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "stream_empty_name", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "input": { + "name": "" + }, + "error": { + "name": "invalid_stream_name", + "code": 1013 + } + }, + { + "case": "user_short_username", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "username": "a" + }, + "error": { + "name": "invalid_username", + "code": 43 + } + }, + { + "case": "user_short_username_and_status_reset", + "scope": "user", + "target": { + "user_id": 5 + }, + "input": { + "username": "a", + "reset": [ + "status" + ] + }, + "error": { + "name": "option_not_resettable", + "code": 4104 + } + } + ], + "create_stream": [ + { + "case": "stream_without_key", + "input": { + "name": "orders" + }, + "wire_options": {}, + "body_hex": "066f7264657273", + "http_body": "{\"name\":\"orders\"}", + "admission_error": null + }, + { + "case": "stream_with_key", + "input": { + "name": "orders", + "encryption_key": { + "raw_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f" + } + }, + "wire_options": { + "encryption_key": { + "value": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", + "kind": "raw", + "explicit": true + } + }, + "body_hex": "066f7264657273020e000000656e6372797074696f6e5f6b65790120000000000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "http_body": "{\"name\":\"orders\",\"options\":{\"encryption_key\":\"AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=\"}}", + "admission_error": null + }, + { + "case": "stream_with_raw_extra_key", + "input": { + "name": "orders", + "raw": { + "future_option": "enabled" + } + }, + "wire_options": { + "future_option": { + "value": "enabled", + "kind": "string", + "explicit": true + } + }, + "body_hex": "066f7264657273020d0000006675747572655f6f7074696f6e0207000000656e61626c6564", + "http_body": "{\"name\":\"orders\",\"options\":{\"future_option\":\"enabled\"}}", + "admission_error": { + "name": "unsupported_option_key", + "code": 4041 + } + } + ], + "create_topic": [ + { + "case": "topic_with_options", + "input": { + "stream_id": 1, + "name": "events", + "partitions_count": 3, + "compression_algorithm": "gzip", + "message_expiry": "604800000000", + "max_topic_size": "9007199254740993", + "segment_size": "134217728", + "durability": "persisted", + "consumer_offset_durability": "replicated" + }, + "wire_options": { + "compression_algorithm": { + "value": "gzip", + "kind": "string", + "explicit": true + }, + "consumer_offset_durability": { + "value": "replicated", + "kind": "string", + "explicit": true + }, + "durability": { + "value": "persisted", + "kind": "string", + "explicit": true + }, + "max_topic_size": { + "value": "9007199254740993", + "kind": "uint64", + "explicit": true + }, + "message_expiry": { + "value": "604800000000", + "kind": "uint64", + "explicit": true + }, + "segment_size": { + "value": "134217728", + "kind": "uint64", + "explicit": true + } + }, + "body_hex": "01040100000003000000066576656e74730215000000636f6d7072657373696f6e5f616c676f726974686d0204000000677a6970021a000000636f6e73756d65725f6f66667365745f6475726162696c697479020a0000007265706c696361746564020a0000006475726162696c6974790209000000706572736973746564020e0000006d61785f746f7069635f73697a650c080000000100000000002000020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000020c0000007365676d656e745f73697a650c080000000000000800000000", + "http_body": "{\"partitions_count\":3,\"name\":\"events\",\"options\":{\"compression_algorithm\":\"gzip\",\"consumer_offset_durability\":\"replicated\",\"durability\":\"persisted\",\"max_topic_size\":\"9007199254740993\",\"message_expiry\":604800000000,\"segment_size\":134217728}}", + "admission_error": null + } + ], + "create_user": [ + { + "case": "user_with_empty_options", + "input": { + "username": "alice", + "password": "s3cret-pass", + "status": "active" + }, + "wire_options": {}, + "body_hex": "05616c6963650b7333637265742d706173730100", + "http_body": "{\"username\":\"alice\",\"password\":\"s3cret-pass\",\"status\":\"active\",\"permissions\":null}", + "admission_error": null + } + ], + "http_update": [ + { + "case": "topic_flat_keys", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"name\":\"events-v2\",\"message_expiry\":\"7 days\"}", + "update_resource_hex": "0101040100000001040200000000020e0000006d6573736167655f657870697279020600000037206461797302040000006e616d6502090000006576656e74732d7632", + "outcome": "accepted", + "error": null + }, + { + "case": "topic_reset_array", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":604800000000,\"reset\":[\"max_topic_size\"]}", + "update_resource_hex": "01010401000000010402000000010e6d61785f746f7069635f73697a65020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000", + "outcome": "accepted", + "error": null + }, + { + "case": "topic_null_ignored", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"name\":null,\"max_topic_size\":null,\"message_expiry\":\"1 day\"}", + "update_resource_hex": "0101040100000001040200000000020e0000006d6573736167655f65787069727902050000003120646179", + "outcome": "accepted", + "error": null + }, + { + "case": "topic_u64_above_2_pow_53_as_string", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"max_topic_size\":\"9007199254740993\"}", + "update_resource_hex": "0101040100000001040200000000020e0000006d61785f746f7069635f73697a65021000000039303037313939323534373430393933", + "outcome": "accepted", + "error": null + }, + { + "case": "stream_rename", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "body": "{\"name\":\"payments-v2\"}", + "update_resource_hex": "020104030000000002040000006e616d65020b0000007061796d656e74732d7632", + "outcome": "accepted", + "error": null + }, + { + "case": "user_username_and_status", + "scope": "user", + "target": { + "user_id": 5 + }, + "body": "{\"username\":\"alice2\",\"status\":\"inactive\"}", + "update_resource_hex": "030104050000000002060000007374617475730208000000696e6163746976650208000000757365726e616d650206000000616c69636532", + "outcome": "accepted", + "error": null + }, + { + "case": "topic_duplicate_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":\"1 day\",\"message_expiry\":\"2 days\"}", + "update_resource_hex": null, + "outcome": "json_refused", + "error": null + }, + { + "case": "topic_nested_object", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":{\"value\":\"1 day\"}}", + "update_resource_hex": null, + "outcome": "json_refused", + "error": null + }, + { + "case": "topic_null_only", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"max_topic_size\":null}", + "update_resource_hex": "0101040100000001040200000000", + "outcome": "decode_refused", + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_zero_sentinel_only", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":0}", + "update_resource_hex": "0101040100000001040200000000020e0000006d6573736167655f6578706972790c080000000000000000000000", + "outcome": "admission_refused", + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_reset_names_set_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":\"1 day\",\"reset\":[\"message_expiry\"]}", + "update_resource_hex": "01010401000000010402000000010e6d6573736167655f657870697279020e0000006d6573736167655f65787069727902050000003120646179", + "outcome": "decode_refused", + "error": { + "name": "invalid_command", + "code": 3 + } + }, + { + "case": "topic_unknown_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"retention\":\"1 day\"}", + "update_resource_hex": "01010401000000010402000000000209000000726574656e74696f6e02050000003120646179", + "outcome": "admission_refused", + "error": { + "name": "unsupported_option_key", + "code": 4041 + } + }, + { + "case": "topic_set_create_only_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"compression_algorithm\":\"gzip\"}", + "update_resource_hex": "01010401000000010402000000000215000000636f6d7072657373696f6e5f616c676f726974686d0204000000677a6970", + "outcome": "admission_refused", + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "topic_reset_create_only_key", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"reset\":[\"compression_algorithm\"]}", + "update_resource_hex": "010104010000000104020000000115636f6d7072657373696f6e5f616c676f726974686d", + "outcome": "admission_refused", + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "stream_reset_name", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "body": "{\"reset\":[\"name\"]}", + "update_resource_hex": "0201040300000001046e616d65", + "outcome": "admission_refused", + "error": { + "name": "option_not_resettable", + "code": 4104 + } + }, + { + "case": "topic_name_as_number", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"name\":5}", + "update_resource_hex": "010104010000000104020000000002040000006e616d650c080000000500000000000000", + "outcome": "admission_refused", + "error": { + "name": "invalid_topic_name", + "code": 2014 + } + }, + { + "case": "stream_empty_name", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "body": "{\"name\":\"\"}", + "update_resource_hex": "020104030000000002040000006e616d65030100000000", + "outcome": "admission_refused", + "error": { + "name": "invalid_stream_name", + "code": 1013 + } + }, + { + "case": "user_status_as_bool", + "scope": "user", + "target": { + "user_id": 5 + }, + "body": "{\"status\":true}", + "update_resource_hex": "03010405000000000206000000737461747573030100000001", + "outcome": "admission_refused", + "error": { + "name": "invalid_user_status", + "code": 45 + } + }, + { + "case": "topic_create_only_key_as_bool", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"segment_size\":true}", + "update_resource_hex": "0101040100000001040200000000020c0000007365676d656e745f73697a65030100000001", + "outcome": "admission_refused", + "error": { + "name": "option_not_updatable", + "code": 4103 + } + }, + { + "case": "stream_read_only_key_as_number", + "scope": "stream", + "target": { + "stream_id": 3 + }, + "body": "{\"encryption_key_source\":1}", + "update_resource_hex": "02010403000000000215000000656e6372797074696f6e5f6b65795f736f757263650c080000000100000000000000", + "outcome": "admission_refused", + "error": { + "name": "read_only_option", + "code": 4105 + } + }, + { + "case": "topic_invalid_value", + "scope": "topic", + "target": { + "stream_id": 1, + "topic_id": 2 + }, + "body": "{\"message_expiry\":\"soon\"}", + "update_resource_hex": "0101040100000001040200000000020e0000006d6573736167655f6578706972790204000000736f6f6e", + "outcome": "admission_refused", + "error": { + "name": "invalid_option_value", + "code": 4042 + } + } + ], + "stream_responses": [ + { + "case": "stream_source_none", + "state": { + "encryption_key_source": "none", + "stored_extra": {} + }, + "response_hex": "0100000000e090663c1306000000000000100000000000000c000000000000000c706c61696e2d73747265616d00000000230000000215000000656e6372797074696f6e5f6b65795f736f7572636502040000006e6f6e65", + "expected": { + "id": 1, + "created_at": 1710000000000000, + "name": "plain-stream", + "topics_count": 0, + "size_bytes": 4096, + "messages_count": 12, + "options": { + "encryption_key_source": { + "value": "none", + "kind": "string", + "explicit": false + } + }, + "encryption_key_source": "none", + "encryption_key_fingerprint": null, + "encryption_kek_fingerprints": [] + } + }, + { + "case": "stream_source_stream_with_unknown_explicit_key", + "state": { + "encryption_key_source": "stream", + "data_key": { + "raw_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f" + }, + "kek_config_keys": [ + "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=" + ], + "stored_extra": { + "future_option": { + "value": "enabled", + "kind": "string", + "explicit": true + } + } + }, + "response_hex": "0200000000e090663c1306000000000000100000000000000c0000000000000011636c69656e742d6b65792d73747265616d1e000000020d0000006675747572655f6f7074696f6e0207000000656e61626c65648d000000021a000000656e6372797074696f6e5f6b656b5f66696e6765727072696e74021000000033396534663066393563333639336337021a000000656e6372797074696f6e5f6b65795f66696e6765727072696e740210000000383434316533383533386362666135370215000000656e6372797074696f6e5f6b65795f736f75726365020600000073747265616d", + "expected": { + "id": 2, + "created_at": 1710000000000000, + "name": "client-key-stream", + "topics_count": 0, + "size_bytes": 4096, + "messages_count": 12, + "options": { + "encryption_kek_fingerprint": { + "value": "39e4f0f95c3693c7", + "kind": "string", + "explicit": false + }, + "encryption_key_fingerprint": { + "value": "8441e38538cbfa57", + "kind": "string", + "explicit": false + }, + "encryption_key_source": { + "value": "stream", + "kind": "string", + "explicit": false + }, + "future_option": { + "value": "enabled", + "kind": "string", + "explicit": true + } + }, + "encryption_key_source": "stream", + "encryption_key_fingerprint": "8441e38538cbfa57", + "encryption_kek_fingerprints": [ + "39e4f0f95c3693c7" + ] + } + }, + { + "case": "stream_source_server_two_keks", + "state": { + "encryption_key_source": "server", + "data_key": { + "raw_hex": "202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f" + }, + "kek_config_keys": [ + "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=", + "AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA=" + ], + "stored_extra": {} + }, + "response_hex": "0300000000e090663c1306000000000000100000000000000c00000000000000117365727665722d6b65792d73747265616d000000009e000000021a000000656e6372797074696f6e5f6b656b5f66696e6765727072696e740221000000333965346630663935633336393363372c64353564343739383239626332393336021a000000656e6372797074696f6e5f6b65795f66696e6765727072696e740210000000306563616666316261323931666530640215000000656e6372797074696f6e5f6b65795f736f757263650206000000736572766572", + "expected": { + "id": 3, + "created_at": 1710000000000000, + "name": "server-key-stream", + "topics_count": 0, + "size_bytes": 4096, + "messages_count": 12, + "options": { + "encryption_kek_fingerprint": { + "value": "39e4f0f95c3693c7,d55d479829bc2936", + "kind": "string", + "explicit": false + }, + "encryption_key_fingerprint": { + "value": "0ecaff1ba291fe0d", + "kind": "string", + "explicit": false + }, + "encryption_key_source": { + "value": "server", + "kind": "string", + "explicit": false + } + }, + "encryption_key_source": "server", + "encryption_key_fingerprint": "0ecaff1ba291fe0d", + "encryption_kek_fingerprints": [ + "39e4f0f95c3693c7", + "d55d479829bc2936" + ] + } + } + ], + "topic_responses": [ + { + "case": "topic_explicit_and_derived", + "response_hex": "0400000000e090663c1306000000000000080000000000000700000000000000066576656e7473430000000215000000636f6d7072657373696f6e5f616c676f726974686d0204000000677a6970020e0000006d6573736167655f6578706972790c0800000000a0e3d08c0000002b010000021a000000636f6e73756d65725f6f66667365745f6475726162696c697479020a0000007265706c696361746564020a0000006475726162696c697479020a0000007265706c696361746564020e0000006d61785f746f7069635f73697a650c08000000ffffffffffffffff02190000006d657373616765735f72657175697265645f746f5f736176650b04000000000400000217000000706172746974696f6e5f726573697a655f706f6c69637902070000006d757461626c650214000000707265616c6c6f636174655f7365676d656e7473030100000000020c0000007365676d656e745f73697a650c080000000000004000000000022100000073697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c080000000000100000000000", + "expected": { + "id": 4, + "created_at": 1710000000000000, + "name": "events", + "partitions_count": 0, + "size_bytes": 2048, + "messages_count": 7, + "options": { + "compression_algorithm": { + "value": "gzip", + "kind": "string", + "explicit": true + }, + "consumer_offset_durability": { + "value": "replicated", + "kind": "string", + "explicit": false + }, + "durability": { + "value": "replicated", + "kind": "string", + "explicit": false + }, + "max_topic_size": { + "value": "18446744073709551615", + "kind": "uint64", + "explicit": false + }, + "message_expiry": { + "value": "604800000000", + "kind": "uint64", + "explicit": true + }, + "messages_required_to_save": { + "value": "1024", + "kind": "uint32", + "explicit": false + }, + "partition_resize_policy": { + "value": "mutable", + "kind": "string", + "explicit": false + }, + "preallocate_segments": { + "value": "false", + "kind": "bool", + "explicit": false + }, + "segment_size": { + "value": "1073741824", + "kind": "uint64", + "explicit": false + }, + "size_of_messages_required_to_save": { + "value": "1048576", + "kind": "uint64", + "explicit": false + } + }, + "compression_algorithm": "gzip", + "message_expiry": "604800000000", + "max_topic_size": "18446744073709551615" + } + } + ], + "user_responses": [ + { + "case": "user_empty_blocks", + "response_hex": "0500000000e090663c1306000105616c696365000000000000000000000000", + "expected": { + "id": 5, + "created_at": 1710000000000000, + "status": "active", + "username": "alice", + "options": {} + } + }, + { + "case": "user_populated_blocks", + "response_hex": "0600000000e090663c1306000203626f621e000000020d0000006675747572655f6f7074696f6e0207000000656e61626c65641e000000020c0000006675747572655f71756f74610c08000000010000000000200000000000", + "expected": { + "id": 6, + "created_at": 1710000000000000, + "status": "inactive", + "username": "bob", + "options": { + "future_option": { + "value": "enabled", + "kind": "string", + "explicit": true + }, + "future_quota": { + "value": "9007199254740993", + "kind": "uint64", + "explicit": false + } + } + } + } + ] +}
diff --git a/foreign/swift/Tests/IggyTests/ModelTests.swift b/foreign/swift/Tests/IggyTests/ModelTests.swift index 08b09dc..e6a5bbc 100644 --- a/foreign/swift/Tests/IggyTests/ModelTests.swift +++ b/foreign/swift/Tests/IggyTests/ModelTests.swift
@@ -373,19 +373,19 @@ struct TopicOptionsTests { @Test func durabilityIsAlwaysSentAndRawMayStrengthenIt() throws { let defaults = try TopicCreateOptions().toResourceOptions() - #expect(defaults[TopicOptionKey.durability] == .explicit("replicated")) - #expect(defaults[TopicOptionKey.consumerOffsetDurability] == .explicit("replicated")) - let raw = try TopicCreateOptions(raw: [TopicOptionKey.durability: "persisted"]).toResourceOptions() - #expect(raw[TopicOptionKey.durability] == .explicit("persisted")) + #expect(defaults[TopicOptionKey.durability.rawValue] == .explicit("replicated")) + #expect(defaults[TopicOptionKey.consumerOffsetDurability.rawValue] == .explicit("replicated")) + let raw = try TopicCreateOptions(raw: [TopicOptionKey.durability.rawValue: "persisted"]).toResourceOptions() + #expect(raw[TopicOptionKey.durability.rawValue] == .explicit("persisted")) // The server parses the policy case-insensitively; the canonical // spelling is what travels. - let spelled = try TopicCreateOptions(raw: [TopicOptionKey.durability: "Persisted"]).toResourceOptions() - #expect(spelled[TopicOptionKey.durability] == .explicit("persisted")) + let spelled = try TopicCreateOptions(raw: [TopicOptionKey.durability.rawValue: "Persisted"]).toResourceOptions() + #expect(spelled[TopicOptionKey.durability.rawValue] == .explicit("persisted")) #expect(throws: IggyError.self) { - try TopicCreateOptions(durability: .persisted, raw: [TopicOptionKey.durability: "replicated"]).toResourceOptions() + try TopicCreateOptions(durability: .persisted, raw: [TopicOptionKey.durability.rawValue: "replicated"]).toResourceOptions() } #expect(throws: IggyError.self) { - try TopicCreateOptions(raw: [TopicOptionKey.durability: "sometimes"]).toResourceOptions() + try TopicCreateOptions(raw: [TopicOptionKey.durability.rawValue: "sometimes"]).toResourceOptions() } } @@ -394,13 +394,17 @@ @Test func expiryMustBePositive() throws { #expect(throws: IggyError(.invalidOptionValue)) { try TopicCreateOptions(messageExpiry: .after(.seconds(-5))).encode() } #expect(throws: IggyError(.invalidOptionValue)) { try TopicCreateOptions(messageExpiry: .after(.zero)).encode() } - #expect(throws: IggyError(.invalidOptionValue)) { try TopicUpdateOptions(messageExpiry: .after(.milliseconds(-1))).encode() } + #expect(throws: IggyError(.invalidOptionValue)) { try TopicUpdateOptions(messageExpiry: .after(.milliseconds(-1))).toPatch() } #expect(throws: IggyError(.invalidOptionValue)) { try TopicCreateOptions(messageExpiry: .after(.nanoseconds(500))).encode() } #expect(throws: IggyError(.invalidOptionValue)) { try TopicCreateOptions(messageExpiry: .after(.seconds(1 << 58))).encode() } let options = try TopicCreateOptions(messageExpiry: .after(.seconds(1))).toResourceOptions() - #expect(options[TopicOptionKey.messageExpiry] == .explicit(.uint64(1_000_000))) - #expect(try TopicCreateOptions(messageExpiry: .after(.microseconds(1))).toResourceOptions()[TopicOptionKey.messageExpiry] == .explicit(.uint64(1))) - #expect(try TopicCreateOptions(messageExpiry: .never).toResourceOptions()[TopicOptionKey.messageExpiry] == .explicit(.uint64(UInt64.max))) - #expect(try TopicCreateOptions(messageExpiry: .serverDefault).toResourceOptions()[TopicOptionKey.messageExpiry] == nil) + let key = TopicOptionKey.messageExpiry.rawValue + #expect(options[key] == .explicit(.uint64(1_000_000))) + #expect(try TopicCreateOptions(messageExpiry: .after(.microseconds(1))).toResourceOptions()[key] == .explicit(.uint64(1))) + #expect(try TopicCreateOptions(messageExpiry: .never).toResourceOptions()[key] == .explicit(.uint64(UInt64.max))) + #expect(try TopicCreateOptions(messageExpiry: .serverDefault).toResourceOptions()[key] == nil) + // On an update the server default changes nothing. A reset restores the default. + #expect( + try TopicUpdateOptions(name: "renamed", messageExpiry: .serverDefault, maxTopicSize: .serverDefault).toResourceOptions().keys.sorted() == ["name"]) } }
diff --git a/foreign/swift/Tests/IggyTests/ModelVectorTests.swift b/foreign/swift/Tests/IggyTests/ModelVectorTests.swift index 3b406a2..2511c81 100644 --- a/foreign/swift/Tests/IggyTests/ModelVectorTests.swift +++ b/foreign/swift/Tests/IggyTests/ModelVectorTests.swift
@@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +import Foundation import Testing @testable import Iggy @@ -100,6 +101,30 @@ #expect(try OptionsBlock.encode(options) == golden.bytes("options.golden")) } + /// The cross-SDK block Rust pins by hand in + /// `core/binary_protocol/src/primitives/options.rs`, as do the Node, + /// Java, Go and C# SDKs. It is in insertion order, not key order, and + /// decoders accept either. + @Test func goldenOptionsBlock() throws { + let preallocate: [UInt8] = [2, 20, 0, 0, 0] + [UInt8]("preallocate_segments".utf8) + [3, 1, 0, 0, 0, 1] + let segmentSize: [UInt8] = [2, 12, 0, 0, 0] + [UInt8]("segment_size".utf8) + [12, 8, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0] + let durabilityKey: [UInt8] = [2, 10, 0, 0, 0] + [UInt8]("durability".utf8) + let durabilityValue: [UInt8] = [2, 9, 0, 0, 0] + [UInt8]("persisted".utf8) + let block: [UInt8] = preallocate + segmentSize + durabilityKey + durabilityValue + #expect(block == golden.bytes("options.golden_block")) + let options = try OptionsBlock.decode(block[...], explicit: true) + #expect( + options == [ + "preallocate_segments": .explicit(.bool(true)), "segment_size": .explicit(.uint64(1_073_741_824)), "durability": .explicit("persisted"), + ]) + } + + @Test func streamKey() throws { + let key = Data((0..<UInt8(EncryptionKey.length)).map { $0 }) + #expect(try StreamCreateOptions(encryptionKey: key).encode() == golden.bytes("options.stream_create.key")) + #expect(try EncryptionKey.fingerprint(key) == golden.vectors["stream_key.fingerprint"]) + } + @Test func topicOptions() throws { let create = TopicCreateOptions( partitionsCount: 3, compressionAlgorithm: .gzip, messageExpiry: .after(.seconds(604_800)), @@ -107,8 +132,8 @@ consumerOffsetDurability: .replicated, messagesRequiredToSave: 7, sizeOfMessagesRequiredToSaveBytes: 4096, preallocateSegments: false) #expect(try create.encode() == golden.bytes("options.topic_create.full")) - let update = TopicUpdateOptions(compressionAlgorithm: CompressionAlgorithm.none, messageExpiry: .never, maxTopicSize: .unlimited) - #expect(try update.encode() == golden.bytes("options.topic_update.full")) + let update = TopicUpdateOptions(messageExpiry: .never, maxTopicSize: .unlimited) + #expect(try OptionsBlock.encode(update.toPatch().set) == golden.bytes("options.topic_update.full")) let raw = TopicCreateOptions(raw: ["durability": "persisted"]) #expect(try raw.encode() == golden.bytes("options.topic_create.raw")) }
diff --git a/foreign/swift/Tests/IggyTests/ProtocolTests.swift b/foreign/swift/Tests/IggyTests/ProtocolTests.swift index 11ad63b..d8d963e 100644 --- a/foreign/swift/Tests/IggyTests/ProtocolTests.swift +++ b/foreign/swift/Tests/IggyTests/ProtocolTests.swift
@@ -71,6 +71,19 @@ #expect(!session.isBound) } + @Test func metadataRequestIdentifiersAreUniqueAndRetainedInRetryFrame() throws { + var session = ConsensusSession(clientID: 7) + try session.bind(42) + let payload: [UInt8] = [1, 2, 3] + let first = try VSRFrame.encodeRequest(session: &session, code: CommandCode.createStream.rawValue, payload: payload) + let second = try VSRFrame.encodeRequest(session: &session, code: CommandCode.createStream.rawValue, payload: payload) + let range = VSRFrame.RequestOffset.requestChecksum..<VSRFrame.RequestOffset.requestChecksum + 16 + #expect(Array(first.bytes[range]) != Array(repeating: 0, count: 16)) + #expect(Array(first.bytes[range]) != Array(second.bytes[range])) + let retry = first + #expect(Array(retry.bytes[range]) == Array(first.bytes[range])) + } + /// A reconnect replays the Register on a bound session and binds to the /// fence the server returns again, without losing the request watermark. @Test func reconnectPreservesBindingAndRequestCounter() throws { @@ -112,11 +125,11 @@ @Suite("Command codes and operations") struct CommandCodeTests { @Test func retiredCodesStayUnknown() { - for code: UInt32 in [14, 102, 103, 104, 123, 124, 205, 305] { + for code: UInt32 in [14, 35, 102, 103, 104, 123, 124, 204, 205, 304, 305] { #expect(CommandCode(rawValue: code) == nil) #expect(ConsensusOperation.forCode(code) == .nonReplicated) } - for operation: UInt8 in [131, 135, 163, 164, 165] { + for operation: UInt8 in [129, 131, 133, 135, 142, 163, 164, 165] { #expect(ConsensusOperation(rawValue: operation) == nil) } } @@ -142,12 +155,17 @@ #expect(!operation.isMetadata) #expect(!operation.isResultFramed) } - for operation in [ConsensusOperation.finalizeSession, .completeLifecycle, .truncatePartition] { + for operation in [ + ConsensusOperation.finalizeSession, .completeLifecycle, .truncatePartition, .updateStreamResolved, .updateTopicResolved, + .updateUserResolved, .rewrapStreamKeys, + ] { #expect(operation.isInternal) #expect(operation.isMetadata) #expect(!operation.isPartition) } - for operation in [ConsensusOperation.sendMessages, .storeConsumerOffset, .createStream, .register] { + #expect(ConsensusOperation.forCode(CommandCode.updateResource.rawValue) == .updateResource) + #expect(ConsensusOperation.updateResource.isMetadata && ConsensusOperation.updateResource.isResultFramed) + for operation in [ConsensusOperation.sendMessages, .storeConsumerOffset, .createStream, .register, .updateResource] { #expect(!operation.isInternal) } }
diff --git a/foreign/swift/Tests/IggyTests/ResourceOptionsTests.swift b/foreign/swift/Tests/IggyTests/ResourceOptionsTests.swift new file mode 100644 index 0000000..c723df7 --- /dev/null +++ b/foreign/swift/Tests/IggyTests/ResourceOptionsTests.swift
@@ -0,0 +1,586 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation +import Testing + +@testable import Iggy + +/// The resource options contract every SDK shares, checked against +/// `Fixtures/resource_options.json` from the Rust encoders. The Swift encoder +/// and decoders are independent, so these bytes are the contract. +@Suite("Resource options match the shared fixture", .serialized) +struct ResourceOptionsTests { + let fixture = ResourceOptionsFixture.shared + + @Test func givenInvalidKeyWhenEncodingShouldRejectBeforeNetworkIo() async throws { + let key = Data([UInt8](hex: fixture.streamKey.keyHex)) + #expect(key.count == EncryptionKey.length) + #expect(try EncryptionKey.decode(base64: fixture.streamKey.keyBase64) == key) + #expect(try StreamCreateOptions(encryptionKey: key).encode().isEmpty == false) + + for invalid in fixture.streamKey.invalidKeys { + #expect(invalid.error.code == IggyErrorCode.invalidEncryptionKey.rawValue, "\(invalid.name)") + if let rawHex = invalid.input.rawHex { + let raw = Data([UInt8](hex: rawHex)) + #expect(throws: IggyError(.invalidEncryptionKey), "\(invalid.name)") { try StreamCreateOptions(encryptionKey: raw).encode() } + #expect(throws: IggyError(.invalidEncryptionKey), "\(invalid.name)") { try EncryptionKey.fingerprint(raw) } + } + if let base64 = invalid.input.base64 { + #expect(throws: IggyError(.invalidEncryptionKey), "\(invalid.name)") { try EncryptionKey.decode(base64: base64) } + #expect(throws: IggyError(.invalidEncryptionKey), "\(invalid.name)") { try EncryptionKey.fingerprint(base64: base64) } + } + } + #expect(throws: IggyError(.invalidEncryptionKey)) { try EncryptionKey.decode(base64: "") } + #expect(throws: IggyError(.invalidEncryptionKey)) { try StreamCreateOptions(encryptionKey: Data()).encode() } + + // Diagnostics the SDK produces never show the key, typed or raw. + let keyed = StreamCreateOptions(encryptionKey: key, raw: [StreamOptionKey.encryptionKey.rawValue: fixture.streamKey.keyBase64]) + var dumped = "" + dump(keyed, to: &dumped) + let short = Data(key.prefix(31)) + var refusal = "" + do { + _ = try StreamCreateOptions(encryptionKey: short).encode() + } catch { + refusal = "\(error) \(String(reflecting: error))" + } + for rendering in [String(describing: keyed), String(reflecting: keyed), dumped, refusal] { + #expect(rendering.contains("redacted") || rendering == refusal) + for secret in [fixture.streamKey.keyHex, fixture.streamKey.keyBase64, String(fixture.streamKey.keyHex.prefix(16)), "\(Array(key))"] { + #expect(!rendering.contains(secret), "\(rendering)") + } + } + + // A refused key never reaches the connection. + let server = try await FakeVSRServer.started() + defer { Task { await server.stop() } } + let client = fakeServerClient(server) + try await client.connect() + for invalid in fixture.streamKey.invalidKeys { + guard let rawHex = invalid.input.rawHex else { continue } + await expectCode(.invalidEncryptionKey) { + _ = try await client.createStream(name: "keyed", options: StreamCreateOptions(encryptionKey: Data([UInt8](hex: rawHex)))) + } + } + #expect(await server.requests.contains { $0.operation == .createStream } == false) + await client.shutdown() + } + + @Test func givenSharedVectorsWhenEncodingShouldMatchServerContract() async throws { + #expect(fixture.schemaVersion == 1) + let constants = fixture.constants + #expect(ProtocolVersion.current.packed == constants.protocolVersion.packed) + #expect(ProtocolVersion.current.description == constants.protocolVersion.text) + #expect(constants.protocolVersionMin.packed == constants.protocolVersion.packed) + + // Command and operation mirrors. + for (name, entry) in constants.commandCodes { + let code = try #require(CommandCode(rawValue: entry.code), "\(name)") + #expect(code.name.isEmpty == false) + if let operation = entry.operation { + #expect(ConsensusOperation.forCode(entry.code).rawValue == operation, "\(name)") + } else { + #expect(ConsensusOperation.forCode(entry.code) == .nonReplicated, "\(name)") + } + } + #expect(CommandCode.updateResource.name == "resource.update") + for (name, code) in constants.retiredCommandCodes { + #expect(CommandCode(rawValue: code) == nil, "\(name)") + #expect(ConsensusOperation.forCode(code) == .nonReplicated, "\(name)") + } + for (name, entry) in constants.operations { + let operation = try #require(ConsensusOperation(rawValue: entry.value), "\(name)") + #expect(operation.isInternal == entry.internal, "\(name)") + #expect(operation.isMetadata, "\(name)") + #expect(operation.isResultFramed, "\(name)") + // A resolved operation answers the generic request it was rewritten from. + #expect([entry.value, ConsensusOperation.updateResource.rawValue].contains(entry.requestOperation), "\(name)") + } + for (name, value) in constants.retiredOperations { + #expect(ConsensusOperation(rawValue: value) == nil, "\(name)") + } + // The rewrap is server-originated: no command code ever maps to it. + #expect(CommandCode.allCases.allSatisfy { ConsensusOperation.forCode($0.rawValue) != .rewrapStreamKeys }) + + for (name, code) in constants.scopeCodes { + #expect(scope(name)?.rawValue == code, "\(name)") + } + for (name, code) in constants.headerKindCodes { + #expect(HeaderKind(name: name)?.rawValue == code, "\(name)") + } + let flags: [String: OptionFlags] = [ + "updatable": .updatable, "resettable": .resettable, "secret": .secret, "read_only": .readOnly, "critical": .critical, + ] + for (name, bit) in constants.optionFlags { + #expect(flags[name]?.rawValue == bit, "\(name)") + } + for (name, code) in constants.userStatusCodes { + #expect(userStatus(name)?.rawValue == code, "\(name)") + } + #expect(constants.streamKeyLength == EncryptionKey.length) + + // Typed keys: every described catalog key of a scope, in catalog + // order, with the catalog's flags. Internal keys have no SDK surface. + func expectParity<Key: CatalogOptionKey>(_: Key.Type, scope: String) { + let rows = fixture.catalog.filter { $0.described && $0.scopes.contains(scope) } + #expect(Key.allCases.map(\.rawValue) == rows.map(\.key), "\(scope)") + for row in rows { + #expect(Key(rawValue: row.key)?.flags.rawValue == row.flags, "\(scope) \(row.key)") + } + for row in fixture.catalog where row.scopes.contains(scope) && !row.described { + #expect(Key(rawValue: row.key) == nil, "\(scope) \(row.key)") + } + } + expectParity(StreamOptionKey.self, scope: "stream") + expectParity(TopicOptionKey.self, scope: "topic") + expectParity(UserOptionKey.self, scope: "user") + + // DescribeOptions. + for vector in fixture.describeOptions { + #expect(Requests.describeOptions(scope: try #require(scope(vector.scope))) == [UInt8](hex: vector.requestHex)) + let specs = try Responses.optionSpecs([UInt8](hex: vector.responseHex)[...]) + #expect(specs.map(\.key) == vector.entries.map(\.key), "\(vector.scope)") + for (spec, entry) in zip(specs, vector.entries) { + #expect(spec.kind.description == entry.kind, "\(entry.key)") + #expect(spec.flags.rawValue == entry.flags, "\(entry.key)") + if let defaultValue = entry.defaultValue { + #expect(text(HeaderValue(uncheckedKind: spec.kind, bytes: spec.defaultValue)) == defaultValue, "\(entry.key)") + } else { + #expect(spec.defaultValue.isEmpty, "\(entry.key)") + } + } + } + + // Fingerprint. + let key = Data([UInt8](hex: fixture.streamKey.keyHex)) + #expect(try EncryptionKey.fingerprint(key) == fixture.streamKey.fingerprint) + #expect(try EncryptionKey.fingerprint(base64: fixture.streamKey.keyBase64) == fixture.streamKey.fingerprint) + + // UpdateResource, through the typed update values. + for vector in fixture.updateResource { + let body = try updateBody(scope: vector.scope, target: vector.target, input: vector.input) + #expect(body == [UInt8](hex: vector.bodyHex), "\(vector.name)") + } + let server = try await FakeVSRServer.started() + defer { Task { await server.stop() } } + let client = fakeServerClient(server) + try await client.connect() + for vector in fixture.updateResourceErrors { + let code = try #require(IggyErrorCode(rawValue: vector.error.code)) + #expect(code.name == vector.error.name, "\(vector.name)") + // A name with no typed reset member, an internal key among them, + // travels as a raw reset name and the server refuses it. + if !rawResetNames(scope: vector.scope, vector.input).isEmpty { + let body = try updateBody(scope: vector.scope, target: vector.target, input: vector.input) + #expect(body.isEmpty == false, "\(vector.name)") + continue + } + #expect(throws: IggyError(code), "\(vector.name)") { + try updateBody(scope: vector.scope, target: vector.target, input: vector.input) + } + await expectCode(code) { try await update(client, scope: vector.scope, target: vector.target, input: vector.input) } + } + #expect(await server.requests.contains { $0.operation == .updateResource } == false) + await client.shutdown() + + // Creates. + for vector in fixture.createStream { + let key = vector.input.encryptionKey?.rawHex.map { Data([UInt8](hex: $0)) } + let options = StreamCreateOptions(encryptionKey: key, raw: vector.input.raw ?? [:]) + #expect(try Requests.createStream(name: vector.input.name, options: options.encode()) == [UInt8](hex: vector.bodyHex), "\(vector.name)") + } + for vector in fixture.createTopic { + let input = vector.input + let options = TopicCreateOptions( + partitionsCount: input.partitionsCount, compressionAlgorithm: CompressionAlgorithm(name: input.compressionAlgorithm), + messageExpiry: IggyExpiry(wireMicroseconds: try #require(UInt64(input.messageExpiry))), + maxTopicSize: MaxTopicSize(wireValue: try #require(UInt64(input.maxTopicSize))), segmentSizeBytes: UInt64(input.segmentSize), + durability: try #require(Durability(rawValue: input.durability)), + consumerOffsetDurability: try #require(Durability(rawValue: input.consumerOffsetDurability))) + let body = try Requests.createTopic( + streamID: identifier(input.streamID), partitionsCount: input.partitionsCount, name: input.name, options: options.encode()) + #expect(body == [UInt8](hex: vector.bodyHex), "\(vector.name)") + } + for vector in fixture.createUser { + let input = vector.input + let body = try Requests.createUser( + username: input.username, password: input.password, status: try #require(userStatus(input.status)), permissions: nil, + options: UserCreateOptions().encode()) + #expect(body == [UInt8](hex: vector.bodyHex), "\(vector.name)") + } + + // Responses and the typed accessors. + let neverReturned = Set(constants.neverReturnedKeys) + for vector in fixture.streamResponses { + let stream = try Responses.streamDetails([UInt8](hex: vector.responseHex)[...]) + let expected = vector.expected + #expect(stream.id == expected.id) + #expect(stream.createdAt.microseconds == expected.createdAt) + #expect(stream.name == expected.name) + #expect(stream.topicsCount == expected.topicsCount) + #expect(stream.sizeBytes == expected.sizeBytes) + #expect(stream.messagesCount == expected.messagesCount) + #expect(fixtureForm(stream.options) == expected.options, "\(vector.name)") + #expect(stream.encryptionKeySource?.rawValue == expected.encryptionKeySource, "\(vector.name)") + #expect(stream.encryptionKeyFingerprint == expected.encryptionKeyFingerprint, "\(vector.name)") + #expect(stream.encryptionKekFingerprints == expected.encryptionKekFingerprints, "\(vector.name)") + #expect(neverReturned.isDisjoint(with: stream.options.keys), "\(vector.name)") + let listed = try #require(try Responses.streams([UInt8](hex: vector.responseHex)[...]).first) + #expect(listed.options == stream.options) + #expect(listed.encryptionKeySource == stream.encryptionKeySource) + #expect(listed.encryptionKeyFingerprint == stream.encryptionKeyFingerprint) + #expect(listed.encryptionKekFingerprints == stream.encryptionKekFingerprints) + } + for vector in fixture.topicResponses { + let topic = try Responses.topicDetails([UInt8](hex: vector.responseHex)[...]) + let expected = vector.expected + #expect(topic.id == expected.id) + #expect(topic.createdAt.microseconds == expected.createdAt) + #expect(topic.name == expected.name) + #expect(topic.partitionsCount == expected.partitionsCount) + #expect(topic.sizeBytes == expected.sizeBytes) + #expect(topic.messagesCount == expected.messagesCount) + #expect(fixtureForm(topic.options) == expected.options, "\(vector.name)") + #expect(topic.compressionAlgorithm.description == expected.compressionAlgorithm) + #expect(try topic.messageExpiry.wireMicroseconds().map(String.init) == expected.messageExpiry) + #expect(String(topic.maxTopicSize.wireValue) == expected.maxTopicSize) + #expect(neverReturned.isDisjoint(with: topic.options.keys)) + } + for vector in fixture.userResponses { + let user = try Responses.userDetails([UInt8](hex: vector.responseHex)[...]) + let expected = vector.expected + #expect(user.id == expected.id) + #expect(user.createdAt.microseconds == expected.createdAt) + #expect(user.status.description == expected.status) + #expect(user.username == expected.username) + #expect(user.permissions == nil) + #expect(fixtureForm(user.options) == expected.options, "\(vector.name)") + #expect(neverReturned.isDisjoint(with: user.options.keys)) + } + } + + /// Every public catalog key has its typed surface: a create field per + /// create-only or updatable map key, an update field per updatable key, + /// a reset member per resettable key, and an accessor per read-only key. + @Test func givenFixtureCatalogWhenBuildingTypedOptionsShouldCoverEveryPublicKey() throws { + func rows(_ scope: String, _ matches: (ResourceOptionsFixture.CatalogRow) -> Bool) -> Set<String> { + Set(fixture.catalog.filter { $0.scopes.contains(scope) && matches($0) }.map(\.key)) + } + func creatable(_ scope: String) -> Set<String> { + rows(scope) { $0.storage == "map" && ["public", "secret"].contains($0.visibility) && $0.mutability != "read_only" } + } + func updatable(_ scope: String) -> Set<String> { + rows(scope) { $0.described && OptionFlags(rawValue: $0.flags).contains(.updatable) } + } + func resettable(_ scope: String) -> Set<String> { + rows(scope) { $0.described && OptionFlags(rawValue: $0.flags).contains(.resettable) } + } + func readOnly(_ scope: String) -> Set<String> { + rows(scope) { $0.described && OptionFlags(rawValue: $0.flags).contains(.readOnly) } + } + + let streamCreate = StreamCreateOptions(encryptionKey: Data([UInt8](hex: fixture.streamKey.keyHex))) + let topicCreate = TopicCreateOptions( + partitionsCount: 3, compressionAlgorithm: .gzip, messageExpiry: .never, maxTopicSize: .bytes(1 << 30), segmentSizeBytes: 1 << 20, + durability: .persisted, consumerOffsetDurability: .persisted, partitionResizePolicy: .fixed, messagesRequiredToSave: 10, + sizeOfMessagesRequiredToSaveBytes: 1 << 10, preallocateSegments: true) + #expect(unsetFields(streamCreate, except: ["raw"]).isEmpty) + #expect(unsetFields(topicCreate, except: ["raw"]).isEmpty) + #expect(Set(try streamCreate.toResourceOptions().keys) == creatable("stream")) + #expect(Set(try topicCreate.toResourceOptions().keys) == creatable("topic")) + #expect(creatable("user").isEmpty, "a user create key needs a typed field on UserCreateOptions") + + let streamUpdate = StreamUpdateOptions(name: "renamed") + let topicUpdate = TopicUpdateOptions(name: "renamed", messageExpiry: .never, maxTopicSize: .bytes(1 << 30)) + let userUpdate = UserUpdateOptions(username: "renamed", status: .inactive) + let untyped: Set<String> = ["reset", "raw", "rawReset"] + #expect(unsetFields(streamUpdate, except: untyped).isEmpty) + #expect(unsetFields(topicUpdate, except: untyped).isEmpty) + #expect(unsetFields(userUpdate, except: untyped).isEmpty) + #expect(Set(try streamUpdate.toPatch().set.keys) == updatable("stream")) + #expect(Set(try topicUpdate.toPatch().set.keys) == updatable("topic")) + #expect(Set(try userUpdate.toPatch().set.keys) == updatable("user")) + + func resetMembers<Key: CatalogOptionKey>(_: Key.Type) -> Set<String> { + Set(Key.allCases.filter { $0.flags.contains(.resettable) }.map(\.rawValue)) + } + #expect(resetMembers(StreamOptionKey.self) == resettable("stream")) + #expect(resetMembers(TopicOptionKey.self) == resettable("topic")) + #expect(resetMembers(UserOptionKey.self) == resettable("user")) + let resets = Set(TopicOptionKey.allCases.filter { $0.flags.contains(.resettable) }) + #expect(Set(try TopicUpdateOptions(reset: resets).toPatch().reset) == resettable("topic")) + + let streamAccessors: [String: (StreamDetails) -> Bool] = [ + StreamOptionKey.encryptionKeySource.rawValue: { $0.encryptionKeySource != nil }, + StreamOptionKey.encryptionKeyFingerprint.rawValue: { $0.encryptionKeyFingerprint != nil }, + StreamOptionKey.encryptionKekFingerprint.rawValue: { !$0.encryptionKekFingerprints.isEmpty }, + ] + #expect(Set(streamAccessors.keys) == readOnly("stream")) + #expect(readOnly("topic").isEmpty, "a read-only topic key needs an accessor on Topic and TopicDetails") + #expect(readOnly("user").isEmpty, "a read-only user key needs an accessor on UserInfo and UserInfoDetails") + for vector in fixture.streamResponses { + let stream = try Responses.streamDetails([UInt8](hex: vector.responseHex)[...]) + for (key, reads) in streamAccessors where stream.options[key] != nil { + #expect(reads(stream), "\(vector.name) \(key)") + } + } + } + + /// A raw key or reset name may be one a newer server knows, so only the + /// shape of such a patch is checked here. A typed-only patch is refused + /// with the category the server would return, values last. + @Test func givenRawKeysWhenPatchingShouldLeaveTheCategoryChecksToTheServer() throws { + let segment = try TopicUpdateOptions(reset: [.name], raw: [TopicOptionKey.segmentSize.rawValue: "1 GiB"]).toPatch() + #expect(segment.reset == [TopicOptionKey.name.rawValue]) + #expect(segment.set[TopicOptionKey.segmentSize.rawValue] == .explicit("1 GiB")) + let source = try StreamUpdateOptions(reset: [.name], raw: [StreamOptionKey.encryptionKeySource.rawValue: "x"]).toPatch() + #expect(source.reset == [StreamOptionKey.name.rawValue]) + let internalReset = try UserUpdateOptions(rawReset: ["critical_keys"]).toPatch() + #expect(internalReset.reset == ["critical_keys"] && internalReset.set.isEmpty) + // Typed and raw reset names travel together in byte order. + let mixed = try TopicUpdateOptions(reset: [.messageExpiry], rawReset: ["future_retention", "message_expiry"]).toPatch() + #expect(mixed.reset == ["future_retention", "message_expiry"]) + + #expect(throws: IggyError(.invalidCommand)) { try TopicUpdateOptions(raw: ["future": "x"], rawReset: ["future"]).toPatch() } + #expect(throws: IggyError(.invalidCommand)) { try StreamUpdateOptions(name: "renamed", rawReset: ["name"]).toPatch() } + #expect(throws: IggyError(.unsupportedOptionKey)) { try UserUpdateOptions(rawReset: [""]).toPatch() } + #expect(throws: IggyError(.unsupportedOptionKey)) { try UserUpdateOptions(rawReset: [String(repeating: "k", count: 256)]).toPatch() } + let tooMany = Set((0...OptionsPatch.maxResetKeys).map { "future_\($0)" }) + #expect(throws: IggyError(.unsupportedOptionKey)) { try TopicUpdateOptions(rawReset: tooMany).toPatch() } + + #expect(throws: IggyError(.invalidCommand)) { try UserUpdateOptions(username: "ab", reset: [.username]).toPatch() } + #expect(throws: IggyError(.optionNotResettable)) { try UserUpdateOptions(username: "ab", reset: [.status]).toPatch() } + #expect(throws: IggyError(.invalidUsername)) { try UserUpdateOptions(username: "ab").toPatch() } + #expect(throws: IggyError(.optionNotUpdatable)) { try TopicUpdateOptions(name: "", reset: [.compressionAlgorithm]).toPatch() } + } + + /// The primary commits an `UpdateResource` under its scope's resolved + /// operation, so the reply names 72, 73 or 74, never the request's 150. + @Test func givenResolvedUpdateReplyWhenCorrelatingShouldAcceptOnlyMatchingRequestIdentity() async throws { + let golden = GoldenFixture.shared + let header = golden.bytes("vsr.reply.update_topic_resolved")[...] + #expect(VSRFrame.readReplyOperation(header) == .updateTopicResolved) + #expect(VSRFrame.hasCommittedResult(header)) + #expect(try VSRFrame.decodeReply(header: header, body: UInt32(0).littleEndianBytes[...]).isEmpty) + + let resolved: [ConsensusOperation] = [.updateStreamResolved, .updateTopicResolved, .updateUserResolved] + #expect(resolved.map(\.rawValue) == [72, 73, 74]) + #expect(ConsensusOperation.rewrapStreamKeys.rawValue == 75) + #expect(ConsensusOperation.rewrapStreamKeys.isInternal) + for operation in resolved { + #expect(operation.isInternal) + #expect(operation.isMetadata) + #expect(operation.isResultFramed) + } + + let server = try await FakeVSRServer.started() + defer { Task { await server.stop() } } + let client = fakeServerClient(server) + try await client.connect() + for operation in resolved { + // The first reply answers another request id: the client drops the + // connection and sends the same request again on the resumed + // session, whose resolved reply it accepts. + let misrouted = Locked(true) + await server.respond { request in + if let reply = FakeVSRServer.sessionReply(request) { return reply } + guard request.operation == .updateResource else { return FakeVSRServer.defaultReply(request, address: server.address) } + return .resolved(operation, requestID: misrouted.exchange(false) ? request.requestID + 7 : nil) + } + let connections = await server.connectionCount + try await client.updateStream(1, options: StreamUpdateOptions(name: "renamed")) + #expect(await server.connectionCount == connections + 1) + let attempts = await server.requests.filter { $0.operation == .updateResource }.suffix(2) + #expect(attempts.count == 2 && Set(attempts.map(\.requestID)).count == 1) + #expect(Set(attempts.map(\.checksum)).count == 1) + try await client.updateTopic(streamID: 1, topicID: 2, options: TopicUpdateOptions(reset: [.maxTopicSize])) + try await client.updateUser(9, options: UserUpdateOptions(status: .inactive)) + } + let updates = await server.requests.filter { $0.operation == .updateResource } + #expect(updates.count == 4 * resolved.count) + #expect( + updates.allSatisfy { + $0.code == CommandCode.updateResource.rawValue + && $0.checksum != 0 + && $0.checksum != XXH3.hash64($0.payload) + }) + await client.shutdown() + } + + /// Entries of a kind this build does not know are skipped, so a resource + /// a newer server wrote still reads back. A known kind with a malformed + /// value fails the read instead of being reinterpreted. + @Test func givenUnknownOptionKindWhenDecodingShouldPreserveSupportedValues() throws { + let derived = + optionEntry("encryption_key_fingerprint", kind: HeaderKind.string.rawValue, value: Array("8441e38538cbfa57".utf8)) + + optionEntry("encryption_key_source", kind: HeaderKind.string.rawValue, value: Array("stream".utf8)) + + optionEntry("future_vector", kind: 99, value: [1, 2, 3]) + let explicit = optionEntry("future_option", kind: HeaderKind.string.rawValue, value: Array("enabled".utf8)) + let stream = try Responses.streamDetails(streamResponse(explicit: explicit, derived: derived)[...]) + #expect(stream.options.keys.sorted() == ["encryption_key_fingerprint", "encryption_key_source", "future_option"]) + #expect(stream.options["future_option"] == .explicit("enabled")) + #expect(stream.encryptionKeySource == .stream) + #expect(stream.encryptionKeyFingerprint == "8441e38538cbfa57") + #expect(stream.encryptionKekFingerprints.isEmpty) + + let malformed = optionEntry("future_quota", kind: HeaderKind.uint64.rawValue, value: [1, 2, 3, 4]) + #expect(throws: IggyError(.invalidHeaderValue)) { + try Responses.streamDetails(streamResponse(explicit: [], derived: derived + malformed)[...]) + } + let unknownSource = optionEntry("encryption_key_source", kind: HeaderKind.string.rawValue, value: Array("vault".utf8)) + let future = try Responses.streamDetails(streamResponse(explicit: [], derived: unknownSource)[...]) + #expect(future.options["encryption_key_source"] == .derived("vault")) + #expect(future.encryptionKeySource == nil) + } + + // MARK: Helpers + + func fakeServerClient(_ server: FakeVSRServer) -> IggyClient { + var configuration = ClientConfiguration(address: server.address, autoLogin: .usernamePassword(username: "iggy", password: "iggy")) + configuration.reconnection = ReconnectionOptions(enabled: true, interval: .milliseconds(20), reestablishAfter: .zero) + configuration.connectTimeout = .seconds(2) + configuration.heartbeatInterval = .seconds(60) + return IggyClient(configuration: configuration, connector: NIOConnector(), logger: testLogger) + } + + func scope(_ name: String) -> OptionsScope? { + [OptionsScope.topic, .stream, .user].first { "\($0)" == name } + } + + func userStatus(_ name: String) -> UserStatus? { + [UserStatus.active, .inactive].first { $0.description == name } + } + + func identifier(_ id: ResourceOptionsFixture.TargetID) throws -> Identifier { + switch id { + case .numeric(let value): Identifier(numeric: value) + case .named(let name): try Identifier(named: name) + } + } + + func target(_ ids: [String: ResourceOptionsFixture.TargetID], _ name: String) throws -> Identifier { + try identifier(try #require(ids[name], "missing \(name)")) + } + + func topicUpdate(_ input: ResourceOptionsFixture.UpdateInput) throws -> TopicUpdateOptions { + TopicUpdateOptions( + name: input.name, messageExpiry: try input.messageExpiry.map { IggyExpiry(wireMicroseconds: try #require(UInt64($0))) }, + maxTopicSize: try input.maxTopicSize.map { MaxTopicSize(wireValue: try #require(UInt64($0))) }, + reset: Set((input.reset ?? []).compactMap(TopicOptionKey.init(rawValue:))), rawReset: rawResetNames(TopicOptionKey.self, input)) + } + + func streamUpdate(_ input: ResourceOptionsFixture.UpdateInput) throws -> StreamUpdateOptions { + StreamUpdateOptions( + name: input.name, reset: Set((input.reset ?? []).compactMap(StreamOptionKey.init(rawValue:))), + rawReset: rawResetNames(StreamOptionKey.self, input)) + } + + func userUpdate(_ input: ResourceOptionsFixture.UpdateInput) throws -> UserUpdateOptions { + UserUpdateOptions( + username: input.username, status: try input.status.map { try #require(userStatus($0)) }, + reset: Set((input.reset ?? []).compactMap(UserOptionKey.init(rawValue:))), rawReset: rawResetNames(UserOptionKey.self, input)) + } + + /// Reset names of the input this SDK has no typed member for. + func rawResetNames<Key: CatalogOptionKey>(_: Key.Type, _ input: ResourceOptionsFixture.UpdateInput) -> Set<String> { + Set((input.reset ?? []).filter { Key(rawValue: $0) == nil }) + } + + func rawResetNames(scope: String, _ input: ResourceOptionsFixture.UpdateInput) -> Set<String> { + switch scope { + case "topic": rawResetNames(TopicOptionKey.self, input) + case "stream": rawResetNames(StreamOptionKey.self, input) + default: rawResetNames(UserOptionKey.self, input) + } + } + + func updateBody(scope: String, target ids: [String: ResourceOptionsFixture.TargetID], input: ResourceOptionsFixture.UpdateInput) throws -> [UInt8] { + switch scope { + case "topic": + try Requests.updateResource(.topic(stream: target(ids, "stream_id"), topic: target(ids, "topic_id")), topicUpdate(input).toPatch()) + case "stream": + try Requests.updateResource(.stream(target(ids, "stream_id")), streamUpdate(input).toPatch()) + default: + try Requests.updateResource(.user(target(ids, "user_id")), userUpdate(input).toPatch()) + } + } + + func update( + _ client: IggyClient, scope: String, target ids: [String: ResourceOptionsFixture.TargetID], input: ResourceOptionsFixture.UpdateInput + ) + async throws + { + switch scope { + case "topic": + try await client.updateTopic(streamID: target(ids, "stream_id"), topicID: target(ids, "topic_id"), options: topicUpdate(input)) + case "stream": + try await client.updateStream(target(ids, "stream_id"), options: streamUpdate(input)) + default: + try await client.updateUser(target(ids, "user_id"), options: userUpdate(input)) + } + } + + /// Stored properties of `value` left nil, so a new typed field cannot + /// slip past the surface checks unset. + func unsetFields(_ value: Any, except skipped: Set<String>) -> [String] { + Mirror(reflecting: value).children.compactMap { child in + guard let label = child.label, !skipped.contains(label) else { return nil } + let field = Mirror(reflecting: child.value) + return field.displayStyle == .optional && field.children.isEmpty ? label : nil + } + } + + /// An option value in the fixture's reversible text form. + func text(_ value: HeaderValue) -> String { + switch value.kind { + case .raw: Data(value.bytes).base64EncodedString() + default: value.description + } + } + + func fixtureForm(_ options: ResourceOptions) -> [String: ResourceOptionsFixture.OptionEntry] { + options.mapValues { ResourceOptionsFixture.OptionEntry(value: text($0.value), kind: $0.value.kind.description, explicit: $0.explicit) } + } + + func optionEntry(_ key: String, kind: UInt8, value: [UInt8]) -> [UInt8] { + var writer = ByteWriter() + writer.write(HeaderKind.string.rawValue) + writer.write(UInt32(key.utf8.count)) + writer.write(Array(key.utf8)) + writer.write(kind) + writer.write(UInt32(value.count)) + writer.write(value) + return writer.bytes + } + + /// A `GetStream` reply for a stream without topics. + func streamResponse(explicit: [UInt8], derived: [UInt8]) throws -> [UInt8] { + var writer = ByteWriter() + writer.write(UInt32(9)) + writer.write(UInt64(1_710_000_000_000_000)) + writer.write(UInt32(0)) + writer.write(UInt64(0)) + writer.write(UInt64(0)) + try writer.writeName("future-stream") + writer.write(UInt32(explicit.count)) + writer.write(explicit) + writer.write(UInt32(derived.count)) + writer.write(derived) + return writer.bytes + } +}
diff --git a/foreign/swift/Tests/IggyTests/ResponseVectorTests.swift b/foreign/swift/Tests/IggyTests/ResponseVectorTests.swift index 9433c59..3702c98 100644 --- a/foreign/swift/Tests/IggyTests/ResponseVectorTests.swift +++ b/foreign/swift/Tests/IggyTests/ResponseVectorTests.swift
@@ -35,13 +35,16 @@ #expect(topic.name == name) #expect(topic.createdAt == createdAt) #expect(topic.partitionsCount == 3) + // The three settings the header used to carry come from the blocks. #expect(topic.messageExpiry == .after(.seconds(604_800))) #expect(topic.compressionAlgorithm == CompressionAlgorithm.none) - #expect(topic.maxTopicSize == .bytes(1_073_741_824)) + #expect(topic.maxTopicSize == .unlimited) #expect(topic.sizeBytes == 4096) #expect(topic.messagesCount == 100) #expect(topic.options["durability"] == .explicit("persisted")) + #expect(topic.options["message_expiry"] == .explicit(.uint64(604_800_000_000))) #expect(topic.options["segment_size"] == .explicit(.uint64(1_073_741_824))) + #expect(topic.options["compression_algorithm"] == .derived("none")) #expect(topic.options["max_topic_size"] == .derived(.uint64(UInt64.max))) } @@ -53,6 +56,10 @@ #expect(details.topicsCount == 2) #expect(details.sizeBytes == 2048) #expect(details.messagesCount == 200) + #expect(details.options.values.allSatisfy { !$0.explicit }) + #expect(details.encryptionKeySource == .stream) + #expect(details.encryptionKeyFingerprint == golden.vectors["stream_key.fingerprint"]) + #expect(details.encryptionKekFingerprints == ["39e4f0f95c3693c7", "d55d479829bc2936"]) #expect(details.topics.count == 2) expectSampleTopic(details.topics[0], id: 1, name: "topic-a") expectSampleTopic(details.topics[1], id: 2, name: "topic-b") @@ -60,7 +67,7 @@ let streams = try Responses.streams(body("response.get_streams")) #expect(streams.map(\.id) == [1, 2]) #expect(streams.map(\.topicsCount) == [2, 0]) - #expect(streams.allSatisfy { $0.options.isEmpty }) + #expect(streams.allSatisfy { $0.options == details.options && $0.encryptionKeySource == .stream }) #expect(try Responses.streams([]).isEmpty) } @@ -68,9 +75,10 @@ let details = try Responses.topicDetails(body("response.get_topic")) expectSampleTopic( Topic( - id: details.id, createdAt: details.createdAt, name: details.name, sizeBytes: details.sizeBytes, messageExpiry: details.messageExpiry, - compressionAlgorithm: details.compressionAlgorithm, maxTopicSize: details.maxTopicSize, messagesCount: details.messagesCount, + id: details.id, createdAt: details.createdAt, name: details.name, sizeBytes: details.sizeBytes, messagesCount: details.messagesCount, partitionsCount: details.partitionsCount, options: details.options), id: 1, name: "my-topic") + #expect(details.messageExpiry == .after(.seconds(604_800))) + #expect(details.maxTopicSize == .unlimited) #expect(details.partitions.map(\.id) == [0, 1, 2]) #expect(details.partitions[0].currentOffset == 99) #expect(details.partitions[0].segmentsCount == 2) @@ -146,6 +154,7 @@ let users = try Responses.users(body("response.get_users")) #expect(users.map(\.username) == ["admin", "alice"]) #expect(users.map(\.status) == [.active, .inactive]) + #expect(users.allSatisfy { $0.options.isEmpty }) } @Test func loginAndIdentity() throws { @@ -222,7 +231,10 @@ @Test func optionSpecs() throws { let specs = try Responses.optionSpecs(body("response.describe_options")) - #expect(specs.map(\.key) == ["segment_size", "durability"]) + #expect(specs.map(\.key) == ["segment_size", "durability", "message_expiry", "encryption_key"]) + #expect(specs.map(\.flags) == [[], [], [.updatable, .resettable], [.secret, .critical]]) + #expect(specs[3].kind == .raw) + #expect(specs[3].defaultValue.isEmpty) #expect(specs[0].kind == .uint64) #expect(specs[0].defaultValue == UInt64(1_073_741_824).littleEndianBytes) #expect(specs[0].description == "Segment size")
diff --git a/foreign/swift/Tests/IggyTests/Support/FakeVSRServer.swift b/foreign/swift/Tests/IggyTests/Support/FakeVSRServer.swift index ae8018d..3f87e0d 100644 --- a/foreign/swift/Tests/IggyTests/Support/FakeVSRServer.swift +++ b/foreign/swift/Tests/IggyTests/Support/FakeVSRServer.swift
@@ -56,6 +56,9 @@ case eviction(VSRFrame.EvictionReason) /// A reply that answers some other request id. case wrongRequestID(UInt64) + /// A committed success under the operation the primary rewrote the + /// request into, answering `requestID` or else the request's own id. + case resolved(ConsensusOperation, requestID: UInt64? = nil) /// Closes the connection instead of answering. case close /// Never answers. @@ -448,6 +451,11 @@ connection: request.connection, operation: operation, code: request.code, requestID: requestID, session: request.session, clientID: request.clientID, checksum: request.checksum, context: request.context, payload: request.payload) write(FakeVSRServer.encodeReply(request: wrong, operation: operation, status: 0, body: framedBody([]))) + case .resolved(let resolved, let requestID): + let answered = FakeVSRServer.Request( + connection: request.connection, operation: resolved, code: request.code, requestID: requestID ?? request.requestID, + session: request.session, clientID: request.clientID, checksum: request.checksum, context: request.context, payload: request.payload) + write(FakeVSRServer.encodeReply(request: answered, operation: resolved, status: 0, body: [0, 0, 0, 0])) case .close: channel.close(promise: nil) case .stall:
diff --git a/foreign/swift/Tests/IggyTests/Support/ResourceOptionsFixture.swift b/foreign/swift/Tests/IggyTests/Support/ResourceOptionsFixture.swift new file mode 100644 index 0000000..8dad5b4 --- /dev/null +++ b/foreign/swift/Tests/IggyTests/Support/ResourceOptionsFixture.swift
@@ -0,0 +1,412 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +/// The cross-SDK resource options fixture, generated by +/// `core/common/examples/resource_options_fixtures.rs` from the real Rust +/// encoders and copied byte for byte into every SDK. Never edit the copy. +struct ResourceOptionsFixture: Decodable, Sendable { + struct Version: Decodable { + let packed: UInt32 + let text: String + } + + struct CommandEntry: Decodable { + let code: UInt32 + let operation: UInt8? + } + + struct OperationEntry: Decodable { + let value: UInt8 + let `internal`: Bool + let requestOperation: UInt8 + + enum CodingKeys: String, CodingKey { + case value + case `internal` + case requestOperation = "request_operation" + } + } + + struct Constants: Decodable { + let protocolVersion: Version + let protocolVersionMin: Version + let commandCodes: [String: CommandEntry] + let retiredCommandCodes: [String: UInt32] + let operations: [String: OperationEntry] + let retiredOperations: [String: UInt8] + let scopeCodes: [String: UInt8] + let headerKindCodes: [String: UInt8] + let optionFlags: [String: UInt8] + let userStatusCodes: [String: UInt8] + let streamKeyLength: Int + let neverReturnedKeys: [String] + + enum CodingKeys: String, CodingKey { + case protocolVersion = "protocol_version" + case protocolVersionMin = "protocol_version_min" + case commandCodes = "command_codes" + case retiredCommandCodes = "retired_command_codes" + case operations + case retiredOperations = "retired_operations" + case scopeCodes = "scope_codes" + case headerKindCodes = "header_kind_codes" + case optionFlags = "option_flags" + case userStatusCodes = "user_status_codes" + case streamKeyLength = "stream_key_length" + case neverReturnedKeys = "never_returned_keys" + } + } + + struct CatalogRow: Decodable { + let key: String + let scopes: [String] + let kind: String + let kindCode: UInt8 + let storage: String + let mutability: String + let visibility: String + let critical: Bool + let flags: UInt8 + let described: Bool + + enum CodingKeys: String, CodingKey { + case key, scopes, kind, storage, mutability, visibility, critical, flags, described + case kindCode = "kind_code" + } + } + + struct DescribeEntry: Decodable { + let key: String + let kind: String + let flags: UInt8 + let defaultValue: String? + + enum CodingKeys: String, CodingKey { + case key, kind, flags + case defaultValue = "default" + } + } + + struct DescribeVector: Decodable { + let scope: String + let requestHex: String + let responseHex: String + let entries: [DescribeEntry] + + enum CodingKeys: String, CodingKey { + case scope, entries + case requestHex = "request_hex" + case responseHex = "response_hex" + } + } + + struct ErrorRef: Decodable { + let name: String + let code: UInt32 + } + + /// A stream key as 32 raw bytes or as base64 text. + struct KeyInput: Decodable { + let rawHex: String? + let base64: String? + + enum CodingKeys: String, CodingKey { + case rawHex = "raw_hex" + case base64 + } + } + + struct InvalidKey: Decodable { + let name: String + let input: KeyInput + let error: ErrorRef + + enum CodingKeys: String, CodingKey { + case name = "case" + case input, error + } + } + + struct StreamKey: Decodable { + let keyHex: String + let keyBase64: String + let fingerprint: String + let invalidKeys: [InvalidKey] + + enum CodingKeys: String, CodingKey { + case fingerprint + case keyHex = "key_hex" + case keyBase64 = "key_base64" + case invalidKeys = "invalid_keys" + } + } + + /// A target id: a number is a numeric id, a string a named one. + enum TargetID: Decodable { + case numeric(UInt32) + case named(String) + + init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + if let numeric = try? container.decode(UInt32.self) { + self = .numeric(numeric) + } else { + self = .named(try container.decode(String.self)) + } + } + } + + /// Typed update input. u64 values are decimal strings. + struct UpdateInput: Decodable { + let name: String? + let username: String? + let status: String? + let messageExpiry: String? + let maxTopicSize: String? + let reset: [String]? + + enum CodingKeys: String, CodingKey { + case name, username, status, reset + case messageExpiry = "message_expiry" + case maxTopicSize = "max_topic_size" + } + } + + struct UpdateVector: Decodable { + let name: String + let scope: String + let target: [String: TargetID] + let input: UpdateInput + let wireReset: [String] + let bodyHex: String + + enum CodingKeys: String, CodingKey { + case name = "case" + case scope, target, input + case wireReset = "wire_reset" + case bodyHex = "body_hex" + } + } + + struct UpdateErrorVector: Decodable { + let name: String + let scope: String + let target: [String: TargetID] + let input: UpdateInput + let error: ErrorRef + + enum CodingKeys: String, CodingKey { + case name = "case" + case scope, target, input, error + } + } + + struct CreateStreamInput: Decodable { + let name: String + let encryptionKey: KeyInput? + let raw: [String: String]? + + enum CodingKeys: String, CodingKey { + case name, raw + case encryptionKey = "encryption_key" + } + } + + struct CreateStreamVector: Decodable { + let name: String + let input: CreateStreamInput + let bodyHex: String + let admissionError: ErrorRef? + + enum CodingKeys: String, CodingKey { + case name = "case" + case input + case bodyHex = "body_hex" + case admissionError = "admission_error" + } + } + + struct CreateTopicInput: Decodable { + let streamID: TargetID + let name: String + let partitionsCount: UInt32 + let compressionAlgorithm: String + let messageExpiry: String + let maxTopicSize: String + let segmentSize: String + let durability: String + let consumerOffsetDurability: String + + enum CodingKeys: String, CodingKey { + case name, durability + case streamID = "stream_id" + case partitionsCount = "partitions_count" + case compressionAlgorithm = "compression_algorithm" + case messageExpiry = "message_expiry" + case maxTopicSize = "max_topic_size" + case segmentSize = "segment_size" + case consumerOffsetDurability = "consumer_offset_durability" + } + } + + struct CreateTopicVector: Decodable { + let name: String + let input: CreateTopicInput + let bodyHex: String + + enum CodingKeys: String, CodingKey { + case name = "case" + case input + case bodyHex = "body_hex" + } + } + + struct CreateUserInput: Decodable { + let username: String + let password: String + let status: String + } + + struct CreateUserVector: Decodable { + let name: String + let input: CreateUserInput + let bodyHex: String + + enum CodingKeys: String, CodingKey { + case name = "case" + case input + case bodyHex = "body_hex" + } + } + + /// One option in the HTTP JSON form: reversible text, kind, provenance. + struct OptionEntry: Decodable, Equatable { + let value: String + let kind: String + let explicit: Bool + } + + struct StreamProjection: Decodable { + let id: UInt32 + let createdAt: UInt64 + let name: String + let topicsCount: UInt32 + let sizeBytes: UInt64 + let messagesCount: UInt64 + let options: [String: OptionEntry] + let encryptionKeySource: String + let encryptionKeyFingerprint: String? + let encryptionKekFingerprints: [String] + + enum CodingKeys: String, CodingKey { + case id, name, options + case createdAt = "created_at" + case topicsCount = "topics_count" + case sizeBytes = "size_bytes" + case messagesCount = "messages_count" + case encryptionKeySource = "encryption_key_source" + case encryptionKeyFingerprint = "encryption_key_fingerprint" + case encryptionKekFingerprints = "encryption_kek_fingerprints" + } + } + + struct TopicProjection: Decodable { + let id: UInt32 + let createdAt: UInt64 + let name: String + let partitionsCount: UInt32 + let sizeBytes: UInt64 + let messagesCount: UInt64 + let options: [String: OptionEntry] + let compressionAlgorithm: String + let messageExpiry: String + let maxTopicSize: String + + enum CodingKeys: String, CodingKey { + case id, name, options + case createdAt = "created_at" + case partitionsCount = "partitions_count" + case sizeBytes = "size_bytes" + case messagesCount = "messages_count" + case compressionAlgorithm = "compression_algorithm" + case messageExpiry = "message_expiry" + case maxTopicSize = "max_topic_size" + } + } + + struct UserProjection: Decodable { + let id: UInt32 + let createdAt: UInt64 + let status: String + let username: String + let options: [String: OptionEntry] + + enum CodingKeys: String, CodingKey { + case id, status, username, options + case createdAt = "created_at" + } + } + + struct ResponseVector<Projection: Decodable & Sendable>: Decodable, Sendable { + let name: String + let responseHex: String + let expected: Projection + + enum CodingKeys: String, CodingKey { + case name = "case" + case expected + case responseHex = "response_hex" + } + } + + let schemaVersion: Int + let constants: Constants + let catalog: [CatalogRow] + let describeOptions: [DescribeVector] + let streamKey: StreamKey + let updateResource: [UpdateVector] + let updateResourceErrors: [UpdateErrorVector] + let createStream: [CreateStreamVector] + let createTopic: [CreateTopicVector] + let createUser: [CreateUserVector] + let streamResponses: [ResponseVector<StreamProjection>] + let topicResponses: [ResponseVector<TopicProjection>] + let userResponses: [ResponseVector<UserProjection>] + + enum CodingKeys: String, CodingKey { + case constants, catalog + case schemaVersion = "schema_version" + case describeOptions = "describe_options" + case streamKey = "stream_key" + case updateResource = "update_resource" + case updateResourceErrors = "update_resource_errors" + case createStream = "create_stream" + case createTopic = "create_topic" + case createUser = "create_user" + case streamResponses = "stream_responses" + case topicResponses = "topic_responses" + case userResponses = "user_responses" + } + + static let shared: ResourceOptionsFixture = { + let url = Bundle.module.url(forResource: "resource_options", withExtension: "json", subdirectory: "Fixtures")! + let data = try! Data(contentsOf: url) + return try! JSONDecoder().decode(ResourceOptionsFixture.self, from: data) + }() +}
diff --git a/foreign/swift/Tests/IggyTests/Support/TestHelpers.swift b/foreign/swift/Tests/IggyTests/Support/TestHelpers.swift index 1dd57cf..b4dc438 100644 --- a/foreign/swift/Tests/IggyTests/Support/TestHelpers.swift +++ b/foreign/swift/Tests/IggyTests/Support/TestHelpers.swift
@@ -60,15 +60,15 @@ return await condition() } -/// Asserts that `body` throws an ``IggyError`` with `code`. -func expectCode(_ code: IggyErrorCode, sourceLocation: SourceLocation = #_sourceLocation, _ body: () async throws -> Void) async { +/// Asserts that `body` throws an ``IggyError`` with one of `codes`. +func expectCode(_ codes: IggyErrorCode..., sourceLocation: SourceLocation = #_sourceLocation, _ body: () async throws -> Void) async { do { try await body() - Issue.record("expected \(code) but nothing was thrown", sourceLocation: sourceLocation) + Issue.record("expected \(codes) but nothing was thrown", sourceLocation: sourceLocation) } catch let error as IggyError { - #expect(error.code == code, "expected \(code), got \(error)", sourceLocation: sourceLocation) + #expect(codes.contains(error.code), "expected \(codes), got \(error)", sourceLocation: sourceLocation) } catch { - Issue.record("expected \(code), got \(error)", sourceLocation: sourceLocation) + Issue.record("expected \(codes), got \(error)", sourceLocation: sourceLocation) } }
diff --git a/foreign/swift/Tests/IggyTests/WireVectorTests.swift b/foreign/swift/Tests/IggyTests/WireVectorTests.swift index d583f25..370cf94 100644 --- a/foreign/swift/Tests/IggyTests/WireVectorTests.swift +++ b/foreign/swift/Tests/IggyTests/WireVectorTests.swift
@@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +import Foundation import Testing @testable import Iggy @@ -61,7 +62,12 @@ #expect(try encode(5) == golden.bytes("request.delete_stream")) #expect(try encode("my-stream") == golden.bytes("request.get_stream")) #expect(golden.bytes("request.get_streams").isEmpty) - #expect(try Requests.updateStream(streamID: "old-name", name: "new-name", options: []) == golden.bytes("request.update_stream")) + let key = Data((0..<UInt8(EncryptionKey.length)).map { $0 }) + #expect( + try Requests.createStream(name: "test-stream", options: StreamCreateOptions(encryptionKey: key).encode()) + == golden.bytes("request.create_stream.key")) + #expect( + try Requests.updateResource(.stream("old-name"), StreamUpdateOptions(name: "new-name").toPatch()) == golden.bytes("request.update_resource.stream")) } @Test func topics() throws { @@ -71,8 +77,13 @@ #expect(try Requests.streamAndTopic(1, 5) == golden.bytes("request.delete_topic")) #expect(try Requests.streamAndTopic("my-stream", "my-topic") == golden.bytes("request.get_topic")) #expect(try encode(42) == golden.bytes("request.get_topics")) - let options = try OptionsBlock.encode(["message_expiry": .explicit(.uint64(604_800_000_000))]) - #expect(try Requests.updateTopic(streamID: 1, topicID: 2, name: "updated-topic", options: options) == golden.bytes("request.update_topic")) + let update = TopicUpdateOptions(name: "updated-topic", messageExpiry: .after(.seconds(604_800)), reset: [.maxTopicSize]) + #expect(try Requests.updateResource(.topic(stream: 1, topic: 2), update.toPatch()) == golden.bytes("request.update_resource.topic")) + // Reset names travel in byte order, whatever order they were named in. + let resets = TopicUpdateOptions(reset: [.maxTopicSize, .messageExpiry]) + #expect( + try Requests.updateResource(.topic(stream: "stream", topic: "topic"), resets.toPatch()) + == golden.bytes("request.update_resource.topic.reset_only")) } @Test func partitionsAndSegments() throws { @@ -135,8 +146,11 @@ #expect(try encode("old-user") == golden.bytes("request.delete_user")) #expect(try encode(42) == golden.bytes("request.get_user")) #expect(golden.bytes("request.get_users").isEmpty) - #expect(try Requests.updateUser(userID: 1, username: "new-name", status: .inactive, options: []) == golden.bytes("request.update_user")) - #expect(try Requests.updateUser(userID: 5, username: nil, status: nil, options: []) == golden.bytes("request.update_user.none")) + #expect( + try Requests.updateResource(.user(1), UserUpdateOptions(username: "new-name", status: .inactive).toPatch()) + == golden.bytes("request.update_resource.user")) + #expect( + try Requests.updateResource(.user("admin"), UserUpdateOptions(status: .active).toPatch()) == golden.bytes("request.update_resource.user.status")) #expect(try Requests.updatePermissions(userID: 1, permissions: samplePermissions) == golden.bytes("request.update_permissions")) #expect(try Requests.updatePermissions(userID: "admin", permissions: nil) == golden.bytes("request.update_permissions.none")) #expect(try Requests.changePassword(userID: 1, currentPassword: "old-pass", newPassword: "new-pass-123") == golden.bytes("request.change_password")) @@ -178,7 +192,7 @@ @Test func protocolVersionMatchesTheCrate() { #expect(ProtocolVersion.current.packed == golden.protocolVersion) #expect(ProtocolVersion.current.description == ProtocolVersion(packed: golden.protocolVersion).description) - #expect(ProtocolVersion(major: 0, minor: 11, patch: 1).description == "0.11.1") + #expect(ProtocolVersion(major: 0, minor: 12, patch: 0).description == "0.12.0") } } @@ -188,13 +202,23 @@ let clientID = UInt128Value(low: 0xFEDC_BA98_7654_3210, high: 0x0123_4567_89AB_CDEF) func header(_ request: VSRFrame.EncodedRequest) -> [UInt8] { - Array(request.bytes[..<VSRFrame.headerSize]) + var bytes = Array(request.bytes[..<VSRFrame.headerSize]) + bytes.replaceSubrange( + VSRFrame.RequestOffset.requestChecksum..<VSRFrame.RequestOffset.requestChecksum + 16, + with: repeatElement(0, count: 16)) + return bytes + } + + func requestIdentifier(_ request: VSRFrame.EncodedRequest) -> UInt128Value { + let range = VSRFrame.RequestOffset.requestChecksum..<VSRFrame.RequestOffset.requestChecksum + 16 + return UInt128Value(littleEndianBytes: request.bytes[range]) } @Test func registerHeader() throws { var session = ConsensusSession(clientID: clientID) let request = try VSRFrame.encodeRequest(session: &session, code: CommandCode.loginRegister.rawValue, payload: [1, 2, 3]) #expect(header(request) == golden.bytes("vsr.request.register")) + #expect(!requestIdentifier(request).isZero) #expect(Array(request.bytes[VSRFrame.headerSize...]) == [1, 2, 3]) #expect(request.requestID == 0) #expect(request.operation == .register) @@ -204,30 +228,47 @@ var session = ConsensusSession(clientID: clientID) let unbound = try VSRFrame.encodeRequest(session: &session, code: CommandCode.ping.rawValue, payload: []) #expect(header(unbound) == golden.bytes("vsr.request.ping")) + #expect(requestIdentifier(unbound).isZero) try session.bind(99) _ = try session.nextRequestID() _ = try session.nextRequestID() let bound = try VSRFrame.encodeRequest(session: &session, code: CommandCode.ping.rawValue, payload: []) #expect(header(bound) == golden.bytes("vsr.request.ping.bound")) + #expect(requestIdentifier(bound).isZero) } - @Test func metadataHeaderStampsTheChecksum() throws { + @Test func metadataHeaderStampsRandomIdentifier() throws { var session = ConsensusSession(clientID: clientID) try session.bind(99) let payload = golden.bytes("payload.create_stream") #expect(payload == (try Requests.createStream(name: "stream"))) let request = try VSRFrame.encodeRequest(session: &session, code: CommandCode.createStream.rawValue, payload: payload) #expect(header(request) == golden.bytes("vsr.request.create_stream")) + #expect(!requestIdentifier(request).isZero) #expect(request.requestID == 1) #expect(UInt64(littleEndianBytes: request.bytes[VSRFrame.RequestOffset.request..<VSRFrame.RequestOffset.request + 8]) == 1) } - @Test func partitionHeaderLeavesTheChecksumZero() throws { + /// The generic update is replicated, so its header carries a random request identifier. + @Test func updateResourceHeaderStampsRandomIdentifier() throws { + var session = ConsensusSession(clientID: clientID) + try session.bind(99) + let payload = golden.bytes("payload.update_resource") + let update = TopicUpdateOptions(name: "updated-topic", messageExpiry: .after(.seconds(604_800)), reset: [.maxTopicSize]) + #expect(payload == (try Requests.updateResource(.topic(stream: 1, topic: 2), update.toPatch()))) + let request = try VSRFrame.encodeRequest(session: &session, code: CommandCode.updateResource.rawValue, payload: payload) + #expect(header(request) == golden.bytes("vsr.request.update_resource")) + #expect(!requestIdentifier(request).isZero) + #expect(request.operation == .updateResource) + } + + @Test func partitionHeaderLeavesTheIdentifierZero() throws { var session = ConsensusSession(clientID: clientID) try session.bind(99) _ = try session.nextRequestID() let request = try VSRFrame.encodeRequest(session: &session, code: CommandCode.sendMessages.rawValue, payload: [9, 9, 9, 9]) #expect(header(request) == golden.bytes("vsr.request.send_messages")) + #expect(requestIdentifier(request).isZero) #expect(request.requestID == 2) } @@ -254,9 +295,11 @@ } let logout = try VSRFrame.encodeRequest(session: &session, code: CommandCode.logoutUser.rawValue, payload: []) #expect(header(logout) == golden.bytes("vsr.request.logout")) + #expect(!requestIdentifier(logout).isZero) #expect(logout.operation == .logout) let unknown = try VSRFrame.encodeRequest(session: &session, code: 60_000, payload: []) #expect(header(unknown) == golden.bytes("vsr.request.unknown_code")) + #expect(requestIdentifier(unknown).isZero) #expect(unknown.operation == .nonReplicated) }
diff --git a/foreign/swift/Tools/golden-vectors/src/main.rs b/foreign/swift/Tools/golden-vectors/src/main.rs index d9f8e2f..7f804a9 100644 --- a/foreign/swift/Tools/golden-vectors/src/main.rs +++ b/foreign/swift/Tools/golden-vectors/src/main.rs
@@ -54,7 +54,7 @@ use iggy_binary_protocol::requests::streams::*; use iggy_binary_protocol::requests::system::*; use iggy_binary_protocol::requests::topics::{ - CreateTopicRequest, DeleteTopicRequest, GetTopicRequest, GetTopicsRequest, UpdateTopicRequest, + CreateTopicRequest, DeleteTopicRequest, GetTopicRequest, GetTopicsRequest, }; use iggy_binary_protocol::requests::users::login_register::BindSecret; use iggy_binary_protocol::requests::users::*; @@ -92,12 +92,14 @@ }; use iggy_common::{ CompressionAlgorithm, Durability, IggyByteSize, IggyDuration, IggyError, IggyExpiry, - MaxTopicSize, NO_ASSIGNED_PARTITION, RESYNC_REQUIRED_PARTITION_SENTINEL, TopicCreateOptions, - TopicUpdateOptions, + MaxTopicSize, NO_ASSIGNED_PARTITION, OptionKey, OptionsPatch, + RESYNC_REQUIRED_PARTITION_SENTINEL, StreamCreateOptions, StreamEncryptionKey, + StreamUpdateOptions, TopicCreateOptions, TopicUpdateOptions, UserStatus, UserUpdateOptions, + stream_key_fingerprint, }; use secrecy::SecretString; use serde::Serialize; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use twox_hash::{XxHash3_64, XxHash32}; /// Hash and error tables are maps keyed by length and code, so each entry is @@ -149,11 +151,50 @@ ]) } +/// The cross-SDK block `core/binary_protocol/src/primitives/options.rs` pins by +/// hand, in its insertion order rather than key order. +fn golden_options_block() -> WireOptions { + options(&[ + (2, b"preallocate_segments", 3, &[1]), + (2, b"segment_size", 12, &1_073_741_824u64.to_le_bytes()), + (2, b"durability", 2, b"persisted"), + ]) +} + +/// Key bytes 0 to 31, the key of the shared fingerprint vector. +fn sample_key() -> [u8; 32] { + std::array::from_fn(|index| index as u8) +} + +/// The read-only projections of a stream whose key the client chose, wrapped +/// under two KEKs during a rotation. +fn stream_projections() -> WireOptions { + let fingerprint = stream_key_fingerprint(&sample_key()).expect("32-byte key"); + options(&[ + ( + 2, + b"encryption_kek_fingerprint", + 2, + b"39e4f0f95c3693c7,d55d479829bc2936", + ), + (2, b"encryption_key_fingerprint", 2, fingerprint.as_bytes()), + (2, b"encryption_key_source", 2, b"stream"), + ]) +} + +fn update_resource(target: ResourceTarget, patch: OptionsPatch) -> UpdateResourceRequest { + UpdateResourceRequest { + target, + reset: patch.reset, + options: patch.set, + } +} + fn version_info() -> ClientVersionInfo { ClientVersionInfo { protocol_version: IGGY_PROTOCOL_VERSION, sdk_name: name("swift-sdk"), - sdk_version: name("0.1.0"), + sdk_version: name("0.1.1"), } } @@ -259,14 +300,18 @@ id, created_at: 1_710_000_000_000_000, partitions_count: 3, - message_expiry: 604_800_000_000, - compression_algorithm: 1, - max_topic_size: 1_073_741_824, size_bytes: 4096, messages_count: 100, name: name(name_value), - options: sample_options(), - derived_options: options(&[(2, b"max_topic_size", 12, &u64::MAX.to_le_bytes())]), + options: options(&[ + (2, b"durability", 2, b"persisted"), + (2, b"message_expiry", 12, &604_800_000_000u64.to_le_bytes()), + (2, b"segment_size", 12, &1_073_741_824u64.to_le_bytes()), + ]), + derived_options: options(&[ + (2, b"compression_algorithm", 2, b"none"), + (2, b"max_topic_size", 12, &u64::MAX.to_le_bytes()), + ]), } } @@ -279,6 +324,7 @@ messages_count: 200, name: name("my-stream"), options: WireOptions::empty(), + derived_options: stream_projections(), } } @@ -294,8 +340,10 @@ session: u64, payload: &[u8], ) -> RequestHeader { - RequestHeader::for_request(code, client, request, session, payload) - .expect("fixture payloads fit a request frame") + let mut header = RequestHeader::for_request(code, client, request, session, payload) + .expect("fixture payloads fit a request frame"); + header.request_checksum = 0; + header } fn main() { @@ -378,6 +426,11 @@ add(&mut vectors, "polling.last", &WirePollingStrategy::last()); add(&mut vectors, "polling.next", &WirePollingStrategy::next()); add(&mut vectors, "options.golden", &sample_options()); + add( + &mut vectors, + "options.golden_block", + &golden_options_block(), + ); add(&mut vectors, "permissions.sample", &sample_permissions()); add(&mut vectors, "version_info", &version_info()); @@ -398,6 +451,21 @@ options: sample_options(), }, ); + let key_options = StreamCreateOptions { + encryption_key: Some(StreamEncryptionKey::try_from(sample_key().as_slice()).expect("key")), + raw: BTreeMap::new(), + } + .to_wire() + .expect("encodes"); + add(&mut vectors, "options.stream_create.key", &key_options); + add( + &mut vectors, + "request.create_stream.key", + &CreateStreamRequest { + name: name("test-stream"), + options: key_options, + }, + ); add( &mut vectors, "request.delete_stream", @@ -415,12 +483,18 @@ add(&mut vectors, "request.get_streams", &GetStreamsRequest); add( &mut vectors, - "request.update_stream", - &UpdateStreamRequest { - stream_id: named("old-name"), - name: name("new-name"), - options: WireOptions::empty(), - }, + "request.update_resource.stream", + &update_resource( + ResourceTarget::Stream { + stream_id: named("old-name"), + }, + StreamUpdateOptions { + name: Some("new-name".to_owned()), + ..Default::default() + } + .to_patch() + .expect("encodes"), + ), ); add( &mut vectors, @@ -455,15 +529,38 @@ stream_id: numeric(42), }, ); - add( - &mut vectors, - "request.update_topic", - &UpdateTopicRequest { + let topic_update = update_resource( + ResourceTarget::Topic { stream_id: numeric(1), topic_id: numeric(2), - name: name("updated-topic"), - options: options(&[(2, b"message_expiry", 12, &604_800_000_000u64.to_le_bytes())]), }, + TopicUpdateOptions { + name: Some("updated-topic".to_owned()), + message_expiry: Some(IggyExpiry::ExpireDuration(IggyDuration::new_from_secs( + 604_800, + ))), + reset: BTreeSet::from([OptionKey::MaxTopicSize]), + ..Default::default() + } + .to_patch() + .expect("encodes"), + ); + add(&mut vectors, "request.update_resource.topic", &topic_update); + add( + &mut vectors, + "request.update_resource.topic.reset_only", + &update_resource( + ResourceTarget::Topic { + stream_id: named("stream"), + topic_id: named("topic"), + }, + TopicUpdateOptions { + reset: BTreeSet::from([OptionKey::MaxTopicSize, OptionKey::MessageExpiry]), + ..Default::default() + } + .to_patch() + .expect("encodes"), + ), ); add( &mut vectors, @@ -674,23 +771,34 @@ add(&mut vectors, "request.get_users", &GetUsersRequest); add( &mut vectors, - "request.update_user", - &UpdateUserRequest { - user_id: numeric(1), - username: Some(name("new-name")), - status: Some(2), - options: WireOptions::empty(), - }, + "request.update_resource.user", + &update_resource( + ResourceTarget::User { + user_id: numeric(1), + }, + UserUpdateOptions { + username: Some("new-name".to_owned()), + status: Some(UserStatus::Inactive), + ..Default::default() + } + .to_patch() + .expect("encodes"), + ), ); add( &mut vectors, - "request.update_user.none", - &UpdateUserRequest { - user_id: numeric(5), - username: None, - status: None, - options: WireOptions::empty(), - }, + "request.update_resource.user.status", + &update_resource( + ResourceTarget::User { + user_id: named("admin"), + }, + UserUpdateOptions { + status: Some(UserStatus::Active), + ..Default::default() + } + .to_patch() + .expect("encodes"), + ), ); add( &mut vectors, @@ -936,6 +1044,22 @@ ))), ); + let update_resource_payload = topic_update.to_bytes(); + vectors.insert( + "payload.update_resource".to_owned(), + hex(&update_resource_payload), + ); + vectors.insert( + "vsr.request.update_resource".to_owned(), + hex(bytemuck::bytes_of(&request_header( + UPDATE_RESOURCE_CODE, + client_id, + 1, + 99, + &update_resource_payload, + ))), + ); + let reply = ReplyHeader { command: Command::Reply, operation: Operation::CreateStream, @@ -953,6 +1077,15 @@ "vsr.reply.committed".to_owned(), hex(bytemuck::bytes_of(&committed)), ); + // The primary commits an `UpdateResource` under its scope's resolved + // operation, so the reply names that operation, not the request's. + let mut resolved = committed; + resolved.operation = Operation::UpdateTopicResolved; + resolved.size = (HEADER_SIZE + 4) as u32; + vectors.insert( + "vsr.reply.update_topic_resolved".to_owned(), + hex(bytemuck::bytes_of(&resolved)), + ); let denied = ReplyHeader { command: Command::Reply, operation: Operation::NonReplicated, @@ -1188,6 +1321,7 @@ status: 1, username: name("admin"), options: WireOptions::empty(), + derived_options: WireOptions::empty(), }; add(&mut vectors, "response.user", &user); add( @@ -1218,6 +1352,7 @@ status: 2, username: name("alice"), options: WireOptions::empty(), + derived_options: WireOptions::empty(), }, ], }, @@ -1342,15 +1477,31 @@ OptionDescriptor { key: name("segment_size"), kind: 12, + flags: 0, default_value: Bytes::copy_from_slice(&1_073_741_824u64.to_le_bytes()), description: "Segment size".to_owned(), }, OptionDescriptor { key: name("durability"), kind: 2, + flags: 0, default_value: Bytes::from_static(b"replicated"), description: String::new(), }, + OptionDescriptor { + key: name("message_expiry"), + kind: 12, + flags: 0b11, + default_value: Bytes::copy_from_slice(&u64::MAX.to_le_bytes()), + description: String::new(), + }, + OptionDescriptor { + key: name("encryption_key"), + kind: 1, + flags: 0b1_0100, + default_value: Bytes::new(), + description: String::new(), + }, ], }, ); @@ -1451,15 +1602,14 @@ &create_options.to_wire().expect("encodes"), ); let update_options = TopicUpdateOptions { - compression_algorithm: Some(CompressionAlgorithm::None), message_expiry: Some(IggyExpiry::NeverExpire), max_topic_size: Some(MaxTopicSize::Unlimited), - raw: BTreeMap::new(), + ..Default::default() }; add( &mut vectors, "options.topic_update.full", - &update_options.to_wire().expect("encodes"), + &update_options.to_patch().expect("encodes").set, ); let mut raw = BTreeMap::new(); raw.insert("durability".to_owned(), "persisted".to_owned()); @@ -1473,6 +1623,11 @@ &raw_options.to_wire().expect("encodes"), ); + vectors.insert( + "stream_key.fingerprint".to_owned(), + stream_key_fingerprint(&sample_key()).expect("32-byte key"), + ); + let golden = Golden { protocol_version: IGGY_PROTOCOL_VERSION, protocol_version_min: IGGY_PROTOCOL_VERSION_MIN,
diff --git a/gateways/kafka/src/bridge/iggy_bridge/cluster_id.rs b/gateways/kafka/src/bridge/iggy_bridge/cluster_id.rs index 141e479..c69b49d 100644 --- a/gateways/kafka/src/bridge/iggy_bridge/cluster_id.rs +++ b/gateways/kafka/src/bridge/iggy_bridge/cluster_id.rs
@@ -26,8 +26,8 @@ use iggy::prelude::{ Consumer, Durability, Identifier, IggyError, IggyExpiry, IggyMessage, MaxTopicSize, - MessageClient, Partitioning, PollingStrategy, StreamClient, TopicClient, TopicCreateOptions, - TopicDetails, + MessageClient, Partitioning, PollingStrategy, StreamClient, StreamCreateOptions, TopicClient, + TopicCreateOptions, TopicDetails, }; use tracing::{info, warn}; @@ -203,8 +203,9 @@ } async fn create_cluster_id_stream(&self, store: &IdStore) -> Result<(), BridgeError> { + let options = StreamCreateOptions::default(); match self - .metadata_write(|| self.client.create_stream(&store.stream_name)) + .metadata_write(|| self.client.create_stream(&store.stream_name, &options)) .await { Ok(_) | Err(BridgeError::Iggy(IggyError::StreamNameAlreadyExists(_))) => Ok(()), @@ -314,11 +315,12 @@ impl From<&TopicDetails> for IdTopicOptions { fn from(topic: &TopicDetails) -> Self { + let resolved = topic.resolved_options(); Self { partitions_count: topic.partitions_count, durability: TopicCreateOptions::from_resource_options(&topic.options).durability, - message_expiry: topic.message_expiry, - max_topic_size: topic.max_topic_size, + message_expiry: resolved.message_expiry, + max_topic_size: resolved.max_topic_size, } } }
diff --git a/gateways/kafka/src/bridge/iggy_bridge/topics.rs b/gateways/kafka/src/bridge/iggy_bridge/topics.rs index ea7d287..8d4225c 100644 --- a/gateways/kafka/src/bridge/iggy_bridge/topics.rs +++ b/gateways/kafka/src/bridge/iggy_bridge/topics.rs
@@ -21,8 +21,8 @@ use std::sync::Arc; use iggy::prelude::{ - Identifier, IggyClient, IggyError, IggyExpiry, PartitionClient, StreamClient, Topic, - TopicClient, TopicCreateOptions, TopicDetails, TopicUpdateOptions, + Identifier, IggyClient, IggyError, IggyExpiry, PartitionClient, StreamClient, + StreamCreateOptions, Topic, TopicClient, TopicCreateOptions, TopicDetails, TopicUpdateOptions, }; use kafka_protocol::protocol::StrBytes; use tokio::time::{Instant, timeout_at}; @@ -354,8 +354,7 @@ /// Sets `message_expiry` on the Iggy topic backing `kafka_topic`. /// - /// Does not create the topic and does not rename it: `name` is the mapped Iggy - /// topic name, which `update_topic` also takes as the topic's new name. + /// Does not create the topic and does not rename it. /// /// # Errors /// @@ -376,11 +375,8 @@ message_expiry: Some(message_expiry), ..TopicUpdateOptions::default() }; - self.metadata_write(|| { - self.client - .update_topic(&stream_id, &topic_id, topic_name, &options) - }) - .await + self.metadata_write(|| self.client.update_topic(&stream_id, &topic_id, &options)) + .await } async fn topics_in_stream(&self, stream_name: &str) -> Result<Vec<Topic>, BridgeError> { @@ -705,9 +701,10 @@ /// this pays one round trip in every case instead of up to two in the common one. async fn ensure_stream(client: &IggyClient, stream_name: &str) -> Result<Identifier, BridgeError> { let identifier = Identifier::named(stream_name).map_err(BridgeError::Iggy)?; + let options = StreamCreateOptions::default(); stream_ensured( stream_name, - metadata_write_on(client, || client.create_stream(stream_name)).await, + metadata_write_on(client, || client.create_stream(stream_name, &options)).await, )?; Ok(identifier) } @@ -831,10 +828,11 @@ .into_iter() .map(|topic| { let message_expiry_explicit = message_expiry_is_explicit(&topic.options); + let message_expiry = topic.resolved_options().message_expiry; ( topic.name, TopicConfigSnapshot { - message_expiry: topic.message_expiry, + message_expiry, message_expiry_explicit, }, )
diff --git a/gateways/kafka/src/protocol/handlers/describe_configs.rs b/gateways/kafka/src/protocol/handlers/describe_configs.rs index 90ac2dc..778296e 100644 --- a/gateways/kafka/src/protocol/handlers/describe_configs.rs +++ b/gateways/kafka/src/protocol/handlers/describe_configs.rs
@@ -334,8 +334,8 @@ use std::str::FromStr; use iggy::prelude::{ - CompressionAlgorithm, HeaderKey, HeaderValue, IggyByteSize, IggyExpiry, IggyTimestamp, - MaxTopicSize, OptionValue, ResourceOptions, TopicDetails, topic_option_keys, + HeaderKey, HeaderValue, IggyByteSize, IggyExpiry, IggyTimestamp, OptionValue, + ResourceOptions, TopicDetails, topic_option_keys, }; use kafka_protocol::messages::describe_configs_response::DescribeConfigsResourceResult; use kafka_protocol::protocol::StrBytes; @@ -347,21 +347,19 @@ fn topic_details(expiry: IggyExpiry, explicit: bool) -> TopicDetails { let mut options = ResourceOptions::new(); - if explicit { - let key = HeaderKey::from_str(topic_option_keys::MESSAGE_EXPIRY).expect("expiry key"); - options.insert( - key, - OptionValue::explicit(HeaderValue::from(u64::from(expiry))), - ); - } + let key = HeaderKey::from_str(topic_option_keys::MESSAGE_EXPIRY).expect("expiry key"); + let value = HeaderValue::from(u64::from(expiry)); + let option = if explicit { + OptionValue::explicit(value) + } else { + OptionValue::derived(value) + }; + options.insert(key, option); TopicDetails { id: 1, created_at: IggyTimestamp::default(), name: "orders".to_string(), size: IggyByteSize::default(), - message_expiry: expiry, - compression_algorithm: CompressionAlgorithm::None, - max_topic_size: MaxTopicSize::default(), messages_count: 0, partitions_count: 1, partitions: Vec::new(), @@ -375,7 +373,7 @@ include_documentation: bool, ) -> (Vec<DescribeConfigsResourceResult>, bool) { described_entries( - details.message_expiry, + details.resolved_options().message_expiry, message_expiry_is_explicit(&details.options), configuration_keys, include_documentation,
diff --git a/gateways/kafka/tests/bridge_iggy_integration_tests.rs b/gateways/kafka/tests/bridge_iggy_integration_tests.rs index 6273c4f..007001d 100644 --- a/gateways/kafka/tests/bridge_iggy_integration_tests.rs +++ b/gateways/kafka/tests/bridge_iggy_integration_tests.rs
@@ -20,6 +20,7 @@ //! exercises directly: `ensure_stream_and_topic` idempotent on repeated calls, and the bridge //! module invoked from a real (non-unit) test rather than only compiled. +use iggy::prelude::StreamCreateOptions; use std::collections::HashMap; use std::str::FromStr; use std::time::Duration; @@ -556,7 +557,7 @@ .await .expect("delete the stream (and its topic with it)"); let recreated = raw - .create_stream("kafka") + .create_stream("kafka", &StreamCreateOptions::default()) .await .expect("recreate a stream under the same name"); if recreated.id == original.id {
diff --git a/gateways/kafka/tests/cluster_id_real_bridge_tests.rs b/gateways/kafka/tests/cluster_id_real_bridge_tests.rs index 3a5cd76..0083d19 100644 --- a/gateways/kafka/tests/cluster_id_real_bridge_tests.rs +++ b/gateways/kafka/tests/cluster_id_real_bridge_tests.rs
@@ -25,8 +25,8 @@ use futures::future::join_all; use iggy::prelude::{ Consumer, Durability, Identifier, IggyExpiry, IggyMessage, MaxTopicSize, MessageClient, - Partitioning, Permissions, PollingStrategy, StreamClient, StreamPermissions, TopicClient, - TopicCreateOptions, UserClient, UserStatus, + Partitioning, Permissions, PollingStrategy, StreamClient, StreamCreateOptions, + StreamPermissions, TopicClient, TopicCreateOptions, UserClient, UserCreateOptions, UserStatus, }; use kafka_protocol::messages::{DescribeClusterResponse, MetadataResponse}; use kafka_protocol::protocol::Decodable; @@ -70,7 +70,10 @@ ) -> IggyBridge { let raw = raw_client(server).await; let stream = raw - .create_stream(&cluster_id_stream(DEFAULT_STREAM)) + .create_stream( + &cluster_id_stream(DEFAULT_STREAM), + &StreamCreateOptions::default(), + ) .await .expect("create stream"); raw.create_user( @@ -81,6 +84,7 @@ streams: Some(BTreeMap::from([(stream.id as usize, permissions)])), ..Permissions::default() }), + &UserCreateOptions::default(), ) .await .expect("create user"); @@ -179,8 +183,9 @@ .expect("get_topic") .expect("the id topic exists"); assert_eq!(topic.partitions_count, 1); - assert_eq!(topic.message_expiry, IggyExpiry::NeverExpire); - assert_eq!(topic.max_topic_size, MaxTopicSize::Unlimited); + let resolved = topic.resolved_options(); + assert_eq!(resolved.message_expiry, IggyExpiry::NeverExpire); + assert_eq!(resolved.max_topic_size, MaxTopicSize::Unlimited); assert_eq!( TopicCreateOptions::from_resource_options(&topic.options).durability, Durability::Persisted @@ -244,7 +249,7 @@ let server = TestServer::spawn(data_dir.path()).await; let raw = raw_client(&server).await; let stream_name = cluster_id_stream(DEFAULT_STREAM); - raw.create_stream(&stream_name) + raw.create_stream(&stream_name, &StreamCreateOptions::default()) .await .expect("create stream"); let stream_id = Identifier::named(&stream_name).expect("stream name");
diff --git a/gateways/kafka/tests/metadata_real_bridge_tests.rs b/gateways/kafka/tests/metadata_real_bridge_tests.rs index df1b813..25b4b85 100644 --- a/gateways/kafka/tests/metadata_real_bridge_tests.rs +++ b/gateways/kafka/tests/metadata_real_bridge_tests.rs
@@ -28,8 +28,8 @@ use bytes::Bytes; use iggy::prelude::{ - GlobalPermissions, Identifier, Permissions, StreamClient, StreamPermissions, UserClient, - UserStatus, + GlobalPermissions, Identifier, Permissions, StreamClient, StreamCreateOptions, + StreamPermissions, UserClient, UserCreateOptions, UserStatus, }; use secrecy::SecretString; use serial_test::serial; @@ -796,7 +796,10 @@ /// Creates the default stream, then a bridge user holding only `on_stream` on it, signed in. async fn limited_bridge(server: &TestServer, on_stream: StreamPermissions) -> IggyBridge { let root = raw_client(server).await; - let stream = root.create_stream("kafka").await.expect("seed the stream"); + let stream = root + .create_stream("kafka", &StreamCreateOptions::default()) + .await + .expect("seed the stream"); let stream_id = usize::try_from(stream.id).expect("stream id fits usize"); let permissions = Permissions { global: GlobalPermissions::default(), @@ -807,6 +810,7 @@ LIMITED_PASSWORD, UserStatus::Active, Some(permissions), + &UserCreateOptions::default(), ) .await .expect("create the bridge user");
diff --git a/gateways/kafka/tests/offset_groups_real_bridge_tests.rs b/gateways/kafka/tests/offset_groups_real_bridge_tests.rs index eb96cd2..47b4c70 100644 --- a/gateways/kafka/tests/offset_groups_real_bridge_tests.rs +++ b/gateways/kafka/tests/offset_groups_real_bridge_tests.rs
@@ -34,7 +34,7 @@ use iggy::prelude::{ Consumer, ConsumerGroupClient, ConsumerOffsetClient, Identifier, Permissions, StreamClient, - StreamPermissions, TopicClient, UserClient, UserStatus, + StreamPermissions, TopicClient, UserClient, UserCreateOptions, UserStatus, }; use secrecy::SecretString; use serial_test::serial; @@ -1338,6 +1338,7 @@ )])), ..Permissions::default() }), + &UserCreateOptions::default(), ) .await .expect("create a bridge user that may read only the stream kafka");
diff --git a/helm/charts/iggy/Chart.yaml b/helm/charts/iggy/Chart.yaml index a521509..c9337c2 100644 --- a/helm/charts/iggy/Chart.yaml +++ b/helm/charts/iggy/Chart.yaml
@@ -20,7 +20,7 @@ name: iggy description: A Helm chart for Apache Iggy server and web-ui type: application -version: 0.6.0 +version: 0.7.0 appVersion: "0.9.0-edge.6" sources: - https://github.com/apache/iggy
diff --git a/helm/charts/iggy/README.md b/helm/charts/iggy/README.md index 9a668bb..68a69c8 100644 --- a/helm/charts/iggy/README.md +++ b/helm/charts/iggy/README.md
@@ -2,7 +2,7 @@ A Helm chart for Apache Iggy server and web-ui -   +   ## Prerequisites @@ -196,13 +196,7 @@ authenticated handshake or be rejected. The key is at least 32 bytes of CSPRNG output. Turning it on or off is a coordinated restart of the whole cluster: a node that authenticates cannot talk to one that does not. -* **`server.encryption`** encrypts message payloads and state commands at rest - with AES-256-GCM, under a 32-byte base64 key. A node holding a different key - cannot read what its peers wrote. The chart selects the legacy configuration - for versioned images through `0.9.0-edge.7`, including its pinned image. - For newer builds with `[encryption]`, set `server.encryption.configVersion=flat`. - Custom tags require an explicit `flat` or `legacy` layout; `legacy` uses - `[system.encryption]`. +* **`server.encryption`** encrypts data at rest with AES-256-GCM under a 32-byte base64 key that every node must hold. A node holding a different key cannot read what its peers wrote. The pinned image uses the legacy `[system.encryption]` layout, encrypts message payloads and state commands under that key, and leaves data unencrypted with `enabled` false. Server builds using `[encryption]`, selected with `server.encryption.configVersion=flat`, give every encrypted stream its own key, stored wrapped under a key derived from the configured one. With `enabled` true, streams created without their own key get a generated one. With `enabled` false and a key set, only streams created with their own key are encrypted. A stream created unencrypted stays unencrypted. A standalone node that cannot open a stored stream key refuses to boot, while a cluster node starts and serves that stream once a wrap it can open has committed. To rotate the server key on such a build, set the new key, put the retiring ones in `server.encryption.previousKeys` (or under `existingSecret.previousKeysKey` in the existing Secret), and follow the procedure in the [server README](https://github.com/apache/iggy/blob/master/core/server/README.md#stream-encryption). The retiring keys reach the server through the Secret, like the key itself. Phase 2 empties `previousKeys` and sets `IGGY_ENCRYPTION_RETIRED_KEK_FINGERPRINTS` to the old key's fingerprint as a JSON array through `server.env`. Once every stream has dropped the old wrap, clear `IGGY_ENCRYPTION_RETIRED_KEK_FINGERPRINTS` from `server.env` on every node. A stream holds at most three wraps, so retire an old fingerprint before adding a fourth key. Fingerprints are not secret, so that entry is a plain value. The chart selects the legacy layout for released images through `0.9.0-edge.7`, including its pinned image. Other image tags require an explicit `flat` or `legacy` layout. * **`server.jwt`** makes HTTP bearer tokens valid on every node and survive a restart. With `cluster.auth` enabled the server already derives a cluster-wide key from the replica PSK, so setting the JWT secrets is an alternative to that @@ -232,6 +226,10 @@ for the same reason. Cluster size is a roster decision, so add a node to `server.cluster.nodes` and install another release instead. +## Upgrading from chart 0.6 + +Chart 0.7 acts on `server.encryption.key` and `server.encryption.existingSecret.name` even when `server.encryption.enabled` is false, because a server build with per-stream keys needs the configured key to accept client stream keys and to open the stream keys it already stored. A key left in the values with `enabled` false now creates the `<release>-secrets` Secret and reaches the server, and a key that does not decode to 32 bytes fails the render instead of being ignored. Remove a leftover key before upgrading if the release should run without one. A `server.encryption.existingSecret.name` left set with `enabled` false now renders a required reference to the `server.encryption.existingSecret.key` entry of that Secret, so the server container fails to start when the Secret lacks the entry. Add the entry or remove the name before upgrading. + ## Uninstallation ```bash @@ -487,7 +485,7 @@ | podSecurityContext | object | `{"seccompProfile":{"type":"Unconfined"}}` | Pod security context (server uses io_uring, requires unconfined seccomp) | | resources | object | `{}` | Resource limits and requests for server | | securityContext | object | `{"capabilities":{"add":["IPC_LOCK"]}}` | Container security context (server requires IPC_LOCK for io_uring) | -| server | object | `{"advertisedAddress":"","affinity":{},"cluster":{"auth":{"enabled":false,"existingSecret":{"name":"","previousSharedSecretKey":"clusterPreviousSharedSecret","sharedSecretKey":"clusterSharedSecret"},"previousSharedSecret":"","sharedSecret":""},"enabled":false,"name":"iggy-cluster","nodes":[],"requireHostNetwork":true,"selfReplicaId":0},"enabled":true,"encryption":{"configVersion":"auto","enabled":false,"existingSecret":{"key":"encryptionKey","name":""},"key":""},"env":[{"name":"RUST_LOG","value":"info"},{"name":"IGGY_HTTP_ADDRESS","value":"0.0.0.0:3000"},{"name":"IGGY_TCP_ADDRESS","value":"0.0.0.0:8090"},{"name":"IGGY_QUIC_ADDRESS","value":"0.0.0.0:8080"},{"name":"IGGY_WEBSOCKET_ADDRESS","value":"0.0.0.0:8092"}],"extraArgs":[],"hostNetwork":false,"image":{"pullPolicy":"Always","repository":"apache/iggy","tag":""},"ingress":{"annotations":{},"className":"","enabled":false,"hosts":[{"host":"chart-example.local","paths":[{"path":"/","pathType":"ImplementationSpecific"}]}],"tls":[]},"jwt":{"decodingSecret":"","encodingSecret":"","existingSecret":{"decodingSecretKey":"jwtDecodingSecret","encodingSecretKey":"jwtEncodingSecret","name":""}},"nodeSelector":{},"persistence":{"accessMode":"ReadWriteOnce","annotations":{},"enabled":false,"existingClaim":"","size":"8Gi","storageClass":""},"ports":{"http":3000,"quic":8080,"tcp":8090,"websocket":8092},"replicaCount":1,"service":{"port":3000,"type":"ClusterIP"},"serviceMonitor":{"additionalLabels":{},"authorization":{},"enabled":false,"honorLabels":false,"interval":"30s","namespace":"","path":"/metrics","scrapeTimeout":"10s"},"strategy":{},"tolerations":[],"users":{"root":{"createSecret":true,"existingSecret":{"name":"","passwordKey":"password","usernameKey":"username"},"password":"changeit","username":"iggy"}}}` | Iggy server configuration | +| server | object | `{"advertisedAddress":"","affinity":{},"cluster":{"auth":{"enabled":false,"existingSecret":{"name":"","previousSharedSecretKey":"clusterPreviousSharedSecret","sharedSecretKey":"clusterSharedSecret"},"previousSharedSecret":"","sharedSecret":""},"enabled":false,"name":"iggy-cluster","nodes":[],"requireHostNetwork":true,"selfReplicaId":0},"enabled":true,"encryption":{"configVersion":"auto","enabled":false,"existingSecret":{"key":"encryptionKey","name":"","previousKeysKey":"encryptionPreviousKeys"},"key":"","previousKeys":[]},"env":[{"name":"RUST_LOG","value":"info"},{"name":"IGGY_HTTP_ADDRESS","value":"0.0.0.0:3000"},{"name":"IGGY_TCP_ADDRESS","value":"0.0.0.0:8090"},{"name":"IGGY_QUIC_ADDRESS","value":"0.0.0.0:8080"},{"name":"IGGY_WEBSOCKET_ADDRESS","value":"0.0.0.0:8092"}],"extraArgs":[],"hostNetwork":false,"image":{"pullPolicy":"Always","repository":"apache/iggy","tag":""},"ingress":{"annotations":{},"className":"","enabled":false,"hosts":[{"host":"chart-example.local","paths":[{"path":"/","pathType":"ImplementationSpecific"}]}],"tls":[]},"jwt":{"decodingSecret":"","encodingSecret":"","existingSecret":{"decodingSecretKey":"jwtDecodingSecret","encodingSecretKey":"jwtEncodingSecret","name":""}},"nodeSelector":{},"persistence":{"accessMode":"ReadWriteOnce","annotations":{},"enabled":false,"existingClaim":"","size":"8Gi","storageClass":""},"ports":{"http":3000,"quic":8080,"tcp":8090,"websocket":8092},"replicaCount":1,"service":{"port":3000,"type":"ClusterIP"},"serviceMonitor":{"additionalLabels":{},"authorization":{},"enabled":false,"honorLabels":false,"interval":"30s","namespace":"","path":"/metrics","scrapeTimeout":"10s"},"strategy":{},"tolerations":[],"users":{"root":{"createSecret":true,"existingSecret":{"name":"","passwordKey":"password","usernameKey":"username"},"password":"changeit","username":"iggy"}}}` | Iggy server configuration | | server.advertisedAddress | string | `""` | Client-facing address published in cluster metadata. Declaring `IGGY_NODE_ADVERTISED_ADDRESS` in `server.env` instead also works, but setting both is refused at render time. Empty falls back to the in-cluster Service DNS name. Ignored in cluster mode, where the address comes from the node's roster entry, so setting both is refused there too. | | server.affinity | object | `{}` | Affinity rules for server pods | | server.cluster.auth | object | `{"enabled":false,"existingSecret":{"name":"","previousSharedSecretKey":"clusterPreviousSharedSecret","sharedSecretKey":"clusterSharedSecret"},"previousSharedSecret":"","sharedSecret":""}` | Replica-to-replica authentication on the consensus port. When enabled every peer must complete an authenticated handshake or be rejected, and a shared secret becomes mandatory. Enabling it on a running cluster is a coordinated-restart change, not a rolling one. | @@ -503,12 +501,14 @@ | server.cluster.requireHostNetwork | bool | `true` | Refuse to render a cluster node without `server.hostNetwork`. The replica listener binds the roster `ip` verbatim, which no pod owns on the cluster network, so the pod would die at boot with `CannotBindToSocket`. Set this to false only when the roster `ip` is an address the pod itself holds. | | server.cluster.selfReplicaId | int | `0` | Which `nodes` entry this release runs, matched against `replicaId`. | | server.enabled | bool | `true` | Enable the Iggy server deployment | -| server.encryption | object | `{"configVersion":"auto","enabled":false,"existingSecret":{"key":"encryptionKey","name":""},"key":""}` | Server-side encryption of message payloads and state commands, using AES-256-GCM. Every node of a cluster must hold the identical key, or it cannot read data another node wrote. | +| server.encryption | object | `{"configVersion":"auto","enabled":false,"existingSecret":{"key":"encryptionKey","name":"","previousKeysKey":"encryptionPreviousKeys"},"key":"","previousKeys":[]}` | Server-side encryption of message data at rest, using AES-256-GCM. Server builds using the `flat` layout give every encrypted stream its own key, stored wrapped under a key derived from this one. Every node of a cluster must hold the identical key, or it cannot read data another node wrote. | | server.encryption.configVersion | string | `"auto"` | Encryption config layout: `auto` selects `legacy` for released image versions through 0.9.0-edge.7. Set `flat` for newer server builds, or `legacy` for custom builds that still use `[system.encryption]`. | -| server.encryption.enabled | bool | `false` | Enable encryption at rest | +| server.encryption.enabled | bool | `false` | Enable encryption at rest. With the `flat` layout, streams created without their own key get a generated one, and a key set while this is false still accepts streams created with their own key. | | server.encryption.existingSecret.key | string | `"encryptionKey"` | Key inside that Secret | | server.encryption.existingSecret.name | string | `""` | Name of an existing Secret holding the encryption key | +| server.encryption.existingSecret.previousKeysKey | string | `"encryptionPreviousKeys"` | Optional key inside that Secret holding the retiring keys, one bare key or a JSON array of at most two. Read with the `flat` layout only. | | server.encryption.key | string | `""` | 32-byte key, base64 encoded. Ignored when `existingSecret.name` is set. Prefer `existingSecret` outside development: a value here is stored in the Helm release and readable by anyone who can read it. | +| server.encryption.previousKeys | list | `[]` | Retiring keys, at most two, base64 encoded, that still unwrap stream keys while a rotation is in flight. Needs the `flat` layout. Stored in the generated Secret. Ignored when `existingSecret.name` is set. Leave empty outside a rotation. | | server.env | list | `[{"name":"RUST_LOG","value":"info"},{"name":"IGGY_HTTP_ADDRESS","value":"0.0.0.0:3000"},{"name":"IGGY_TCP_ADDRESS","value":"0.0.0.0:8090"},{"name":"IGGY_QUIC_ADDRESS","value":"0.0.0.0:8080"},{"name":"IGGY_WEBSOCKET_ADDRESS","value":"0.0.0.0:8092"}]` | Environment variables for the server container | | server.extraArgs | list | `[]` | Extra command-line arguments appended to the server entrypoint, e.g. `["--with-default-root-credentials"]` for a throwaway development install. `--replica-id` is not one of them: the chart passes it already whenever `cluster.enabled` is set. | | server.hostNetwork | bool | `false` | Run the server pod in the host network namespace. Required for cluster mode, where the replica listener binds the roster IP verbatim. |
diff --git a/helm/charts/iggy/README.md.gotmpl b/helm/charts/iggy/README.md.gotmpl index 8a87a4f..aa07e09 100644 --- a/helm/charts/iggy/README.md.gotmpl +++ b/helm/charts/iggy/README.md.gotmpl
@@ -214,13 +214,7 @@ authenticated handshake or be rejected. The key is at least 32 bytes of CSPRNG output. Turning it on or off is a coordinated restart of the whole cluster: a node that authenticates cannot talk to one that does not. -* **`server.encryption`** encrypts message payloads and state commands at rest - with AES-256-GCM, under a 32-byte base64 key. A node holding a different key - cannot read what its peers wrote. The chart selects the legacy configuration - for versioned images through `0.9.0-edge.7`, including its pinned image. - For newer builds with `[encryption]`, set `server.encryption.configVersion=flat`. - Custom tags require an explicit `flat` or `legacy` layout; `legacy` uses - `[system.encryption]`. +* **`server.encryption`** encrypts data at rest with AES-256-GCM under a 32-byte base64 key that every node must hold. A node holding a different key cannot read what its peers wrote. The pinned image uses the legacy `[system.encryption]` layout, encrypts message payloads and state commands under that key, and leaves data unencrypted with `enabled` false. Server builds using `[encryption]`, selected with `server.encryption.configVersion=flat`, give every encrypted stream its own key, stored wrapped under a key derived from the configured one. With `enabled` true, streams created without their own key get a generated one. With `enabled` false and a key set, only streams created with their own key are encrypted. A stream created unencrypted stays unencrypted. A standalone node that cannot open a stored stream key refuses to boot, while a cluster node starts and serves that stream once a wrap it can open has committed. To rotate the server key on such a build, set the new key, put the retiring ones in `server.encryption.previousKeys` (or under `existingSecret.previousKeysKey` in the existing Secret), and follow the procedure in the [server README](https://github.com/apache/iggy/blob/master/core/server/README.md#stream-encryption). The retiring keys reach the server through the Secret, like the key itself. Phase 2 empties `previousKeys` and sets `IGGY_ENCRYPTION_RETIRED_KEK_FINGERPRINTS` to the old key's fingerprint as a JSON array through `server.env`. Once every stream has dropped the old wrap, clear `IGGY_ENCRYPTION_RETIRED_KEK_FINGERPRINTS` from `server.env` on every node. A stream holds at most three wraps, so retire an old fingerprint before adding a fourth key. Fingerprints are not secret, so that entry is a plain value. The chart selects the legacy layout for released images through `0.9.0-edge.7`, including its pinned image. Other image tags require an explicit `flat` or `legacy` layout. * **`server.jwt`** makes HTTP bearer tokens valid on every node and survive a restart. With `cluster.auth` enabled the server already derives a cluster-wide key from the replica PSK, so setting the JWT secrets is an alternative to that @@ -250,6 +244,10 @@ for the same reason. Cluster size is a roster decision, so add a node to `server.cluster.nodes` and install another release instead. +## Upgrading from chart 0.6 + +Chart 0.7 acts on `server.encryption.key` and `server.encryption.existingSecret.name` even when `server.encryption.enabled` is false, because a server build with per-stream keys needs the configured key to accept client stream keys and to open the stream keys it already stored. A key left in the values with `enabled` false now creates the `<release>-secrets` Secret and reaches the server, and a key that does not decode to 32 bytes fails the render instead of being ignored. Remove a leftover key before upgrading if the release should run without one. A `server.encryption.existingSecret.name` left set with `enabled` false now renders a required reference to the `server.encryption.existingSecret.key` entry of that Secret, so the server container fails to start when the Secret lacks the entry. Add the entry or remove the name before upgrading. + ## Uninstallation ```bash
diff --git a/helm/charts/iggy/examples/cluster-3-node.yaml b/helm/charts/iggy/examples/cluster-3-node.yaml index 6578a1d..274e326 100644 --- a/helm/charts/iggy/examples/cluster-3-node.yaml +++ b/helm/charts/iggy/examples/cluster-3-node.yaml
@@ -71,9 +71,8 @@ storageClass: "" size: 8Gi - # Encrypts message payloads and state commands at rest with AES-256-GCM. The - # key is cluster-wide: a node that holds a different one cannot read what its - # peers wrote. + # Encrypts message data at rest with AES-256-GCM. The key is cluster-wide: a + # node that holds a different one cannot read what its peers wrote. encryption: enabled: true existingSecret:
diff --git a/helm/charts/iggy/templates/_helpers.tpl b/helm/charts/iggy/templates/_helpers.tpl index 5dbc9dd..568edac 100644 --- a/helm/charts/iggy/templates/_helpers.tpl +++ b/helm/charts/iggy/templates/_helpers.tpl
@@ -227,7 +227,7 @@ {{- define "iggy.createsSecret" -}} {{- $server := .Values.server }} {{- $create := false }} - {{- if and $server.encryption.enabled (not $server.encryption.existingSecret.name) }} + {{- if and $server.encryption.key (not $server.encryption.existingSecret.name) }} {{- $create = true }} {{- end }} {{- if and (or $server.jwt.encodingSecret $server.jwt.decodingSecret) (not $server.jwt.existingSecret.name) }} @@ -259,13 +259,31 @@ {{- fail "server.users.root has neither createSecret nor existingSecret.name, so the container receives no IGGY_ROOT_USERNAME or IGGY_ROOT_PASSWORD. The chart requires both for initial root creation. Point existingSecret.name at a Secret holding them, set createSecret with a username and password, or for a throwaway development install add --with-default-root-credentials to server.extraArgs." }} {{- end }} {{- end }} - {{- if $server.encryption.enabled }} - {{- if and (not $server.encryption.key) (not $server.encryption.existingSecret.name) }} - {{- fail "server.encryption.enabled is true but no key was given. Set server.encryption.key to a base64-encoded 32-byte key, or point server.encryption.existingSecret.name at a Secret holding one." }} + {{- if and $server.encryption.enabled (not $server.encryption.key) (not $server.encryption.existingSecret.name) }} + {{- fail "server.encryption.enabled is true but no key was given. Set server.encryption.key to a base64-encoded 32-byte key, or point server.encryption.existingSecret.name at a Secret holding one." }} + {{- end }} + {{- if and $server.encryption.key (not $server.encryption.existingSecret.name) }} + {{- if ne (len (b64dec $server.encryption.key)) 32 }} + {{- fail (printf "server.encryption.key decodes to %d bytes. AES-256-GCM takes a base64-encoded 32-byte key, and the server refuses the config on anything else, enabled or not." (len (b64dec $server.encryption.key))) }} {{- end }} - {{- if and $server.encryption.key (not $server.encryption.existingSecret.name) }} - {{- if ne (len (b64dec $server.encryption.key)) 32 }} - {{- fail (printf "server.encryption.key decodes to %d bytes. AES-256-GCM takes a base64-encoded 32-byte key, and the server fails the boot with 'Invalid encryption key' on anything else." (len (b64dec $server.encryption.key))) }} + {{- end }} + {{- if and $server.encryption.previousKeys (not $server.encryption.existingSecret.name) }} + {{- $previousKeys := $server.encryption.previousKeys }} + {{- if not $server.encryption.key }} + {{- fail "server.encryption.previousKeys is set without server.encryption.key. A rotation needs the new key in key and the retiring ones in previousKeys." }} + {{- end }} + {{- if gt (len $previousKeys) 2 }} + {{- fail (printf "server.encryption.previousKeys holds %d keys. The server accepts at most two." (len $previousKeys)) }} + {{- end }} + {{- if ne (len (uniq $previousKeys)) (len $previousKeys) }} + {{- fail "server.encryption.previousKeys lists the same key twice, which the server rejects." }} + {{- end }} + {{- range $previousKeys }} + {{- if ne (len (b64dec .)) 32 }} + {{- fail (printf "A server.encryption.previousKeys entry decodes to %d bytes. Each must be a base64-encoded 32-byte key." (len (b64dec .))) }} + {{- end }} + {{- if eq . $server.encryption.key }} + {{- fail "server.encryption.previousKeys repeats server.encryption.key, which the server rejects. Leave previousKeys empty outside a rotation." }} {{- end }} {{- end }} {{- end }} @@ -297,7 +315,7 @@ {{- define "iggy.secretEnv" -}} {{- $server := .Values.server }} {{- $generated := include "iggy.secretName" . }} - {{- if $server.encryption.enabled }} + {{- if or $server.encryption.enabled $server.encryption.key $server.encryption.existingSecret.name }} {{- $configVersion := $server.encryption.configVersion | default "auto" }} {{- if eq $configVersion "auto" }} {{- $imageTag := $server.image.tag | default .Chart.AppVersion }} @@ -314,12 +332,25 @@ {{- fail "server.encryption.configVersion must be auto, legacy, or flat." }} {{- end }} - name: {{ $prefix }}_ENABLED - value: "true" + value: {{ $server.encryption.enabled | quote }} - name: {{ $prefix }}_KEY valueFrom: secretKeyRef: name: {{ default $generated $server.encryption.existingSecret.name }} key: {{ ternary $server.encryption.existingSecret.key "encryptionKey" (ne $server.encryption.existingSecret.name "") }} + {{- if and $server.encryption.previousKeys (not $server.encryption.existingSecret.name) }} + {{- if eq $configVersion "legacy" }} + {{- fail "server.encryption.previousKeys needs a server build using [encryption]. The legacy [system.encryption] layout has no key rotation. Set server.encryption.configVersion to flat for such a build." }} + {{- end }} + {{- end }} + {{- if and (eq $configVersion "flat") (or $server.encryption.previousKeys $server.encryption.existingSecret.name) }} +- name: IGGY_ENCRYPTION_PREVIOUS_KEYS + valueFrom: + secretKeyRef: + name: {{ default $generated $server.encryption.existingSecret.name }} + key: {{ ternary $server.encryption.existingSecret.previousKeysKey "encryptionPreviousKeys" (ne $server.encryption.existingSecret.name "") }} + optional: true + {{- end }} {{- end }} {{- if or $server.jwt.existingSecret.name $server.jwt.encodingSecret }} - name: IGGY_HTTP_JWT_ENCODING_SECRET
diff --git a/helm/charts/iggy/templates/server-secrets.yaml b/helm/charts/iggy/templates/server-secrets.yaml index e75e6e4..a93acd3 100644 --- a/helm/charts/iggy/templates/server-secrets.yaml +++ b/helm/charts/iggy/templates/server-secrets.yaml
@@ -24,8 +24,11 @@ {{- include "iggy.labels" . | nindent 4 }} type: Opaque stringData: - {{- if and .Values.server.encryption.enabled (not .Values.server.encryption.existingSecret.name) }} + {{- if and .Values.server.encryption.key (not .Values.server.encryption.existingSecret.name) }} encryptionKey: {{ .Values.server.encryption.key | quote }} + {{- with .Values.server.encryption.previousKeys }} + encryptionPreviousKeys: {{ toJson . | quote }} + {{- end }} {{- end }} {{- if not .Values.server.jwt.existingSecret.name }} {{- with .Values.server.jwt.encodingSecret }}
diff --git a/helm/charts/iggy/values.yaml b/helm/charts/iggy/values.yaml index 4c80b6b..816df33 100644 --- a/helm/charts/iggy/values.yaml +++ b/helm/charts/iggy/values.yaml
@@ -103,11 +103,15 @@ # -- PVC storage size size: 8Gi - # -- Server-side encryption of message payloads and state commands, using - # AES-256-GCM. Every node of a cluster must hold the identical key, or it - # cannot read data another node wrote. + # -- Server-side encryption of message data at rest, using AES-256-GCM. + # Server builds using the `flat` layout give every encrypted stream its own + # key, stored wrapped under a key derived from this one. Every node of a + # cluster must hold the identical key, or it cannot read data another node + # wrote. encryption: - # -- Enable encryption at rest + # -- Enable encryption at rest. With the `flat` layout, streams created + # without their own key get a generated one, and a key set while this is + # false still accepts streams created with their own key. enabled: false # -- Encryption config layout: `auto` selects `legacy` for released image # versions through 0.9.0-edge.7. Set `flat` for newer server builds, or @@ -117,11 +121,19 @@ # Prefer `existingSecret` outside development: a value here is stored in the # Helm release and readable by anyone who can read it. key: "" + # -- Retiring keys, at most two, base64 encoded, that still unwrap stream + # keys while a rotation is in flight. Needs the `flat` layout. Stored in the + # generated Secret. Ignored when `existingSecret.name` is set. Leave empty + # outside a rotation. + previousKeys: [] existingSecret: # -- Name of an existing Secret holding the encryption key name: "" # -- Key inside that Secret key: encryptionKey + # -- Optional key inside that Secret holding the retiring keys, one bare + # key or a JSON array of at most two. Read with the `flat` layout only. + previousKeysKey: encryptionPreviousKeys # -- Secrets used to sign and validate HTTP bearer tokens. Left unset, each # node generates a random secret on every start, which invalidates tokens
diff --git a/scripts/ci/test-helm.sh b/scripts/ci/test-helm.sh index e459d35..36e520a 100755 --- a/scripts/ci/test-helm.sh +++ b/scripts/ci/test-helm.sh
@@ -56,6 +56,9 @@ # A well-formed 32-byte AES-256-GCM key, base64 encoded. Render-only: it never # reaches a running server. HELM_TEST_ENCRYPTION_KEY="AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" +HELM_TEST_PREVIOUS_ENCRYPTION_KEY="ICEiIyQlJicoKSorLC0uLzAxMjM0NTY3ODk6Ozw9Pj8=" +HELM_TEST_OLDER_ENCRYPTION_KEY="QEFCQ0RFRkdISUpLTE1OT1BRUlNUVVZXWFlaW1xdXl8=" +HELM_TEST_OLDEST_ENCRYPTION_KEY="YGFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6e3x9fn8=" # HELM_SMOKE_GATEWAY_NAMESPACE and HELM_SMOKE_GATEWAY_NAME must match the # defaults in scripts/ci/setup-helm-smoke-cluster.sh - the HTTPRoute @@ -313,6 +316,10 @@ --set server.encryption.existingSecret.name=shared-key \ > "$HELM_RENDER_DIR/legacy-encryption.yaml" grep -q 'name: IGGY_SYSTEM_ENCRYPTION_KEY' "$HELM_RENDER_DIR/legacy-encryption.yaml" + if grep -q 'PREVIOUS_KEYS' "$HELM_RENDER_DIR/legacy-encryption.yaml"; then + echo "Error: the legacy encryption layout has no previous keys" >&2 + exit 1 + fi helm template iggy "$CHART_DIR" \ --set-string server.image.tag=custom-flat \ @@ -346,6 +353,52 @@ --set server.encryption.configVersion=invalid \ --set server.encryption.existingSecret.name=shared-key + # given_disabled_default_with_key_when_rendering_should_supply_key_and_false_enabled + helm template iggy "$CHART_DIR" \ + --set-string server.image.tag=custom-flat \ + --set server.encryption.configVersion=flat \ + --set-string server.encryption.key="$HELM_TEST_ENCRYPTION_KEY" \ + > "$HELM_RENDER_DIR/disabled-with-key.yaml" + grep -q "^ encryptionKey: \"${HELM_TEST_ENCRYPTION_KEY}\"$" "$HELM_RENDER_DIR/disabled-with-key.yaml" + grep -A1 'name: IGGY_ENCRYPTION_ENABLED' "$HELM_RENDER_DIR/disabled-with-key.yaml" | grep -q 'value: "false"' + grep -q 'name: IGGY_ENCRYPTION_KEY' "$HELM_RENDER_DIR/disabled-with-key.yaml" + + helm template iggy "$CHART_DIR" \ + --set-string server.image.tag=custom-flat \ + --set server.encryption.configVersion=flat \ + --set server.encryption.existingSecret.name=shared-key \ + > "$HELM_RENDER_DIR/disabled-with-secret.yaml" + grep -A1 'name: IGGY_ENCRYPTION_ENABLED' "$HELM_RENDER_DIR/disabled-with-secret.yaml" | grep -q 'value: "false"' + grep -q '^ name: shared-key$' "$HELM_RENDER_DIR/disabled-with-secret.yaml" + grep -A5 'name: IGGY_ENCRYPTION_PREVIOUS_KEYS' "$HELM_RENDER_DIR/disabled-with-secret.yaml" \ + | grep -q '^ key: encryptionPreviousKeys$' + grep -A5 'name: IGGY_ENCRYPTION_PREVIOUS_KEYS' "$HELM_RENDER_DIR/disabled-with-secret.yaml" \ + | grep -q '^ optional: true$' + if grep -qE '^ +encryptionKey:' "$HELM_RENDER_DIR/disabled-with-secret.yaml"; then + echo "Error: an existing encryption Secret must not be inlined" >&2 + exit 1 + fi + + helm template iggy "$CHART_DIR" \ + --set-string server.image.tag=custom-flat \ + --set server.encryption.configVersion=flat \ + --set-string server.encryption.key="$HELM_TEST_ENCRYPTION_KEY" \ + --set-string "server.encryption.previousKeys[0]=$HELM_TEST_PREVIOUS_ENCRYPTION_KEY" \ + > "$HELM_RENDER_DIR/previous-keys.yaml" + grep -qxF " encryptionPreviousKeys: \"[\\\"${HELM_TEST_PREVIOUS_ENCRYPTION_KEY}\\\"]\"" "$HELM_RENDER_DIR/previous-keys.yaml" + grep -A5 'name: IGGY_ENCRYPTION_PREVIOUS_KEYS' "$HELM_RENDER_DIR/previous-keys.yaml" \ + | grep -q '^ key: encryptionPreviousKeys$' + if [ "$(grep -cF "$HELM_TEST_PREVIOUS_ENCRYPTION_KEY" "$HELM_RENDER_DIR/previous-keys.yaml")" -ne 1 ]; then + echo "Error: a previous encryption key must reach the server only through the Secret" >&2 + exit 1 + fi + + helm template iggy "$CHART_DIR" > "$HELM_RENDER_DIR/no-encryption.yaml" + if grep -q 'ENCRYPTION_' "$HELM_RENDER_DIR/no-encryption.yaml"; then + echo "Error: no encryption value set, yet encryption variables were rendered" >&2 + exit 1 + fi + assert_render_rejected "server.replicaCount=3" --set server.replicaCount=3 assert_render_rejected "a single node without root credentials" \ --set server.users.root.createSecret=false @@ -357,6 +410,30 @@ assert_render_rejected "an encryption key that is not 32 bytes" \ --set server.encryption.enabled=true \ --set-string server.encryption.key=bm90LTMyLWJ5dGVz + assert_render_rejected "a disabled encryption key that is not 32 bytes" \ + --set-string server.encryption.key=bm90LTMyLWJ5dGVz + assert_render_rejected "previous encryption keys on the legacy layout" \ + --set-string server.encryption.key="$HELM_TEST_ENCRYPTION_KEY" \ + --set-string "server.encryption.previousKeys[0]=$HELM_TEST_PREVIOUS_ENCRYPTION_KEY" + assert_render_rejected "previous encryption keys without a key" \ + --set server.encryption.configVersion=flat \ + --set-string "server.encryption.previousKeys[0]=$HELM_TEST_PREVIOUS_ENCRYPTION_KEY" + assert_render_rejected "more than two previous encryption keys" \ + --set server.encryption.configVersion=flat \ + --set-string server.encryption.key="$HELM_TEST_ENCRYPTION_KEY" \ + --set-string "server.encryption.previousKeys={${HELM_TEST_PREVIOUS_ENCRYPTION_KEY},${HELM_TEST_OLDER_ENCRYPTION_KEY},${HELM_TEST_OLDEST_ENCRYPTION_KEY}}" + assert_render_rejected "a previous encryption key listed twice" \ + --set server.encryption.configVersion=flat \ + --set-string server.encryption.key="$HELM_TEST_ENCRYPTION_KEY" \ + --set-string "server.encryption.previousKeys={${HELM_TEST_PREVIOUS_ENCRYPTION_KEY},${HELM_TEST_PREVIOUS_ENCRYPTION_KEY}}" + assert_render_rejected "a previous encryption key equal to the key" \ + --set server.encryption.configVersion=flat \ + --set-string server.encryption.key="$HELM_TEST_ENCRYPTION_KEY" \ + --set-string "server.encryption.previousKeys[0]=$HELM_TEST_ENCRYPTION_KEY" + assert_render_rejected "a previous encryption key that is not 32 bytes" \ + --set server.encryption.configVersion=flat \ + --set-string server.encryption.key="$HELM_TEST_ENCRYPTION_KEY" \ + --set-string "server.encryption.previousKeys[0]=bm90LTMyLWJ5dGVz" assert_render_rejected "replica auth without a secret" \ -f "$CHART_DIR/examples/cluster-3-node.yaml" \ --set server.cluster.auth.existingSecret.name="" \
diff --git a/scripts/run-bdd-tests.sh b/scripts/run-bdd-tests.sh index 78ce589..e56f032 100755 --- a/scripts/run-bdd-tests.sh +++ b/scripts/run-bdd-tests.sh
@@ -36,7 +36,7 @@ log "Usage: $0 [--coverage] <sdk> [feature]" log "" log " sdk: rust | python | php | go | go-race | node | csharp | java | cpp | all | clean (default: all)" - log " feature: basic_messaging | leader_redirection | raw_command | session_contract | stream_crud | all (default: all)" + log " feature: basic_messaging | leader_redirection | raw_command | session_contract | stream_crud | stream_options | resource_options_patch | all (default: all)" log "" log " Every suite runs against iggy-server, taken from IGGY_SERVER_PATH" log " (default: target/debug/iggy-server) with an iggy CLI at IGGY_CLI_PATH." @@ -49,7 +49,7 @@ } case "$FEATURE" in - basic_messaging|leader_redirection|raw_command|session_contract|stream_crud|all) ;; + basic_messaging|leader_redirection|raw_command|session_contract|stream_crud|stream_options|resource_options_patch|all) ;; *) log "Unknown feature: ${FEATURE}" usage @@ -70,7 +70,7 @@ COMPOSE_FILES=(-f docker-compose.yml) case "$FEATURE" in - basic_messaging|leader_redirection|raw_command|session_contract|stream_crud|all) + basic_messaging|leader_redirection|raw_command|session_contract|stream_crud|stream_options|resource_options_patch|all) COMPOSE_FILES+=(-f docker-compose.server.yml) ;; esac case "$FEATURE" in @@ -131,6 +131,11 @@ esac fi + if [ "$FEATURE" = "resource_options_patch" ] && [ "$svc" = "php-bdd" ]; then + unsupported "$FEATURE" "$svc" || return 1 + return 0 + fi + log "${emoji} ${label}..." local code=0 docker compose "${COMPOSE_FILES[@]}" \
diff --git a/web/docs/server-api.http b/web/docs/server-api.http index 47b09f5..6fe2cf0 100644 --- a/web/docs/server-api.http +++ b/web/docs/server-api.http
@@ -65,24 +65,62 @@ Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJqdGkiOiJjMmQ5ZWZmNS03OTQ5LTQ1MTEtYmM2YS1hZGY3MmQwN2VjNDkiLCJpc3MiOiJpZ2d5LnJzIiwiYXVkIjoiaWdneS5ycyIsInN1YiI6MSwiaWF0IjoxNjk5OTg0MzU4LCJleHAiOjE2OTk5ODc5NTgsIm5iZiI6MTY5OTk4NDM1OH0.-C96N0kTHKnjxkUXobGnfkejkvUIHrQa8yEKHD-N5cU { - "stream_id": 29, "name":"stream29" } +### Create stream with its own encryption key, 32 random bytes as standard base64 (never reuse this sample) +POST {{url}}/streams +Content-Type: application/json +Authorization: Bearer {{access_token}} + +{ + "name": "payments", + "options": { + "encryption_key": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + } +} + +### Update stream +PUT {{url}}/streams/29 +Content-Type: application/json +Authorization: Bearer {{access_token}} + +{ + "name": "stream29-renamed" +} + ### Create topic POST {{url}}/streams/11/topics Content-Type: application/json Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJqdGkiOiJmODM0NTI5MC05NzBlLTQ3YzAtOTMwOS02YWM5MmZiMWEyYTYiLCJpc3MiOiJpZ2d5LnJzIiwiYXVkIjoiaWdneS5ycyIsInN1YiI6MSwiaWF0IjoxNjk5OTgxMTU4LCJleHAiOjE2OTk5ODQ3NTgsIm5iZiI6MTY5OTk4MTE1OH0.6WxgKsOAZQfeAPciwHOcE1oR3iJXM0VORX-RtsvT2hU { - "topic_id": 16, "name": "topic16", "partitions_count": 10, - "message_expiry": 0, - "compression_algorithm": "none", - "max_topic_size": "1000000000" + "options": { + "message_expiry": "7 days", + "compression_algorithm": "none", + "max_topic_size": "1000000000" + } } +### Update topic +# A value sets a key, an absent key or null keeps it, a key in reset returns to its default. +# compression_algorithm and the other create-only keys are refused here. +PUT {{url}}/streams/11/topics/16 +Content-Type: application/json +Authorization: Bearer {{access_token}} + +{ + "name": "topic16-renamed", + "message_expiry": "1 day", + "reset": ["max_topic_size"] +} + +### Describe the topic option catalog (scope: stream, topic or user) +GET {{url}}/options/topic +Authorization: Bearer {{access_token}} + ### Create partitions POST {{url}}/streams/3/topics/3/partitions @@ -122,6 +160,15 @@ Authorization: Bearer {{access_token}} Content-Type: application/json +### Update User +PUT {{url}}/users/37 +Authorization: Bearer {{access_token}} +Content-Type: application/json + +{ + "status": "inactive" +} + ### Delete User DELETE {{url}}/users/2 Authorization: Bearer {{access_token}}
diff --git a/web/package-lock.json b/web/package-lock.json index d3c5b39..fbfb5df 100644 --- a/web/package-lock.json +++ b/web/package-lock.json
@@ -1,12 +1,12 @@ { "name": "iggy-web-ui", - "version": "0.4.1-edge.1", + "version": "0.4.1-edge.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "iggy-web-ui", - "version": "0.4.1-edge.1", + "version": "0.4.1-edge.2", "dependencies": { "@floating-ui/dom": "^1.8.0", "@tailwindcss/postcss": "^4.3.3",
diff --git a/web/package.json b/web/package.json index 0add559..7017404 100644 --- a/web/package.json +++ b/web/package.json
@@ -1,6 +1,6 @@ { "name": "iggy-web-ui", - "version": "0.4.1-edge.1", + "version": "0.4.1-edge.2", "private": true, "scripts": { "dev": "vite dev --port 3050",
diff --git a/web/src/lib/api/ApiSchema.ts b/web/src/lib/api/ApiSchema.ts index ac0b66d..d9f6d0a 100644 --- a/web/src/lib/api/ApiSchema.ts +++ b/web/src/lib/api/ApiSchema.ts
@@ -15,6 +15,14 @@ // specific language governing permissions and limitations // under the License. +import type { + OptionInputValue, + StreamCreateBody, + StreamUpdateBody, + TopicCreateBody, + TopicUpdateBody, + UserUpdateBody +} from '#lib/api/resourceOptions.js'; import type { GlobalPermissions, StreamPermissions } from '#lib/domain/Permissions.js'; import type { KeysToSnakeCase } from '#lib/utils/utilTypes.js'; @@ -34,6 +42,7 @@ password: string; status: 'active' | 'inactive'; permissions: Permissions | null; + options?: Record<string, OptionInputValue>; }; } | { @@ -45,13 +54,9 @@ path: `/users/${number}`; } | { - method: 'GET'; + method: 'PUT'; path: `/users/${number}`; - body: { - username: string; - status: 'active' | 'inactive'; - permissions: Permissions | null; - }; + body: UserUpdateBody; } | { method: 'PUT'; @@ -85,16 +90,12 @@ | { method: 'POST'; path: '/streams'; - body: { - name: string; - }; + body: StreamCreateBody; } | { method: 'PUT'; path: `/streams/${number}`; - body: { - name: string; - }; + body: StreamUpdateBody; } | { method: 'DELETE'; @@ -117,23 +118,12 @@ | { method: 'POST'; path: `/streams/${number}/topics`; - body: { - compression_algorithm: 'none' | 'gzip'; - max_topic_size: number; - message_expiry: number; - name: string; - partitions_count: number; - }; + body: TopicCreateBody; } | { method: 'PUT'; path: `/streams/${number}/topics/${number}`; - body: { - name: string; - message_expiry?: number; - compression_algorithm?: 'none' | 'gzip'; - max_topic_size?: number; - }; + body: TopicUpdateBody; } | { method: 'DELETE';
diff --git a/web/src/lib/api/resourceOptions.test.mjs b/web/src/lib/api/resourceOptions.test.mjs new file mode 100644 index 0000000..cece2ed --- /dev/null +++ b/web/src/lib/api/resourceOptions.test.mjs
@@ -0,0 +1,159 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { + UINT64_MAX, + isOptionExplicit, + isStreamEncryptionKey, + optionText, + optionUint64, + streamCreate, + streamUpdate, + topicCreate, + topicUpdate, + userUpdate +} from './resourceOptions.ts'; + +const currentTopic = { name: 'orders', messageExpiry: 0, maxTopicSize: '1073741824' }; +const streamKey = Buffer.alloc(32, 7).toString('base64'); + +test('given_topic_reset_when_building_update_should_list_only_reset_keys', () => { + const body = topicUpdate({ + current: currentTopic, + next: { name: 'orders', messageExpiry: 604_800_000_000, maxTopicSize: '2 GiB' }, + reset: ['max_topic_size', 'max_topic_size'] + }); + + assert.deepEqual(body, { message_expiry: '604800000000', reset: ['max_topic_size'] }); + assert.ok(Object.values(body).every((value) => value !== null)); +}); + +test('given_changed_topic_when_building_update_should_send_only_changed_keys', () => { + assert.deepEqual( + topicUpdate({ + current: { ...currentTopic, messageExpiry: 60_000_000 }, + next: { name: 'orders-v2', messageExpiry: 0, maxTopicSize: ' 1073741824 ' } + }), + { name: 'orders-v2', message_expiry: 'unlimited' } + ); + assert.deepEqual(topicUpdate({ current: currentTopic, next: currentTopic }), {}); +}); + +test('given_topic_settings_when_building_create_should_encode_options_as_update_does', () => { + const settings = { name: 'orders', messageExpiry: 3_600_000_000, maxTopicSize: ' 1 GiB ' }; + const created = topicCreate({ ...settings, partitionsCount: 3, compressionAlgorithm: 'gzip' }); + const updated = topicUpdate({ + current: { name: 'orders', messageExpiry: 1, maxTopicSize: '1' }, + next: settings + }); + + assert.deepEqual(created, { + name: 'orders', + partitions_count: 3, + options: { + compression_algorithm: 'gzip', + message_expiry: '3600000000', + max_topic_size: '1 GiB' + } + }); + assert.equal(created.options.message_expiry, updated.message_expiry); + assert.equal(created.options.max_topic_size, updated.max_topic_size); +}); + +test('given_default_topic_settings_when_building_create_should_omit_default_options', () => { + const settings = { name: 'orders', partitionsCount: 1, messageExpiry: 0 }; + + assert.deepEqual(topicCreate({ ...settings, compressionAlgorithm: 'none', maxTopicSize: ' ' }), { + name: 'orders', + partitions_count: 1, + options: {} + }); + assert.deepEqual( + topicCreate({ ...settings, compressionAlgorithm: 'none', maxTopicSize: 'unlimited' }).options, + { max_topic_size: 'unlimited' } + ); +}); + +test('given_stream_key_when_building_create_should_use_nested_options_without_persistence', () => { + assert.equal(isStreamEncryptionKey(streamKey), true); + assert.deepEqual(streamCreate('orders', streamKey), { + name: 'orders', + options: { encryption_key: streamKey } + }); + assert.deepEqual(streamCreate('orders'), { name: 'orders', options: {} }); + + const invalidKeys = [ + '', + 'not base64 at all', + Buffer.alloc(31, 7).toString('base64'), + Buffer.alloc(33, 7).toString('base64'), + streamKey.slice(0, -1), + `${streamKey} `, + `${streamKey.slice(0, -2)}d=`, + `${Buffer.alloc(32, 0xff).toString('base64url')}=` + ]; + for (const invalidKey of invalidKeys) { + assert.equal(isStreamEncryptionKey(invalidKey), false, `Accepted key: ${invalidKey}`); + } + for (const invalidKey of invalidKeys.slice(1)) { + assert.throws(() => streamCreate('orders', invalidKey), TypeError); + } +}); + +test('given_u64_option_above_2_pow_53_when_reading_should_keep_precision', () => { + const { options } = JSON.parse( + '{"options":{' + + '"max_topic_size":{"value":"9007199254740993","kind":"uint64","explicit":true},' + + '"message_expiry":{"value":"18446744073709551615","kind":"uint64","explicit":false},' + + '"compression_algorithm":{"value":"gzip","kind":"string","explicit":true}}}' + ); + + assert.equal(optionUint64(options, 'max_topic_size'), 9_007_199_254_740_993n); + assert.equal(optionUint64(options, 'message_expiry'), UINT64_MAX); + assert.equal(optionUint64(options, 'compression_algorithm'), undefined); + assert.equal(optionUint64(options, 'segment_size'), undefined); + assert.equal(optionUint64(undefined, 'max_topic_size'), undefined); +}); + +test('given_option_entries_when_reading_should_return_text_and_explicit_flag', () => { + const options = { + compression_algorithm: { value: 'gzip', kind: 'string', explicit: true }, + encryption_key_source: { value: 'server', kind: 'string', explicit: false } + }; + + assert.equal(optionText(options, 'compression_algorithm'), 'gzip'); + assert.equal(optionText(options, 'constructor'), undefined); + assert.equal(isOptionExplicit(options, 'compression_algorithm'), true); + assert.equal(isOptionExplicit(options, 'encryption_key_source'), false); + assert.equal(isOptionExplicit(options, 'max_topic_size'), false); +}); + +test('given_stream_and_user_changes_when_building_update_should_send_only_changed_keys', () => { + assert.deepEqual(streamUpdate({ current: { name: 'orders' }, next: { name: 'sales' } }), { + name: 'sales' + }); + assert.deepEqual(streamUpdate({ current: { name: 'orders' }, next: { name: 'orders' } }), {}); + assert.deepEqual( + userUpdate({ + current: { username: 'alice', status: 'active' }, + next: { username: 'alice', status: 'inactive' } + }), + { status: 'inactive' } + ); +});
diff --git a/web/src/lib/api/resourceOptions.ts b/web/src/lib/api/resourceOptions.ts new file mode 100644 index 0000000..5cdeb0d --- /dev/null +++ b/web/src/lib/api/resourceOptions.ts
@@ -0,0 +1,146 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import type { Stream } from '#lib/domain/Stream.js'; +import type { Topic } from '#lib/domain/Topic.js'; +import type { User } from '#lib/domain/User.js'; + +export const UINT64_MAX = 18_446_744_073_709_551_615n; +export const UNLIMITED = 'unlimited'; +export const INVALID_STREAM_KEY_MESSAGE = + 'Encryption key must be standard padded base64 of exactly 32 bytes'; + +const STREAM_KEY_BYTES = 32; +const STANDARD_BASE64 = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/; + +export type OptionEntry = { value: string; kind: string; explicit: boolean }; +export type ResourceOptions = Record<string, OptionEntry>; +export type OptionInputValue = string | number | boolean; +export type CompressionAlgorithm = 'none' | 'gzip'; +export type TopicResettableKey = 'message_expiry' | 'max_topic_size'; + +export type TopicCreateBody = { + name: string; + partitions_count: number; + options: { + compression_algorithm?: CompressionAlgorithm; + message_expiry?: string; + max_topic_size?: string; + }; +}; + +export type TopicUpdateBody = { + name?: string; + message_expiry?: string; + max_topic_size?: string; + reset?: TopicResettableKey[]; +}; + +export type StreamCreateBody = { name: string; options: { encryption_key?: string } }; +export type StreamUpdateBody = { name?: string }; +export type UserUpdateBody = { username?: string; status?: User['status'] }; + +type Change<T> = { current: T; next: T }; +type TopicUpdatable = Pick<Topic, 'name' | 'messageExpiry' | 'maxTopicSize'>; + +export type TopicCreateInput = TopicUpdatable & { + partitionsCount: number; + compressionAlgorithm: CompressionAlgorithm; +}; + +export function optionText(options: ResourceOptions | undefined, key: string): string | undefined { + return options && Object.prototype.hasOwnProperty.call(options, key) + ? options[key].value + : undefined; +} + +export function optionUint64( + options: ResourceOptions | undefined, + key: string +): bigint | undefined { + const text = optionText(options, key); + return text !== undefined && /^\d+$/.test(text) ? BigInt(text) : undefined; +} + +export function isOptionExplicit(options: ResourceOptions | undefined, key: string): boolean { + return ( + options !== undefined && + Object.prototype.hasOwnProperty.call(options, key) && + options[key].explicit === true + ); +} + +export function isStreamEncryptionKey(text: string): boolean { + if (!STANDARD_BASE64.test(text)) return false; + const bytes = atob(text); + // Re-encoding refuses nonzero trailing bits, which the server also refuses. + return bytes.length === STREAM_KEY_BYTES && btoa(bytes) === text; +} + +export function streamCreate(name: string, encryptionKey = ''): StreamCreateBody { + if (encryptionKey === '') return { name, options: {} }; + if (!isStreamEncryptionKey(encryptionKey)) throw new TypeError(INVALID_STREAM_KEY_MESSAGE); + return { name, options: { encryption_key: encryptionKey } }; +} + +// An omitted key resolves to the catalog default and stays reported as a default. +export function topicCreate(input: TopicCreateInput): TopicCreateBody { + const { name, partitionsCount, compressionAlgorithm, messageExpiry, maxTopicSize } = input; + const options: TopicCreateBody['options'] = {}; + const size = maxTopicSize.trim(); + if (compressionAlgorithm !== 'none') options.compression_algorithm = compressionAlgorithm; + if (messageExpiry > 0) options.message_expiry = messageExpiryText(messageExpiry); + if (size !== '') options.max_topic_size = size; + return { name, partitions_count: partitionsCount, options }; +} + +export function topicUpdate({ + current, + next, + reset = [] +}: Change<TopicUpdatable> & { reset?: TopicResettableKey[] }): TopicUpdateBody { + const body: TopicUpdateBody = {}; + const maxTopicSize = next.maxTopicSize.trim(); + if (next.name !== current.name) body.name = next.name; + if (!reset.includes('message_expiry') && next.messageExpiry !== current.messageExpiry) { + body.message_expiry = messageExpiryText(next.messageExpiry); + } + if (!reset.includes('max_topic_size') && maxTopicSize !== current.maxTopicSize) { + body.max_topic_size = maxTopicSize; + } + if (reset.length > 0) body.reset = [...new Set(reset)]; + return body; +} + +export function streamUpdate({ current, next }: Change<Pick<Stream, 'name'>>): StreamUpdateBody { + return next.name !== current.name ? { name: next.name } : {}; +} + +export function userUpdate({ + current, + next +}: Change<Pick<User, 'username' | 'status'>>): UserUpdateBody { + const body: UserUpdateBody = {}; + if (next.username !== current.username) body.username = next.username; + if (next.status !== current.status) body.status = next.status; + return body; +} + +// Zero means never in the form, but the server reads zero as its default. +function messageExpiryText(micros: number): string { + return micros > 0 ? String(micros) : UNLIMITED; +}
diff --git a/web/src/lib/components/Modals/AddStreamModal.svelte b/web/src/lib/components/Modals/AddStreamModal.svelte index 7c8265e..17f259c 100644 --- a/web/src/lib/components/Modals/AddStreamModal.svelte +++ b/web/src/lib/components/Modals/AddStreamModal.svelte
@@ -24,9 +24,15 @@ import { z } from 'zod'; import Button from '../Button.svelte'; import Input from '../Input.svelte'; + import PasswordInput from '../PasswordInput.svelte'; import ModalBase from './ModalBase.svelte'; import { showToast } from '../AppToasts.svelte'; import { fetchRouteApi } from '#lib/api/fetchRouteApi.js'; + import { + INVALID_STREAM_KEY_MESSAGE, + isStreamEncryptionKey, + streamCreate + } from '#lib/api/resourceOptions.js'; import { customInvalidateAll } from '../PeriodicInvalidator.svelte'; interface Props { @@ -35,6 +41,10 @@ let { closeModal }: Props = $props(); + // Kept out of the superform so the key never enters a store. + let encryptionKey = $state(''); + let encryptionKeyError = $state(''); + const schema = z.object({ name: z .string() @@ -50,12 +60,19 @@ async onUpdate({ form }) { if (!form.valid) return; + encryptionKeyError = ''; + if (encryptionKey !== '' && !isStreamEncryptionKey(encryptionKey)) { + encryptionKeyError = INVALID_STREAM_KEY_MESSAGE; + return; + } + + const body = streamCreate(form.data.name, encryptionKey); + encryptionKey = ''; + const { data, ok } = await fetchRouteApi({ method: 'POST', path: '/streams', - body: { - name: form.data.name - } + body }); if (!ok) { @@ -101,9 +118,23 @@ </script> <ModalBase {closeModal} title="Add new stream"> - <form method="POST" class="flex flex-col h-[300px] gap-4" use:enhance> + <form method="POST" class="flex flex-col h-[400px] gap-4" use:enhance> <Input name="name" label="Name" bind:value={$form.name} errorMessage={$errors.name?.[0]} /> + <div class="flex flex-col gap-2"> + <PasswordInput + label="Encryption key (optional)" + name="streamEncryptionKey" + autocomplete="new-password" + spellcheck="false" + bind:value={encryptionKey} + errorMessage={encryptionKeyError} + /> + <span class="text-xs text-shade-d200 dark:text-shade-l700 ml-1"> + Standard base64 of 32 random bytes. The server must have an encryption key configured. + </span> + </div> + <div class="flex justify-end gap-3 mt-auto w-full"> <Button type="button" variant="text" class="w-2/5" onclick={() => closeModal()}>Cancel</Button >
diff --git a/web/src/lib/components/Modals/AddTopicModal.svelte b/web/src/lib/components/Modals/AddTopicModal.svelte index fe4bfd5..4c4822a 100644 --- a/web/src/lib/components/Modals/AddTopicModal.svelte +++ b/web/src/lib/components/Modals/AddTopicModal.svelte
@@ -28,6 +28,7 @@ import type { CloseModalFn } from '#lib/types/utilTypes.js'; import type { StreamDetails } from '#lib/domain/StreamDetails.js'; import { fetchRouteApi } from '#lib/api/fetchRouteApi.js'; + import { topicCreate } from '#lib/api/resourceOptions.js'; import DurationInput from '../DurationInput.svelte'; import { numberSizes } from '#lib/utils/constants/numberSizes.js'; import { showToast } from '../AppToasts.svelte'; @@ -48,7 +49,7 @@ partitions_count: z.number().min(0).max(numberSizes.max.u32).default(1), message_expiry: z.number().min(0).max(Number.MAX_SAFE_INTEGER).default(0), compression_algorithm: z.enum(['none', 'gzip']).default('none'), - max_topic_size: z.number().min(0).max(numberSizes.max.u32).default(2_000_000_000) + max_topic_size: z.string().trim().default('') }); const { form, errors, enhance, constraints, submitting } = superForm(defaults(zod4(schema)), { @@ -61,13 +62,13 @@ const { data, ok } = await fetchRouteApi({ method: 'POST', path: `/streams/${streamDetails.id}/topics`, - body: { + body: topicCreate({ name: form.data.name, - partitions_count: form.data.partitions_count, - message_expiry: form.data.message_expiry, - compression_algorithm: form.data.compression_algorithm, - max_topic_size: form.data.max_topic_size - } + partitionsCount: form.data.partitions_count, + compressionAlgorithm: form.data.compression_algorithm, + messageExpiry: form.data.message_expiry, + maxTopicSize: form.data.max_topic_size + }) }); if (!ok) { @@ -139,7 +140,7 @@ <DurationInput label="Message expiry" bind:value={$form.message_expiry} /> <Select - label="Compression Algorithm" + label="Compression algorithm" type="text" name="compressionAlgorithm" options={['none', 'gzip']} @@ -148,14 +149,18 @@ errorMessage={$errors.compression_algorithm?.[0]} /> - <Input - label="Max topic size" - type="number" - name="maxTopicSize" - bind:value={$form.max_topic_size} - {...$constraints.max_topic_size} - errorMessage={$errors.max_topic_size?.[0]} - /> + <div class="flex flex-col gap-2"> + <Input + label="Max topic size" + name="maxTopicSize" + placeholder="Unlimited, or e.g. 1 GiB" + bind:value={$form.max_topic_size} + errorMessage={$errors.max_topic_size?.[0]} + /> + <span class="text-xs text-shade-d200 dark:text-shade-l700 ml-1"> + Bytes or a size such as 1 GiB. Leave empty for unlimited. + </span> + </div> <div class="flex justify-end gap-3 mt-auto"> <Button variant="text" type="button" class="w-2/5" onclick={() => closeModal()}>Cancel</Button
diff --git a/web/src/lib/components/Modals/EditUserModal.svelte b/web/src/lib/components/Modals/EditUserModal.svelte index ba77a96..cacc6c1 100644 --- a/web/src/lib/components/Modals/EditUserModal.svelte +++ b/web/src/lib/components/Modals/EditUserModal.svelte
@@ -19,21 +19,121 @@ <script lang="ts"> import type { CloseModalFn } from '#lib/types/utilTypes.js'; - + import type { User } from '#lib/domain/User.js'; + import { z } from 'zod'; + import { setError, superForm, defaults } from 'sveltekit-superforms/client'; + import { zod4 } from 'sveltekit-superforms/adapters'; + import Button from '../Button.svelte'; + import Input from '../Input.svelte'; + import Listbox from '../Listbox.svelte'; import ModalBase from './ModalBase.svelte'; + import { fetchRouteApi } from '#lib/api/fetchRouteApi.js'; + import { userUpdate } from '#lib/api/resourceOptions.js'; + import { showToast } from '../AppToasts.svelte'; + import { customInvalidateAll } from '../PeriodicInvalidator.svelte'; interface Props { closeModal: CloseModalFn; + user: User; } - let { closeModal }: Props = $props(); + let { closeModal, user }: Props = $props(); + + // userMapper pads the username and marks root with a display suffix. + const username = (user.id === 0 ? user.username.replace(/\(root\)$/, '') : user.username).trim(); + + const schema = z.object({ + username: z.string().min(1, 'Username is required').trim().default(username), + status: z.enum(['active', 'inactive']).default(user.status) + }); + + const { form, errors, enhance, constraints, submitting, tainted } = superForm( + defaults(zod4(schema)), + { + SPA: true, + validators: zod4(schema), + invalidateAll: false, + taintedMessage: false, + async onUpdate({ form }) { + if (!form.valid) return; + + const body = userUpdate({ current: { username, status: user.status }, next: form.data }); + if (Object.keys(body).length === 0) return closeModal(); + + const { data, ok } = await fetchRouteApi({ + method: 'PUT', + path: `/users/${user.id}`, + body + }); + + if (!ok) { + // Handle API errors + if (data?.field && data?.reason) { + // Field-specific error - show in form + return setError(form, data.field, data.reason); + } else if (data?.reason) { + // General error with reason - show toast + let errorMessage = data.reason; + if (data.code && data.id) { + errorMessage += `\n${data.code} (${data.id})`; + } else if (data.code) { + errorMessage += `\n${data.code}`; + } + showToast({ + type: 'error', + description: errorMessage, + duration: 5000 + }); + } else { + // Fallback error message + showToast({ + type: 'error', + description: 'Failed to update user', + duration: 5000 + }); + } + return; + } + + closeModal(async () => { + await customInvalidateAll(); + showToast({ + type: 'success', + description: `User ${form.data.username} has been updated.`, + duration: 3500 + }); + }); + } + } + ); </script> <ModalBase {closeModal} title="Edit user"> - <div class="h-[300px] flex flex-col"> - <!-- <div class="flex justify-end gap-3 mt-auto"> - <Button variant="text" class="w-2/5" onclick={closeModal}>Cancel</Button> - <Button type="submit" variant="contained" class="w-2/5">Save</Button> - </div> --> - </div> + <form method="POST" class="flex flex-col h-[300px] gap-4" use:enhance> + <Input + label="Username" + name="username" + bind:value={$form.username} + {...$constraints.username} + errorMessage={$errors.username?.[0]} + /> + + <Listbox + label="Status" + options={[ + { name: 'Active', value: 'active' }, + { name: 'Inactive', value: 'inactive' } + ]} + selectedValue={$form.status} + on:selectedValue={(e) => ($form.status = e.detail as 'active' | 'inactive')} + /> + + <div class="flex justify-end gap-3 mt-auto w-full"> + <Button type="button" variant="text" class="w-2/5" onclick={() => closeModal()}>Cancel</Button + > + <Button type="submit" variant="contained" class="w-2/5" disabled={$submitting || !$tainted} + >Update</Button + > + </div> + </form> </ModalBase>
diff --git a/web/src/lib/components/Modals/StreamSettingsModal.svelte b/web/src/lib/components/Modals/StreamSettingsModal.svelte index 83f1dc4..f814280 100644 --- a/web/src/lib/components/Modals/StreamSettingsModal.svelte +++ b/web/src/lib/components/Modals/StreamSettingsModal.svelte
@@ -28,6 +28,7 @@ import { setError, superForm, defaults } from 'sveltekit-superforms/client'; import { zod4 } from 'sveltekit-superforms/adapters'; import { fetchRouteApi } from '#lib/api/fetchRouteApi.js'; + import { streamUpdate } from '#lib/api/resourceOptions.js'; import { goto } from '$app/navigation'; import { showToast } from '../AppToasts.svelte'; import ModalConfirmation from '../ModalConfirmation.svelte'; @@ -62,12 +63,13 @@ async onUpdate({ form }) { if (!form.valid) return; + const body = streamUpdate({ current: stream, next: { name: form.data.name } }); + if (Object.keys(body).length === 0) return closeModal(); + const { data, ok } = await fetchRouteApi({ method: 'PUT', path: `/streams/${stream.id}`, - body: { - name: form.data.name - } + body }); if (!ok) { @@ -176,10 +178,28 @@ {/snippet} </ModalConfirmation> - <div class="h-[350px] flex flex-col"> - <form method="POST" class="flex flex-col h-[300px] gap-4 flex-3 pb-5" use:enhance> + <div class="h-[460px] flex flex-col"> + <form method="POST" class="flex flex-col h-[410px] gap-4 flex-3 pb-5" use:enhance> <Input name="name" label="Name" bind:value={$form.name} errorMessage={$errors.name?.[0]} /> + <div class="flex flex-col gap-1 ml-1"> + <span class="text-sm text-color">Encryption</span> + <span class="text-xs text-shade-d200 dark:text-shade-l700"> + Key source: {stream.encryptionKeySource ?? 'unknown'} + </span> + {#if stream.encryptionKeyFingerprint} + <span class="text-xs text-shade-d200 dark:text-shade-l700"> + Key fingerprint: <span class="font-mono">{stream.encryptionKeyFingerprint}</span> + </span> + {/if} + {#if stream.encryptionKekFingerprints.length > 0} + <span class="text-xs text-shade-d200 dark:text-shade-l700"> + Server key fingerprints: + <span class="font-mono">{stream.encryptionKekFingerprints.join(', ')}</span> + </span> + {/if} + </div> + <div class="flex justify-end gap-3 w-full mt-auto"> <Button type="button" variant="text" class="w-2/5" onclick={() => closeModal()} >Cancel</Button
diff --git a/web/src/lib/components/Modals/TopicSettingsModal.svelte b/web/src/lib/components/Modals/TopicSettingsModal.svelte index 1e198d0..51f4792 100644 --- a/web/src/lib/components/Modals/TopicSettingsModal.svelte +++ b/web/src/lib/components/Modals/TopicSettingsModal.svelte
@@ -23,12 +23,14 @@ import { z } from 'zod'; import Button from '../Button.svelte'; + import Checkbox from '../Checkbox.svelte'; import Icon from '../Icon.svelte'; import Input from '../Input.svelte'; import ModalBase from './ModalBase.svelte'; import { setError, superForm, defaults } from 'sveltekit-superforms/client'; import { zod4 } from 'sveltekit-superforms/adapters'; import { fetchRouteApi } from '#lib/api/fetchRouteApi.js'; + import { topicUpdate, type TopicResettableKey } from '#lib/api/resourceOptions.js'; import { goto } from '$app/navigation'; import { resolve } from '$app/paths'; import type { Path } from '$app/types'; @@ -48,6 +50,8 @@ let { topic, closeModal, onDeleteRedirectPath }: Props = $props(); let confirmationOpen = $state(false); + let resetMessageExpiry = $state(false); + let resetMaxTopicSize = $state(false); const schema = z.object({ name: z @@ -55,7 +59,12 @@ .min(1, 'Name must contain at least 1 character') .max(255, 'Name must not exceed 255 characters') .default(topic.name), - message_expiry: z.number().min(0).max(Number.MAX_SAFE_INTEGER).default(topic.messageExpiry) + message_expiry: z.number().min(0).max(Number.MAX_SAFE_INTEGER).default(topic.messageExpiry), + max_topic_size: z + .string() + .trim() + .min(1, 'Max topic size is required') + .default(topic.maxTopicSize) }); const { form, errors, enhance, submitting, tainted } = superForm(defaults(zod4(schema)), { @@ -67,13 +76,25 @@ if (!form.valid) return; if (!page.params.streamId) return; + const reset: TopicResettableKey[] = []; + if (resetMessageExpiry) reset.push('message_expiry'); + if (resetMaxTopicSize) reset.push('max_topic_size'); + + const body = topicUpdate({ + current: topic, + next: { + name: form.data.name, + messageExpiry: form.data.message_expiry, + maxTopicSize: form.data.max_topic_size + }, + reset + }); + if (Object.keys(body).length === 0) return closeModal(); + const { data, ok } = await fetchRouteApi({ method: 'PUT', path: `/streams/${+page.params.streamId}/topics/${topic.id}`, - body: { - name: form.data.name, - message_expiry: form.data.message_expiry - } + body }); if (!ok) { @@ -111,7 +132,7 @@ await customInvalidateAll(); showToast({ type: 'success', - description: `Stream ${form.data.name} has been updated.`, + description: `Topic ${form.data.name} has been updated.`, duration: 3500 }); }); @@ -159,18 +180,79 @@ {/snippet} </ModalConfirmation> - <div class="h-[400px] flex flex-col"> + <div class="min-h-[400px] flex flex-col"> <form method="POST" class="flex flex-col gap-4 flex-3 pb-5" use:enhance> <Input name="name" label="Name" bind:value={$form.name} errorMessage={$errors.name?.[0]} /> - <DurationInput label="Message expiry" bind:value={$form.message_expiry} /> + {#if resetMessageExpiry} + <div class="flex flex-col gap-2"> + <span class="text-sm text-color ml-1">Message expiry</span> + <span class="text-xs text-shade-d200 dark:text-shade-l700 ml-1"> + Returns to the server default on update. + </span> + </div> + {:else} + <DurationInput label="Message expiry" bind:value={$form.message_expiry} /> + {/if} + + {#if topic.messageExpiryExplicit} + <div class="flex items-center gap-2"> + <Checkbox bind:checked={resetMessageExpiry} value="message_expiry" /> + <span class="text-sm text-color select-none">Reset message expiry to server default</span> + </div> + {/if} + + {#if resetMaxTopicSize} + <div class="flex flex-col gap-2"> + <span class="text-sm text-color ml-1">Max topic size</span> + <span class="text-xs text-shade-d200 dark:text-shade-l700 ml-1"> + Returns to the server default on update. + </span> + </div> + {:else} + <div class="flex flex-col gap-2"> + <Input + name="maxTopicSize" + label="Max topic size" + placeholder="e.g. 1 GiB or unlimited" + bind:value={$form.max_topic_size} + errorMessage={$errors.max_topic_size?.[0]} + /> + <span class="text-xs text-shade-d200 dark:text-shade-l700 ml-1"> + Bytes, a size such as 1 GiB, or unlimited. + </span> + </div> + {/if} + + {#if topic.maxTopicSizeExplicit} + <div class="flex items-center gap-2"> + <Checkbox bind:checked={resetMaxTopicSize} value="max_topic_size" /> + <span class="text-sm text-color select-none">Reset max topic size to server default</span> + </div> + {/if} + + <div class="flex flex-col gap-2"> + <Input + name="compressionAlgorithm" + label="Compression algorithm" + value={topic.compressionAlgorithm} + readonly + /> + <span class="text-xs text-shade-d200 dark:text-shade-l700 ml-1"> + Set when the topic is created and cannot be changed. + </span> + </div> <div class="flex justify-end gap-3 w-full mt-auto"> <Button type="button" variant="text" class="w-2/5" onclick={() => closeModal()} >Cancel</Button > - <Button type="submit" variant="contained" class="w-2/5" disabled={$submitting || !$tainted} + <Button + type="submit" + variant="contained" + class="w-2/5" + disabled={$submitting || (!$tainted && !resetMessageExpiry && !resetMaxTopicSize)} >Update</Button > </div>
diff --git a/web/src/lib/components/RouteComponents/Settings/UsersTab.svelte b/web/src/lib/components/RouteComponents/Settings/UsersTab.svelte index 8e9610b..7ac4ade 100644 --- a/web/src/lib/components/RouteComponents/Settings/UsersTab.svelte +++ b/web/src/lib/components/RouteComponents/Settings/UsersTab.svelte
@@ -50,8 +50,8 @@ { label: 'Edit', icon: 'editPen', - action: () => { - openModal('EditUserModal'); + action: (user: User) => { + openModal('EditUserModal', { user }); } }, { @@ -68,7 +68,7 @@ openModal('DeleteUserModal'); } } - ] satisfies { label: string; icon: iconType; action: VoidFunction }[]; + ] satisfies { label: string; icon: iconType; action: (_user: User) => void }[]; const toggleAllChecked = (e: Event) => { const { checked } = e.target as HTMLInputElement; @@ -179,7 +179,7 @@ {#each userActions as { action, icon, label } (label)} <button onclick={() => { - action(); + action(row); close(); }} class={twMerge(
diff --git a/web/src/lib/domain/Stream.ts b/web/src/lib/domain/Stream.ts index 3ee091d..42002af 100644 --- a/web/src/lib/domain/Stream.ts +++ b/web/src/lib/domain/Stream.ts
@@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +import { optionText } from '#lib/api/resourceOptions.js'; + export type Stream = { id: number; name: string; @@ -23,9 +25,13 @@ messagesCount: number; topicsCount: number; createdAt: number; + encryptionKeySource: string | undefined; + encryptionKeyFingerprint: string | undefined; + encryptionKekFingerprints: string[]; }; export function streamMapper(item: any): Stream { + const kekFingerprints = optionText(item.options, 'encryption_kek_fingerprint'); return { id: item.id, name: item.name, @@ -33,7 +39,10 @@ sizeFormatted: item.size, messagesCount: item.messages_count, topicsCount: item.topics_count, - createdAt: item.created_at + createdAt: item.created_at, + encryptionKeySource: optionText(item.options, 'encryption_key_source'), + encryptionKeyFingerprint: optionText(item.options, 'encryption_key_fingerprint'), + encryptionKekFingerprints: kekFingerprints ? kekFingerprints.split(',') : [] }; }
diff --git a/web/src/lib/domain/Topic.ts b/web/src/lib/domain/Topic.ts index 7109ca0..daf7e52 100644 --- a/web/src/lib/domain/Topic.ts +++ b/web/src/lib/domain/Topic.ts
@@ -15,6 +15,13 @@ // specific language governing permissions and limitations // under the License. +import { + UINT64_MAX, + UNLIMITED, + isOptionExplicit, + optionText, + optionUint64 +} from '#lib/api/resourceOptions.js'; import { bytesFormatter } from '#lib/utils/formatters/bytesFormatter.js'; import { formatDate } from '#lib/utils/formatters/dateFormatter.js'; import { durationFormatter } from '#lib/utils/formatters/durationFormatter.js'; @@ -27,16 +34,20 @@ messagesCount: number; messageExpiry: number; messageExpiryFormatted: string; + messageExpiryExplicit: boolean; partitionsCount: number; createdAt: string; - compressionAlgorithm: number; - maxTopicSize: number; + compressionAlgorithm: string; + maxTopicSize: string; + maxTopicSizeExplicit: boolean; }; export function topicMapper(item: any): Topic { - const micros: number = item.message_expiry ?? 0; + const micros = optionUint64(item.options, 'message_expiry') ?? 0n; + const maxTopicSize = optionUint64(item.options, 'max_topic_size') ?? UINT64_MAX; - const messageExpiry = micros === 0 || micros >= Number.MAX_SAFE_INTEGER ? 0 : micros; + const messageExpiry = + micros === 0n || micros >= BigInt(Number.MAX_SAFE_INTEGER) ? 0 : Number(micros); return { id: item.id, name: item.name, @@ -44,11 +55,13 @@ sizeFormatted: bytesFormatter(item.size), messageExpiry, messageExpiryFormatted: formatExpiry(messageExpiry), + messageExpiryExplicit: isOptionExplicit(item.options, 'message_expiry'), messagesCount: item.messages_count, partitionsCount: item.partitions_count, createdAt: formatDate(item.created_at), - compressionAlgorithm: item.compression_algorithm ?? 0, - maxTopicSize: item.max_topic_size + compressionAlgorithm: optionText(item.options, 'compression_algorithm') ?? 'none', + maxTopicSize: maxTopicSize === UINT64_MAX ? UNLIMITED : maxTopicSize.toString(), + maxTopicSizeExplicit: isOptionExplicit(item.options, 'max_topic_size') }; }
diff --git a/web/src/routes/dashboard/settings/users/+page.svelte b/web/src/routes/dashboard/settings/users/+page.svelte index 304de6f..46c89b1 100644 --- a/web/src/routes/dashboard/settings/users/+page.svelte +++ b/web/src/routes/dashboard/settings/users/+page.svelte
@@ -36,6 +36,7 @@ import SettingsLayout from '#lib/components/Layouts/SettingsLayout.svelte'; import { fade } from 'svelte/transition'; import Input from '#lib/components/Input.svelte'; + import type { User } from '#lib/domain/User.js'; interface Props { data: any; @@ -52,8 +53,8 @@ { label: 'Edit', icon: 'editPen', - action: () => { - openModal('EditUserModal'); + action: (user: User) => { + openModal('EditUserModal', { user }); } }, { @@ -70,7 +71,7 @@ openModal('DeleteUserModal'); } } - ] satisfies { label: string; icon: iconType; action: VoidFunction }[]; + ] satisfies { label: string; icon: iconType; action: (_user: User) => void }[]; const toggleAllChecked = (e: Event) => { const { checked } = e.target as HTMLInputElement; @@ -226,7 +227,7 @@ {#each userActions as { action, icon, label } (action)} <button onclick={() => { - action(); + action(row); close(); }} class={twMerge(