diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 415c2b96859..59bbd6ce598 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,8 +59,20 @@ jobs: - name: Setup just uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 + - name: Install pnpm + uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda # v4 + + - name: Install Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 20 + cache: pnpm + + - name: Install Node.js dependencies + run: pnpm install + - name: Run unit tests - run: just test-unit --verbose + run: just test-unit runner-tests: name: Subgraph Runner integration tests @@ -113,7 +125,7 @@ jobs: run: pnpm install - name: Run runner tests - run: just test-runner --verbose + run: just test-runner integration-tests: name: Run integration tests @@ -171,13 +183,15 @@ jobs: run: pnpm install - name: Start anvil - run: anvil --gas-limit 100000000000 --base-fee 1 --block-time 2 --timestamp 1743944919 --port 3021 & + run: anvil --gas-limit 100000000000 --base-fee 1 --timestamp 1743944919 --port 3021 & - name: Build graph-node run: just build --test integration_tests - name: Run integration tests - run: just test-integration --verbose + run: | + export PATH="${{ github.workspace }}/node_modules/.bin:$PATH" + just test-integration - name: Cat graph-node.log if: always() diff --git a/.github/workflows/gnd-binary-build.yml b/.github/workflows/gnd-binary-build.yml index 753388733d2..237de3342b5 100644 --- a/.github/workflows/gnd-binary-build.yml +++ b/.github/workflows/gnd-binary-build.yml @@ -1,7 +1,18 @@ name: Build gnd Binaries on: + release: + types: [published] workflow_dispatch: + inputs: + release_tag: + description: 'Existing release tag to upload binaries to and publish as on npm (e.g. v0.44.0). Leave empty to only build artifacts.' + type: string + default: '' + dry_run: + description: 'Dry-run npm publish (no actual publish)' + type: boolean + default: false jobs: build: @@ -18,7 +29,7 @@ jobs: runner: ubuntu-22.04 asset_name: gnd-linux-aarch64 - target: x86_64-apple-darwin - runner: macos-13 + runner: macos-14 asset_name: gnd-macos-x86_64 - target: aarch64-apple-darwin runner: macos-latest @@ -29,45 +40,60 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 + + - name: Cache built binary + id: bin-cache + uses: actions/cache@v5 + with: + path: | + ${{ matrix.asset_name }}.gz + ${{ matrix.asset_name }}.zip + key: gnd-${{ matrix.target }}-${{ hashFiles('Cargo.lock', '**/Cargo.toml', '**/*.rs') }} - name: Install Rust toolchain + if: steps.bin-cache.outputs.cache-hit != 'true' run: | rustup toolchain install stable rustup target add ${{ matrix.target }} rustup default stable - name: Rust Cache + if: steps.bin-cache.outputs.cache-hit != 'true' uses: Swatinem/rust-cache@v2 with: key: ${{ matrix.target }} - name: Install dependencies (Ubuntu) - if: startsWith(matrix.runner, 'ubuntu') + if: steps.bin-cache.outputs.cache-hit != 'true' && startsWith(matrix.runner, 'ubuntu') run: | sudo apt-get update sudo apt-get install -y protobuf-compiler musl-tools + if [ "${{ matrix.target }}" = "aarch64-unknown-linux-gnu" ]; then + sudo apt-get install -y gcc-aarch64-linux-gnu + fi - name: Install dependencies (macOS) - if: startsWith(matrix.runner, 'macos') + if: steps.bin-cache.outputs.cache-hit != 'true' && startsWith(matrix.runner, 'macos') run: | brew install protobuf - name: Install protobuf (Windows) - if: startsWith(matrix.runner, 'windows') + if: steps.bin-cache.outputs.cache-hit != 'true' && startsWith(matrix.runner, 'windows') run: choco install protoc - - name: Build gnd binary (Unix/Mac) - if: ${{ !startsWith(matrix.runner, 'windows') }} + if: steps.bin-cache.outputs.cache-hit != 'true' && !startsWith(matrix.runner, 'windows') run: cargo build --bin gnd --release --target ${{ matrix.target }} + env: + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc - name: Build gnd binary (Windows) - if: startsWith(matrix.runner, 'windows') + if: steps.bin-cache.outputs.cache-hit != 'true' && startsWith(matrix.runner, 'windows') run: cargo build --bin gnd --release --target ${{ matrix.target }} - name: Sign macOS binary - if: startsWith(matrix.runner, 'macos') + if: steps.bin-cache.outputs.cache-hit != 'true' && startsWith(matrix.runner, 'macos') uses: lando/code-sign-action@v3 with: file: target/${{ matrix.target }}/release/gnd @@ -77,7 +103,7 @@ jobs: options: --options runtime --entitlements entitlements.plist - name: Notarize macOS binary - if: startsWith(matrix.runner, 'macos') + if: steps.bin-cache.outputs.cache-hit != 'true' && startsWith(matrix.runner, 'macos') uses: lando/notarize-action@v2 with: product-path: target/${{ matrix.target }}/release/gnd @@ -86,20 +112,20 @@ jobs: appstore-connect-team-id: ${{ secrets.APPLE_TEAM_ID }} - name: Prepare binary (Unix) - if: ${{ !startsWith(matrix.runner, 'windows') }} + if: steps.bin-cache.outputs.cache-hit != 'true' && !startsWith(matrix.runner, 'windows') run: | cp target/${{ matrix.target }}/release/gnd ${{ matrix.asset_name }} chmod +x ${{ matrix.asset_name }} gzip ${{ matrix.asset_name }} - name: Prepare binary (Windows) - if: startsWith(matrix.runner, 'windows') + if: steps.bin-cache.outputs.cache-hit != 'true' && startsWith(matrix.runner, 'windows') run: | copy target\${{ matrix.target }}\release\gnd.exe ${{ matrix.asset_name }} 7z a -tzip ${{ matrix.asset_name }}.zip ${{ matrix.asset_name }} - name: Upload artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: ${{ matrix.asset_name }} path: | @@ -110,11 +136,11 @@ jobs: release: name: Create Release needs: build - if: startsWith(github.ref, 'refs/tags/') + if: startsWith(github.ref, 'refs/tags/') || inputs.release_tag != '' runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Setup GitHub CLI run: | @@ -124,7 +150,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Download all artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: path: artifacts @@ -133,22 +159,197 @@ jobs: - name: Upload Assets to Release run: | - # Extract version from ref (remove refs/tags/ prefix) - VERSION=${GITHUB_REF#refs/tags/} - - # Upload Linux x86_64 asset - gh release upload $VERSION artifacts/gnd-linux-x86_64/gnd-linux-x86_64.gz --repo $GITHUB_REPOSITORY - - # Upload Linux ARM64 asset - gh release upload $VERSION artifacts/gnd-linux-aarch64/gnd-linux-aarch64.gz --repo $GITHUB_REPOSITORY - - # Upload macOS x86_64 asset - gh release upload $VERSION artifacts/gnd-macos-x86_64/gnd-macos-x86_64.gz --repo $GITHUB_REPOSITORY - - # Upload macOS ARM64 asset - gh release upload $VERSION artifacts/gnd-macos-aarch64/gnd-macos-aarch64.gz --repo $GITHUB_REPOSITORY - - # Upload Windows x86_64 asset - gh release upload $VERSION artifacts/gnd-windows-x86_64.exe/gnd-windows-x86_64.exe.zip --repo $GITHUB_REPOSITORY + VERSION="${{ inputs.release_tag != '' && inputs.release_tag || github.ref_name }}" + + gh release upload $VERSION --clobber --repo $GITHUB_REPOSITORY \ + artifacts/gnd-linux-x86_64/gnd-linux-x86_64.gz \ + artifacts/gnd-linux-aarch64/gnd-linux-aarch64.gz \ + artifacts/gnd-macos-x86_64/gnd-macos-x86_64.gz \ + artifacts/gnd-macos-aarch64/gnd-macos-aarch64.gz \ + artifacts/gnd-windows-x86_64.exe/gnd-windows-x86_64.exe.zip env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + publish-npm: + name: Publish npm package for ${{ matrix.platform }} + needs: release + if: startsWith(github.ref, 'refs/tags/') || inputs.release_tag != '' + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + strategy: + fail-fast: false + matrix: + include: + - platform: linux-x64 + asset: gnd-linux-x86_64.gz + os_field: linux + cpu_field: x64 + extract: gunzip + - platform: linux-arm64 + asset: gnd-linux-aarch64.gz + os_field: linux + cpu_field: arm64 + extract: gunzip + - platform: darwin-x64 + asset: gnd-macos-x86_64.gz + os_field: darwin + cpu_field: x64 + extract: gunzip + - platform: darwin-arm64 + asset: gnd-macos-aarch64.gz + os_field: darwin + cpu_field: arm64 + extract: gunzip + - platform: win32-x64 + asset: gnd-windows-x86_64.exe.zip + os_field: win32 + cpu_field: x64 + extract: unzip + steps: + - uses: actions/setup-node@v6 + with: + node-version: 24 + + - name: Download gnd binary + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release download "${{ inputs.release_tag != '' && inputs.release_tag || github.ref_name }}" \ + --repo "${{ github.repository }}" \ + --pattern "${{ matrix.asset }}" \ + --output ./binary-archive + + - name: Extract binary + run: | + mkdir -p pkg/bin + if [ "${{ matrix.extract }}" = "gunzip" ]; then + gunzip -c ./binary-archive > pkg/bin/gnd + chmod +x pkg/bin/gnd + else + unzip ./binary-archive -d pkg/bin + mv pkg/bin/*.exe pkg/bin/gnd.exe + fi + + - name: Determine version and npm tag + id: version + shell: bash + run: | + VERSION="${{ inputs.release_tag != '' && inputs.release_tag || github.ref_name }}" + VERSION="${VERSION#v}" + echo "version=${VERSION}" >> $GITHUB_OUTPUT + # Prerelease versions (e.g. 0.42.2-dev.1) need an explicit --tag + if [[ "$VERSION" == *-* ]]; then + # Extract prerelease identifier (e.g. "dev" from "0.42.2-dev.1") + PRE="${VERSION#*-}" + TAG="${PRE%%.*}" + echo "tag=${TAG}" >> $GITHUB_OUTPUT + else + echo "tag=latest" >> $GITHUB_OUTPUT + fi + + - name: Create package.json + shell: bash + run: | + if [ "${{ matrix.os_field }}" = "win32" ]; then + BIN_PATH="./bin/gnd.exe" + else + BIN_PATH="./bin/gnd" + fi + + cat > pkg/package.json << EOF + { + "name": "@graphprotocol/gnd-${{ matrix.platform }}", + "version": "${{ steps.version.outputs.version }}", + "description": "gnd binary for ${{ matrix.platform }}", + "os": ["${{ matrix.os_field }}"], + "cpu": ["${{ matrix.cpu_field }}"], + "bin": { + "gnd": "${BIN_PATH}" + }, + "publishConfig": { + "access": "public", + "provenance": true + }, + "license": "(Apache-2.0 OR MIT)", + "repository": { + "type": "git", + "url": "https://github.com/graphprotocol/graph-node.git" + } + } + EOF + + - name: Publish + run: npm publish --provenance --access public --tag ${{ steps.version.outputs.tag }} ${{ inputs.dry_run && '--dry-run' || '' }} + working-directory: pkg + + publish-npm-wrapper: + name: Publish @graphprotocol/gnd wrapper + needs: publish-npm + if: startsWith(github.ref, 'refs/tags/') || inputs.release_tag != '' + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-node@v6 + with: + node-version: 24 + + - name: Determine version and npm tag + id: version + shell: bash + run: | + VERSION="${{ inputs.release_tag != '' && inputs.release_tag || github.ref_name }}" + VERSION="${VERSION#v}" + echo "version=${VERSION}" >> $GITHUB_OUTPUT + if [[ "$VERSION" == *-* ]]; then + PRE="${VERSION#*-}" + TAG="${PRE%%.*}" + echo "tag=${TAG}" >> $GITHUB_OUTPUT + else + echo "tag=latest" >> $GITHUB_OUTPUT + fi + + - name: Create wrapper package + shell: bash + run: | + VERSION="${{ steps.version.outputs.version }}" + + mkdir -p pkg/bin + cp gnd/npm/bin/gnd.js pkg/bin/gnd.js + cp gnd/npm/README.md pkg/README.md + + cat > pkg/package.json << EOF + { + "name": "@graphprotocol/gnd", + "version": "${VERSION}", + "description": "Graph Node Development CLI", + "bin": { + "gnd": "./bin/gnd.js" + }, + "optionalDependencies": { + "@graphprotocol/gnd-darwin-arm64": "${VERSION}", + "@graphprotocol/gnd-darwin-x64": "${VERSION}", + "@graphprotocol/gnd-linux-arm64": "${VERSION}", + "@graphprotocol/gnd-linux-x64": "${VERSION}", + "@graphprotocol/gnd-win32-x64": "${VERSION}" + }, + "publishConfig": { + "access": "public", + "provenance": true + }, + "license": "(Apache-2.0 OR MIT)", + "repository": { + "type": "git", + "url": "https://github.com/graphprotocol/graph-node.git" + } + } + EOF + + - name: Publish + run: npm publish --provenance --access public --tag ${{ steps.version.outputs.tag }} ${{ inputs.dry_run && '--dry-run' || '' }} + working-directory: pkg \ No newline at end of file diff --git a/.gitignore b/.gitignore index 038afe1d530..0480ebcf19b 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ lcov.info # Node dependencies node_modules/ +.pnpm-store/ # Docker volumes and debug logs .postgres @@ -37,3 +38,4 @@ logfile # Local claude settings .claude/settings.local.json +.claude/ralph-loop.local.md diff --git a/CLAUDE.md b/CLAUDE.md index 52c08a65342..b8ec983428b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -84,21 +84,23 @@ When filtering for specific tests, ensure the intended test name(s) appear in th 3. Anvil running on localhost:3021 4. PNPM 5. Foundry (for smart contract compilation) -6. **Built graph-node binary** (integration tests require the compiled binary) The environment dependencies and environment setup are operated by the human. **Running Integration Tests:** ```bash -# REQUIRED: Build graph-node binary before running integration tests -just build - -# Run all integration tests +# Run all integration tests (automatically builds graph-node and gnd) just test-integration # Run a specific integration test case (e.g., "grafted" test case) TEST_CASE=grafted just test-integration + +# (Optional) Use graph-cli instead of gnd for compatibility testing +GRAPH_CLI=node_modules/.bin/graph just test-integration + +# Override ports if using different service ports (e.g., for local development) +POSTGRES_TEST_PORT=5432 ETHEREUM_TEST_PORT=8545 IPFS_TEST_PORT=5001 just test-integration ``` **⚠️ Test Verification Requirements:** @@ -112,6 +114,8 @@ TEST_CASE=grafted just test-integration - Integration tests take significant time (several minutes) - Tests automatically reset the database between runs - Logs are written to `tests/integration-tests/graph-node.log` +- **If a test hangs for >10 minutes**, it's likely stuck - kill with `pkill -9 integration_tests` and check logs +- CI uses the default ports (3011, 3021, 3001) - local development can override with environment variables ### Code Quality @@ -258,10 +262,7 @@ just test-runner # PostgreSQL: localhost:3011, IPFS: localhost:3001, Anvil: localhost:3021 nix run .#integration -# Claude: Build graph-node binary before running integration tests -just build - -# Claude: Run integration tests +# Claude: Run integration tests (automatically builds graph-node and gnd) just test-integration ``` diff --git a/Cargo.lock b/Cargo.lock index a0c04723ed1..5e204e6ad4d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,18 +14,18 @@ dependencies = [ [[package]] name = "addr2line" -version = "0.25.1" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" +checksum = "59317f77929f0e679d39364702289274de2f0f0b22cbf50b2b8cff2169a0b27a" dependencies = [ "gimli", ] [[package]] -name = "adler" -version = "1.0.2" +name = "adler2" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] name = "ahash" @@ -33,7 +33,7 @@ version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "const-random", "getrandom 0.3.1", "once_cell", @@ -50,6 +50,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -58,9 +73,9 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "alloy" -version = "1.3.0" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf23ee5a0d40c75ade22bf33f117058461fc30a95e84d48b01c845c28f4ea7c5" +checksum = "5e1e915a830ea2ee123c9d3886bfec3302a7ddcd73a973ab165ed45aca2e42c3" dependencies = [ "alloy-consensus", "alloy-contract", @@ -91,14 +106,14 @@ checksum = "ef3a72a2247c34a8545ee99e562b1b9b69168e5000567257ae51e91b4e6b1193" dependencies = [ "alloy-primitives", "num_enum", - "strum", + "strum 0.27.2", ] [[package]] name = "alloy-consensus" -version = "1.3.0" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41e46a465e50a339a817070ec23f06eb3fc9fbb8af71612868367b875a9d49e3" +checksum = "83447eeb17816e172f1dfc0db1f9dc0b7c5d069bd1f7cecbecceb382bf931015" dependencies = [ "alloy-eips", "alloy-primitives", @@ -110,11 +125,11 @@ dependencies = [ "auto_impl", "borsh", "c-kzg", - "derive_more 2.1.1", + "derive_more", "either", "k256", "once_cell", - "rand 0.8.5", + "rand 0.8.6", "secp256k1 0.30.0", "serde", "serde_json", @@ -124,9 +139,9 @@ dependencies = [ [[package]] name = "alloy-consensus-any" -version = "1.3.0" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07001b1693af794c7526aab400b42e38075f986ef8fef78841e5ebc745473e56" +checksum = "5406343e306856dc2be762700e98a16904de45dee14a07f233e742ce68daff2f" dependencies = [ "alloy-consensus", "alloy-eips", @@ -139,9 +154,9 @@ dependencies = [ [[package]] name = "alloy-contract" -version = "1.3.0" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ef1b07c3ff5bf4fab5b8e6c46190cd40b2f2fd2cd72b5b02527a38125d0bff4" +checksum = "3ae8b60d71b92824e095b4003ff01fd2bc923017b7568997c5f459240e83499c" dependencies = [ "alloy-consensus", "alloy-dyn-abi", @@ -158,13 +173,14 @@ dependencies = [ "futures-util", "serde_json", "thiserror 2.0.18", + "tracing", ] [[package]] name = "alloy-core" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a651e1d9e50e6d0a78bd23cd08facb70459a94501c4036c7799a093e569a310" +checksum = "62ddde5968de6044d67af107ad835bc0069a7ca245870b94c5958a7d8712b184" dependencies = [ "alloy-dyn-abi", "alloy-json-abi", @@ -175,9 +191,9 @@ dependencies = [ [[package]] name = "alloy-dyn-abi" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d48a9101f4a67c22fae57489f1ddf3057b8ab4a368d8eac3be088b6e9d9c9d9" +checksum = "a475bb02d9cef2dbb99065c1664ab3fe1f9352e21d6d5ed3f02cdbfc06ed1abc" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -188,7 +204,7 @@ dependencies = [ "proptest", "serde", "serde_json", - "winnow 0.7.13", + "winnow 1.0.0", ] [[package]] @@ -201,7 +217,7 @@ dependencies = [ "alloy-rlp", "arbitrary", "crc", - "rand 0.8.5", + "rand 0.8.6", "serde", "thiserror 2.0.18", ] @@ -216,7 +232,7 @@ dependencies = [ "alloy-rlp", "arbitrary", "borsh", - "rand 0.8.5", + "rand 0.8.6", "serde", ] @@ -231,20 +247,34 @@ dependencies = [ "arbitrary", "borsh", "k256", - "rand 0.8.5", + "rand 0.8.6", "serde", "thiserror 2.0.18", ] +[[package]] +name = "alloy-eip7928" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3231de68d5d6e75332b7489cfcc7f4dfabeba94d990a10e4b923af0e6623540" +dependencies = [ + "alloy-primitives", + "alloy-rlp", + "arbitrary", + "borsh", + "serde", +] + [[package]] name = "alloy-eips" -version = "1.3.0" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707337efeb051ddbaece17a73eaec5150945a5a5541112f4146508248edc2e40" +checksum = "0dca4c89ace90684b4b77366d00631ed498c9af962079af2a5dbc593a0618a77" dependencies = [ "alloy-eip2124", "alloy-eip2930", "alloy-eip7702", + "alloy-eip7928", "alloy-primitives", "alloy-rlp", "alloy-serde", @@ -252,19 +282,18 @@ dependencies = [ "auto_impl", "borsh", "c-kzg", - "derive_more 2.1.1", + "derive_more", "either", "serde", "serde_with", - "sha2", - "thiserror 2.0.18", + "sha2 0.10.9", ] [[package]] name = "alloy-genesis" -version = "1.3.0" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64ba7afffa225272cf50c62ff04ac574adc7bfa73af2370db556340f26fcff5c" +checksum = "ab0e0fe9e6d1120ad7bb9254c3fc2b9bc80a8df42a033fb626be6559c13d5153" dependencies = [ "alloy-eips", "alloy-primitives", @@ -277,9 +306,9 @@ dependencies = [ [[package]] name = "alloy-json-abi" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9914c147bb9b25f440eca68a31dc29f5c22298bfa7754aa802965695384122b0" +checksum = "7c36c9d7f9021601b04bfef14a4b64849f6d73116a4e91e071d7fbfe10247901" dependencies = [ "alloy-primitives", "alloy-sol-type-parser", @@ -289,13 +318,13 @@ dependencies = [ [[package]] name = "alloy-json-rpc" -version = "1.4.3" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b626409c98ba43aaaa558361bca21440c88fd30df7542c7484b9c7a1489cdb" +checksum = "ec0a82e56b1843bce483942d54fcadea92e676f1bde162e93c7d3b621fabc4e1" dependencies = [ "alloy-primitives", "alloy-sol-types", - "http 1.4.0", + "http 1.4.2", "serde", "serde_json", "thiserror 2.0.18", @@ -304,9 +333,9 @@ dependencies = [ [[package]] name = "alloy-network" -version = "1.3.0" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "364a5eaa598437d7a57bcbcb4b7fcb0518e192cf809a19b09b2b5cf73b9ba1cd" +checksum = "a7db7b095b0b1db1d18ce7e91dcd2e82007f2d52bfb8125e6b64633a74a06bc3" dependencies = [ "alloy-consensus", "alloy-consensus-any", @@ -321,7 +350,7 @@ dependencies = [ "alloy-sol-types", "async-trait", "auto_impl", - "derive_more 2.1.1", + "derive_more", "futures-utils-wasm", "serde", "serde_json", @@ -330,9 +359,9 @@ dependencies = [ [[package]] name = "alloy-network-primitives" -version = "1.3.0" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21af5255bd276e528ee625d97033884916e879a1c6edcd5b70a043bd440c0710" +checksum = "cd28d9bfd11729037d194f2b1d43db8642eb3f342032691f4ca96bb745479c3c" dependencies = [ "alloy-consensus", "alloy-eips", @@ -343,39 +372,39 @@ dependencies = [ [[package]] name = "alloy-primitives" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7db950a29746be9e2f2c6288c8bd7a6202a81f999ce109a2933d2379970ec0fa" +checksum = "4885c1409b6936c4898e646ef58baf6ec54edaf6d8179f79df805a7b85b7cf3e" dependencies = [ "alloy-rlp", "arbitrary", "bytes", - "cfg-if 1.0.0", + "cfg-if", "const-hex", - "derive_more 2.1.1", + "derive_more", "foldhash 0.2.0", - "hashbrown 0.16.1", - "indexmap 2.11.4", + "hashbrown 0.17.0", + "indexmap 2.14.0", "itoa", "k256", "keccak-asm", "paste", "proptest", - "proptest-derive 0.6.0", - "rand 0.9.2", + "proptest-derive 0.8.0", + "rand 0.9.3", "rapidhash", "ruint", - "rustc-hash 2.1.1", + "rustc-hash", + "secp256k1 0.31.1", "serde", "sha3", - "tiny-keccak 2.0.2", ] [[package]] name = "alloy-provider" -version = "1.3.0" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc919fe241f9dd28c4c7f7dcff9e66e550c280bafe3545e1019622e1239db38" +checksum = "8955ab30418343de57b356de2ea60200f9fb8016a7ea3bc7f5c6176f01a8b1cf" dependencies = [ "alloy-chains", "alloy-consensus", @@ -407,7 +436,7 @@ dependencies = [ "lru", "parking_lot", "pin-project", - "reqwest", + "reqwest 0.13.2", "serde", "serde_json", "thiserror 2.0.18", @@ -419,9 +448,9 @@ dependencies = [ [[package]] name = "alloy-pubsub" -version = "1.4.3" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94813abbd7baa30c700ea02e7f92319dbcb03bff77aeea92a3a9af7ba19c5c70" +checksum = "7cd85cfea1fa8ebd20d3475e961fe3a3624c0eb4659ea137715c0c83c8aeaff0" dependencies = [ "alloy-json-rpc", "alloy-primitives", @@ -446,7 +475,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f70d83b765fdc080dbcd4f4db70d8d23fe4761f2f02ebfa9146b833900634b4" dependencies = [ "alloy-rlp-derive", - "arrayvec 0.7.4", + "arrayvec", "bytes", ] @@ -458,14 +487,14 @@ checksum = "64b728d511962dda67c1bc7ea7c03736ec275ed2cf4c35d9585298ac9ccf3b73" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] name = "alloy-rpc-client" -version = "1.4.3" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff01723afc25ec4c5b04de399155bef7b6a96dfde2475492b1b7b4e7a4f46445" +checksum = "24f461f091dc8f657e73b5dea18fd63d5c7049720cd252f1eade4a7ebed6a7e1" dependencies = [ "alloy-json-rpc", "alloy-primitives", @@ -476,7 +505,7 @@ dependencies = [ "alloy-transport-ws", "futures 0.3.31", "pin-project", - "reqwest", + "reqwest 0.13.2", "serde", "serde_json", "tokio", @@ -489,9 +518,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types" -version = "1.3.0" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3000edc72a300048cf461df94bfa29fc5d7760ddd88ca7d56ea6fc8b28729" +checksum = "052c031d1f7c5611997056bbcb8814e5cbf20f7efeee8c3de690555172038cf2" dependencies = [ "alloy-primitives", "alloy-rpc-types-anvil", @@ -506,9 +535,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-anvil" -version = "1.3.0" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1207e852f30297d6918f91df3e76f758fa7b519ea1e49fbd7d961ce796663f9" +checksum = "2ff111a54268dc0bbd3b17f98571a7e27cc661dc081ad2999d91888647eb2e11" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -518,32 +547,37 @@ dependencies = [ [[package]] name = "alloy-rpc-types-any" -version = "1.3.0" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ebc96cf29095c10a183fb7106a097fe12ca8dd46733895582da255407f54b29" +checksum = "0a6561ed4759c974d9c144500a59e3fb8c1d87327a12900d5ce455c0cae6dcb6" dependencies = [ "alloy-consensus-any", + "alloy-network-primitives", + "alloy-primitives", "alloy-rpc-types-eth", "alloy-serde", + "serde", + "serde_json", ] [[package]] name = "alloy-rpc-types-debug" -version = "1.4.3" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab1ebed118b701c497e6541d2d11dfa6f3c6ae31a3c52999daa802fcdcc16b7" +checksum = "48b9ad6eee93dd35a9ec0a6c1c6b180892a900ee17a6ed6500921552dd71e846" dependencies = [ "alloy-primitives", - "derive_more 2.1.1", + "alloy-rlp", + "derive_more", "serde", "serde_with", ] [[package]] name = "alloy-rpc-types-engine" -version = "1.3.0" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f35af673cc14e89813ab33671d79b6e73fe38788c5f3a8ec3a75476b58225f53" +checksum = "7eba59e1c069f168a01982f42a57797736923b76aa854194df4930be17867e1c" dependencies = [ "alloy-consensus", "alloy-eips", @@ -551,17 +585,17 @@ dependencies = [ "alloy-rlp", "alloy-serde", "arbitrary", - "derive_more 2.1.1", - "rand 0.8.5", + "derive_more", + "rand 0.8.6", "serde", - "strum", + "strum 0.27.2", ] [[package]] name = "alloy-rpc-types-eth" -version = "1.3.0" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cc3f354a5079480acca0a6533d1d3838177a03ea494ef0ae8d1679efea88274" +checksum = "175a2a5b6017d7f61b5e4b800d21215fe8e94fe729d00828e13bb6d93dcf3492" dependencies = [ "alloy-consensus", "alloy-consensus-any", @@ -572,7 +606,7 @@ dependencies = [ "alloy-serde", "alloy-sol-types", "arbitrary", - "itertools 0.14.0", + "itertools 0.13.0", "serde", "serde_json", "serde_with", @@ -581,9 +615,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-trace" -version = "1.3.0" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d782d80221dfaa5a2f8a7bf277370bdec10e4e8119f5a60d2e2b1adb2e806ca" +checksum = "514b4b1ce3354f65067b4fc7eb75358e0f2ec8be3340c96dea65d6894f9ca435" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -595,9 +629,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-txpool" -version = "1.3.0" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3076c226bb4365f9c3ac0cd4082ba86208aaa1485cbf664383a90aba7c36b26" +checksum = "76e34a42ebb4a71ab0bfdebc6d2f3c7bf809f01edf154d08fed159d10d1ef1d4" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -607,9 +641,9 @@ dependencies = [ [[package]] name = "alloy-serde" -version = "1.3.0" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a438ce4cd49ec4bc213868c1fe94f2fe103d4c3f22f6a42073db974f9c0962da" +checksum = "cc21a8772af7d78bba286726aa245bd2ff81cd9abe230afea2e91578996831c9" dependencies = [ "alloy-primitives", "arbitrary", @@ -619,9 +653,9 @@ dependencies = [ [[package]] name = "alloy-signer" -version = "1.4.3" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb10ccd49d0248df51063fce6b716f68a315dd912d55b32178c883fd48b4021d" +checksum = "8ffbce94c50dd9d4d1f83e044c5595bbd3ada981bd3057ce28b3a5470e77385d" dependencies = [ "alloy-primitives", "async-trait", @@ -634,9 +668,9 @@ dependencies = [ [[package]] name = "alloy-signer-local" -version = "1.3.0" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69c260e78b9c104c444f8a202f283d5e8c6637e6fa52a83f649ad6aaa0b91fd0" +checksum = "e48366d2c42b8d95ef951101bafa28486590f21b7a1e68b6b2d069746557bbe3" dependencies = [ "alloy-consensus", "alloy-network", @@ -644,48 +678,48 @@ dependencies = [ "alloy-signer", "async-trait", "k256", - "rand 0.8.5", + "rand 0.8.6", "thiserror 2.0.18", ] [[package]] name = "alloy-sol-macro" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3b96d5f5890605ba9907ce1e2158e2701587631dc005bfa582cf92dd6f21147" +checksum = "840128ed2b2971d6d4668a553fe403a82683d3acc646c73e75887e7157408033" dependencies = [ "alloy-sol-macro-expander", "alloy-sol-macro-input", "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] name = "alloy-sol-macro-expander" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8247b7cca5cde556e93f8b3882b01dbd272f527836049083d240c57bf7b4c15" +checksum = "63ec265e5d65d725175f6ca7711c970824c90ef9c0d1f1973711d4150ee612dd" dependencies = [ "alloy-json-abi", "alloy-sol-macro-input", "const-hex", "heck 0.5.0", - "indexmap 2.11.4", + "indexmap 2.14.0", "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.114", + "sha3", + "syn 2.0.118", "syn-solidity", - "tiny-keccak 2.0.2", ] [[package]] name = "alloy-sol-macro-input" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cd54f38512ac7bae10bbc38480eefb1b9b398ca2ce25db9cc0c048c6411c4f1" +checksum = "89bf01077f18650876cfa682eb1f949967b5cde03f1a51c955c469d2c9b4aa67" dependencies = [ "alloy-json-abi", "const-hex", @@ -695,25 +729,25 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "syn 2.0.114", + "syn 2.0.118", "syn-solidity", ] [[package]] name = "alloy-sol-type-parser" -version = "1.5.2" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af67a0b0dcebe14244fc92002cd8d96ecbf65db4639d479f5fcd5805755a4c27" +checksum = "857b470ecdd2ed38beaf82ad1a38c516a8ff75266750f38b9eeed001d575241b" dependencies = [ "serde", - "winnow 0.7.13", + "winnow 1.0.0", ] [[package]] name = "alloy-sol-types" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1038284171df8bfd48befc0c7b78f667a7e2be162f45f07bd1c378078ebe58" +checksum = "384cf252de0db2dec52821eac037a7f57e2aa33fe5b900ce6fe39973402341f1" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -723,14 +757,14 @@ dependencies = [ [[package]] name = "alloy-transport" -version = "1.4.3" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f50a9516736d22dd834cc2240e5bf264f338667cc1d9e514b55ec5a78b987ca" +checksum = "86052fdcec72d37ca4aa4b66254601e7453c45a6e1c70aa4561033d002fb80cc" dependencies = [ "alloy-json-rpc", "auto_impl", - "base64 0.22.1", - "derive_more 2.1.1", + "base64", + "derive_more", "futures 0.3.31", "futures-utils-wasm", "parking_lot", @@ -746,13 +780,14 @@ dependencies = [ [[package]] name = "alloy-transport-http" -version = "1.4.3" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a18b541a6197cf9a084481498a766fdf32fefda0c35ea6096df7d511025e9f1" +checksum = "b273587487921274f4f5d0ef2c7ef36944dcbb75a4e2318e69eae822bd263f91" dependencies = [ "alloy-json-rpc", "alloy-transport", - "reqwest", + "itertools 0.13.0", + "reqwest 0.13.2", "serde_json", "tower 0.5.2", "tracing", @@ -761,9 +796,9 @@ dependencies = [ [[package]] name = "alloy-transport-ipc" -version = "1.4.3" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8075911680ebc537578cacf9453464fd394822a0f68614884a9c63f9fbaf5e89" +checksum = "bfb89df168b24773ef603af14f2449c05a7d3f27d05d3eceaea6bf96cccae168" dependencies = [ "alloy-json-rpc", "alloy-pubsub", @@ -775,24 +810,26 @@ dependencies = [ "serde", "serde_json", "tokio", - "tokio-util 0.7.18", + "tokio-util", "tracing", ] [[package]] name = "alloy-transport-ws" -version = "1.4.3" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "921d37a57e2975e5215f7dd0f28873ed5407c7af630d4831a4b5c737de4b0b8b" +checksum = "33e32e0b47d3b3bf5770b7c132090c614b008d307c5e1544f1925f5b7e9e9af6" dependencies = [ "alloy-pubsub", "alloy-transport", "futures 0.3.31", - "http 1.4.0", + "http 1.4.2", + "rustls", "serde_json", "tokio", - "tokio-tungstenite 0.26.2", + "tokio-tungstenite 0.28.0", "tracing", + "url", "ws_stream_wasm", ] @@ -805,9 +842,9 @@ dependencies = [ "alloy-primitives", "alloy-rlp", "arbitrary", - "arrayvec 0.7.4", + "arrayvec", "derive_arbitrary", - "derive_more 2.1.1", + "derive_more", "nybbles", "proptest", "proptest-derive 0.5.1", @@ -818,14 +855,14 @@ dependencies = [ [[package]] name = "alloy-tx-macros" -version = "1.3.0" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99dac443033e83b14f68fac56e8c27e76421f1253729574197ceccd06598f3ef" +checksum = "01a0035943b75fe1e249f52e688492d7a1b1826bc2d19b8e1d5d3c24a2ad8f50" dependencies = [ - "darling 0.21.3", + "darling 0.23.0", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -839,12 +876,27 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.14" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +dependencies = [ + "anstyle", + "anstyle-parse 0.2.4", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstream" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" dependencies = [ "anstyle", - "anstyle-parse", + "anstyle-parse 1.0.0", "anstyle-query", "anstyle-wincon", "colorchoice", @@ -854,9 +906,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.7" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" [[package]] name = "anstyle-parse" @@ -867,6 +919,15 @@ dependencies = [ "utf8parse", ] +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + [[package]] name = "anstyle-query" version = "1.1.0" @@ -878,19 +939,20 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "3.0.3" +version = "3.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", - "windows-sys 0.52.0", + "once_cell_polyfill", + "windows-sys 0.60.2", ] [[package]] name = "anyhow" -version = "1.0.100" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" [[package]] name = "arbitrary" @@ -903,9 +965,12 @@ dependencies = [ [[package]] name = "arc-swap" -version = "1.7.1" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +dependencies = [ + "rustversion", +] [[package]] name = "arcstr" @@ -961,7 +1026,7 @@ dependencies = [ "ark-ff-macros 0.5.0", "ark-serialize 0.5.0", "ark-std 0.5.0", - "arrayvec 0.7.4", + "arrayvec", "digest 0.10.7", "educe", "itertools 0.13.0", @@ -998,7 +1063,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62945a2f7e6de02a31fe400aa489f0e0f5b2502e69f95f853adb82a96c7a6b60" dependencies = [ "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -1036,7 +1101,7 @@ dependencies = [ "num-traits", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -1067,7 +1132,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f4d068aaf107ebcd7dfb52bc748f8030e0fc930ac8e360146ca54c1203088f7" dependencies = [ "ark-std 0.5.0", - "arrayvec 0.7.4", + "arrayvec", "digest 0.10.7", "num-bigint 0.4.6", ] @@ -1079,7 +1144,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1df2c09229cbc5a028b1d70e00fdb2acee28b1055dfb5ca73eea49c5a25c4e7c" dependencies = [ "num-traits", - "rand 0.8.5", + "rand 0.8.6", ] [[package]] @@ -1089,7 +1154,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94893f1e0c6eeab764ade8dc4c0db24caf4fe7cbbaafc0eba0a9030f447b5185" dependencies = [ "num-traits", - "rand 0.8.5", + "rand 0.8.6", ] [[package]] @@ -1099,21 +1164,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "246a225cc6131e9ee4f24619af0f19d67761fff15d7ccc22e42b80846e69449a" dependencies = [ "num-traits", - "rand 0.8.5", + "rand 0.8.6", ] -[[package]] -name = "arrayref" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" - -[[package]] -name = "arrayvec" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" - [[package]] name = "arrayvec" version = "0.7.4" @@ -1125,9 +1178,9 @@ dependencies = [ [[package]] name = "arrow" -version = "57.0.0" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4df8bb5b0bd64c0b9bc61317fcc480bad0f00e56d3bc32c69a4c8dada4786bae" +checksum = "ffaaa3e009861fd829d0a24dd6f115aa8e4634324bb092147d43baafe69ca4a7" dependencies = [ "arrow-arith", "arrow-array", @@ -1146,9 +1199,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "57.2.0" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "288015089e7931843c80ed4032c5274f02b37bcb720c4a42096d50b390e70372" +checksum = "3ac95125e1d71c4a252b5a9c729aef111e80418f08aaa6dbabd1ba66918247fc" dependencies = [ "arrow-array", "arrow-buffer", @@ -1160,9 +1213,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "57.2.0" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65ca404ea6191e06bf30956394173337fa9c35f445bd447fe6c21ab944e1a23c" +checksum = "0c60c79628e9a97cb90d7a0dc3e944f216a902f837d4ecabc14d524bddbbc137" dependencies = [ "ahash", "arrow-buffer", @@ -1170,7 +1223,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.0", "num-complex", "num-integer", "num-traits", @@ -1178,9 +1231,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "57.2.0" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36356383099be0151dacc4245309895f16ba7917d79bdb71a7148659c9206c56" +checksum = "6026f638c400e9878c1b1cc05c3cfd46fbf381285916ab408678701c1df46c1a" dependencies = [ "bytes", "half", @@ -1190,9 +1243,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "57.2.0" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8e372ed52bd4ee88cc1e6c3859aa7ecea204158ac640b10e187936e7e87074" +checksum = "c82c236c3caf8df5664284f3f1fbe89938852163998c3fdbf37e84ac220445e9" dependencies = [ "arrow-array", "arrow-buffer", @@ -1201,7 +1254,7 @@ dependencies = [ "arrow-schema", "arrow-select", "atoi", - "base64 0.22.1", + "base64", "chrono", "half", "lexical-core", @@ -1211,9 +1264,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "57.2.0" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e4100b729fe656f2e4fb32bc5884f14acf9118d4ad532b7b33c1132e4dce896" +checksum = "12714e5fb7954159af1e26d4e0d37108bcf1a2ad5ee5c5bf02a944d564d588b7" dependencies = [ "arrow-array", "arrow-cast", @@ -1226,9 +1279,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "57.2.0" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf87f4ff5fc13290aa47e499a8b669a82c5977c6a1fedce22c7f542c1fd5a597" +checksum = "7bd568aa70c4ec5947027b0d5caee94877433b661a0bb9e8ddceeeb5f0c9b1ab" dependencies = [ "arrow-buffer", "arrow-schema", @@ -1239,9 +1292,9 @@ dependencies = [ [[package]] name = "arrow-flight" -version = "57.0.0" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f70bb56412a007b0cfc116d15f24dda6adeed9611a213852a004cda20085a3b9" +checksum = "68365401e834743d708094927e2ca727a32d639fe900df04b936e07a36701b74" dependencies = [ "arrow-arith", "arrow-array", @@ -1254,7 +1307,7 @@ dependencies = [ "arrow-schema", "arrow-select", "arrow-string", - "base64 0.22.1", + "base64", "bytes", "futures 0.3.31", "once_cell", @@ -1267,9 +1320,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "57.2.0" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3ca63edd2073fcb42ba112f8ae165df1de935627ead6e203d07c99445f2081" +checksum = "e57ee4d470eab1a021bc4b63fa2b2c15d572892bf227b0a982d3b755a6c662b5" dependencies = [ "arrow-array", "arrow-buffer", @@ -1281,18 +1334,19 @@ dependencies = [ [[package]] name = "arrow-json" -version = "57.2.0" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a36b2332559d3310ebe3e173f75b29989b4412df4029a26a30cc3f7da0869297" +checksum = "38f47e0e7a284e1f3707a780dc8cd5451b1614e9e398ea2d9ca03c7a2fe9a9ed" dependencies = [ "arrow-array", "arrow-buffer", "arrow-cast", - "arrow-data", + "arrow-ord", "arrow-schema", + "arrow-select", "chrono", "half", - "indexmap 2.11.4", + "indexmap 2.14.0", "itoa", "lexical-core", "memchr", @@ -1305,9 +1359,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "57.2.0" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c4e0530272ca755d6814218dffd04425c5b7854b87fa741d5ff848bf50aa39" +checksum = "a79cf73ad2eba8686ec2aa9bbf8671208e509025f166afc040cedbd94ffe4983" dependencies = [ "arrow-array", "arrow-buffer", @@ -1318,9 +1372,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "57.2.0" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b07f52788744cc71c4628567ad834cadbaeb9f09026ff1d7a4120f69edf7abd3" +checksum = "cea0f7d8ed6182f14952761e2c0f989852d5aa334fcbc49f73a9f2247c25b879" dependencies = [ "arrow-array", "arrow-buffer", @@ -1331,15 +1385,15 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "57.2.0" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bb63203e8e0e54b288d0d8043ca8fa1013820822a27692ef1b78a977d879f2c" +checksum = "80b3e786a0dd9103acd583a6fb486dbf2f3268466cc0bd571dcf34cef231c1f1" [[package]] name = "arrow-select" -version = "57.2.0" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c96d8a1c180b44ecf2e66c9a2f2bbcb8b1b6f14e165ce46ac8bde211a363411b" +checksum = "067a67e0361f6c31f4a7248759f36ca4ca71b187a941ed4d49da1c7d3d4db624" dependencies = [ "ahash", "arrow-array", @@ -1351,9 +1405,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "57.2.0" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8ad6a81add9d3ea30bf8374ee8329992c7fd246ffd8b7e2f48a3cea5aa0cc9a" +checksum = "99bc95847f3ff62a2b03d6f8ce2e3e78f01362060549a2a311898dd442f6256d" dependencies = [ "arrow-array", "arrow-buffer", @@ -1382,6 +1436,20 @@ dependencies = [ "serde_json", ] +[[package]] +name = "async-compression" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddb939d66e4ae03cee6091612804ba446b12878410cfa17f785f4dd67d4014e8" +dependencies = [ + "brotli", + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", +] + [[package]] name = "async-graphql" version = "7.2.1" @@ -1394,15 +1462,15 @@ dependencies = [ "async-io", "async-trait", "asynk-strim", - "base64 0.22.1", + "base64", "bytes", "chrono", "fast_chemail", "fnv", "futures-util", "handlebars", - "http 1.4.0", - "indexmap 2.11.4", + "http 1.4.2", + "indexmap 2.14.0", "mime", "multer", "num-traits", @@ -1429,7 +1497,7 @@ dependencies = [ "serde_json", "tokio", "tokio-stream", - "tokio-util 0.7.18", + "tokio-util", "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1445,8 +1513,8 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "strum", - "syn 2.0.114", + "strum 0.27.2", + "syn 2.0.118", "thiserror 2.0.18", ] @@ -1469,7 +1537,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e3ef112905abea9dea592fc868a6873b10ebd3f983e83308f995d6284e9ba41" dependencies = [ "bytes", - "indexmap 2.11.4", + "indexmap 2.14.0", "serde", "serde_json", ] @@ -1481,15 +1549,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "456b8a8feb6f42d237746d4b3e9a178494627745c3c56c6ea55d92ba50d026fc" dependencies = [ "autocfg", - "cfg-if 1.0.0", + "cfg-if", "concurrent-queue", "futures-io", "futures-lite", "parking", "polling", - "rustix 1.0.7", + "rustix 1.1.4", "slab", - "windows-sys 0.61.1", + "windows-sys 0.61.2", +] + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", ] [[package]] @@ -1500,7 +1579,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -1522,18 +1601,18 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] name = "async-trait" -version = "0.1.81" +version = "0.1.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -1574,20 +1653,9 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "atomic_refcell" -version = "0.1.13" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41e67cd8309bbd06cd603a9e693a784ac2e5d1e955f11286e355089fcab3047c" - -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi 0.1.19", - "libc", - "winapi", -] +checksum = "21e4227379beff4205943696e6c3e0cd809bacdf3f0edd6e3dd153e2269571a4" [[package]] name = "auto_impl" @@ -1597,7 +1665,7 @@ checksum = "ffdcb70bdbc4d478427380519163274ac86e52916e10f0a8889adf0f96d3fee7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -1606,21 +1674,43 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +[[package]] +name = "aws-lc-rs" +version = "1.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "axum" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" dependencies = [ "axum-core", - "base64 0.22.1", + "base64", "bytes", "form_urlencoded", "futures-util", - "http 1.4.0", - "http-body 1.0.0", + "http 1.4.2", + "http-body", "http-body-util", - "hyper 1.8.1", + "hyper", "hyper-util", "itoa", "matchit", @@ -1632,10 +1722,10 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_urlencoded", - "sha1", + "sha1 0.10.6", "sync_wrapper", "tokio", - "tokio-tungstenite 0.28.0", + "tokio-tungstenite 0.29.0", "tower 0.5.2", "tower-layer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1650,8 +1740,8 @@ checksum = "59446ce19cd142f8833f856eb31f3eb097812d1479ab224f54d72428ca21ea22" dependencies = [ "bytes", "futures-core", - "http 1.4.0", - "http-body 1.0.0", + "http 1.4.2", + "http-body", "http-body-util", "mime", "pin-project-lite", @@ -1682,18 +1772,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - -[[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - [[package]] name = "base64" version = "0.22.1" @@ -1706,15 +1784,6 @@ version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d809780667f4410e7c41b07f52439b94d2bdf8528eeedc287fa38d3b7f95d82" -[[package]] -name = "beef" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" -dependencies = [ - "serde", -] - [[package]] name = "bigdecimal" version = "0.1.2" @@ -1727,17 +1796,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bigdecimal" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6773ddc0eafc0e509fb60e48dff7f450f8e674a0686ae8605e8d9901bd5eefa" -dependencies = [ - "num-bigint 0.4.6", - "num-integer", - "num-traits", -] - [[package]] name = "bimap" version = "0.6.3" @@ -1783,9 +1841,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.9.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" [[package]] name = "bitvec" @@ -1801,48 +1859,33 @@ dependencies = [ [[package]] name = "blake3" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b64485778c4f16a6a5a9d335e80d449ac6c70cdd6a06d2af18a6f6f775a125b3" -dependencies = [ - "arrayref", - "arrayvec 0.5.2", - "cc", - "cfg-if 0.1.10", - "constant_time_eq 0.1.5", - "crypto-mac", - "digest 0.9.0", -] - -[[package]] -name = "blake3" -version = "1.8.2" +version = "1.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3888aaa89e4b2a40fca9848e400f6a658a5a3978de7be858e209cafa8be9a4a0" +checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" dependencies = [ - "arrayref", - "arrayvec 0.7.4", + "arrayvec", "cc", - "cfg-if 1.0.0", - "constant_time_eq 0.3.1", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.0", ] [[package]] name = "block-buffer" -version = "0.9.0" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ "generic-array", ] [[package]] name = "block-buffer" -version = "0.10.4" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be" dependencies = [ - "generic-array", + "hybrid-array", ] [[package]] @@ -1877,7 +1920,28 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", +] + +[[package]] +name = "brotli" +version = "8.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", ] [[package]] @@ -1907,9 +1971,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.16.0" +version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" dependencies = [ "allocator-api2", ] @@ -1928,9 +1992,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.10.1" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" dependencies = [ "serde", ] @@ -1953,9 +2017,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.43" +version = "1.2.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "739eb0f94557554b3ca9a86d2d37bebd49c5e6d0c1d2bda35ba5bdac830befc2" +checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" dependencies = [ "find-msvc-tools", "jobserver", @@ -1964,10 +2028,10 @@ dependencies = [ ] [[package]] -name = "cfg-if" -version = "0.1.10" +name = "cesu8" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" [[package]] name = "cfg-if" @@ -1981,30 +2045,40 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.0", +] + [[package]] name = "chrono" -version = "0.4.43" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fac4744fb15ae8337dc853fee7fb3f4e48c0fbaa23d0afe49c447b4fab126118" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "js-sys", "num-traits", "serde", "wasm-bindgen", - "windows-link 0.2.0", + "windows-link 0.2.1", ] [[package]] name = "cid" -version = "0.11.1" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3147d8272e8fa0ccd29ce51194dd98f79ddfb8191ba9e3409884e751798acf3a" +checksum = "21a304f95f84d169a6f31c4d0a30d784643aaa0bbc9c1e449a2c23e963ec4971" dependencies = [ - "core2", "multibase", "multihash", - "unsigned-varint 0.8.0", + "unsigned-varint", ] [[package]] @@ -2023,13 +2097,22 @@ version = "4.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1c09dd5ada6c6c78075d6fd0da3f90d8080651e2d6cc8eb2f1aaa4034ced708" dependencies = [ - "anstream", + "anstream 0.6.21", "anstyle", "clap_lex", "strsim", "terminal_size", ] +[[package]] +name = "clap_complete" +version = "4.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b4be9c4c4b1f30b78d8a750e0822b6a6102d97e62061c583a6c1dea2dfb33ae" +dependencies = [ + "clap", +] + [[package]] name = "clap_derive" version = "4.5.8" @@ -2039,7 +2122,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -2048,11 +2131,23 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + [[package]] name = "cobs" -version = "0.2.3" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror 2.0.18", +] [[package]] name = "colorchoice" @@ -2071,7 +2166,7 @@ dependencies = [ "memchr", "pin-project-lite", "tokio", - "tokio-util 0.7.18", + "tokio-util", ] [[package]] @@ -2096,14 +2191,26 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "console" +version = "0.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fe5f465a4f6fee88fad41b85d990f84c835335e85b5d9e6e63e0d06d28cba7c" +dependencies = [ + "encode_unicode", + "libc", + "unicode-width", + "windows-sys 0.61.2", +] + [[package]] name = "const-hex" version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3bb320cac8a0750d7f25280aa97b09c26edfe161164238ecbbb31092b079e735" dependencies = [ - "cfg-if 1.0.0", - "cpufeatures", + "cfg-if", + "cpufeatures 0.2.12", "proptest", "serde_core", ] @@ -2114,6 +2221,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "const-random" version = "0.1.18" @@ -2131,26 +2244,14 @@ checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ "getrandom 0.2.15", "once_cell", - "tiny-keccak 2.0.2", + "tiny-keccak", ] [[package]] name = "constant_time_eq" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" - -[[package]] -name = "constant_time_eq" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" - -[[package]] -name = "convert_case" -version = "0.4.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" [[package]] name = "convert_case" @@ -2187,22 +2288,13 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "core2" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505" -dependencies = [ - "memchr", -] - [[package]] name = "cpp_demangle" version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96e58d342ad113c2b878f16d5d034c03be492ae460cdbc02b7f0f2284d310c7d" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -2214,48 +2306,59 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "cranelift-assembler-x64" -version = "0.123.5" +version = "0.133.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57cc4ac031157d0206cf6a8faa48284034721cd367a45e004c4e06329f51e106" +checksum = "e06aeba2c965fc446d13c56a6ccb2631b78445d7544543dd9a25289977630914" dependencies = [ "cranelift-assembler-x64-meta", ] [[package]] name = "cranelift-assembler-x64-meta" -version = "0.123.5" +version = "0.133.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a121c08faeeca04c85280dbddb19521e3ed7169430fd6abc34496e656c18b20" +checksum = "ee2d2dde4ec1352715595b5cfa6fe2e5b8ebb9da3457b3ee8db0aa2808c069aa" dependencies = [ "cranelift-srcgen", ] [[package]] name = "cranelift-bforest" -version = "0.123.5" +version = "0.133.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39f2b2cd8224147b4e193c2de68cf0085b693b242bb766c594828db3907151cb" +checksum = "03b4982ef9fa54ec9eee841e891e7ddc5434be1250e88de31572e000c888f30b" dependencies = [ "cranelift-entity", + "wasmtime-internal-core", ] [[package]] name = "cranelift-bitset" -version = "0.123.5" +version = "0.133.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f7468865d7cf72637a30d5fb97c4fc38b6ea82ab54ca913c81e7403274802be" +checksum = "529143118c4eeb58c39ecb02319557d512be6c61348486422974ab8e3906b8a8" dependencies = [ "serde", "serde_derive", + "wasmtime-internal-core", ] [[package]] name = "cranelift-codegen" -version = "0.123.5" +version = "0.133.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e96c94a373ec1a35fb889730525f3fd220e66b1cf222b3426f5eb6e0404718e5" +checksum = "b7780677247ad3577e3a6a3ebf43f39b325a11d6393db72b2c9968a910d4d13d" dependencies = [ "bumpalo", "cranelift-assembler-x64", @@ -2267,22 +2370,26 @@ dependencies = [ "cranelift-entity", "cranelift-isle", "gimli", - "hashbrown 0.15.2", + "hashbrown 0.17.0", + "libm", "log", + "postcard", "pulley-interpreter", "regalloc2", - "rustc-hash 2.1.1", + "rustc-hash", "serde", + "serde_derive", + "sha2 0.10.9", "smallvec", "target-lexicon", - "wasmtime-internal-math", + "wasmtime-internal-core", ] [[package]] name = "cranelift-codegen-meta" -version = "0.123.5" +version = "0.133.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5904cbc4e8d4f8a69129a365da30d6f9f0e6ca024c4e0728d5da615e8db3c44" +checksum = "ac9645250416cbf92454fe61160e17e026e0ce405906a54500b114f923ddffc9" dependencies = [ "cranelift-assembler-x64-meta", "cranelift-codegen-shared", @@ -2293,37 +2400,39 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" -version = "0.123.5" +version = "0.133.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1009f9e206d5fba4add039539f3e16378815a53b8477bd2d1fc8e3bde6ea93a" +checksum = "20ee8d222ff0fd3681791979afbf88586ac9f49010d3db96b3cbe4c96759aee3" [[package]] name = "cranelift-control" -version = "0.123.5" +version = "0.133.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0c5e3cc40402febecdba0a9e45999b1ab9aef8b120833182b08830b7be292fb" +checksum = "591abe6f5312bd2c4220f1b3bead56c2ad00257c52668015ba013b85dcf2a17a" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.123.5" +version = "0.133.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d58a1de9bdab836734c42902ce948a5cdcc923ae8ce30b29a24dbe76098df659" +checksum = "a5300c49cf940526fe771517b3b3eabd5d0ff164ee61698579cf403fe8d3af3c" dependencies = [ "cranelift-bitset", "serde", "serde_derive", + "wasmtime-internal-core", ] [[package]] name = "cranelift-frontend" -version = "0.123.5" +version = "0.133.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd3423b326097e627a378c106eb57d5ddb3f303d4deb87d29bf8b982dd1d6afc" +checksum = "da4adbf760207fdbbe130f1191cce01cdef66831a9f648b1f39ff2800d126d45" dependencies = [ "cranelift-codegen", + "hashbrown 0.17.0", "log", "smallvec", "target-lexicon", @@ -2331,15 +2440,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.123.5" +version = "0.133.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f56f0e7abec391b94314ab2e9a1002c5a0aed6e29e4709318a7e33315767bed7" +checksum = "8315b21ff018226a42a60a4702c2dd75f6447cac26e9bca622e14c22088c2ff5" [[package]] name = "cranelift-native" -version = "0.123.5" +version = "0.133.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e528d9c791306c55c3bef6c70a77cc9712ca9a32b12bae86924224e65604cb69" +checksum = "d506ef23a60715bde451b06620b14402166ded3b648454fccbf04f3e46a4aa70" dependencies = [ "cranelift-codegen", "libc", @@ -2348,9 +2457,9 @@ dependencies = [ [[package]] name = "cranelift-srcgen" -version = "0.123.5" +version = "0.133.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8558dda6bd86b48c7b31b46555b5eed24b55c839e554a42765c23bf98de62997" +checksum = "48ed47e602652e3410f9387fc0db70fefadcee4d78a78881421aabcab4e26b89" [[package]] name = "crc" @@ -2373,7 +2482,7 @@ version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -2432,6 +2541,33 @@ version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +[[package]] +name = "crossterm" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" +dependencies = [ + "bitflags 2.11.1", + "crossterm_winapi", + "derive_more", + "document-features", + "mio", + "parking_lot", + "rustix 1.1.4", + "signal-hook", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + [[package]] name = "crunchy" version = "0.2.2" @@ -2461,13 +2597,12 @@ dependencies = [ ] [[package]] -name = "crypto-mac" -version = "0.8.0" +name = "crypto-common" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" +checksum = "77727bb15fa921304124b128af125e7e3b968275d1b108b379190264f4423710" dependencies = [ - "generic-array", - "subtle", + "hybrid-array", ] [[package]] @@ -2532,7 +2667,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -2545,9 +2680,8 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "serde", "strsim", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -2559,8 +2693,9 @@ dependencies = [ "ident_case", "proc-macro2", "quote", + "serde", "strsim", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -2571,7 +2706,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core 0.20.10", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -2582,7 +2717,7 @@ checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ "darling_core 0.21.3", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -2593,7 +2728,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core 0.23.0", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -2602,7 +2737,7 @@ version = "6.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "crossbeam-utils", "hashbrown 0.14.5", "lock_api", @@ -2642,17 +2777,34 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0be2b1d1d6ec8d846f05e137292d0b89133caf95ef33695424c09568bdd39b1b" dependencies = [ - "deadpool-runtime", + "deadpool-runtime 0.1.4", "lazy_static", "num_cpus", "tokio", ] +[[package]] +name = "deadpool" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883466cb8db62725aee5f4a6011e8a5d42912b42632df32aad57fc91127c6e04" +dependencies = [ + "deadpool-runtime 0.3.1", + "num_cpus", + "tokio", +] + [[package]] name = "deadpool-runtime" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" + +[[package]] +name = "deadpool-runtime" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2657f61fb1dd8bf37a8d51093cc7cee4e77125b22f7753f49b289f831bec2bae" dependencies = [ "tokio", ] @@ -2678,18 +2830,18 @@ version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "const-oid", + "const-oid 0.9.6", "zeroize", ] [[package]] name = "deranged" -version = "0.3.11" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" dependencies = [ "powerfmt", - "serde", + "serde_core", ] [[package]] @@ -2711,7 +2863,7 @@ checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -2732,7 +2884,7 @@ dependencies = [ "darling 0.20.10", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -2742,20 +2894,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn 2.0.114", -] - -[[package]] -name = "derive_more" -version = "0.99.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3da29a38df43d6f156149c9b43ded5e018ddff2a855cf2cfd62e8cd7d079c69f" -dependencies = [ - "convert_case 0.4.0", - "proc-macro2", - "quote", - "rustc_version 0.4.0", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -2773,28 +2912,28 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" dependencies = [ - "convert_case 0.10.0", + "convert_case", "proc-macro2", "quote", "rustc_version 0.4.0", - "syn 2.0.114", + "syn 2.0.118", "unicode-xid", ] [[package]] name = "diesel" -version = "2.3.2" +version = "2.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8496eeb328dce26ee9d9b73275d396d9bddb433fa30106cf6056dd8c3c2764c" +checksum = "9940fb8467a0a06312218ed384185cb8536aa10d8ec017d0ce7fad2c1bd882d5" dependencies = [ - "bigdecimal 0.3.1", - "bitflags 2.9.0", + "bigdecimal", + "bitflags 2.11.1", "byteorder", "chrono", "diesel_derives", "downcast-rs", "itoa", - "num-bigint 0.4.6", + "num-bigint 0.2.6", "num-integer", "num-traits", "pq-sys", @@ -2804,15 +2943,15 @@ dependencies = [ [[package]] name = "diesel-async" -version = "0.7.4" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13096fb8dae53f2d411c4b523bec85f45552ed3044a2ab4d85fb2092d9cb4f34" +checksum = "dd39af30158d444884f166fe4c58f35dc40ad71ad017bb59408a3448526ff4bd" dependencies = [ - "deadpool", + "deadpool 0.13.0", "diesel", "futures-core", "futures-util", - "scoped-futures", + "pin-project-lite", "tokio", "tokio-postgres", ] @@ -2826,7 +2965,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -2840,22 +2979,22 @@ dependencies = [ [[package]] name = "diesel_derives" -version = "2.3.6" +version = "2.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c30b2969f923fa1f73744b92bb7df60b858df8832742d9a3aceb79236c0be1d2" +checksum = "d1817b7f4279b947fc4cafddec12b0e5f8727141706561ce3ac94a60bddd1cf5" dependencies = [ "diesel_table_macro_syntax", "dsl_auto_type", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] name = "diesel_migrations" -version = "2.3.1" +version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "745fd255645f0f1135f9ec55c7b00e0882192af9683ab4731e4bba3da82b8f9c" +checksum = "28d0f4a98124ba6d4ca75da535f65984badec16a003b6e2f94a01e31a79490b8" dependencies = [ "diesel", "migrations_internals", @@ -2868,7 +3007,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe2444076b48641147115697648dc743c2c00b61adade0f01ce67133c7babe8c" dependencies = [ - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -2899,18 +3038,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", - "const-oid", - "crypto-common", + "const-oid 0.9.6", + "crypto-common 0.1.6", "subtle", ] +[[package]] +name = "digest" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4850db49bf08e663084f7fb5c87d202ef91a3907271aff24a94eb97ff039153c" +dependencies = [ + "block-buffer 0.12.0", + "const-oid 0.10.2", + "crypto-common 0.2.1", +] + [[package]] name = "directories-next" version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "339ee130d97a610ea5a5872d2bbb130fdf68884ff09d3028b81bec8a1ac23bbc" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "dirs-sys-next", ] @@ -2929,7 +3079,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "dirs-sys-next", ] @@ -2964,7 +3114,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -2973,6 +3123,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aac81fa3e28d21450aa4d2ac065992ba96a1d7303efbce51a95f4fd175b67562" +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs", +] + [[package]] name = "downcast-rs" version = "2.0.2" @@ -2990,7 +3149,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -2999,6 +3158,12 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + [[package]] name = "ecdsa" version = "0.16.9" @@ -3023,7 +3188,7 @@ dependencies = [ "enum-ordinalize", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -3079,7 +3244,7 @@ version = "0.8.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -3099,14 +3264,14 @@ checksum = "8ca9601fb2d62598ee17836250842873a413586e5d7ed88b356e38ddbb0ec631" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] name = "env_filter" -version = "0.1.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a009aa4810eb158359dda09d0c87378e4bbb89b5a801f016885a4707ba24f7ea" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" dependencies = [ "log", "regex", @@ -3114,11 +3279,11 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.11.8" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" dependencies = [ - "anstream", + "anstream 1.0.0", "anstyle", "env_filter", "jiff", @@ -3142,7 +3307,7 @@ checksum = "452c458dfb890a2fea64e4c894f83daad61689fb9bbe84c382e1ce6d7536710b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -3171,47 +3336,24 @@ dependencies = [ ] [[package]] -name = "ethabi" -version = "17.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4966fba78396ff92db3b817ee71143eccd98acf0f876b8d600e585a670c5d1b" -dependencies = [ - "ethereum-types", - "hex", - "once_cell", - "regex", - "serde", - "serde_json", - "sha3", - "thiserror 1.0.61", - "uint 0.9.5", -] - -[[package]] -name = "ethbloom" -version = "0.12.1" +name = "event-listener" +version = "5.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11da94e443c60508eb62cf256243a64da87304c2802ac2528847f79d750007ef" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" dependencies = [ - "crunchy", - "fixed-hash 0.7.0", - "impl-rlp", - "impl-serde", - "tiny-keccak 2.0.2", + "concurrent-queue", + "parking", + "pin-project-lite", ] [[package]] -name = "ethereum-types" -version = "0.13.1" +name = "event-listener-strategy" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2827b94c556145446fcce834ca86b7abf0c39a805883fe20e72c5bfdb5a0dc6" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" dependencies = [ - "ethbloom", - "fixed-hash 0.7.0", - "impl-rlp", - "impl-serde", - "primitive-types 0.11.1", - "uint 0.9.5", + "event-listener", + "pin-project-lite", ] [[package]] @@ -3237,9 +3379,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.1.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" [[package]] name = "fastrlp" @@ -3247,7 +3389,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "139834ddba373bbdd213dffe02c8d110508dcf1726c2be27e8d1f7d7e1856418" dependencies = [ - "arrayvec 0.7.4", + "arrayvec", "auto_impl", "bytes", ] @@ -3258,7 +3400,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce8dba4714ef14b8274c371879b175aa55b16b30f269663f19d576f380018dc4" dependencies = [ - "arrayvec 0.7.4", + "arrayvec", "auto_impl", "bytes", ] @@ -3275,9 +3417,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.4" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "firestorm" @@ -3293,27 +3435,21 @@ checksum = "2c5f6c2c942da57e2aaaa84b8a521489486f14e75e7fa91dab70aba913975f98" [[package]] name = "fixed-hash" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcf0ed7fe52a17a03854ec54a9f76d6d84508d1c0e66bc1793301c73fc8493c" +checksum = "835c052cb0c08c1acf6ffd71c022172e18723949c8282f2b9f27efbc51e64534" dependencies = [ "byteorder", - "rand 0.8.5", + "rand 0.8.6", "rustc-hex", "static_assertions", ] [[package]] -name = "fixed-hash" -version = "0.8.0" +name = "fixedbitset" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "835c052cb0c08c1acf6ffd71c022172e18723949c8282f2b9f27efbc51e64534" -dependencies = [ - "byteorder", - "rand 0.8.5", - "rustc-hex", - "static_assertions", -] +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "fixedbitset" @@ -3327,18 +3463,19 @@ version = "25.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.11.1", "rustc_version 0.4.0", ] [[package]] name = "flate2" -version = "1.0.30" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -3383,6 +3520,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "fsevent-sys" version = "4.1.0" @@ -3470,7 +3613,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -3485,12 +3628,6 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" -[[package]] -name = "futures-timer" -version = "3.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" - [[package]] name = "futures-util" version = "0.3.31" @@ -3517,24 +3654,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42012b0f064e01aa58b545fe3727f90f7dd4020f4a3ea735b50344965f5a57e9" [[package]] -name = "fxhash" -version = "0.2.1" +name = "fuzzy-matcher" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +checksum = "54614a3312934d066701a80f20f15fa3b56d67ac7722b39eea5b4c9dd1d66c94" dependencies = [ - "byteorder", + "thread_local", ] [[package]] name = "fxprof-processed-profile" -version = "0.6.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27d12c0aed7f1e24276a241aadc4cb8ea9f83000f34bc062b7cc2d51e3b0fabd" +checksum = "25234f20a3ec0a962a61770cfe39ecf03cb529a6e474ad8cff025ed497eda557" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.11.1", "debugid", - "fxhash", + "rustc-hash", "serde", + "serde_derive", "serde_json", ] @@ -3555,9 +3693,11 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", + "js-sys", "libc", "wasi 0.11.0+wasi-snapshot-preview1", + "wasm-bindgen", ] [[package]] @@ -3566,20 +3706,37 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", + "js-sys", "libc", "wasi 0.13.3+wasi-0.2.2", + "wasm-bindgen", "windows-targets 0.52.6", ] +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "rand_core 0.10.0", + "wasip2", + "wasip3", +] + [[package]] name = "gimli" -version = "0.32.3" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" +checksum = "0bf7f043f89559805f8c7cacc432749b2fa0d0a0a9ee46ce47164ed5ba7f126c" dependencies = [ - "fallible-iterator 0.3.0", - "indexmap 2.11.4", + "fnv", + "hashbrown 0.16.1", + "indexmap 2.14.0", "stable_deref_trait", ] @@ -3601,7 +3758,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", "time", ] @@ -3626,27 +3783,53 @@ dependencies = [ [[package]] name = "gnd" -version = "0.36.0" +version = "0.45.0" dependencies = [ + "Inflector", "anyhow", + "async-trait", "clap", + "clap_complete", + "console 0.16.4", "env_logger", "git-testament", "globset", "graph", + "graph-chain-ethereum", "graph-core", + "graph-graphql", "graph-node", + "graph-store-postgres", + "graphql-tools", + "hex", + "indicatif", + "inquire", "lazy_static", "notify", + "open", "pgtemp", "pq-sys", + "regex", + "reqwest 0.12.23", + "semver 1.0.28", + "serde", + "serde_json", + "serde_yaml", + "sha1 0.11.0", + "similar", + "tempfile", + "thiserror 2.0.18", "tokio", - "tokio-util 0.7.18", + "tokio-util", + "tower 0.5.2", + "url", + "walkdir", + "wasmparser 0.118.2", ] [[package]] name = "graph" -version = "0.36.0" +version = "0.45.0" dependencies = [ "Inflector", "ahash", @@ -3657,9 +3840,8 @@ dependencies = [ "async-stream", "async-trait", "atomic_refcell", - "atty", - "base64 0.21.7", - "bigdecimal 0.1.2", + "base64", + "bigdecimal", "bs58 0.5.1", "bytes", "chrono", @@ -3667,11 +3849,10 @@ dependencies = [ "clap", "csv", "defer", - "derivative", + "derive_more", "diesel", "diesel_derives", "envconfig", - "ethabi", "futures 0.1.31", "futures 0.3.31", "graph_derive", @@ -3680,12 +3861,12 @@ dependencies = [ "hex", "hex-literal", "http 0.2.12", - "http 1.4.0", + "http 1.4.2", "http-body-util", - "hyper 1.8.1", + "hyper", "hyper-util", "indoc", - "itertools 0.14.0", + "itertools 0.15.0", "lazy-regex", "lazy_static", "lru_time_cache", @@ -3695,40 +3876,41 @@ dependencies = [ "num-traits", "object_store", "parking_lot", - "petgraph", + "petgraph 0.8.3", "portable-atomic", "priority-queue", "prometheus", "prost", "prost-types", - "rand 0.9.2", + "rand 0.9.3", "redis", "regex", - "reqwest", - "semver 1.0.27", + "reqwest 0.12.23", + "rustls", + "semver 1.0.28", "serde", "serde_derive", "serde_json", "serde_plain", "serde_regex", "serde_yaml", - "sha2", + "sha2 0.11.0", "slog", "slog-async", "slog-envlogger", "slog-term", "sqlparser 0.57.0", - "sqlparser 0.60.0", + "sqlparser 0.62.0", "stable-hash 0.3.4", "stable-hash 0.4.4", - "strum_macros", + "strum_macros 0.28.0", + "tempfile", "thiserror 2.0.18", - "tiny-keccak 1.5.0", "tokio", "tokio-retry", "tokio-stream", - "tokio-util 0.7.18", - "toml 0.9.11+spec-1.1.0", + "tokio-util", + "toml 1.1.2+spec-1.1.0", "tonic", "tonic-prost", "tonic-prost-build", @@ -3739,7 +3921,7 @@ dependencies = [ [[package]] name = "graph-chain-common" -version = "0.36.0" +version = "0.45.0" dependencies = [ "anyhow", "heck 0.5.0", @@ -3749,24 +3931,24 @@ dependencies = [ [[package]] name = "graph-chain-ethereum" -version = "0.36.0" +version = "0.45.0" dependencies = [ "anyhow", "async-trait", - "base64 0.22.1", + "base64", "envconfig", + "futures 0.3.31", "graph", "graph-runtime-derive", "graph-runtime-wasm", "hex", - "itertools 0.14.0", + "itertools 0.15.0", "jsonrpc-core", "prost", "prost-types", - "semver 1.0.27", + "semver 1.0.28", "serde", "thiserror 2.0.18", - "tiny-keccak 1.5.0", "tokio", "tokio-stream", "tonic-prost-build", @@ -3775,7 +3957,7 @@ dependencies = [ [[package]] name = "graph-chain-near" -version = "0.36.0" +version = "0.45.0" dependencies = [ "anyhow", "async-trait", @@ -3792,7 +3974,7 @@ dependencies = [ [[package]] name = "graph-core" -version = "0.36.0" +version = "0.45.0" dependencies = [ "alloy", "anyhow", @@ -3806,16 +3988,16 @@ dependencies = [ "graph-chain-near", "graph-runtime-wasm", "indoc", - "itertools 0.14.0", + "itertools 0.15.0", "parking_lot", "prometheus", "serde_yaml", "slog", - "strum", + "strum 0.28.0", "thiserror 2.0.18", "tokio", "tokio-retry", - "tokio-util 0.7.18", + "tokio-util", "tower 0.5.3", "tower-test", "wiremock", @@ -3823,7 +4005,7 @@ dependencies = [ [[package]] name = "graph-graphql" -version = "0.36.0" +version = "0.45.0" dependencies = [ "anyhow", "async-recursion", @@ -3839,10 +4021,11 @@ dependencies = [ [[package]] name = "graph-node" -version = "0.36.0" +version = "0.45.0" dependencies = [ "anyhow", "clap", + "console 0.16.4", "diesel", "diesel-async", "env_logger", @@ -3859,7 +4042,8 @@ dependencies = [ "graph-store-postgres", "graphman", "graphman-server", - "itertools 0.14.0", + "indicatif", + "itertools 0.15.0", "json-structural-diff", "lazy_static", "prometheus", @@ -3867,35 +4051,35 @@ dependencies = [ "shellexpand", "termcolor", "tokio", - "tokio-util 0.7.18", + "tokio-util", "url", ] [[package]] name = "graph-runtime-derive" -version = "0.36.0" +version = "0.45.0" dependencies = [ "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] name = "graph-runtime-test" -version = "0.36.0" +version = "0.45.0" dependencies = [ "async-trait", "graph", "graph-chain-ethereum", "graph-runtime-wasm", - "rand 0.9.2", - "semver 1.0.27", + "rand 0.9.3", + "semver 1.0.28", "test-store", "wasmtime", ] [[package]] name = "graph-runtime-wasm" -version = "0.36.0" +version = "0.45.0" dependencies = [ "anyhow", "async-trait", @@ -3905,7 +4089,7 @@ dependencies = [ "hex", "never", "parity-wasm", - "semver 1.0.27", + "semver 1.0.28", "serde_yaml", "wasm-instrument", "wasmtime", @@ -3913,7 +4097,7 @@ dependencies = [ [[package]] name = "graph-server-http" -version = "0.36.0" +version = "0.45.0" dependencies = [ "async-trait", "graph", @@ -3924,10 +4108,10 @@ dependencies = [ [[package]] name = "graph-server-index-node" -version = "0.36.0" +version = "0.45.0" dependencies = [ "async-trait", - "blake3 1.8.2", + "blake3", "git-testament", "graph", "graph-chain-ethereum", @@ -3937,32 +4121,37 @@ dependencies = [ [[package]] name = "graph-server-json-rpc" -version = "0.36.0" +version = "0.45.0" dependencies = [ + "axum", "graph", - "jsonrpsee", "serde", + "serde_json", + "slog", + "thiserror 2.0.18", + "tokio", ] [[package]] name = "graph-server-metrics" -version = "0.36.0" +version = "0.45.0" dependencies = [ "graph", ] [[package]] name = "graph-store-postgres" -version = "0.36.0" +version = "0.45.0" dependencies = [ "Inflector", "anyhow", + "arrow", "async-trait", - "blake3 1.8.2", + "blake3", "chrono", "clap", - "deadpool", - "derive_more 2.1.1", + "deadpool 0.13.0", + "derive_more", "diesel", "diesel-async", "diesel-derive-enum", @@ -3974,18 +4163,19 @@ dependencies = [ "graph", "graphman-store", "hex", - "itertools 0.14.0", + "itertools 0.15.0", "lazy_static", "lru_time_cache", "openssl", + "parquet", "postgres", "postgres-openssl", "pretty_assertions", - "rand 0.9.2", + "rand 0.9.3", "serde", "serde_json", "serde_yaml", - "sqlparser 0.60.0", + "sqlparser 0.62.0", "stable-hash 0.3.4", "thiserror 2.0.18", "tokio", @@ -3994,7 +4184,7 @@ dependencies = [ [[package]] name = "graph-tests" -version = "0.36.0" +version = "0.45.0" dependencies = [ "anyhow", "assert-json-diff", @@ -4008,29 +4198,29 @@ dependencies = [ "graph-runtime-wasm", "graph-server-index-node", "graph-store-postgres", - "secp256k1 0.21.3", "serde", "serde_yaml", "slog", + "slog-async", + "slog-term", "tokio", "tokio-stream", - "tokio-util 0.7.18", - "web3", + "tokio-util", ] [[package]] name = "graph_derive" -version = "0.36.0" +version = "0.45.0" dependencies = [ "proc-macro-utils", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] name = "graphman" -version = "0.36.0" +version = "0.45.0" dependencies = [ "anyhow", "diesel", @@ -4038,14 +4228,14 @@ dependencies = [ "graph", "graph-store-postgres", "graphman-store", - "itertools 0.14.0", + "itertools 0.15.0", "thiserror 2.0.18", "tokio", ] [[package]] name = "graphman-server" -version = "0.36.0" +version = "0.45.0" dependencies = [ "anyhow", "async-graphql", @@ -4059,32 +4249,32 @@ dependencies = [ "graphman", "graphman-store", "lazy_static", - "reqwest", + "reqwest 0.12.23", "serde", "serde_json", "slog", "test-store", "thiserror 2.0.18", "tokio", - "tower-http", + "tower-http 0.7.0", ] [[package]] name = "graphman-store" -version = "0.36.0" +version = "0.45.0" dependencies = [ "anyhow", "async-trait", "chrono", "diesel", - "strum", + "strum 0.28.0", ] [[package]] name = "graphql-tools" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a378ccbe227983c1f4aeaafd088d0de9c5e3d5b6cd8e7aa9b0c682582a6c067" +checksum = "9b1ae57fa544e67a661c97805be3e36b97e7d15d65d67bb34beac24d940f987e" dependencies = [ "combine", "itoa", @@ -4109,39 +4299,20 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http 0.2.12", - "indexmap 2.11.4", - "slab", - "tokio", - "tokio-util 0.7.18", - "tracing", -] - -[[package]] -name = "h2" -version = "0.4.5" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" dependencies = [ "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "http 1.4.0", - "indexmap 2.11.4", + "http 1.4.2", + "indexmap 2.14.0", "slab", "tokio", - "tokio-util 0.7.18", + "tokio-util", "tracing", ] @@ -4151,7 +4322,7 @@ version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "crunchy", "num-traits", "zerocopy", @@ -4191,10 +4362,7 @@ version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" dependencies = [ - "allocator-api2", - "equivalent", "foldhash 0.1.5", - "serde", ] [[package]] @@ -4202,6 +4370,17 @@ name = "hashbrown" version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "hashbrown" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" dependencies = [ "foldhash 0.2.0", "serde", @@ -4218,30 +4397,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "headers" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322106e6bd0cba2d5ead589ddb8150a13d7c4217cf80d7c4f682ca994ccc6aa9" -dependencies = [ - "base64 0.21.7", - "bytes", - "headers-core", - "http 1.4.0", - "httpdate", - "mime", - "sha1", -] - -[[package]] -name = "headers-core" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4" -dependencies = [ - "http 1.4.0", -] - [[package]] name = "heck" version = "0.4.1" @@ -4254,15 +4409,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - [[package]] name = "hermit-abi" version = "0.3.9" @@ -4287,7 +4433,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fda06d18ac606267c40c04e41b9947729bf8b9efe74bd4e82b61a5f26a510b9f" dependencies = [ - "arrayvec 0.7.4", + "arrayvec", ] [[package]] @@ -4327,25 +4473,14 @@ dependencies = [ [[package]] name = "http" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" dependencies = [ "bytes", "itoa", ] -[[package]] -name = "http-body" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" -dependencies = [ - "bytes", - "http 0.2.12", - "pin-project-lite", -] - [[package]] name = "http-body" version = "1.0.0" @@ -4353,7 +4488,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" dependencies = [ "bytes", - "http 1.4.0", + "http 1.4.2", ] [[package]] @@ -4364,8 +4499,8 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http 1.4.0", - "http-body 1.0.0", + "http 1.4.2", + "http-body", "pin-project-lite", ] @@ -4388,47 +4523,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" [[package]] -name = "hyper" -version = "0.14.29" +name = "hybrid-array" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f361cde2f109281a220d4307746cdfd5ee3f410da58a70377762396775634b33" +checksum = "1a79f2aff40c18ab8615ddc5caa9eb5b96314aef18fe5823090f204ad988e813" dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2 0.3.26", - "http 0.2.12", - "http-body 0.4.6", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2 0.5.7", - "tokio", - "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "tracing", - "want", + "typenum", ] [[package]] name = "hyper" -version = "1.8.1" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" dependencies = [ "atomic-waker", "bytes", "futures-channel", "futures-core", - "h2 0.4.5", - "http 1.4.0", - "http-body 1.0.0", + "h2", + "http 1.4.2", + "http-body", "httparse", "httpdate", "itoa", "pin-project-lite", - "pin-utils", "smallvec", "tokio", "want", @@ -4441,16 +4560,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" dependencies = [ "futures-util", - "http 1.4.0", - "hyper 1.8.1", + "http 1.4.2", + "hyper", "hyper-util", "rustls", - "rustls-native-certs 0.7.1", "rustls-pki-types", "tokio", "tokio-rustls", "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "webpki-roots 0.26.11", ] [[package]] @@ -4459,7 +4576,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 1.8.1", + "hyper", "hyper-util", "pin-project-lite", "tokio", @@ -4474,7 +4591,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.8.1", + "hyper", "hyper-util", "native-tls", "tokio", @@ -4484,23 +4601,22 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.19" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "futures-channel", - "futures-core", "futures-util", - "http 1.4.0", - "http-body 1.0.0", - "hyper 1.8.1", + "http 1.4.2", + "http-body", + "hyper", "ipnet", "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.0", + "socket2", "system-configuration", "tokio", "tower-layer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4538,9 +4654,9 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d1fcc7f316b2c079dde77564a1360639c1a956a23fa96122732e416cb10717bb" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "num-traits", - "rand 0.8.5", + "rand 0.8.6", "static_assertions", ] @@ -4659,14 +4775,14 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] name = "id-arena" -version = "2.2.1" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25a2bc672d1148e28034f176e01fffebb08b35768468cc954630da77a1449005" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" [[package]] name = "ident_case" @@ -4674,17 +4790,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" -[[package]] -name = "idna" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" -dependencies = [ - "matches", - "unicode-bidi", - "unicode-normalization", -] - [[package]] name = "idna" version = "1.1.0" @@ -4715,24 +4820,6 @@ dependencies = [ "parity-scale-codec", ] -[[package]] -name = "impl-rlp" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28220f89297a075ddc7245cd538076ee98b01f2a9c23a53a4f1105d5a322808" -dependencies = [ - "rlp", -] - -[[package]] -name = "impl-serde" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4551f042f3438e64dbd6226b20527fc84a6e1fe65688b58746a2f53623f25f5c" -dependencies = [ - "serde", -] - [[package]] name = "impl-trait-for-tuples" version = "0.2.2" @@ -4757,17 +4844,30 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.11.4" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "arbitrary", "equivalent", - "hashbrown 0.16.1", + "hashbrown 0.17.0", "serde", "serde_core", ] +[[package]] +name = "indicatif" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993f007684f2e9727160da8b960ec161264703bfd1af084fd2e34d040c9a0dd4" +dependencies = [ + "console 0.16.4", + "portable-atomic", + "unicode-width", + "unit-prefix", + "web-time", +] + [[package]] name = "indoc" version = "2.0.7" @@ -4783,7 +4883,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f37dccff2791ab604f9babef0ba14fbe0be30bd368dc541e2b08d07c8aa908f3" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.11.1", "inotify-sys", "libc", ] @@ -4797,6 +4897,20 @@ dependencies = [ "libc", ] +[[package]] +name = "inquire" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6654738b8024300cf062d04a1c13c10c8e2cea598ec1c47dc9b6641159429756" +dependencies = [ + "bitflags 2.11.1", + "crossterm", + "dyn-clone", + "fuzzy-matcher", + "unicode-segmentation", + "unicode-width", +] + [[package]] name = "interprocess" version = "2.2.3" @@ -4819,13 +4933,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] -name = "iri-string" -version = "0.7.8" +name = "is-docker" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2" +checksum = "928bae27f42bc99b60d9ac7334e3a21d10ad8f1835a4e12ec3ec0464765ed1b3" dependencies = [ - "memchr", - "serde", + "once_cell", ] [[package]] @@ -4839,6 +4952,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "is-wsl" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "173609498df190136aa7dea1a91db051746d339e18476eed5ca40521f02d7aa5" +dependencies = [ + "is-docker", + "once_cell", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.0" @@ -4872,6 +4995,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.17" @@ -4900,143 +5032,117 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.15" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be1f93b8b1eb69c77f24bbb0afdf66f54b632ee39af40ca21c4365a1d7347e49" +checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" dependencies = [ "jiff-static", "log", "portable-atomic", "portable-atomic-util", - "serde", + "serde_core", ] [[package]] name = "jiff-static" -version = "0.2.15" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03343451ff899767262ec32146f6d559dd759fdadf42ff0e227c7c48f72594b4" +checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] -name = "jobserver" -version = "0.1.31" +name = "jni" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" dependencies = [ - "libc", + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror 1.0.61", + "walkdir", + "windows-sys 0.45.0", ] [[package]] -name = "js-sys" -version = "0.3.77" +name = "jni-sys" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" dependencies = [ - "once_cell", - "wasm-bindgen", + "jni-sys 0.4.1", ] [[package]] -name = "json-structural-diff" -version = "0.2.0" +name = "jni-sys" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e878e36a8a44c158505c2c818abdc1350413ad83dcb774a0459f6a7ef2b65cbf" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" dependencies = [ - "console", - "difflib", - "regex", - "serde_json", + "jni-sys-macros", ] [[package]] -name = "jsonrpc-core" -version = "18.0.0" +name = "jni-sys-macros" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14f7f76aef2d054868398427f6c54943cf3d1caa9a7ec7d0c38d69df97a965eb" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ - "futures 0.3.31", - "futures-executor", - "futures-util", - "log", - "serde", - "serde_derive", - "serde_json", + "quote", + "syn 2.0.118", ] [[package]] -name = "jsonrpsee" -version = "0.15.1" +name = "jobserver" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bd0d559d5e679b1ab2f869b486a11182923863b1b3ee8b421763cdd707b783a" +checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" dependencies = [ - "jsonrpsee-core", - "jsonrpsee-http-server", - "jsonrpsee-types", + "libc", ] [[package]] -name = "jsonrpsee-core" -version = "0.15.1" +name = "js-sys" +version = "0.3.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3dc3e9cf2ba50b7b1d7d76a667619f82846caa39e8e8daa8a4962d74acaddca" +checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" dependencies = [ - "anyhow", - "arrayvec 0.7.4", - "async-trait", - "beef", - "futures-channel", + "cfg-if", "futures-util", - "globset", - "http 0.2.12", - "hyper 0.14.29", - "jsonrpsee-types", - "lazy_static", - "parking_lot", - "rand 0.8.5", - "rustc-hash 1.1.0", - "serde", - "serde_json", - "thiserror 1.0.61", - "tokio", - "tracing", - "unicase", + "wasm-bindgen", ] [[package]] -name = "jsonrpsee-http-server" -version = "0.15.1" +name = "json-structural-diff" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03802f0373a38c2420c70b5144742d800b509e2937edc4afb116434f07120117" +checksum = "e878e36a8a44c158505c2c818abdc1350413ad83dcb774a0459f6a7ef2b65cbf" dependencies = [ - "futures-channel", - "futures-util", - "hyper 0.14.29", - "jsonrpsee-core", - "jsonrpsee-types", - "serde", + "console 0.15.11", + "difflib", + "regex", "serde_json", - "tokio", - "tracing", - "tracing-futures", ] [[package]] -name = "jsonrpsee-types" -version = "0.15.1" +name = "jsonrpc-core" +version = "18.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e290bba767401b646812f608c099b922d8142603c9e73a50fb192d3ac86f4a0d" +checksum = "14f7f76aef2d054868398427f6c54943cf3d1caa9a7ec7d0c38d69df97a965eb" dependencies = [ - "anyhow", - "beef", + "futures 0.3.31", + "futures-executor", + "futures-util", + "log", "serde", + "serde_derive", "serde_json", - "thiserror 1.0.61", - "tracing", ] [[package]] @@ -5045,28 +5151,29 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6e3919bbaa2945715f0bb6d3934a173d1e9a59ac23767fbaaef277265a7411b" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "ecdsa", "elliptic-curve", "once_cell", "serdect", - "sha2", + "sha2 0.10.9", ] [[package]] name = "keccak" -version = "0.1.5" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" +checksum = "9e24a010dd405bd7ed803e5253182815b41bf2e6a80cc3bfc066658e03a198aa" dependencies = [ - "cpufeatures", + "cfg-if", + "cpufeatures 0.3.0", ] [[package]] name = "keccak-asm" -version = "0.1.4" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "505d1856a39b200489082f90d897c3f07c455563880bc5952e38eabf731c83b6" +checksum = "1766b89733097006f3a1388a02849865d6bc98c89273cb622e29fdd209922183" dependencies = [ "digest 0.10.7", "sha3-asm", @@ -5094,9 +5201,9 @@ dependencies = [ [[package]] name = "lazy-regex" -version = "3.5.1" +version = "3.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5c13b6857ade4c8ee05c3c3dc97d2ab5415d691213825b90d3211c425c1f907" +checksum = "6bae91019476d3ec7147de9aa291cadb6d870abf2f3015d2da73a90325ac1496" dependencies = [ "lazy-regex-proc_macros", "once_cell", @@ -5105,14 +5212,14 @@ dependencies = [ [[package]] name = "lazy-regex-proc_macros" -version = "3.5.1" +version = "3.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a95c68db5d41694cea563c86a4ba4dc02141c16ef64814108cb23def4d5438" +checksum = "4de9c1e1439d8b7b3061b2d209809f447ca33241733d9a3c01eabf2dc8d94358" dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -5192,15 +5299,15 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.175" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libm" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libredox" @@ -5208,7 +5315,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.11.1", "libc", ] @@ -5220,9 +5327,9 @@ checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "linux-raw-sys" -version = "0.9.4" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" @@ -5230,6 +5337,12 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856" +[[package]] +name = "litrs" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" + [[package]] name = "lock_api" version = "0.4.14" @@ -5247,13 +5360,19 @@ checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "lru" -version = "0.13.0" +version = "0.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "227748d55f2f0ab4735d87fd623798cb6b664512fe979705f829c9f81c934465" +checksum = "a1dc47f592c06f33f8e3aea9591776ec7c9f9e4124778ff8a3c3b87159f7e593" dependencies = [ - "hashbrown 0.15.2", + "hashbrown 0.16.1", ] +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "lru_time_cache" version = "0.11.11" @@ -5261,23 +5380,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9106e1d747ffd48e6be5bb2d97fa706ed25b144fbee4d5c02eae110cd8d6badd" [[package]] -name = "mach2" -version = "0.4.2" +name = "lz4_flex" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709" +checksum = "db9a0d582c2874f68138a16ce1867e0ffde6c0bb0a0df85e1f36d04146db488a" dependencies = [ - "libc", + "twox-hash", ] [[package]] -name = "macro-string" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b27834086c65ec3f9387b096d66e99f221cf081c2b738042aa252bcd41204e3" +name = "mach2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dae608c151f68243f2b000364e1f7b186d9c29845f7d2d85bd31b9ad77ad552b" + +[[package]] +name = "macro-string" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59a9dbbfc75d2688ed057456ce8a3ee3f48d12eec09229f560f3643b9f275653" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -5286,12 +5411,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" -[[package]] -name = "matches" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" - [[package]] name = "matchit" version = "0.8.4" @@ -5304,7 +5423,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "digest 0.10.7", ] @@ -5316,11 +5435,11 @@ checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memfd" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2cffa4ad52c6f791f4f8b15f0c05f9824b2ced1160e88cc393d64fff9a8ac64" +checksum = "ad38eb12aea514a0466ea40a80fd8cc83637065948eb4a426e4aa46261175227" dependencies = [ - "rustix 0.38.34", + "rustix 1.1.4", ] [[package]] @@ -5330,7 +5449,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "36c791ecdf977c99f45f23280405d7723727470f6689a5e6dbf513ac547ae10d" dependencies = [ "serde", - "toml 0.9.11+spec-1.1.0", + "toml 0.9.12+spec-1.1.0", ] [[package]] @@ -5362,11 +5481,12 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.7.4" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ - "adler", + "adler2", + "simd-adler32", ] [[package]] @@ -5390,7 +5510,7 @@ dependencies = [ "bytes", "encoding_rs", "futures-util", - "http 1.4.0", + "http 1.4.2", "httparse", "memchr", "mime", @@ -5411,12 +5531,11 @@ dependencies = [ [[package]] name = "multihash" -version = "0.19.1" +version = "0.19.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "076d548d76a0e2a0d4ab471d0b1c36c577786dfc4471242035d97a12a735c492" +checksum = "577c63b00ad74d57e8c9aa870b5fccebf2fd64a308a5aee9f1bb88e4aea19447" dependencies = [ - "core2", - "unsigned-varint 0.7.2", + "unsigned-varint", ] [[package]] @@ -5448,13 +5567,25 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c96aba5aa877601bb3f6dd6a63a969e1f82e60646e81e71b14496995e9853c91" +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.11.1", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "notify" version = "8.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d3d07927151ff8575b7087f245456e549fea62edf0ec4e565a5ee50c8402bc3" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.11.1", "fsevent-sys", "inotify", "kqueue", @@ -5505,9 +5636,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" [[package]] name = "num-integer" @@ -5571,7 +5702,7 @@ checksum = "ff32365de1b6743cb203b710788263c44a03de03802daf96092f2da4fe6ba4d7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -5582,7 +5713,7 @@ checksum = "7b5676b5c379cf5b03da1df2b3061c4a4e2aa691086a56ac923e08c143f53f59" dependencies = [ "alloy-rlp", "arbitrary", - "cfg-if 1.0.0", + "cfg-if", "proptest", "ruint", "serde", @@ -5591,40 +5722,43 @@ dependencies = [ [[package]] name = "object" -version = "0.37.3" +version = "0.39.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" dependencies = [ "crc32fast", - "hashbrown 0.15.2", - "indexmap 2.11.4", + "hashbrown 0.17.0", + "indexmap 2.14.0", "memchr", ] [[package]] name = "object_store" -version = "0.12.4" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c1be0c6c22ec0817cdc77d3842f721a17fd30ab6965001415b5402a74e6b740" +checksum = "765784b4390c6bcf80316e5a22f4e3661b639c9d8c83246856643c27d8ce9dbe" dependencies = [ "async-trait", - "base64 0.22.1", + "aws-lc-rs", + "base64", "bytes", "chrono", "form_urlencoded", - "futures 0.3.31", - "http 1.4.0", + "futures-channel", + "futures-core", + "futures-util", + "http 1.4.2", "http-body-util", "humantime", - "hyper 1.8.1", - "itertools 0.14.0", + "hyper", + "itertools 0.15.0", + "nix", "parking_lot", "percent-encoding", "quick-xml", - "rand 0.9.2", - "reqwest", - "ring", - "rustls-pemfile", + "rand 0.10.1", + "reqwest 0.13.2", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", @@ -5635,6 +5769,7 @@ dependencies = [ "walkdir", "wasm-bindgen-futures", "web-time", + "windows-sys 0.61.2", ] [[package]] @@ -5644,22 +5779,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] -name = "opaque-debug" -version = "0.3.1" +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "open" +version = "5.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" +checksum = "cd8d3b65c44123a56e0133d2cd06ce4361bd3ca99d41198b2f25e3c3db9b8b4a" +dependencies = [ + "is-wsl", + "libc", +] [[package]] name = "openssl" -version = "0.10.75" +version = "0.10.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" dependencies = [ - "bitflags 2.9.0", - "cfg-if 1.0.0", + "bitflags 2.11.1", + "cfg-if", "foreign-types", "libc", - "once_cell", "openssl-macros", "openssl-sys", ] @@ -5672,7 +5816,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -5692,9 +5836,9 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.111" +version = "0.9.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" dependencies = [ "cc", "libc", @@ -5715,7 +5859,7 @@ version = "3.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "306800abfa29c7f16596b5970a588435e3d5b3149683d00c12b699cc19f895ee" dependencies = [ - "arrayvec 0.7.4", + "arrayvec", "bitvec", "byte-slice-cast", "impl-trait-for-tuples", @@ -5763,11 +5907,43 @@ version = "0.9.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "libc", "redox_syscall 0.5.2", "smallvec", - "windows-link 0.2.0", + "windows-link 0.2.1", +] + +[[package]] +name = "parquet" +version = "59.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "970dff83e97d953c827ae8176f6bf4e9f77bf62daacc01ec5df348ec5eacd913" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", + "base64", + "brotli", + "bytes", + "chrono", + "flate2", + "half", + "hashbrown 0.17.0", + "lz4_flex", + "num-bigint 0.4.6", + "num-integer", + "num-traits", + "paste", + "seq-macro", + "simdutf8", + "snap", + "twox-hash", + "zstd", ] [[package]] @@ -5813,7 +5989,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -5824,7 +6000,17 @@ checksum = "a941429fea7e08bedec25e4f6785b6ffaacc6b755da98df5ef3e7dcf4a124c4f" dependencies = [ "once_cell", "pest", - "sha2", + "sha2 0.10.9", +] + +[[package]] +name = "petgraph" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset 0.4.2", + "indexmap 2.14.0", ] [[package]] @@ -5833,9 +6019,9 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ - "fixedbitset", + "fixedbitset 0.5.7", "hashbrown 0.15.2", - "indexmap 2.11.4", + "indexmap 2.14.0", "serde", ] @@ -5896,14 +6082,14 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] name = "pin-project-lite" -version = "0.2.14" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "pin-utils" @@ -5933,19 +6119,19 @@ version = "3.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "concurrent-queue", "hermit-abi 0.5.2", "pin-project-lite", - "rustix 1.0.7", - "windows-sys 0.61.1", + "rustix 1.1.4", + "windows-sys 0.61.2", ] [[package]] name = "portable-atomic" -version = "1.11.1" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" [[package]] name = "portable-atomic-util" @@ -5958,9 +6144,9 @@ dependencies = [ [[package]] name = "postcard" -version = "1.1.1" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "170a2601f67cc9dba8edd8c4870b15f71a6a2dc196daec8c83f72b59dff628a8" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" dependencies = [ "cobs", "embedded-io 0.4.0", @@ -5984,9 +6170,9 @@ dependencies = [ [[package]] name = "postgres-openssl" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f86f073ad570f76e9e278ce6f05775fc723eed7daa6b4f9c2aa078080a564a0" +checksum = "06743eefaa1a5c0ef2ccb6d9abf6528790a229eabd62ddcabf9b2a3aeff09fa4" dependencies = [ "openssl", "tokio", @@ -6000,15 +6186,15 @@ version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbef655056b916eb868048276cfd5d6a7dea4f81560dfd047f97c8c6fe3fcfd4" dependencies = [ - "base64 0.22.1", + "base64", "byteorder", "bytes", "fallible-iterator 0.2.0", "hmac", "md-5", "memchr", - "rand 0.9.2", - "sha2", + "rand 0.9.3", + "sha2 0.10.9", "stringprep", ] @@ -6074,20 +6260,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" dependencies = [ "proc-macro2", - "syn 2.0.114", -] - -[[package]] -name = "primitive-types" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28720988bff275df1f51b171e1b2a18c30d194c4d2b61defdacecd625a5d94a" -dependencies = [ - "fixed-hash 0.7.0", - "impl-codec", - "impl-rlp", - "impl-serde", - "uint 0.9.5", + "syn 2.0.118", ] [[package]] @@ -6096,7 +6269,7 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b34d9fd68ae0b74a41b21c03c2f62847aa0ffea044eee893b4c140b37e244e2" dependencies = [ - "fixed-hash 0.8.0", + "fixed-hash", "impl-codec", "uint 0.9.5", ] @@ -6108,7 +6281,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93980406f12d9f8140ed5abe7155acb10bb1e69ea55c88960b9c2f117445ef96" dependencies = [ "equivalent", - "indexmap 2.11.4", + "indexmap 2.14.0", "serde", ] @@ -6118,7 +6291,7 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284" dependencies = [ - "toml_edit 0.21.1", + "toml_edit", ] [[package]] @@ -6140,7 +6313,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -6156,9 +6329,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.105" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "535d180e0ecab6268a3e718bb9fd44db66bbbc256257165fc699dadf70d16fe7" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" dependencies = [ "unicode-ident", ] @@ -6169,14 +6342,14 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ca5326d8d0b950a9acd87e6a3f94745394f62e4dae1b1ee22b2bc0c394af43a" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "fnv", "lazy_static", "libc", "memchr", "parking_lot", "protobuf", - "reqwest", + "reqwest 0.12.23", "thiserror 2.0.18", ] @@ -6188,9 +6361,9 @@ checksum = "bee689443a2bd0a16ab0348b52ee43e3b2d1b1f931c8aa5c9f8de4c86fbe8c40" dependencies = [ "bit-set", "bit-vec", - "bitflags 2.9.0", + "bitflags 2.11.1", "num-traits", - "rand 0.9.2", + "rand 0.9.3", "rand_chacha 0.9.0", "rand_xorshift", "regex-syntax", @@ -6207,25 +6380,25 @@ checksum = "4ee1c9ac207483d5e7db4940700de86a9aae46ef90c48b57f99fe7edb8345e49" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] name = "proptest-derive" -version = "0.6.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "095a99f75c69734802359b682be8daaf8980296731f6470434ea2c652af1dd30" +checksum = "c57924a81864dddafba92e1bf92f9bf82f97096c44489548a60e888e1547549b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] name = "prost" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" dependencies = [ "bytes", "prost-derive", @@ -6238,38 +6411,38 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ "heck 0.5.0", - "itertools 0.14.0", + "itertools 0.13.0", "log", "multimap", - "petgraph", + "petgraph 0.8.3", "prettyplease", "prost", "prost-types", "pulldown-cmark", "pulldown-cmark-to-cmark", "regex", - "syn 2.0.114", + "syn 2.0.118", "tempfile", ] [[package]] name = "prost-derive" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] name = "prost-types" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" dependencies = [ "prost", ] @@ -6292,7 +6465,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4aeaa1f2460f1d348eeaeed86aea999ce98c1bded6f089ff8514c9d9dbdc973" dependencies = [ "anyhow", - "indexmap 2.11.4", + "indexmap 2.14.0", "log", "protobuf", "protobuf-support", @@ -6325,7 +6498,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e8bbe1a966bd2f362681a44f6edce3c2310ac21e4d5067a6e7ec396297a6ea0" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.11.1", "memchr", "unicase", ] @@ -6341,25 +6514,25 @@ dependencies = [ [[package]] name = "pulley-interpreter" -version = "36.0.5" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56a1abe1fcec21c32b62000af24b8b6db11b87609b64fd1c9a9e17c42422225" +checksum = "38b92604caae1a1899b6a5b54967289dd538177c626004c91accf9d0ec7e4a12" dependencies = [ "cranelift-bitset", "log", "pulley-macros", - "wasmtime-internal-math", + "wasmtime-internal-core", ] [[package]] name = "pulley-macros" -version = "36.0.5" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0d56dac306fbee0e990d4bac359c86d58f60f058e1e2d1aee1b7928689f08d3" +checksum = "5a7ac85c0bb3fb351f10d531230aaa5e366b46d7c4e5328e5f02801d6dac1165" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -6370,9 +6543,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quick-xml" -version = "0.38.2" +version = "0.40.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d200a41a7797e6461bd04e4e95c3347053a731c32c87f066f2f0dda22dbdbba8" +checksum = "2474bd2e5029e7ccb6abb2ba48cf2383a333851dedf495901544281590c7da7f" dependencies = [ "memchr", "serde", @@ -6380,60 +6553,75 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.2" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4ceeeeabace7857413798eb1ffa1e9c905a9946a57d81fb69b4b71c4d8eb3ad" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" dependencies = [ "bytes", + "cfg_aliases", "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash 1.1.0", + "rustc-hash", "rustls", - "thiserror 1.0.61", + "socket2", + "thiserror 2.0.18", "tokio", "tracing", + "web-time", ] [[package]] name = "quinn-proto" -version = "0.11.8" +version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" dependencies = [ + "aws-lc-rs", "bytes", - "rand 0.8.5", + "getrandom 0.3.1", + "lru-slab", + "rand 0.9.3", "ring", - "rustc-hash 2.1.1", + "rustc-hash", "rustls", + "rustls-pki-types", "slab", - "thiserror 1.0.61", + "thiserror 2.0.18", "tinyvec", "tracing", + "web-time", ] [[package]] name = "quinn-udp" -version = "0.5.2" +version = "0.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9096629c45860fc7fb143e125eb826b5e721e10be3263160c7d60ca832cf8c46" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" dependencies = [ + "cfg_aliases", "libc", "once_cell", - "socket2 0.5.7", + "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] name = "quote" -version = "1.0.44" +version = "1.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "r2d2" version = "0.8.10" @@ -6453,9 +6641,9 @@ checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" [[package]] name = "rand" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -6465,15 +6653,26 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "7ec095654a25171c2124e9e3393a930bddbffdc939556c914957a4c3e0a87166" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.3", "serde", ] +[[package]] +name = "rand" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core 0.10.0", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -6513,6 +6712,12 @@ dependencies = [ "serde", ] +[[package]] +name = "rand_core" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba" + [[package]] name = "rand_xorshift" version = "0.4.0" @@ -6568,7 +6773,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" dependencies = [ "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -6579,15 +6784,16 @@ checksum = "d3edd4d5d42c92f0a659926464d4cce56b562761267ecf0f469d85b7de384175" [[package]] name = "redis" -version = "1.0.2" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfe20977fe93830c0e9817a16fbf1ed1cfd8d4bba366087a1841d2c6033c251" +checksum = "a12e6b5f4d8ef33944e833e2b1859ad478deab6e431d7337b30ee2efe21f7543" dependencies = [ "arc-swap", "arcstr", + "async-lock", "backon", "bytes", - "cfg-if 1.0.0", + "cfg-if", "combine", "futures-channel", "futures-util", @@ -6597,9 +6803,9 @@ dependencies = [ "pin-project-lite", "ryu", "sha1_smol", - "socket2 0.6.0", + "socket2", "tokio", - "tokio-util 0.7.18", + "tokio-util", "url", "xxhash-rust", ] @@ -6619,7 +6825,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.11.1", ] [[package]] @@ -6633,25 +6839,46 @@ dependencies = [ "thiserror 1.0.61", ] +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "regalloc2" -version = "0.12.2" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5216b1837de2149f8bc8e6d5f88a9326b63b8c836ed58ce4a0a29ec736a59734" +checksum = "de2c52737737f8609e94f975dee22854a2d5c125772d4b1cf292120f4d45c186" dependencies = [ "allocator-api2", "bumpalo", - "hashbrown 0.15.2", + "hashbrown 0.17.0", "log", - "rustc-hash 2.1.1", + "rustc-hash", + "serde", "smallvec", ] [[package]] name = "regex" -version = "1.12.2" +version = "1.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" dependencies = [ "aho-corasick", "memchr", @@ -6672,9 +6899,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.8" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "reqwest" @@ -6682,17 +6909,18 @@ version = "0.12.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d429f34c8092b2d42c7c93cec323bb4adeb7c67698f70839adec842ec10c7ceb" dependencies = [ - "base64 0.22.1", + "async-compression", + "base64", "bytes", "encoding_rs", "futures-channel", "futures-core", "futures-util", - "h2 0.4.5", - "http 1.4.0", - "http-body 1.0.0", + "h2", + "http 1.4.2", + "http-body", "http-body-util", - "hyper 1.8.1", + "hyper", "hyper-rustls", "hyper-tls", "hyper-util", @@ -6703,9 +6931,6 @@ dependencies = [ "native-tls", "percent-encoding", "pin-project-lite", - "quinn", - "rustls", - "rustls-native-certs 0.8.1", "rustls-pki-types", "serde", "serde_json", @@ -6713,17 +6938,56 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-native-tls", + "tokio-util", + "tower 0.5.2", + "tower-http 0.6.11", + "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams 0.4.0", + "web-sys", +] + +[[package]] +name = "reqwest" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab3f43e3283ab1488b624b44b0e988d0acea0b3214e694730a055cb6b2efa801" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http 1.4.2", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "serde", + "serde_json", + "sync_wrapper", + "tokio", "tokio-rustls", - "tokio-util 0.7.18", + "tokio-util", "tower 0.5.2", - "tower-http", + "tower-http 0.6.11", "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", + "wasm-streams 0.5.0", "web-sys", - "webpki-roots 1.0.5", ] [[package]] @@ -6743,7 +7007,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70ac5d832aa16abd7d1def883a8545280c20a60f523a370aa3a9617c2b8550ee" dependencies = [ "cc", - "cfg-if 1.0.0", + "cfg-if", "getrandom 0.2.15", "libc", "untrusted", @@ -6778,10 +7042,10 @@ dependencies = [ "num-integer", "num-traits", "parity-scale-codec", - "primitive-types 0.12.2", + "primitive-types", "proptest", - "rand 0.8.5", - "rand 0.9.2", + "rand 0.8.6", + "rand 0.9.3", "rlp", "ruint-macro", "serde_core", @@ -6801,12 +7065,6 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - [[package]] name = "rustc-hash" version = "2.1.1" @@ -6834,7 +7092,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.27", + "semver 1.0.28", ] [[package]] @@ -6843,7 +7101,7 @@ version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.11.1", "errno", "libc", "linux-raw-sys 0.4.14", @@ -6852,26 +7110,26 @@ dependencies = [ [[package]] name = "rustix" -version = "1.0.7" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.11.1", "errno", "libc", - "linux-raw-sys 0.9.4", - "windows-sys 0.59.0", + "linux-raw-sys 0.12.1", + "windows-sys 0.52.0", ] [[package]] name = "rustls" -version = "0.23.28" +version = "0.23.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7160e3e10bf4535308537f3c4e1641468cd0e485175d6163087c0393c7d46643" +checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" dependencies = [ + "aws-lc-rs", "log", "once_cell", - "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -6880,51 +7138,60 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.7.1" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88d6d420651b496bdd98684116959239430022a115c1240e6c3993be0b15fba" +checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" dependencies = [ "openssl-probe", - "rustls-pemfile", "rustls-pki-types", "schannel", - "security-framework 2.11.0", + "security-framework 3.7.0", ] [[package]] -name = "rustls-native-certs" -version = "0.8.1" +name = "rustls-pki-types" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" dependencies = [ - "openssl-probe", - "rustls-pki-types", - "schannel", - "security-framework 3.2.0", + "web-time", + "zeroize", ] [[package]] -name = "rustls-pemfile" -version = "2.1.2" +name = "rustls-platform-verifier" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" +checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" dependencies = [ - "base64 0.22.1", - "rustls-pki-types", + "core-foundation 0.10.0", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework 3.7.0", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.52.0", ] [[package]] -name = "rustls-pki-types" -version = "1.11.0" +name = "rustls-platform-verifier-android" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.3" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4a72fe2bcf7a6ac6fd7d0b9e5cb68aeb7d4c0a0271730218b3e92d43b4eb435" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -6982,12 +7249,27 @@ dependencies = [ ] [[package]] -name = "scoped-futures" -version = "0.1.4" +name = "schemars" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b24aae2d0636530f359e9d5ef0c04669d11c5e756699b27a6a6d845d8329091" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" dependencies = [ - "pin-project-lite", + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "schemars" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", ] [[package]] @@ -7013,39 +7295,41 @@ dependencies = [ [[package]] name = "secp256k1" -version = "0.21.3" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c42e6f1735c5f00f51e43e28d6634141f2bcad10931b2609ddd74a86d751260" +checksum = "b50c5943d326858130af85e049f2661ba3c78b26589b8ab98e65e80ae44a1252" dependencies = [ - "secp256k1-sys 0.4.2", + "bitcoin_hashes", + "rand 0.8.6", + "secp256k1-sys 0.10.1", + "serde", ] [[package]] name = "secp256k1" -version = "0.30.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b50c5943d326858130af85e049f2661ba3c78b26589b8ab98e65e80ae44a1252" +checksum = "2c3c81b43dc2d8877c216a3fccf76677ee1ebccd429566d3e67447290d0c42b2" dependencies = [ "bitcoin_hashes", - "rand 0.8.5", - "secp256k1-sys 0.10.1", - "serde", + "rand 0.9.3", + "secp256k1-sys 0.11.0", ] [[package]] name = "secp256k1-sys" -version = "0.4.2" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957da2573cde917463ece3570eab4a0b3f19de6f1646cde62e6fd3868f566036" +checksum = "d4387882333d3aa8cb20530a17c69a3752e97837832f34f6dccc760e715001d9" dependencies = [ "cc", ] [[package]] name = "secp256k1-sys" -version = "0.10.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4387882333d3aa8cb20530a17c69a3752e97837832f34f6dccc760e715001d9" +checksum = "dcb913707158fadaf0d8702c2db0e857de66eb003ccfdda5924b5f5ac98efb38" dependencies = [ "cc", ] @@ -7056,7 +7340,7 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.11.1", "core-foundation 0.9.4", "core-foundation-sys", "libc", @@ -7065,11 +7349,11 @@ dependencies = [ [[package]] name = "security-framework" -version = "3.2.0" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.11.1", "core-foundation 0.10.0", "core-foundation-sys", "libc", @@ -7078,9 +7362,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.14.0" +version = "2.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" dependencies = [ "core-foundation-sys", "libc", @@ -7097,9 +7381,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" dependencies = [ "serde", "serde_core", @@ -7120,11 +7404,17 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + [[package]] name = "serde" -version = "1.0.226" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dca6411025b24b60bfa7ec1fe1f8e710ac09782dca409ee8237ba74b51295fd" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ "serde_core", "serde_derive", @@ -7132,33 +7422,35 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.226" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba2ba63999edb9dac981fb34b3e5c0d111a69b0924e253ed29d83f7c99e966a4" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.226" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8db53ae22f34573731bafa1db20f04027b2d25e02d8205921b569171699cdb33" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] name = "serde_json" -version = "1.0.120" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ "itoa", - "ryu", + "memchr", "serde", + "serde_core", + "zmij", ] [[package]] @@ -7182,9 +7474,9 @@ dependencies = [ [[package]] name = "serde_regex" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8136f1a4ea815d7eac4101cfd0b16dc0cb5e1fe1b8609dfd728058656b7badf" +checksum = "bafc8d0c5330cecff10f16b459b479fd9acaa5b4acd7167301414e21b0057012" dependencies = [ "regex", "serde", @@ -7192,20 +7484,11 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.6" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" dependencies = [ - "serde", -] - -[[package]] -name = "serde_spanned" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8bbf91e5a4d6315eee45e704372590b30e260ee83af6639d64557f51b067776" -dependencies = [ - "serde_core", + "serde_core", ] [[package]] @@ -7222,17 +7505,19 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.12.0" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6b6f7f2fcb69f747921f79f3926bd1e203fce4fef62c268dd3abfb6d86029aa" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" dependencies = [ - "base64 0.22.1", + "base64", + "bs58 0.5.1", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.11.4", - "serde", - "serde_derive", + "indexmap 2.14.0", + "schemars 0.9.0", + "schemars 1.2.1", + "serde_core", "serde_json", "serde_with_macros", "time", @@ -7240,14 +7525,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.12.0" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d00caa5193a3c8362ac2b73be6b9e768aa5a4b2f721d8f4b339600c3cb51f8e" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" dependencies = [ - "darling 0.20.10", + "darling 0.23.0", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -7256,7 +7541,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.11.4", + "indexmap 2.14.0", "itoa", "ryu", "serde", @@ -7274,27 +7559,25 @@ dependencies = [ ] [[package]] -name = "sha-1" -version = "0.9.8" +name = "sha1" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99cd6713db3cf16b6c84e06321e049a9b9f699826e16096d23bbcc44d15d51a6" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ - "block-buffer 0.9.0", - "cfg-if 1.0.0", - "cpufeatures", - "digest 0.9.0", - "opaque-debug", + "cfg-if", + "cpufeatures 0.2.12", + "digest 0.10.7", ] [[package]] name = "sha1" -version = "0.10.6" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" dependencies = [ - "cfg-if 1.0.0", - "cpufeatures", - "digest 0.10.7", + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.2", ] [[package]] @@ -7309,36 +7592,47 @@ version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ - "cfg-if 1.0.0", - "cpufeatures", + "cfg-if", + "cpufeatures 0.2.12", "digest 0.10.7", ] +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.2", +] + [[package]] name = "sha3" -version = "0.10.8" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" +checksum = "be176f1a57ce4e3d31c1a166222d9768de5954f811601fb7ca06fc8203905ce1" dependencies = [ - "digest 0.10.7", + "digest 0.11.2", "keccak", ] [[package]] name = "sha3-asm" -version = "0.1.4" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28efc5e327c837aa837c59eae585fc250715ef939ac32881bcc11677cd02d46" +checksum = "9f3f15d4e239ebe08413eed880e0f9b5af4b40ee0472543320efa91d488e96a7" dependencies = [ "cc", - "cfg-if 1.0.0", + "cfg-if", ] [[package]] name = "shellexpand" -version = "3.1.1" +version = "3.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b1fdf65dd6331831494dd616b30351c38e96e45921a27745cf98490458b90bb" +checksum = "32824fab5e16e6c4d86dc1ba84489390419a39f97699852b66480bb87d297ed8" dependencies = [ "dirs", ] @@ -7349,6 +7643,27 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc" +dependencies = [ + "libc", + "mio", + "signal-hook", +] + [[package]] name = "signal-hook-registry" version = "1.4.2" @@ -7368,12 +7683,27 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd-adler32" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" + [[package]] name = "simdutf8" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +[[package]] +name = "similar" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6505efef05804732ed8a3f2d4f279429eb485bd69d5b0cc6b19cc02005cda16" +dependencies = [ + "bstr", +] + [[package]] name = "siphasher" version = "1.0.1" @@ -7465,23 +7795,19 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.13.2" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" dependencies = [ "arbitrary", "serde", ] [[package]] -name = "socket2" -version = "0.5.7" +name = "snap" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] +checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" [[package]] name = "socket2" @@ -7493,21 +7819,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "soketto" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d1c5305e39e09653383c2c7244f2f78b3bcae37cf50c64cb4789c9f5096ec2" -dependencies = [ - "base64 0.13.1", - "bytes", - "futures 0.3.31", - "httparse", - "log", - "rand 0.8.5", - "sha-1", -] - [[package]] name = "spin" version = "0.9.8" @@ -7537,13 +7848,13 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.60.0" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "505aa16b045c4c1375bf5f125cce3813d0176325bfe9ffc4a903f423de7774ff" +checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" dependencies = [ "log", "recursive", - "sqlparser_derive 0.4.0", + "sqlparser_derive 0.5.0", ] [[package]] @@ -7554,26 +7865,26 @@ checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] name = "sqlparser_derive" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "028e551d5e270b31b9f3ea271778d9d827148d4287a5d96167b6bb9787f5cc38" +checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] name = "stable-hash" version = "0.3.4" -source = "git+https://github.com/graphprotocol/stable-hash?branch=old#7af76261e8098c58bfadd5b7c31810e1c0fdeccb" +source = "git+https://github.com/graphprotocol/stable-hash?rev=42b85d936c6b9f1b7a4b8385d2f081d5564daa69#42b85d936c6b9f1b7a4b8385d2f081d5564daa69" dependencies = [ - "blake3 0.3.8", + "blake3", "firestorm 0.4.6", "ibig", "lazy_static", @@ -7584,15 +7895,15 @@ dependencies = [ [[package]] name = "stable-hash" version = "0.4.4" -source = "git+https://github.com/graphprotocol/stable-hash?branch=main#e50aabef55b8c4de581ca5c4ffa7ed8beed7e998" +source = "git+https://github.com/graphprotocol/stable-hash?rev=70300f974e61529675aef3f20803f26a23f0e0ea#70300f974e61529675aef3f20803f26a23f0e0ea" dependencies = [ - "blake3 0.3.8", + "blake3", "firestorm 0.5.1", "ibig", "lazy_static", "leb128", "num-traits", - "uint 0.8.5", + "uint 0.10.1", "xxhash-rust", ] @@ -7609,7 +7920,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1f8b29fb42aafcea4edeeb6b2f2d7ecd0d969c48b4cf0d2e64aafc471dd6e59" dependencies = [ "cc", - "cfg-if 1.0.0", + "cfg-if", "libc", "psm", "windows-sys 0.59.0", @@ -7650,7 +7961,16 @@ version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" dependencies = [ - "strum_macros", + "strum_macros 0.27.2", +] + +[[package]] +name = "strum" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" +dependencies = [ + "strum_macros 0.28.0", ] [[package]] @@ -7662,7 +7982,19 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", +] + +[[package]] +name = "strum_macros" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] @@ -7684,9 +8016,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.114" +version = "2.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" dependencies = [ "proc-macro2", "quote", @@ -7695,14 +8027,14 @@ dependencies = [ [[package]] name = "syn-solidity" -version = "1.5.2" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f92d01b5de07eaf324f7fca61cc6bd3d82bbc1de5b6c963e6fe79e86f36580d" +checksum = "ec005042c7d952febc1a3ef5b0f6674e9054aa836877a31c90b20e25b3d31744" dependencies = [ "paste", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -7722,16 +8054,16 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] name = "system-configuration" -version = "0.6.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.11.1", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -7760,19 +8092,20 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "target-lexicon" -version = "0.13.2" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" [[package]] name = "tempfile" -version = "3.10.1" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ - "cfg-if 1.0.0", "fastrand", - "rustix 0.38.34", + "getrandom 0.4.2", + "once_cell", + "rustix 1.1.4", "windows-sys 0.52.0", ] @@ -7808,7 +8141,7 @@ dependencies = [ [[package]] name = "test-store" -version = "0.36.0" +version = "0.45.0" dependencies = [ "async-trait", "diesel", @@ -7853,7 +8186,7 @@ checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -7864,7 +8197,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -7873,7 +8206,7 @@ version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "once_cell", ] @@ -7888,44 +8221,35 @@ dependencies = [ [[package]] name = "time" -version = "0.3.36" +version = "0.3.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" dependencies = [ "deranged", "itoa", "num-conv", "powerfmt", - "serde", + "serde_core", "time-core", "time-macros", ] [[package]] name = "time-core" -version = "0.1.2" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" [[package]] name = "time-macros" -version = "0.2.18" +version = "0.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" dependencies = [ "num-conv", "time-core", ] -[[package]] -name = "tiny-keccak" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d8a021c69bb74a44ccedb824a046447e2c84a01df9e5c20779750acb38e11b2" -dependencies = [ - "crunchy", -] - [[package]] name = "tiny-keccak" version = "2.0.2" @@ -7962,9 +8286,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.49.0" +version = "1.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" +checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" dependencies = [ "bytes", "libc", @@ -7972,9 +8296,9 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.6.0", + "socket2", "tokio-macros", - "windows-sys 0.61.1", + "windows-sys 0.61.2", ] [[package]] @@ -7985,7 +8309,7 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -8029,21 +8353,21 @@ dependencies = [ "pin-project-lite", "postgres-protocol", "postgres-types", - "rand 0.9.2", - "socket2 0.6.0", + "rand 0.9.3", + "socket2", "tokio", - "tokio-util 0.7.18", + "tokio-util", "whoami", ] [[package]] name = "tokio-retry" -version = "0.3.0" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f" +checksum = "4a129d95275ebf4c493ec53bf0f8cd95f5ac161bc4f381700809a54f595d4470" dependencies = [ - "pin-project", - "rand 0.8.5", + "pin-project-lite", + "rand 0.10.1", "tokio", ] @@ -8066,7 +8390,7 @@ dependencies = [ "futures-core", "pin-project-lite", "tokio", - "tokio-util 0.7.18", + "tokio-util", ] [[package]] @@ -8084,9 +8408,9 @@ dependencies = [ [[package]] name = "tokio-tungstenite" -version = "0.26.2" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a9daff607c6d2bf6c16fd681ccb7eecc83e4e2cdc1ca067ffaadfca5de7f084" +checksum = "d25a406cddcc431a75d3d9afc6a7c0f7428d4891dd973e4d54c56b46127bf857" dependencies = [ "futures-util", "log", @@ -8094,35 +8418,20 @@ dependencies = [ "rustls-pki-types", "tokio", "tokio-rustls", - "tungstenite 0.26.2", + "tungstenite 0.28.0", "webpki-roots 0.26.11", ] [[package]] name = "tokio-tungstenite" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d25a406cddcc431a75d3d9afc6a7c0f7428d4891dd973e4d54c56b46127bf857" +checksum = "8f72a05e828585856dacd553fba484c242c46e391fb0e58917c942ee9202915c" dependencies = [ "futures-util", "log", "tokio", - "tungstenite 0.28.0", -] - -[[package]] -name = "tokio-util" -version = "0.6.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507" -dependencies = [ - "bytes", - "futures-core", - "futures-io", - "futures-sink", - "log", - "pin-project-lite", - "tokio", + "tungstenite 0.29.0", ] [[package]] @@ -8141,29 +8450,32 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.15" +version = "0.9.12+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac2caab0bf757388c6c0ae23b3293fdb463fee59434529014f85e3263b995c28" +checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" dependencies = [ - "serde", - "serde_spanned 0.6.6", - "toml_datetime 0.6.6", - "toml_edit 0.22.16", + "indexmap 2.14.0", + "serde_core", + "serde_spanned", + "toml_datetime 0.7.5+spec-1.1.0", + "toml_parser", + "toml_writer", + "winnow 0.7.13", ] [[package]] name = "toml" -version = "0.9.11+spec-1.1.0" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3afc9a848309fe1aaffaed6e1546a7a14de1f935dc9d89d32afd9a44bab7c46" +checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee" dependencies = [ - "indexmap 2.11.4", + "indexmap 2.14.0", "serde_core", - "serde_spanned 1.0.4", - "toml_datetime 0.7.5+spec-1.1.0", + "serde_spanned", + "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", "toml_writer", - "winnow 0.7.13", + "winnow 1.0.0", ] [[package]] @@ -8171,9 +8483,6 @@ name = "toml_datetime" version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" -dependencies = [ - "serde", -] [[package]] name = "toml_datetime" @@ -8185,66 +8494,62 @@ dependencies = [ ] [[package]] -name = "toml_edit" -version = "0.21.1" +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" dependencies = [ - "indexmap 2.11.4", - "toml_datetime 0.6.6", - "winnow 0.5.40", + "serde_core", ] [[package]] name = "toml_edit" -version = "0.22.16" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "278f3d518e152219c994ce877758516bca5e118eaed6996192a774fb9fbf0788" +checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" dependencies = [ - "indexmap 2.11.4", - "serde", - "serde_spanned 0.6.6", + "indexmap 2.14.0", "toml_datetime 0.6.6", - "winnow 0.6.13", + "winnow 0.5.40", ] [[package]] name = "toml_parser" -version = "1.0.6+spec-1.1.0" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3198b4b0a8e11f09dd03e133c0280504d0801269e9afa46362ffde1cbeebf44" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ - "winnow 0.7.13", + "winnow 1.0.0", ] [[package]] name = "toml_writer" -version = "1.0.6+spec-1.1.0" +version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab16f14aed21ee8bfd8ec22513f7287cd4a91aa92e44edfe2c17ddd004e92607" +checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" [[package]] name = "tonic" -version = "0.14.3" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a286e33f82f8a1ee2df63f4fa35c0becf4a85a0cb03091a15fd7bf0b402dc94a" +checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" dependencies = [ "async-trait", "axum", - "base64 0.22.1", + "base64", "bytes", "flate2", - "h2 0.4.5", - "http 1.4.0", - "http-body 1.0.0", + "h2", + "http 1.4.2", + "http-body", "http-body-util", - "hyper 1.8.1", + "hyper", "hyper-timeout", "hyper-util", "percent-encoding", "pin-project", - "rustls-native-certs 0.8.1", - "socket2 0.6.0", + "rustls-native-certs", + "socket2", "sync_wrapper", "tokio", "tokio-rustls", @@ -8257,21 +8562,21 @@ dependencies = [ [[package]] name = "tonic-build" -version = "0.14.3" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27aac809edf60b741e2d7db6367214d078856b8a5bff0087e94ff330fb97b6fc" +checksum = "c68f61875ac5293cf72e6c8cf0158086428c82c37229e98c840878f1706b0322" dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] name = "tonic-prost" -version = "0.14.3" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6c55a2d6a14174563de34409c9f92ff981d006f56da9c6ecd40d9d4a31500b0" +checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" dependencies = [ "bytes", "prost", @@ -8280,16 +8585,16 @@ dependencies = [ [[package]] name = "tonic-prost-build" -version = "0.14.3" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4556786613791cfef4ed134aa670b61a85cfcacf71543ef33e8d801abae988f" +checksum = "654e5643eff75d7f8c99197ce1440ed19a3474eada74c12bbac488b2cafdae27" dependencies = [ "prettyplease", "proc-macro2", "prost-build", "prost-types", "quote", - "syn 2.0.114", + "syn 2.0.118", "tempfile", "tonic-build", ] @@ -8303,12 +8608,12 @@ dependencies = [ "futures-core", "futures-util", "hdrhistogram", - "indexmap 2.11.4", + "indexmap 2.14.0", "pin-project-lite", "slab", "sync_wrapper", "tokio", - "tokio-util 0.7.18", + "tokio-util", "tower-layer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "tracing", @@ -8317,17 +8622,17 @@ dependencies = [ [[package]] name = "tower" version = "0.5.3" -source = "git+https://github.com/tower-rs/tower.git#c1e28af2a9af852f0261883531bbc182121a35fd" +source = "git+https://github.com/tower-rs/tower.git#df06d70dbea345facbffb5881fe8647f53bf424d" dependencies = [ "futures-core", "futures-util", "hdrhistogram", - "indexmap 2.11.4", + "indexmap 2.14.0", "pin-project-lite", "slab", "sync_wrapper", "tokio", - "tokio-util 0.7.18", + "tokio-util", "tower-layer 0.3.3 (git+https://github.com/tower-rs/tower.git)", "tower-service 0.3.3 (git+https://github.com/tower-rs/tower.git)", "tracing", @@ -8335,20 +8640,35 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.8" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.11.1", "bytes", "futures-util", - "http 1.4.0", - "http-body 1.0.0", - "iri-string", + "http 1.4.2", + "http-body", "pin-project-lite", "tower 0.5.2", "tower-layer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "url", +] + +[[package]] +name = "tower-http" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b11f75e912b0c2be01b63d8cf8057b8c3f97cf34abb3d431a3a4c8675498e233" +dependencies = [ + "bitflags 2.11.1", + "bytes", + "http 1.4.2", + "percent-encoding", + "pin-project-lite", + "tower-layer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -8360,7 +8680,7 @@ checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" [[package]] name = "tower-layer" version = "0.3.3" -source = "git+https://github.com/tower-rs/tower.git#c1e28af2a9af852f0261883531bbc182121a35fd" +source = "git+https://github.com/tower-rs/tower.git#df06d70dbea345facbffb5881fe8647f53bf424d" [[package]] name = "tower-service" @@ -8371,12 +8691,12 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tower-service" version = "0.3.3" -source = "git+https://github.com/tower-rs/tower.git#c1e28af2a9af852f0261883531bbc182121a35fd" +source = "git+https://github.com/tower-rs/tower.git#df06d70dbea345facbffb5881fe8647f53bf424d" [[package]] name = "tower-test" version = "0.4.1" -source = "git+https://github.com/tower-rs/tower.git#c1e28af2a9af852f0261883531bbc182121a35fd" +source = "git+https://github.com/tower-rs/tower.git#df06d70dbea345facbffb5881fe8647f53bf424d" dependencies = [ "pin-project-lite", "tokio", @@ -8405,7 +8725,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -8417,16 +8737,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "tracing-futures" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" -dependencies = [ - "pin-project", - "tracing", -] - [[package]] name = "try-lock" version = "0.2.5" @@ -8435,40 +8745,45 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tungstenite" -version = "0.26.2" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4793cb5e56680ecbb1d843515b23b6de9a75eb04b66643e256a396d43be33c13" +checksum = "8628dcc84e5a09eb3d8423d6cb682965dea9133204e8fb3efee74c2a0c259442" dependencies = [ "bytes", "data-encoding", - "http 1.4.0", + "http 1.4.2", "httparse", "log", - "rand 0.9.2", + "rand 0.9.3", "rustls", "rustls-pki-types", - "sha1", + "sha1 0.10.6", "thiserror 2.0.18", "utf-8", ] [[package]] name = "tungstenite" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8628dcc84e5a09eb3d8423d6cb682965dea9133204e8fb3efee74c2a0c259442" +checksum = "6c01152af293afb9c7c2a57e4b559c5620b421f6d133261c60dd2d0cdb38e6b8" dependencies = [ "bytes", "data-encoding", - "http 1.4.0", + "http 1.4.2", "httparse", "log", - "rand 0.9.2", - "sha1", + "rand 0.9.3", + "sha1 0.10.6", "thiserror 2.0.18", - "utf-8", ] +[[package]] +name = "twox-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" + [[package]] name = "typenum" version = "1.17.0" @@ -8483,21 +8798,21 @@ checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" [[package]] name = "uint" -version = "0.8.5" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9db035e67dfaf7edd9aebfe8676afcd63eed53c8a4044fed514c8cccf1835177" +checksum = "76f64bba2c53b04fcab63c01a7d7427eadc821e3bc48c34dc9ba29c501164b52" dependencies = [ "byteorder", "crunchy", - "rustc-hex", + "hex", "static_assertions", ] [[package]] name = "uint" -version = "0.9.5" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76f64bba2c53b04fcab63c01a7d7427eadc821e3bc48c34dc9ba29c501164b52" +checksum = "6f9227a75a5a540a464c832ad4a4195dbdbecd8787610a56262721fde6f04f90" dependencies = [ "byteorder", "crunchy", @@ -8566,16 +8881,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" [[package]] -name = "unsafe-libyaml" -version = "0.2.11" +name = "unit-prefix" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" +checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" [[package]] -name = "unsigned-varint" -version = "0.7.2" +name = "unsafe-libyaml" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" [[package]] name = "unsigned-varint" @@ -8596,9 +8911,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" dependencies = [ "form_urlencoded", - "idna 1.1.0", + "idna", "percent-encoding", "serde", + "serde_derive", ] [[package]] @@ -8692,6 +9008,24 @@ dependencies = [ "wit-bindgen-rt", ] +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen", +] + [[package]] name = "wasite" version = "0.1.0" @@ -8700,47 +9034,32 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.100" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "once_cell", "rustversion", "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" -dependencies = [ - "bumpalo", - "log", - "proc-macro2", - "quote", - "syn 2.0.114", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.42" +version = "0.4.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280" dependencies = [ - "cfg-if 1.0.0", "js-sys", "wasm-bindgen", - "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.100" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -8748,34 +9067,41 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.100" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" dependencies = [ + "bumpalo", "proc-macro2", "quote", - "syn 2.0.114", - "wasm-bindgen-backend", + "syn 2.0.118", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.100" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" dependencies = [ "unicode-ident", ] [[package]] -name = "wasm-encoder" -version = "0.236.1" +name = "wasm-compose" +version = "0.251.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "724fccfd4f3c24b7e589d333fc0429c68042897a7e8a5f8694f31792471841e7" +checksum = "b089037d7eb453ed57b560fe7833de0707411c8b9fdc429745ced77e2a1bacb9" dependencies = [ - "leb128fmt", - "wasmparser 0.236.1", + "anyhow", + "heck 0.5.0", + "indexmap 2.14.0", + "log", + "petgraph 0.6.5", + "smallvec", + "wasm-encoder 0.251.0", + "wasmparser 0.251.0", + "wat", ] [[package]] @@ -8788,15 +9114,47 @@ dependencies = [ "wasmparser 0.244.0", ] +[[package]] +name = "wasm-encoder" +version = "0.251.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a879a421bd17c528b74721b2abf4c62e8f1d1889c2ba8c3c50d02deaf2ce395" +dependencies = [ + "leb128fmt", + "wasmparser 0.251.0", +] + +[[package]] +name = "wasm-encoder" +version = "0.252.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8185ae345fa5687c054626ff9a50e7089797a343d9904d1dc9820eb4c4d3196f" +dependencies = [ + "leb128fmt", + "wasmparser 0.252.0", +] + [[package]] name = "wasm-instrument" -version = "0.2.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bca81f5279342b38b17d9acbf007a46ddeb73144e2bd5f0a21bfa9fc5d4ab3e" +checksum = "2a47ecb37b9734d1085eaa5ae1a81e60801fd8c28d4cabdd8aedb982021918bc" dependencies = [ "parity-wasm", ] +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap 2.14.0", + "wasm-encoder 0.244.0", + "wasmparser 0.244.0", +] + [[package]] name = "wasm-streams" version = "0.4.0" @@ -8810,69 +9168,92 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasmparser" version = "0.118.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77f1154f1ab868e2a01d9834a805faca7bf8b50d041b4ca714d005d0dab1c50c" dependencies = [ - "indexmap 2.11.4", - "semver 1.0.27", + "indexmap 2.14.0", + "semver 1.0.28", ] [[package]] name = "wasmparser" -version = "0.236.1" +version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9b1e81f3eb254cf7404a82cee6926a4a3ccc5aad80cc3d43608a070c67aa1d7" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.11.1", "hashbrown 0.15.2", - "indexmap 2.11.4", - "semver 1.0.27", + "indexmap 2.14.0", + "semver 1.0.28", +] + +[[package]] +name = "wasmparser" +version = "0.251.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "437970b35b1a85cfde9c74b2398352d8d653f3bd8e3a3db0c063ea8f5b4b36ff" +dependencies = [ + "bitflags 2.11.1", + "hashbrown 0.17.0", + "indexmap 2.14.0", + "semver 1.0.28", "serde", ] [[package]] name = "wasmparser" -version = "0.244.0" +version = "0.252.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +checksum = "d3eb099dcadcde5be9eef55e3a337128efd4e44b4c93122487e4d2e4e1c6627c" dependencies = [ - "bitflags 2.9.0", - "indexmap 2.11.4", - "semver 1.0.27", + "bitflags 2.11.1", + "indexmap 2.14.0", + "semver 1.0.28", ] [[package]] name = "wasmprinter" -version = "0.236.1" +version = "0.251.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2df225df06a6df15b46e3f73ca066ff92c2e023670969f7d50ce7d5e695abbb1" +checksum = "8798c1a699bd25648b6708eefe94d97c6f9891febb94b42cca1f7a4b086ea64e" dependencies = [ "anyhow", "termcolor", - "wasmparser 0.236.1", + "wasmparser 0.251.0", ] [[package]] name = "wasmtime" -version = "36.0.5" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "901adbcfe03e3ad9db86f5665d6e00d54c904d4b81235c375635991596dfef3b" +checksum = "c4213d2f019a5e44aa8a61d8826dd33a505bff79f749b14a8bafd67321cb9351" dependencies = [ "addr2line", - "anyhow", "async-trait", - "bitflags 2.9.0", + "bitflags 2.11.1", "bumpalo", "cc", - "cfg-if 1.0.0", + "cfg-if", "encoding_rs", + "futures 0.3.31", "fxprof-processed-profile", "gimli", - "hashbrown 0.15.2", - "indexmap 2.11.4", "ittapi", "libc", "log", @@ -8883,118 +9264,124 @@ dependencies = [ "postcard", "pulley-interpreter", "rayon", - "rustix 1.0.7", - "semver 1.0.27", + "rustix 1.1.4", + "semver 1.0.28", "serde", "serde_derive", "serde_json", "smallvec", "target-lexicon", - "wasm-encoder 0.236.1", - "wasmparser 0.236.1", + "tempfile", + "wasm-compose", + "wasm-encoder 0.251.0", + "wasmparser 0.251.0", "wasmtime-environ", - "wasmtime-internal-asm-macros", "wasmtime-internal-cache", "wasmtime-internal-component-macro", "wasmtime-internal-component-util", + "wasmtime-internal-core", "wasmtime-internal-cranelift", "wasmtime-internal-fiber", "wasmtime-internal-jit-debug", "wasmtime-internal-jit-icache-coherence", - "wasmtime-internal-math", - "wasmtime-internal-slab", "wasmtime-internal-unwinder", "wasmtime-internal-versioned-export-macros", - "wasmtime-internal-winch", "wat", - "windows-sys 0.60.2", + "windows-sys 0.61.2", + "wit-parser 0.251.0", ] [[package]] name = "wasmtime-environ" -version = "36.0.5" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00984333e84fa259b72b5bc113e1699d04f20c3ac191bf3e268e32bd93e493fd" +checksum = "d45863de41977ec6453e859cf843d456fa3fcb45a659b66d16e794f90ec4f5b7" dependencies = [ "anyhow", "cpp_demangle", + "cranelift-bforest", "cranelift-bitset", "cranelift-entity", "gimli", - "indexmap 2.11.4", + "hashbrown 0.17.0", + "indexmap 2.14.0", "log", "object", "postcard", "rustc-demangle", - "semver 1.0.27", + "semver 1.0.28", "serde", "serde_derive", + "sha2 0.10.9", "smallvec", "target-lexicon", - "wasm-encoder 0.236.1", - "wasmparser 0.236.1", + "wasm-encoder 0.251.0", + "wasmparser 0.251.0", "wasmprinter", "wasmtime-internal-component-util", -] - -[[package]] -name = "wasmtime-internal-asm-macros" -version = "36.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f42078a2603132bb5d7f2d5114ce57992e0fa344a9521385dc159c63472a9a" -dependencies = [ - "cfg-if 1.0.0", + "wasmtime-internal-core", ] [[package]] name = "wasmtime-internal-cache" -version = "36.0.5" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ae7355594563bec11af97afb429ea8576f497960a78d59684ffabd11a862b8" +checksum = "438bc7dc45fb75297d75f79a9a0ce852345d13ebc6a6863f6f688f013836a9dd" dependencies = [ - "anyhow", - "base64 0.22.1", + "base64", "directories-next", "log", "postcard", - "rustix 1.0.7", + "rustix 1.1.4", "serde", "serde_derive", - "sha2", - "toml 0.8.15", - "windows-sys 0.60.2", + "sha2 0.10.9", + "toml 0.9.12+spec-1.1.0", + "wasmtime-environ", + "windows-sys 0.61.2", "zstd", ] [[package]] name = "wasmtime-internal-component-macro" -version = "36.0.5" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c271bbe00cf374564ee37bee4fe298f8ad10e267aba07b2ec598d376677f062" +checksum = "f1e48f8d4966d62a10b6d70722bc432c1e163890be2801d3b5784589ad36ffc3" dependencies = [ "anyhow", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", "wasmtime-internal-component-util", "wasmtime-internal-wit-bindgen", - "wit-parser", + "wit-parser 0.251.0", ] [[package]] name = "wasmtime-internal-component-util" -version = "36.0.5" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a61fa3310d28256440f4bad63bd9462f364ee9faca7c3dcfccc7c2a48db46005" +checksum = "819ad5abd5822a22dbf4014475cdfd1fe790707761cd732d74aaa3ba4d5ba489" [[package]] -name = "wasmtime-internal-cranelift" -version = "36.0.5" +name = "wasmtime-internal-core" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6401e096bfbb50e75a00bd83162fee68b1800d65937364463a4ad43da3f140f8" +checksum = "3fc28372e36eaf8cf70faa83b5779137f7e99c8d18569a125d1580e735cc9e4d" dependencies = [ "anyhow", - "cfg-if 1.0.0", + "hashbrown 0.17.0", + "libm", + "serde", +] + +[[package]] +name = "wasmtime-internal-cranelift" +version = "46.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a433efc6e35112a5457e1dc8bc4d8d39820ac7722267e89bc04e5df641f32124" +dependencies = [ + "cfg-if", "cranelift-codegen", "cranelift-control", "cranelift-entity", @@ -9008,119 +9395,87 @@ dependencies = [ "smallvec", "target-lexicon", "thiserror 2.0.18", - "wasmparser 0.236.1", + "wasmparser 0.251.0", "wasmtime-environ", - "wasmtime-internal-math", + "wasmtime-internal-core", + "wasmtime-internal-unwinder", "wasmtime-internal-versioned-export-macros", ] [[package]] name = "wasmtime-internal-fiber" -version = "36.0.5" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8dd592465c4fffd866fc6f50db2cc7ae0c73d2742699e351b3680b5f84f21ede" +checksum = "18a1d3a39d0d210f6b8574ee96a4315e0a14c67f3a1fc3cd5372cb10d2fb4422" dependencies = [ - "anyhow", "cc", - "cfg-if 1.0.0", + "cfg-if", "libc", - "rustix 1.0.7", - "wasmtime-internal-asm-macros", + "rustix 1.1.4", + "wasmtime-environ", "wasmtime-internal-versioned-export-macros", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] name = "wasmtime-internal-jit-debug" -version = "36.0.5" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6c60a180c90eea53266a6627c353a8101090f1e084f59e1bd4666f5c55e405" +checksum = "9f667288cb4dfa68a4639ffac4d5628535dda64ebdc2b990526efb12b30ba803" dependencies = [ "cc", "object", - "rustix 1.0.7", + "rustix 1.1.4", "wasmtime-internal-versioned-export-macros", ] [[package]] name = "wasmtime-internal-jit-icache-coherence" -version = "36.0.5" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efe8de0903b246b59b112f2a7116f3d2315c41a9252ab78de90dae93b9cab50e" +checksum = "eba651d44ab0faad4c58106b3adb45068189fb65ef50f0c404b6d9e3bf81a357" dependencies = [ - "anyhow", - "cfg-if 1.0.0", + "cfg-if", "libc", - "windows-sys 0.60.2", + "wasmtime-internal-core", + "windows-sys 0.61.2", ] -[[package]] -name = "wasmtime-internal-math" -version = "36.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e1d143a7388e4adfae7c1d6c6ceb44325b4b45b2e393e39b25ddaf563e7e587" -dependencies = [ - "libm", -] - -[[package]] -name = "wasmtime-internal-slab" -version = "36.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de954a96e144df5b22805367f91a1754237f6bf99918f087d0ea1970be3b6365" - [[package]] name = "wasmtime-internal-unwinder" -version = "36.0.5" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9923ac3d2b967e8ecbfefddaf19909b6a9a03b5b969b2a71af52300e3e404419" +checksum = "2ecc52563b0558af2a7487eb710de07cc4532564b55528876129238e83118cb1" dependencies = [ - "anyhow", - "cfg-if 1.0.0", + "cfg-if", "cranelift-codegen", "log", "object", + "wasmtime-environ", ] [[package]] name = "wasmtime-internal-versioned-export-macros" -version = "36.0.5" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef2c0062b75377b8d0a20239436b06df2e01a3521e9f14af6ea9b438c60fc030" +checksum = "e747f4a074699ba1b4e4d841fb263f9b7df5bd1555181c4752bf5990d21ba676" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", -] - -[[package]] -name = "wasmtime-internal-winch" -version = "36.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8542e7cfd5b77ad33ac4cab866cb2b2eca350c7c34ac73e13fe78e83871ad3d7" -dependencies = [ - "anyhow", - "cranelift-codegen", - "gimli", - "object", - "target-lexicon", - "wasmparser 0.236.1", - "wasmtime-environ", - "wasmtime-internal-cranelift", - "winch-codegen", + "syn 2.0.118", ] [[package]] name = "wasmtime-internal-wit-bindgen" -version = "36.0.5" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04c8be7f99d674c7af47ceba83ee4dc4e36132798c920a7bfd7ca44ce7733f99" +checksum = "80009f46991622814196d96fac6fc0a938f46b5cba737a8f4e21e24e5a03856f" dependencies = [ "anyhow", - "bitflags 2.9.0", + "bitflags 2.11.1", "heck 0.5.0", - "indexmap 2.11.4", - "wit-parser", + "indexmap 2.14.0", + "wit-parser 0.251.0", ] [[package]] @@ -9139,31 +9494,31 @@ dependencies = [ [[package]] name = "wast" -version = "244.0.0" +version = "252.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2e7b9f9e23311275920e3d6b56d64137c160cf8af4f84a7283b36cfecbf4acb" +checksum = "942a3449d6a593fccc111a6241c8df52bda168af30e40bf9580d4394d7374c65" dependencies = [ "bumpalo", "leb128fmt", "memchr", "unicode-width", - "wasm-encoder 0.244.0", + "wasm-encoder 0.252.0", ] [[package]] name = "wat" -version = "1.244.0" +version = "1.252.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbf35b87ed352f9ab6cd0732abde5a67dd6153dfd02c493e61459218b19456fa" +checksum = "c72a4ba7088f7bac94cf516e49882bdf97068904a563768cf249efc839ec42cb" dependencies = [ "wast", ] [[package]] name = "web-sys" -version = "0.3.69" +version = "0.3.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d" dependencies = [ "js-sys", "wasm-bindgen", @@ -9180,50 +9535,12 @@ dependencies = [ ] [[package]] -name = "web3" -version = "0.19.0-graph" -source = "git+https://github.com/graphprotocol/rust-web3?branch=graph-patches-onto-0.18#f9f27f45ce23bf489d8bd010b50b2b207eb316cb" -dependencies = [ - "arrayvec 0.7.4", - "base64 0.13.1", - "bytes", - "derive_more 0.99.19", - "ethabi", - "ethereum-types", - "futures 0.3.31", - "futures-timer", - "headers", - "hex", - "idna 0.2.3", - "jsonrpc-core", - "log", - "once_cell", - "parking_lot", - "pin-project", - "reqwest", - "rlp", - "secp256k1 0.21.3", - "serde", - "serde_json", - "soketto", - "tiny-keccak 2.0.2", - "tokio", - "tokio-stream", - "tokio-util 0.6.10", - "url", - "web3-async-native-tls", -] - -[[package]] -name = "web3-async-native-tls" -version = "0.4.0" +name = "webpki-root-certs" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f6d8d1636b2627fe63518d5a9b38a569405d9c9bc665c43c9c341de57227ebb" +checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c" dependencies = [ - "native-tls", - "thiserror 1.0.61", - "tokio", - "url", + "rustls-pki-types", ] [[package]] @@ -9304,26 +9621,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "winch-codegen" -version = "36.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ce0c15cfd084585ed8f5519d4f405de98ff530f6afe31b88a5560688879c85e" -dependencies = [ - "anyhow", - "cranelift-assembler-x64", - "cranelift-codegen", - "gimli", - "regalloc2", - "smallvec", - "target-lexicon", - "thiserror 2.0.18", - "wasmparser 0.236.1", - "wasmtime-environ", - "wasmtime-internal-cranelift", - "wasmtime-internal-math", -] - [[package]] name = "windows-core" version = "0.52.0" @@ -9341,9 +9638,9 @@ checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" [[package]] name = "windows-link" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45e46c0661abb7180e7b9c281db115305d49ca1709ab8242adf09666d2173c65" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" [[package]] name = "windows-registry" @@ -9374,6 +9671,15 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -9412,11 +9718,26 @@ dependencies = [ [[package]] name = "windows-sys" -version = "0.61.1" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f109e41dd4a3c848907eb83d5a42ea98b3769495597450cf6d153507b166f0f" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "windows-link 0.2.0", + "windows-link 0.2.1", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", ] [[package]] @@ -9467,6 +9788,12 @@ dependencies = [ "windows_x86_64_msvc 0.53.0", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -9485,6 +9812,12 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -9503,6 +9836,12 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -9533,6 +9872,12 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -9551,6 +9896,12 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -9569,6 +9920,12 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -9587,6 +9944,12 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -9616,18 +9979,15 @@ dependencies = [ [[package]] name = "winnow" -version = "0.6.13" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59b5e5f6c299a3c7890b876a2a587f3115162487e704907d9b6cd29473052ba1" -dependencies = [ - "memchr", -] +checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf" [[package]] name = "winnow" -version = "0.7.13" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf" +checksum = "a90e88e4667264a994d34e6d1ab2d26d398dcdca8b7f52bec8668957517fc7d8" dependencies = [ "memchr", ] @@ -9639,12 +9999,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08db1edfb05d9b3c1542e521aea074442088292f00b5f28e435c714a98f85031" dependencies = [ "assert-json-diff", - "base64 0.22.1", - "deadpool", + "base64", + "deadpool 0.12.3", "futures 0.3.31", - "http 1.4.0", + "http 1.4.2", "http-body-util", - "hyper 1.8.1", + "hyper", "hyper-util", "log", "once_cell", @@ -9655,31 +10015,120 @@ dependencies = [ "url", ] +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck 0.5.0", + "wit-parser 0.244.0", +] + [[package]] name = "wit-bindgen-rt" version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.11.1", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck 0.5.0", + "indexmap 2.14.0", + "prettyplease", + "syn 2.0.118", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.118", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags 2.11.1", + "indexmap 2.14.0", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder 0.244.0", + "wasm-metadata", + "wasmparser 0.244.0", + "wit-parser 0.244.0", ] [[package]] name = "wit-parser" -version = "0.236.1" +version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16e4833a20cd6e85d6abfea0e63a399472d6f88c6262957c17f546879a80ba15" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" dependencies = [ "anyhow", "id-arena", - "indexmap 2.11.4", + "indexmap 2.14.0", "log", - "semver 1.0.27", + "semver 1.0.28", "serde", "serde_derive", "serde_json", "unicode-xid", - "wasmparser 0.236.1", + "wasmparser 0.244.0", +] + +[[package]] +name = "wit-parser" +version = "0.251.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e960732e824fab95099971a09e638979347c94ca48568d3c854c945729196947" +dependencies = [ + "anyhow", + "hashbrown 0.17.0", + "id-arena", + "indexmap 2.14.0", + "log", + "semver 1.0.28", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser 0.251.0", ] [[package]] @@ -9754,7 +10203,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", "synstructure", ] @@ -9775,7 +10224,7 @@ checksum = "2c7962b26b0a8685668b671ee4b54d007a67d4eaf05fda79ac0ecf41e32270f1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -9795,7 +10244,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", "synstructure", ] @@ -9816,7 +10265,7 @@ checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] [[package]] @@ -9838,9 +10287,21 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.118", ] +[[package]] +name = "zlib-rs" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c745c48e1007337ed136dc99df34128b9faa6ed542d80a1c673cf55a6d7236c8" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + [[package]] name = "zstd" version = "0.13.3" diff --git a/Cargo.toml b/Cargo.toml index bd9808622a9..a956a514a95 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,8 +26,8 @@ members = [ ] [workspace.package] -version = "0.36.0" -edition = "2021" +version = "0.45.0" +edition = "2024" authors = ["The Graph core developers & contributors"] readme = "README.md" homepage = "https://thegraph.com" @@ -35,17 +35,24 @@ repository = "https://github.com/graphprotocol/graph-node" license = "MIT OR Apache-2.0" [workspace.dependencies] -alloy = { version = "1.0.33", features = ["dyn-abi", "json-abi", "full", "arbitrary", "json-rpc", "serde"] } -alloy-rpc-types = "1.0.33" +alloy = { version = "2.0.5", features = ["dyn-abi", "json-abi", "full", "arbitrary", "json-rpc", "serde"] } +alloy-rpc-types = "2.0.0" +# rustls is pulled in transitively by alloy (aws_lc_rs) and object_store via +# reqwest (ring). With both providers linked, rustls 0.23 requires an explicit +# default provider to be installed before any TLS use. We install aws_lc_rs +# explicitly in each binary/test entry point. +rustls = { version = "0.23", default-features = false, features = ["aws_lc_rs"] } anyhow = "1.0" async-graphql = { version = "7.2.1", features = ["chrono"] } async-graphql-axum = "7.2.1" async-trait = "0.1.74" -axum = "0.8.8" -chrono = "0.4.43" +axum = "0.8.9" +chrono = "0.4.44" bs58 = "0.5.1" clap = { version = "4.5.4", features = ["derive", "env", "wrap_help"] } -derivative = "2.2.0" +clap_complete = "4" +console = "0.16" +derive_more = { version = "2.1.1", default-features = false } diesel = { version = "2.2.7", features = [ "postgres", "serde_json", @@ -54,64 +61,73 @@ diesel = { version = "2.2.7", features = [ "chrono", "i-implement-a-third-party-backend-and-opt-into-breaking-changes", ] } -diesel-async = { version = "0.7.4", features = ["deadpool", "async-connection-wrapper", "tokio", "postgres"] } +diesel-async = { version = "0.9.0", features = ["deadpool", "async-connection-wrapper", "tokio", "postgres"] } diesel-derive-enum = { version = "2.1.0", features = ["postgres"] } diesel-dynamic-schema = { version = "0.2.3", features = ["postgres"] } -diesel_derives = "2.3.6" -diesel_migrations = "2.3.1" +diesel_derives = "2.3.9" +diesel_migrations = "2.3.2" +env_logger = "0.11.10" envconfig = "0.11.1" +git-testament = "0.2" graph = { path = "./graph" } graph-core = { path = "./core" } graph-store-postgres = { path = "./store/postgres" } graphman-server = { path = "./server/graphman" } graphman = { path = "./core/graphman" } graphman-store = { path = "./core/graphman_store" } -graphql-tools = "0.5.0" -itertools = "0.14.0" +graphql-tools = "0.5.1" +indicatif = "0.18" +Inflector = "0.11.3" +itertools = "0.15.0" lazy_static = "1.5.0" prost = "0.14" prost-types = "0.14" -redis = { version = "1.0.2", features = [ +redis = { version = "1.2.2", features = [ "aio", "connection-manager", "tokio-comp", ] } regex = "1.5.4" reqwest = "0.12.23" +semver = { version = "1", features = ["serde"] } serde = { version = "1.0.126", features = ["rc"] } serde_derive = "1.0.125" serde_json = { version = "1.0", features = ["arbitrary_precision"] } serde_regex = "1.1.0" serde_yaml = "0.9.21" slog = { version = "2.8.2", features = ["release_max_level_trace", "max_level_trace"] } -sqlparser = { version = "0.60.0", features = ["visitor"] } -strum = { version = "0.27", features = ["derive"] } -syn = { version = "2.0.114", features = ["full"] } +slog-async = "2.5.0" +slog-term = "2.7.0" +sqlparser = { version = "0.62.0", features = ["visitor"] } +strum = { version = "0.28", features = ["derive"] } +syn = { version = "2.0.117", features = ["full"] } test-store = { path = "./store/test-store" } thiserror = "2.0.18" -deadpool = { version = "0.12", features = ["rt_tokio_1", "managed"] } -tokio = { version = "1.49.0", features = ["full"] } +deadpool = { version = "0.13", features = ["rt_tokio_1", "managed"] } +tokio = { version = "1.50.0", features = ["full"] } tokio-stream = { version = "0.1.18", features = ["sync"] } tokio-retry = "0.3.0" tonic = { version = "0.14", features = ["tls-native-roots", "gzip"] } tonic-prost = "0.14" tonic-prost-build = "0.14" -tower-http = { version = "0.6.8", features = ["cors"] } +tower-http = { version = "0.7.0", features = ["cors"] } tower = { version = "0.5.1", features = ["full"] } wasmparser = "0.118.1" -wasmtime = { version = "36.0.5", features = ["async"] } +wasmtime = { version = "46.0.0", features = ["async"] } rand = { version = "0.9.2", features = ["os_rng"] } prometheus = "0.14.0" +url = "2.5.8" # Dependencies related to Amp subgraphs ahash = "0.8.11" -arrow = { version = "=57.0.0" } -arrow-flight = { version = "=57.0.0", features = ["flight-sql-experimental"] } +arrow = { version = "=59.0.0" } +arrow-flight = { version = "=59.0.0", features = ["flight-sql-experimental"] } +parquet = { version = "=59.0.0" } futures = "0.3.31" half = "2.7.1" indoc = "2.0.7" -lazy-regex = "3.4.1" +lazy-regex = "3.6.0" parking_lot = "0.12.4" sqlparser-latest = { version = "0.57.0", package = "sqlparser", features = ["visitor"] } tokio-util = "0.7.15" diff --git a/NEWS.md b/NEWS.md index 719d2f12e49..8068ecbc9fa 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,320 @@ # NEWS +## v0.45.0 + +``` +$ docker pull graphprotocol/graph-node:v0.45.0 +``` + +### Breaking Changes + +- **`GRAPH_GETH_ETH_CALL_ERRORS` renamed to `GRAPH_RPC_ETH_CALL_ERRORS`.** The env var (and internal config field) controlling which `eth_call` error messages are treated as reverts is renamed for provider-neutral naming. Update it before upgrading, or the setting is silently ignored and defaults apply. ([#6487](https://github.com/graphprotocol/graph-node/pull/6487)) + +### What's New + +- **Postponed attribute index creation re-enabled, now built in the background.** `GRAPH_POSTPONE_ATTRIBUTE_INDEX_CREATION` (introduced in v0.44.0 but inert — hardcoded off) now actually defers non-critical attribute indexes until a new deployment nears chain head (`GRAPH_POSTPONE_INDEXES_CREATION_THRESHOLD`, default 10000 blocks from head). Index creation now runs in a background task instead of blocking block processing, and interrupted `CREATE INDEX CONCURRENTLY` runs are detected and rebuilt on retry instead of being silently skipped. ([#6608](https://github.com/graphprotocol/graph-node/pull/6608)) +- **`ethereum.decodeParams` host function.** Decodes calldata/event data whose top-level type is a dynamic tuple (e.g. Gnosis Safe's `execTransaction`), which `ethereum.decode` cannot handle since it expects a single ABI value with a leading offset word. Gated behind apiVersion 0.0.10; `ethereum.decode` itself is unchanged. ([#6649](https://github.com/graphprotocol/graph-node/pull/6649)) + +### Bug Fixes + +- Fixed `EthereumLogFilter` merging per-handler `receipt` flags with overwrite instead of OR semantics, so a handler declaring `receipt: true` could silently run with `event.receipt == null` if another handler on the same filter key declared `receipt: false`. ([#6558](https://github.com/graphprotocol/graph-node/pull/6558)) +- Fixed grafts silently succeeding when the graft block was below the base deployment's earliest available block, leaving the destination missing pre-graft entity versions and producing `unexpected null in handler` errors on every event; `Graft::validate` now rejects these upfront. ([#6610](https://github.com/graphprotocol/graph-node/pull/6610)) +- Fixed `_logs` queries returning an empty result (indistinguishable from a subgraph producing no logs) when no log store is configured; now returns a `NotSupported` error. ([#6637](https://github.com/graphprotocol/graph-node/pull/6637)) +- Fixed startup panicking when a shard is configured with `pool_size = 0`; such shards' chains are now skipped instead of erroring. ([#6648](https://github.com/graphprotocol/graph-node/pull/6648)) +- Fixed the Firehose extended-blocks check matching on `optimism-mainnet` instead of the `optimism` chain ID used by Firehose endpoints, so Optimism's extended block data went undetected. ([#6662](https://github.com/graphprotocol/graph-node/pull/6662)) +- Fixed fulltext (`tsvector`) search columns being silently dropped when copying or grafting a subgraph, leaving the destination's fulltext index empty. ([#6687](https://github.com/graphprotocol/graph-node/pull/6687)) +- Fixed Reth `StackUnderflow`/`StackOverflow`/`OpcodeNotFound` EVM halts being misclassified as non-deterministic — Reth surfaces them via its `Debug` format (e.g. `EVM error: StackUnderflow`), which didn't match graph-node's space-separated deterministic-error patterns, so subgraphs indexing via Reth-backed RPC providers stalled and retried instead of recording a deterministic revert. ([#6645](https://github.com/graphprotocol/graph-node/pull/6645)) + +### Graphman + +- Fixed `graphman dump` panicking on tables with more than ~2 GiB in a single string/binary column; batches are now split into byte-bounded slices before conversion to Arrow. ([#6646](https://github.com/graphprotocol/graph-node/pull/6646)) + +### gnd (Graph Node Dev) + +- Fixed `gnd add` producing a subgraph that failed `codegen`/`build` because it never wrote new event entities to `schema.graphql`; unified `add` and `init` onto one scaffolding path, which also fixes event-name collisions across data sources, overloaded-event disambiguation, small-integer type mapping, and reserved-word parameter names. ([#6660](https://github.com/graphprotocol/graph-node/pull/6660)) + +### Upgrade Notes + +- If you set `GRAPH_GETH_ETH_CALL_ERRORS`, rename it to `GRAPH_RPC_ETH_CALL_ERRORS` before upgrading. ([#6487](https://github.com/graphprotocol/graph-node/pull/6487)) +- `GRAPH_POSTPONE_ATTRIBUTE_INDEX_CREATION` now actually takes effect (it was a no-op in v0.44.0). If you already set it expecting no behavior change, be aware new deployments will now defer some attribute indexes until near chain head. ([#6608](https://github.com/graphprotocol/graph-node/pull/6608)) +- If a subgraph was grafted from a base block below the base deployment's earliest available block before this release, it may be missing pre-graft entity versions; re-create the graft from a valid earliest block. ([#6610](https://github.com/graphprotocol/graph-node/pull/6610)) + +### Contributors + +Thanks to all contributors for this release: @cargopete, @dimitrovmaksim, @fordN, @incrypto32, @lutter, @YaroShkvorets + +**Full Changelog**: https://github.com/graphprotocol/graph-node/compare/v0.44.0...v0.45.0 + +## v0.44.0 + +``` +$ docker pull graphprotocol/graph-node:v0.44.0 +``` + +### Critical Fix + +- **`EntityCache::load_related` returned wrong derived-collection membership on same-block parent reassignment.** Membership was decided against intermediate cache layers rather than the entity's final state, so entities reassigned to a new parent within the same block could be left in the old collection, omitted from the new one, or wrongly retained after a cross-handler revert. See Upgrade Notes. ([#6548](https://github.com/graphprotocol/graph-node/pull/6548)) + +### Breaking Changes + +- **Elasticsearch CLI flags removed.** `--elasticsearch-url`, `--elasticsearch-user`, `--elasticsearch-password` (and the matching `ELASTICSEARCH_*` env vars) are gone. Configure Elasticsearch under the new `[log_store]` section in `graph-node.toml`. See Upgrade Notes. ([#6278](https://github.com/graphprotocol/graph-node/pull/6278)) + +### What's New + +- **Query subgraph logs over GraphQL.** A new `_logs` field exposes subgraph-emitted logs (mapping `log.*` calls, runtime, system) with filters for level, timestamp range, and text search, plus pagination and `orderDirection`. The old Elasticsearch sink for subgraph logs is replaced by a unified `LogStore` abstraction with three backends configured under a new `[log_store]` section in `graph-node.toml`: File (JSON Lines), Loki, or Elasticsearch. Omit `[log_store]` to disable; logs still go to stdout/stderr. ([#6278](https://github.com/graphprotocol/graph-node/pull/6278)) + + + + +### Improvements + +- Revert entity versions during copy instead of in a separate post-copy pass, speeding up subgraph copy and graft. ([#6472](https://github.com/graphprotocol/graph-node/pull/6472)) + +### Bug Fixes + +- Fixed `trace_filter` deserialization failing on providers that omit `result.output` (notably Sonic), which caused repeated trace ingestion retries. Fixed upstream in alloy 2.0.5 ([alloy#3931](https://github.com/alloy-rs/alloy/pull/3931)) and picked up via the bump in this release. ([#6576](https://github.com/graphprotocol/graph-node/pull/6576)) +- Fixed runner panics on out-of-range `BigInt` values: `to_signed_u256` replaced by fallible `to_i256`, and `to_unsigned_u256` now errors on values `>= 2^256` instead of panicking. ([#6560](https://github.com/graphprotocol/graph-node/pull/6560)) +- Fixed `graphman chain change-shard` failing with a duplicate-key error on revert to the original shard. Backups now use unique names and revert reuses the existing `-old`. ([#6199](https://github.com/graphprotocol/graph-node/pull/6199)) + +### Upgrade Notes + +- v0.44.0 changes `EntityCache::load_related` results for same-block parent-reassignment edge cases ([#6548](https://github.com/graphprotocol/graph-node/pull/6548)). Subgraphs that read derived fields within handlers may diverge in POI from v0.43.0 on blocks exhibiting this pattern; resync from before the affected blocks if affected. The v0.44.0 result is canonical. +- If you set `--elasticsearch-*` flags or `ELASTICSEARCH_*` env vars, migrate to a `[log_store]` section in `graph-node.toml` before upgrading. See `docs/log-store.md`. ([#6278](https://github.com/graphprotocol/graph-node/pull/6278)) + +### Contributors + +Thanks to all contributors for this release: @erayack, @fordN, @incrypto32, @lutter + +**Full Changelog**: https://github.com/graphprotocol/graph-node/compare/v0.43.0...v0.44.0 + +## v0.43.0 + +### What's New + +- **`skipDuplicates` for immutable entities.** A new `skipDuplicates` parameter on the `@entity` directive (`@entity(immutable: true, skipDuplicates: true)`) silently skips duplicate inserts instead of failing the subgraph. This unblocks subgraph composition on Amp-powered subgraphs where SQL queries can produce the same entities across block ranges. ([#6458](https://github.com/graphprotocol/graph-node/pull/6458)) +- **Per-chain RPC settings via TOML config.** Chain-specific tuning parameters (`json_rpc_timeout`, `request_retries`, `max_block_range_size`, `polling_interval`, `block_batch_size`, etc.) can now be set per chain in `config.toml` instead of relying on global environment variables. Fully backwards compatible — env vars remain the fallback. ([#6459](https://github.com/graphprotocol/graph-node/pull/6459)) +- **RPC provider failover for block ingestor.** When the current RPC provider becomes unreachable during block ingestion polling, graph-node now automatically switches to a healthy alternative provider. ([#6430](https://github.com/graphprotocol/graph-node/pull/6430)) +- Warn on startup when running a debug build, with `[DEBUG-BUILD]` prefix on all log lines. ([#6488](https://github.com/graphprotocol/graph-node/pull/6488)) + +### Improvements + +- **Call cache eviction rewrite.** Replaced the old per-contract iteration approach with ctid-based join deletes against the `call_meta` table, significantly faster on large `call_cache` tables. The old `--ttl-max-contracts` flag has been replaced by `--max-contracts`, which computes an effective TTL cutoff instead of limiting per-invocation iteration. Eviction now also returns stats on the number of contracts and entries removed. ([#6476](https://github.com/graphprotocol/graph-node/pull/6476)) ([#6477](https://github.com/graphprotocol/graph-node/pull/6477)) +- Reduced unnecessary `eth_getBlockByHash` and `eth_getBlockByNumber` RPC calls by checking the block cache first in `block_pointer_from_number` and `fetch_full_block_with_rpc`. ([#6491](https://github.com/graphprotocol/graph-node/pull/6491), partially reverted by [#6537](https://github.com/graphprotocol/graph-node/pull/6537)) +- Header-only `ChainStore` query methods (`ancestor_block_ptr`, `block_parent_ptr`) skip deserializing the full block `data` JSONB column when only hash/number/parent are needed. ([#6456](https://github.com/graphprotocol/graph-node/pull/6456)) +- Batch checking for update attempts on immutable entities, reducing per-entity overhead. +- Node name is now used as the PostgreSQL `application_name` when `PGAPPNAME` is not set, making it easier to identify graph-node connections in `pg_stat_activity`. + +### Bug Fixes + +- Fixed dropped block trigger when `once` and `polling` filters match the same block — only one trigger type was firing. ([#6530](https://github.com/graphprotocol/graph-node/pull/6530)) +- Fixed block stream ignoring configured `endBlock` in two cases: the block-skip optimization bypassing the `max_end_block` check, and `max_end_block` being set to `None` when multiple data sources share the same `endBlock`. ([#6474](https://github.com/graphprotocol/graph-node/pull/6474)) +- Fixed GraphQL introspection not returning `isDeprecated: false` for `__InputValue`, which caused some client libraries to fail. ([#6475](https://github.com/graphprotocol/graph-node/pull/6475)) +- Fixed IPC provider connections failing when configured with `ipc://` or `file://` URLs — the URL was passed directly to the transport instead of extracting the file path. ([#6443](https://github.com/graphprotocol/graph-node/pull/6443)) +- Fixed `graphman config pools` not working due to hardcoded pool size override. ([#6444](https://github.com/graphprotocol/graph-node/pull/6444)) +- Fixed unfail retry mechanism stopping after the first attempt when the deployment head was still behind the error block. ([#6529](https://github.com/graphprotocol/graph-node/pull/6529)) + +### Note on `ethereum.decode()` whitespace handling + +The migration from `ethabi` to `alloy` in v0.42.0 ([#6063](https://github.com/graphprotocol/graph-node/pull/6063)) incidentally fixed a long-standing parsing bug in `ethabi` where type strings containing whitespace before a type name (e.g. `" address"` with a leading space) were silently decoded as `Uint(8)` instead of the intended type. `alloy` parses these correctly. + +Subgraphs that relied on the incorrect `Uint(8)` decoding to subsequently call `.toBigInt()` on what is actually an `Address` value will abort on v0.42.0+ with: + +``` +Mapping aborted ... Ethereum value is not an int or uint. +``` + +This is not a graph-node regression. Recompile the subgraph with the correct accessor (`.toAddress()` for addresses) to fix. See [#6461](https://github.com/graphprotocol/graph-node/issues/6461) for details. + +### gnd (Graph Node Dev) + +- `gnd indexer` command that delegates to `graph-indexer`, allowing indexer management (allocations, rules, cost models, status) directly through gnd. ([#6492](https://github.com/graphprotocol/graph-node/pull/6492)) +- `gnd deploy` now prompts for `--version-label` in interactive mode and requires it in non-interactive mode. ([#6532](https://github.com/graphprotocol/graph-node/pull/6532)) +- Test framework improvements: partial receipt.logs support, mock IPFS/Arweave clients for file data source testing. ([#6442](https://github.com/graphprotocol/graph-node/pull/6442)) + +### Contributors + +Thanks to all contributors for this release: @aayushbaluni, @dimitrovmaksim, @incrypto32, @isum, @lutter, @suntzu + +**Full Changelog**: https://github.com/graphprotocol/graph-node/compare/v0.42.1...v0.43.0 + +## v0.42.1 + +### Bug Fixes + +- Fixed multi-shard deployments becoming unresponsive due to `mirror_primary_tables` holding locks indefinitely, exhausting the connection pool. Added `statement_timeout` and `lock_timeout` to prevent stuck queries from cascading into a full lockup. ([#6446](https://github.com/graphprotocol/graph-node/pull/6446)) +- Fixed the `ingestor` setting in `[chains.*]` config sections having no effect — block ingestion ran regardless of its value. ([#6447](https://github.com/graphprotocol/graph-node/pull/6447)) + +## v0.42.0 + +### Breaking Changes + +- **Substreams support removed.** Substreams have been unsupported on the network for some time. All substreams-related code has been removed, simplifying the codebase significantly. Substreams-based subgraphs will no longer work. ([#6261](https://github.com/graphprotocol/graph-node/pull/6261)) +- **Migrated from rust-web3 to alloy.** The Ethereum RPC layer now uses the alloy crate instead of rust-web3. This should be transparent to most users, but chains that do not return EIP-2718 typed transaction fields may need the `no_eip2718` provider feature flag. ([#6063](https://github.com/graphprotocol/graph-node/pull/6063)) ([#6317](https://github.com/graphprotocol/graph-node/pull/6317)) + +### What's New + +- **Amp-powered subgraphs (experimental).** A new kind of subgraph powered by [Amp](https://github.com/edgeandnode/amp), the blockchain native database. Amp-powered subgraphs index data directly from Amp servers instead of processing blocks individually, reducing indexing time from days/weeks to minutes/hours. See `docs/amp-powered-subgraphs.md` for details. ([#6218](https://github.com/graphprotocol/graph-node/pull/6218)) ([#6293](https://github.com/graphprotocol/graph-node/pull/6293)) ([#6369](https://github.com/graphprotocol/graph-node/pull/6369)) +- **SQL query interface (experimental).** A new experimental SQL query interface allows querying subgraph data using SQL syntax via GraphQL. Documentation in `docs/implementation/sql-interface.md`. ([#6172](https://github.com/graphprotocol/graph-node/pull/6172)) +- **Async store.** All store and database interactions are now fully async using `diesel_async`, eliminating blocking database calls from tokio tasks. This improves throughput and reduces the risk of thread starvation under heavy load. ([#6194](https://github.com/graphprotocol/graph-node/pull/6194)) ([#6185](https://github.com/graphprotocol/graph-node/pull/6185)) +- **RPC request compression.** Configurable compression for outgoing JSON-RPC requests to upstream providers, supporting gzip, brotli, and deflate. Configured via provider `features` in the config file. ([#6084](https://github.com/graphprotocol/graph-node/pull/6084)) +- Query the current (partially filled) aggregation bucket with `current: include` in GraphQL requests. ([#6293](https://github.com/graphprotocol/graph-node/pull/6293)) +- Extended `first` and `last` aggregation support to `String`, `Bytes`, and entity reference types. ([#6225](https://github.com/graphprotocol/graph-node/pull/6225)) +- Automated account-like table optimization — an optional background job that detects and marks tables with high entity update ratios as account-like. Enabled via `GRAPH_STORE_ACCOUNT_LIKE_SCAN_INTERVAL_HOURS`, `GRAPH_STORE_ACCOUNT_LIKE_MIN_VERSIONS_COUNT`, and `GRAPH_STORE_ACCOUNT_LIKE_MAX_UNIQUE_RATIO`. ([#6209](https://github.com/graphprotocol/graph-node/pull/6209)) +- Subgraph table sizes and row counts are now exposed in `SubgraphIndexingStatus` via the index-node API. ([#6201](https://github.com/graphprotocol/graph-node/pull/6201)) +- Configurable database setup timeout via `GRAPH_STORE_SETUP_TIMEOUT`. +- Option to disable the store call cache via `GRAPH_STORE_DISABLE_CALL_CACHE`, useful for indexers with locally hosted RPC nodes. ([#6214](https://github.com/graphprotocol/graph-node/pull/6214)) +- Docker: added profiling support to the `graph-node-debug` image with frame pointers and `linux-perf`. ([#6418](https://github.com/graphprotocol/graph-node/pull/6418)) + +### Improvements + +- Optimized WASM trigger instantiation by caching linker and compilation artifacts, reducing per-trigger overhead. ([#6364](https://github.com/graphprotocol/graph-node/pull/6364)) +- Optimized log filter matching and trigger allocation, reducing CPU usage during block processing. ([#6419](https://github.com/graphprotocol/graph-node/pull/6419)) +- Reduced unnecessary entity clones during block processing. ([#6362](https://github.com/graphprotocol/graph-node/pull/6362)) +- Replaced `std::sync::RwLock` with `parking_lot::RwLock` across multiple components for better performance under contention. +- Deadlock prevention under heavy load, especially during index node startup — connection pool and store semaphore improvements. ([#6224](https://github.com/graphprotocol/graph-node/pull/6224)) +- Raw subgraph manifests are now loaded from the store cache during startup, reducing IPFS dependency. ([#6223](https://github.com/graphprotocol/graph-node/pull/6223)) +- Deployment hashes are now logged when removing a subgraph. ([#6405](https://github.com/graphprotocol/graph-node/pull/6405)) + +### Bug Fixes + +- Fixed duplicate VID when multiple offchain triggers (file/IPFS data sources) fire in the same block, causing unique constraint violations. Also fixed in `ipfs.map()` callbacks. ([#6336](https://github.com/graphprotocol/graph-node/pull/6336)) ([#6416](https://github.com/graphprotocol/graph-node/pull/6416)) +- Fixed incorrect `nocase` filter generation for non-String types (e.g., `Bytes`), which caused SQL errors like `operator does not exist: bytea ~~* bytea`. ([#6351](https://github.com/graphprotocol/graph-node/pull/6351)) +- Fixed `can_copy_from` failures not fully rolling back deployment creation, leaving orphaned records. ([#6228](https://github.com/graphprotocol/graph-node/pull/6228)) +- Fixed multi-column index validation bug where column order mismatch caused valid indexes to be silently dropped during subgraph copy/graft. ([#6341](https://github.com/graphprotocol/graph-node/pull/6341)) +- Fixed grafting failure ("Unexpected null for non-null column") when source tables have too few rows for PostgreSQL to generate `histogram_bounds` statistics. ([#6275](https://github.com/graphprotocol/graph-node/pull/6275)) +- Fixed panic when converting a negative number to `U256` in the runtime — now returns an error instead. ([#6219](https://github.com/graphprotocol/graph-node/pull/6219)) +- Fixed `graphman copy` copying pruned `earliest_block_number` from source, which could leave the destination deployment in an invalid state. ([#6384](https://github.com/graphprotocol/graph-node/pull/6384)) +- Fixed calls returning `0x` being incorrectly cached in the call cache, preventing stale empty results from persisting. ([#6187](https://github.com/graphprotocol/graph-node/pull/6187)) + +### Graphman + +- **`graphman dump` and `graphman restore` (experimental).** New commands for exporting and importing subgraph data in Parquet format, enabling backup and migration of deployment data across shards. Supports incremental dumps and progress reporting. See `docs/dump.md` for details. ([#6397](https://github.com/graphprotocol/graph-node/pull/6397)) +- `graphman chain call-cache remove` now supports `--ttl-days` to remove stale call cache entries that haven't been accessed within a specified number of days. ([#6186](https://github.com/graphprotocol/graph-node/pull/6186)) + +### gnd (Graph Node Dev) + +- **[experimental]** Major expansion of `gnd` as a drop-in replacement for `graph-cli`: `init`, `add`, `codegen`, `build`, `publish`, `deploy`, `create`, `remove`, `auth`, `clean`, and `test` commands. ([#6282](https://github.com/graphprotocol/graph-node/pull/6282)) + +### Contributors + +Thanks to all contributors for this release: @DaMandal0rian, @dimitrovmaksim, @fubhy, @hudsonhrh, @incrypto32, @isum, @lutter, @shiyasmohd + +**Full Changelog**: https://github.com/graphprotocol/graph-node/compare/v0.41.2...v0.42.0 + +## v0.41.2 + +### Bug Fixes + +- Fixed entity corruption after rewinding pruned subgraphs — rewinding a subgraph with `history_blocks` (pruning) enabled could leave frequently-updated entities with no open version, causing deterministic `unexpected null` errors and making the subgraph unrecoverable without a full re-index ([#6340](https://github.com/graphprotocol/graph-node/pull/6340)) +- Fixed `graphman rewind` and `graphman truncate` targeting the wrong deployment when multiple instances of the same subgraph exist ([#6299](https://github.com/graphprotocol/graph-node/pull/6299)) +- Fixed batch write memoization bug where entity lookups could incorrectly return `None`, causing missed entity updates during indexing ([#6314](https://github.com/graphprotocol/graph-node/pull/6314)) +- Expanded deterministic RPC error handling for Reth and Erigon — added `vm execution error`, `invalidjump`, `notactivated`, and `invalidfeopcode` to recognized deterministic errors. Indexers using these clients no longer need to set `GRAPH_GETH_ETH_CALL_ERRORS` manually for these cases ([#6355](https://github.com/graphprotocol/graph-node/pull/6355)) + +**Full changelog:** + +## v0.41.1 + +### Bug Fixes + +- Fixed a regression in v0.41.0 where the indexing status endpoint would return an empty list when querying all subgraphs without specific deployment filters ([#6210](https://github.com/graphprotocol/graph-node/pull/6210)) + +**Full changelog:** + +## v0.41.0 + +### Bug Fixes + +- Fix non-deterministic handling of transaction receipts in event handlers; receipts are only processed when explicitly declared ([#6200](https://github.com/graphprotocol/graph-node/pull/6200)) + +### New Features + +- Added support for struct field access in declarative calls — enables accessing nested struct fields by field name using dot notation in manifest call declarations ([#6099](https://github.com/graphprotocol/graph-node/pull/6099)) +- New standalone `gnd` (Graph Node Dev) crate — a graph-node binary optimized for local development with minimal setup required ([#6167](https://github.com/graphprotocol/graph-node/pull/6167)) +- Extended IPFS usage metrics and logging capabilities ([#6058](https://github.com/graphprotocol/graph-node/pull/6058)) +- Extended support for additional IPFS content path formats ([#6058](https://github.com/graphprotocol/graph-node/pull/6058)) + +### Improvements + +- Deferred IPFS manifest fetching when starting subgraphs — fixes issue where slow IPFS responses could block assignment event processing ([#6170](https://github.com/graphprotocol/graph-node/pull/6170)) +- Added Content Identifier (CID) to IPFS retry logs for better debugging ([#6144](https://github.com/graphprotocol/graph-node/pull/6144)) +- Fixed handling of empty arrays in indexing status queries ([#6143](https://github.com/graphprotocol/graph-node/pull/6143)) +- Fixed case-insensitive array inputs in queries ([#6075](https://github.com/graphprotocol/graph-node/pull/6075)) +- Fixed panic when handling empty lists in `list_values` ([#6100](https://github.com/graphprotocol/graph-node/pull/6100)) +- Enhanced logging for subgraph assignment processing ([#6169](https://github.com/graphprotocol/graph-node/pull/6169)) +- Added logging for `MAX_BLOCKING_THREADS` configuration setting ([#6161](https://github.com/graphprotocol/graph-node/pull/6161)) + +**Full changelog:** + +## v0.40.1 + +### Improvements + +- Enhanced IPFS logging to include Content Identifiers (CIDs) in operation names for better debugging + +**Full changelog:** + +## v0.40.0 + +### Critical Bugfix + +- Fixed GraphQL query panic with empty arrays: Resolved a critical bug where GraphQL queries using `_in` filters with empty arrays would cause the graph-node to panic ([#6100](https://github.com/graphprotocol/graph-node/pull/6100)) + +### New Features + +- IPFS files can now be cached to disk using `GRAPH_IPFS_CACHE_LOCATION` — reduces IPFS requests after restarts ([#6031](https://github.com/graphprotocol/graph-node/pull/6031)) + +### Improvements + +- Speed up appending changes to batch operations ([#6025](https://github.com/graphprotocol/graph-node/pull/6025)) +- Improved backoff strategy for offchain data sources, configurable via `GRAPH_FDS_MAX_BACKOFF` ([#6043](https://github.com/graphprotocol/graph-node/pull/6043)) +- Better error messages for OR operator usage with column filters ([#6078](https://github.com/graphprotocol/graph-node/pull/6078)) +- Update Wasmtime version for improved WebAssembly performance ([#6050](https://github.com/graphprotocol/graph-node/pull/6050)) + +### Bug Fixes + +- Fixed subgraph composition sync failures with different ID types ([#6080](https://github.com/graphprotocol/graph-node/pull/6080)) +- Fixed pruning status reporting accuracy ([#6062](https://github.com/graphprotocol/graph-node/pull/6062)) + +**Full changelog:** + +## v0.39.1 + +### Critical Fix + +- Reverted `event.transactionLogIndex` behavior change from v0.38.0 that caused subgraph failures with duplicate ID errors and POI divergence between indexers ([#6042](https://github.com/graphprotocol/graph-node/pull/6042)) + +**Full changelog:** + +## v0.39.0 + +### Breaking Changes + +- **Database schema migration**: `subgraphs.subgraph_deployment` table split into `subgraphs.head` and `subgraphs.deployment`. External tools accessing this table will need updates. See [documentation](https://github.com/graphprotocol/graph-node/blob/master/docs/implementation/metadata.md#subgraphshead) ([#6003](https://github.com/graphprotocol/graph-node/pull/6003)) +- **Arweave blockchain support removed** — Arweave subgraphs are no longer supported (file data sources remain unaffected) ([#5951](https://github.com/graphprotocol/graph-node/pull/5951)) +- **`graphman drop` removed** — Use `graphman remove` followed by `graphman unused record && graphman unused remove` ([#5974](https://github.com/graphprotocol/graph-node/pull/5974)) +- **Failed subgraphs are now paused instead of unassigned** ([#5971](https://github.com/graphprotocol/graph-node/pull/5971)) + +### New Features + +- Pruning status tracking with new graphman commands: `prune status`, `prune run`, `prune set` ([#5949](https://github.com/graphprotocol/graph-node/pull/5949)) +- Pruning timeout protection via `GRAPH_STORE_BATCH_TIMEOUT` ([#6002](https://github.com/graphprotocol/graph-node/pull/6002)) +- `graphman chain ingest` for manually ingesting specific blocks ([#5945](https://github.com/graphprotocol/graph-node/pull/5945)) +- Configurable IPFS retry limits via `GRAPH_IPFS_MAX_ATTEMPTS` and `GRAPH_IPFS_REQUEST_TIMEOUT` ([#5998](https://github.com/graphprotocol/graph-node/pull/5998)) + +### Bug Fixes + +- Fixed VID sequence naming to comply with PostgreSQL 63-character limit +- Fixed pruning of tables with VID sequences using CASCADE drops ([#5968](https://github.com/graphprotocol/graph-node/pull/5968)) +- Fixed numerical precision issues with large VID values ([#5970](https://github.com/graphprotocol/graph-node/pull/5970)) + +**Full changelog:** + ## v0.38.0 ### What's new diff --git a/README.md b/README.md index 118a7c8a846..667c606a045 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ # Graph Node [![Build Status](https://github.com/graphprotocol/graph-node/actions/workflows/ci.yml/badge.svg)](https://github.com/graphprotocol/graph-node/actions/workflows/ci.yml?query=branch%3Amaster) -[![Getting Started Docs](https://img.shields.io/badge/docs-getting--started-brightgreen.svg)](docs/getting-started.md) +[![Docs](https://img.shields.io/badge/docs-graph--node-green.svg)](docs/) +[![Subgraphs](https://img.shields.io/badge/docs-subgraphs-green.svg)](https://thegraph.com/docs/en/subgraphs/quick-start/) ## Overview @@ -29,7 +30,7 @@ This is usually only needed for developers who want to contribute to `graph-node To build and run this project, you need to have the following installed on your system: - Rust (latest stable): Follow [How to install - Rust](https://www.rust-lang.org/en-US/install.html). Run `rustup install + Rust](https://rust-lang.org/tools/install/). Run `rustup install stable` in _this directory_ to make sure all required components are installed. The `graph-node` code assumes that the latest available `stable` compiler is used. @@ -114,6 +115,44 @@ Very large `graph-node` instances can also be configured using a the `graph-node` needs to connect to multiple chains or if the work of indexing and querying needs to be split across [multiple databases](./docs/config.md). +#### Log Storage + +`graph-node` supports storing and querying subgraph logs through multiple backends: + +- **File**: Local JSON Lines files (recommended for local development) +- **Elasticsearch**: Enterprise-grade search and analytics (for production) +- **Loki**: Grafana's log aggregation system (for production) +- **Disabled**: No log storage (default) + +**Quick example (file-based logs for local development):** +```bash +mkdir -p ./graph-logs + +cargo run -p graph-node --release -- \ + --postgres-url $POSTGRES_URL \ + --ethereum-rpc mainnet:archive:https://... \ + --ipfs 127.0.0.1:5001 \ + --log-store-backend file \ + --log-store-file-dir ./graph-logs +``` + +Logs are queried via GraphQL at `http://localhost:8000/graphql`: +```graphql +query { + _logs(subgraphId: "QmYourSubgraphHash", level: ERROR, first: 10) { + timestamp + level + text + } +} +``` + +**For complete documentation**, see the **[Log Store Guide](./docs/log-store.md)**, which covers: +- How to configure each backend (Elasticsearch, Loki, File) +- Complete GraphQL query examples +- Choosing the right backend for your use case +- Performance considerations and best practices + ## Contributing Please check [CONTRIBUTING.md](CONTRIBUTING.md) for development flow and conventions we use. diff --git a/chain/common/src/lib.rs b/chain/common/src/lib.rs index b8f2ae47eb4..395dd440c84 100644 --- a/chain/common/src/lib.rs +++ b/chain/common/src/lib.rs @@ -2,13 +2,13 @@ use std::collections::HashMap; use std::fmt::Debug; use anyhow::Error; -use protobuf::descriptor::field_descriptor_proto::Label; -use protobuf::descriptor::field_descriptor_proto::Type; +use protobuf::Message; +use protobuf::UnknownValueRef; use protobuf::descriptor::DescriptorProto; use protobuf::descriptor::FieldDescriptorProto; use protobuf::descriptor::OneofDescriptorProto; -use protobuf::Message; -use protobuf::UnknownValueRef; +use protobuf::descriptor::field_descriptor_proto::Label; +use protobuf::descriptor::field_descriptor_proto::Type; use std::convert::From; use std::path::Path; diff --git a/chain/common/tests/test-acme.rs b/chain/common/tests/test-acme.rs index cd0b32b8582..2945fea337e 100644 --- a/chain/common/tests/test-acme.rs +++ b/chain/common/tests/test-acme.rs @@ -7,8 +7,8 @@ fn check_repeated_type_ok() { let types = parse_proto_file(PROTO_FILE).expect("Unable to read proto file!"); let array_types = types - .iter() - .flat_map(|(_, t)| t.fields.iter()) + .values() + .flat_map(|t| t.fields.iter()) .filter(|t| t.is_array) .map(|t| t.type_name.clone()) .collect::>(); diff --git a/chain/ethereum/Cargo.toml b/chain/ethereum/Cargo.toml index 4caaef9b668..a9c9d42a124 100644 --- a/chain/ethereum/Cargo.toml +++ b/chain/ethereum/Cargo.toml @@ -12,15 +12,15 @@ serde = { workspace = true } prost = { workspace = true } prost-types = { workspace = true } anyhow = "1.0" -tiny-keccak = "1.5.0" hex = "0.4.3" -semver = "1.0.27" +semver = { workspace = true } thiserror = { workspace = true } +futures = { workspace = true } tokio = { workspace = true } tokio-stream = { workspace = true } tower = { workspace = true } -itertools = "0.14.0" +itertools = "0.15.0" graph-runtime-wasm = { path = "../../runtime/wasm" } graph-runtime-derive = { path = "../../runtime/derive" } diff --git a/chain/ethereum/examples/firehose.rs b/chain/ethereum/examples/firehose.rs index 45199a0ec89..3dd21859653 100644 --- a/chain/ethereum/examples/firehose.rs +++ b/chain/ethereum/examples/firehose.rs @@ -4,7 +4,7 @@ use graph::{ env::env_var, firehose::{self, FirehoseEndpoint, SubgraphLimit}, log::logger, - prelude::{prost, tokio, tonic, MetricsRegistry}, + prelude::{MetricsRegistry, prost, tokio, tonic}, }; use graph_chain_ethereum::codec; use hex::ToHex; diff --git a/chain/ethereum/src/adapter.rs b/chain/ethereum/src/adapter.rs index d161462b933..d37d5ba81f7 100644 --- a/chain/ethereum/src/adapter.rs +++ b/chain/ethereum/src/adapter.rs @@ -9,6 +9,7 @@ use graph::data_source::common::ContractCall; use graph::firehose::CallToFilter; use graph::firehose::CombinedFilter; use graph::firehose::LogFilter; +use graph::prelude::alloy::primitives::keccak256; use graph::prelude::alloy::primitives::{Address, B256}; use graph::prelude::alloy::rpc::types::Log; use graph::prelude::alloy::transports::{RpcError, TransportErrorKind}; @@ -18,8 +19,8 @@ use prost_types::Any; use std::cmp; use std::collections::{HashMap, HashSet}; use std::fmt; +use std::hash::Hash; use thiserror::Error; -use tiny_keccak::keccak256; use graph::prelude::*; use graph::{ @@ -35,7 +36,7 @@ const COMBINED_FILTER_TYPE_URL: &str = use crate::capabilities::NodeCapabilities; use crate::data_source::{BlockHandlerFilter, DataSource}; -use crate::{Chain, Mapping, ENV_VARS}; +use crate::{Chain, ENV_VARS, Mapping}; pub type EventSignature = B256; pub type FunctionSelector = [u8; 4]; @@ -351,20 +352,118 @@ pub struct EthereumLogFilter { /// Log filters can be represented as a bipartite graph between contracts and events. An edge /// exists between a contract and an event if a data source for the contract has a trigger for /// the event. - /// Edges are of `bool` type and indicates when a trigger requires a transaction receipt. - contracts_and_events_graph: GraphMap, + /// Edge weights are booleans indicating whether the trigger requires a transaction receipt. + contracts_and_events_graph: MergeGraph, /// Event sigs with no associated address, matching on all addresses. - /// Maps to a boolean representing if a trigger requires a transaction receipt. - wildcard_events: HashMap, - /// Events with any of the topic filters set - /// Maps to a boolean representing if a trigger requires a transaction receipt. - events_with_topic_filters: HashMap, + /// Values are booleans indicating whether the trigger requires a transaction receipt. + wildcard_events: MergeMap, + /// Events with any of the topic filters set. + /// Values are booleans indicating whether the trigger requires a transaction receipt. + events_with_topic_filters: MergeMap, +} + +/// `HashMap` wrapper whose values are OR-merged on every write. +/// +/// The only mutator that writes values is [`MergeMap::or_insert`] — the inner +/// `HashMap` is private so callers cannot bypass the merge via +/// `HashMap::insert`. Used by `EthereumLogFilter` to track per-key receipt +/// requirements where any handler asking for a receipt at a given key forces +/// receipt fetching. +#[derive(Clone, Debug)] +struct MergeMap(HashMap); + +impl Default for MergeMap { + fn default() -> Self { + Self(HashMap::new()) + } +} + +impl MergeMap { + fn or_insert(&mut self, k: K, v: bool) { + self.0.entry(k).and_modify(|e| *e |= v).or_insert(v); + } + + fn get(&self, k: &K) -> Option<&bool> { + self.0.get(k) + } + + fn contains_key(&self, k: &K) -> bool { + self.0.contains_key(k) + } + + fn iter(&self) -> impl Iterator + '_ { + self.0.iter() + } + + fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +impl IntoIterator for MergeMap { + type Item = (K, bool); + type IntoIter = std::collections::hash_map::IntoIter; + fn into_iter(self) -> Self::IntoIter { + self.0.into_iter() + } +} + +/// `GraphMap` wrapper that OR-merges edge weights on +/// every write. +/// +/// The only mutator that writes edge weights is [`MergeGraph::or_add_edge`] — +/// the inner `GraphMap` is private so callers cannot bypass the merge via +/// `GraphMap::add_edge`. +#[derive(Clone, Debug)] +struct MergeGraph(GraphMap); + +impl Default for MergeGraph { + fn default() -> Self { + Self(GraphMap::new()) + } +} + +impl MergeGraph { + fn or_add_edge(&mut self, a: LogFilterNode, b: LogFilterNode, v: bool) { + // Short-circuit on `v == true`: a `true` weight always wins over any + // prior weight, so we can skip the edge_weight lookup entirely. + let merged = v || self.0.edge_weight(a, b).copied().unwrap_or(false); + self.0.add_edge(a, b, merged); + } + + fn edge_weight(&self, a: LogFilterNode, b: LogFilterNode) -> Option<&bool> { + self.0.edge_weight(a, b) + } + + fn contains_edge(&self, a: LogFilterNode, b: LogFilterNode) -> bool { + self.0.contains_edge(a, b) + } + + fn edge_count(&self) -> usize { + self.0.edge_count() + } + + fn nodes(&self) -> impl Iterator + '_ { + self.0.nodes() + } + + fn neighbors(&self, n: LogFilterNode) -> impl Iterator + '_ { + self.0.neighbors(n) + } + + fn remove_node(&mut self, n: LogFilterNode) -> bool { + self.0.remove_node(n) + } + + fn all_edges(&self) -> impl Iterator { + self.0.all_edges() + } } impl From for Vec { fn from(val: EthereumLogFilter) -> Self { - val.eth_get_logs_filters() + val.eth_get_logs_filters(ENV_VARS.get_logs_max_contracts) .map( |EthGetLogsFilter { contracts, @@ -399,8 +498,7 @@ impl EthereumLogFilter { let contract = LogFilterNode::Contract(log.address()); let event = LogFilterNode::Event(*sig); self.contracts_and_events_graph - .all_edges() - .any(|(s, t, _)| (s == contract && t == event) || (t == contract && s == event)) + .contains_edge(contract, event) || self.wildcard_events.contains_key(sig) || self .events_with_topic_filters @@ -439,13 +537,12 @@ impl EthereumLogFilter { let contract_node = LogFilterNode::Contract(*address); let event_node = LogFilterNode::Event(*event_signature); - // Directly iterate over all edges and return true if a matching edge that requires a receipt is found. - for (s, t, &r) in self.contracts_and_events_graph.all_edges() { - if r && ((s == contract_node && t == event_node) - || (t == contract_node && s == event_node)) - { - return true; - } + if self + .contracts_and_events_graph + .edge_weight(contract_node, event_node) + == Some(&true) + { + return true; } } @@ -460,14 +557,14 @@ impl EthereumLogFilter { let event_sig = event_handler.topic0(); match ds.address { Some(contract) if !event_handler.has_additional_topics() => { - this.contracts_and_events_graph.add_edge( + this.contracts_and_events_graph.or_add_edge( LogFilterNode::Contract(contract), LogFilterNode::Event(event_sig), event_handler.receipt, ); } Some(contract) => { - this.events_with_topic_filters.insert( + this.events_with_topic_filters.or_insert( EventSignatureWithTopics::new( Some(contract), event_sig, @@ -481,11 +578,11 @@ impl EthereumLogFilter { None if (!event_handler.has_additional_topics()) => { this.wildcard_events - .insert(event_sig, event_handler.receipt); + .or_insert(event_sig, event_handler.receipt); } None => { - this.events_with_topic_filters.insert( + this.events_with_topic_filters.or_insert( EventSignatureWithTopics::new( ds.address, event_sig, @@ -507,12 +604,18 @@ impl EthereumLogFilter { for event_handler in &mapping.event_handlers { let signature = event_handler.topic0(); this.wildcard_events - .insert(signature, event_handler.receipt); + .or_insert(signature, event_handler.receipt); } this } /// Extends this log filter with another one. + /// + /// The `receipt` flag stored at each filter key is OR-combined across the + /// two filters: a key whose receipt requirement is `true` in either filter + /// stays `true` in the merged result. Overwriting with the incoming value + /// would drop a prior `true` and silently downgrade receipt fetching for + /// any handler that declared `receipt: true` at that key. pub fn extend(&mut self, other: EthereumLogFilter) { if other.is_empty() { return; @@ -525,11 +628,14 @@ impl EthereumLogFilter { events_with_topic_filters, } = other; for (s, t, e) in contracts_and_events_graph.all_edges() { - self.contracts_and_events_graph.add_edge(s, t, *e); + self.contracts_and_events_graph.or_add_edge(s, t, *e); + } + for (k, v) in wildcard_events { + self.wildcard_events.or_insert(k, v); + } + for (k, v) in events_with_topic_filters { + self.events_with_topic_filters.or_insert(k, v); } - self.wildcard_events.extend(wildcard_events); - self.events_with_topic_filters - .extend(events_with_topic_filters); } /// An empty filter is one that never matches. @@ -548,21 +654,24 @@ impl EthereumLogFilter { /// Filters for `eth_getLogs` calls. The filters will not return false positives. This attempts /// to balance between having granular filters but too many calls and having few calls but too /// broad filters causing the Ethereum endpoint to timeout. - pub fn eth_get_logs_filters(self) -> impl Iterator { + pub fn eth_get_logs_filters( + self, + get_logs_max_contracts: usize, + ) -> impl Iterator { let mut filters = Vec::new(); // Start with the wildcard event filters. filters.extend( self.wildcard_events - .into_keys() - .map(EthGetLogsFilter::from_event), + .into_iter() + .map(|(k, _)| EthGetLogsFilter::from_event(k)), ); // Handle events with topic filters. filters.extend( self.events_with_topic_filters - .into_keys() - .map(EthGetLogsFilter::from_event_with_topics), + .into_iter() + .map(|(k, _)| EthGetLogsFilter::from_event_with_topics(k)), ); // The current algorithm is to repeatedly find the maximum cardinality vertex and turn all @@ -598,7 +707,7 @@ impl EthereumLogFilter { for neighbor in g.neighbors(max_vertex) { match neighbor { LogFilterNode::Contract(address) => { - if filter.contracts.len() == ENV_VARS.get_logs_max_contracts { + if filter.contracts.len() == get_logs_max_contracts { // The batch size was reached, register the filter and start a new one. let event = filter.event_signatures[0]; push_filter(filter); @@ -1176,11 +1285,16 @@ pub trait EthereumAdapter: Send + Sync + 'static { address: Address, block_ptr: BlockPtr, ) -> Result; + + /// Returns a boolean indicating whether the adapter can reach + /// the RPC provider it is configured to use. + /// This is used to determine if a provider should be considered healthy. + async fn is_reachable(&self) -> bool; } #[cfg(test)] mod tests { - use crate::adapter::{FunctionSelector, COMBINED_FILTER_TYPE_URL}; + use crate::adapter::{COMBINED_FILTER_TYPE_URL, FunctionSelector}; use super::{EthereumBlockFilter, LogFilterNode}; use super::{EthereumCallFilter, EthereumLogFilter, TriggerFilter}; @@ -1188,9 +1302,8 @@ mod tests { use base64::prelude::*; use graph::blockchain::TriggerFilter as _; use graph::firehose::{CallToFilter, CombinedFilter, LogFilter, MultiLogFilter}; - use graph::petgraph::graphmap::GraphMap; - use graph::prelude::alloy::primitives::{Address, Bytes, B256, U256}; use graph::prelude::EthereumCall; + use graph::prelude::alloy::primitives::{Address, B256, Bytes, U256}; use hex::ToHex; use itertools::Itertools; use prost::Message; @@ -1258,9 +1371,11 @@ mod tests { assert_eq!(sig, actual_sig); let filter = LogFilter { - addresses: vec![Address::from_str(hex_addr) - .expect("failed to parse address") - .to_vec()], + addresses: vec![ + Address::from_str(hex_addr) + .expect("failed to parse address") + .to_vec(), + ], event_signatures: vec![fs.to_vec()], }; @@ -1276,11 +1391,7 @@ mod tests { fn ethereum_trigger_filter_to_firehose() { let sig = |value: u64| B256::from(U256::from(value)); let mut filter = TriggerFilter { - log: EthereumLogFilter { - contracts_and_events_graph: GraphMap::new(), - wildcard_events: HashMap::new(), - events_with_topic_filters: HashMap::new(), - }, + log: EthereumLogFilter::default(), call: EthereumCallFilter { contract_addresses_function_signatures: HashMap::from_iter(vec![ (address(0), (0, HashSet::from_iter(vec![[0u8; 4]]))), @@ -1329,17 +1440,17 @@ mod tests { }, ]; - filter.log.contracts_and_events_graph.add_edge( + filter.log.contracts_and_events_graph.or_add_edge( LogFilterNode::Contract(address(10)), LogFilterNode::Event(sig(100)), false, ); - filter.log.contracts_and_events_graph.add_edge( + filter.log.contracts_and_events_graph.or_add_edge( LogFilterNode::Contract(address(10)), LogFilterNode::Event(sig(101)), false, ); - filter.log.contracts_and_events_graph.add_edge( + filter.log.contracts_and_events_graph.or_add_edge( LogFilterNode::Contract(address(20)), LogFilterNode::Event(sig(100)), false, @@ -1399,11 +1510,7 @@ mod tests { let address = |value: u64| Address::left_padding_from(&value.to_le_bytes()); let sig = |value: u64| B256::left_padding_from(&value.to_le_bytes()); let mut filter = TriggerFilter { - log: EthereumLogFilter { - contracts_and_events_graph: GraphMap::new(), - wildcard_events: HashMap::new(), - events_with_topic_filters: HashMap::new(), - }, + log: EthereumLogFilter::default(), call: EthereumCallFilter { contract_addresses_function_signatures: HashMap::new(), wildcard_signatures: HashSet::new(), @@ -1415,7 +1522,7 @@ mod tests { }, }; - filter.log.contracts_and_events_graph.add_edge( + filter.log.contracts_and_events_graph.or_add_edge( LogFilterNode::Contract(address(10)), LogFilterNode::Event(sig(101)), false, @@ -1645,8 +1752,8 @@ mod tests { } #[test] - fn extending_ethereum_block_filter_every_block_in_base_and_merge_contract_addresses_and_polling_intervals( - ) { + fn extending_ethereum_block_filter_every_block_in_base_and_merge_contract_addresses_and_polling_intervals() + { let mut base = EthereumBlockFilter { polling_intervals: HashSet::from_iter(vec![(10, 3)]), contract_addresses: HashSet::from_iter(vec![(10, address(2))]), @@ -1751,10 +1858,10 @@ fn complete_log_filter() { let contracts: BTreeSet<_> = (0..j).map(|n| Address::from([n as u8; 20])).collect(); // Construct the complete bipartite graph with i events and j contracts. - let mut contracts_and_events_graph = GraphMap::new(); + let mut filter = EthereumLogFilter::default(); for &contract in &contracts { for &event in &events { - contracts_and_events_graph.add_edge( + filter.contracts_and_events_graph.or_add_edge( LogFilterNode::Contract(contract), LogFilterNode::Event(event), false, @@ -1763,13 +1870,9 @@ fn complete_log_filter() { } // Run `eth_get_logs_filters`, which is what we want to test. - let logs_filters: Vec<_> = EthereumLogFilter { - contracts_and_events_graph, - wildcard_events: HashMap::new(), - events_with_topic_filters: HashMap::new(), - } - .eth_get_logs_filters() - .collect(); + let logs_filters: Vec<_> = filter + .eth_get_logs_filters(ENV_VARS.get_logs_max_contracts) + .collect(); // Assert that a contract or event is filtered on iff it was present in the graph. assert_eq!( @@ -1834,14 +1937,6 @@ fn log_filter_require_transacion_receipt_method() { let wildcard_event_with_receipt = b256(6); let wildcard_event_without_receipt = b256(7); - let wildcard_events = [ - (wildcard_event_with_receipt, true), - (wildcard_event_without_receipt, false), - ] - .into_iter() - .collect(); - - let events_with_topic_filters = HashMap::new(); // TODO(krishna): Test events with topic filters let alien_event_signature = b256(8); // those will not be inserted in the graph let alien_contract_address = address(9); @@ -1869,25 +1964,28 @@ fn log_filter_require_transacion_receipt_method() { // event_b -- contract_a [ receipt=false ] // } // ``` - let mut contracts_and_events_graph = GraphMap::new(); - - let event_a_id = contracts_and_events_graph.add_node(event_a_node); - let event_b_id = contracts_and_events_graph.add_node(event_b_node); - let event_c_id = contracts_and_events_graph.add_node(event_c_node); - let contract_a_id = contracts_and_events_graph.add_node(contract_a_node); - let contract_b_id = contracts_and_events_graph.add_node(contract_b_node); - let contract_c_id = contracts_and_events_graph.add_node(contract_c_node); - contracts_and_events_graph.add_edge(event_a_id, contract_a_id, true); - contracts_and_events_graph.add_edge(event_b_id, contract_b_id, true); - contracts_and_events_graph.add_edge(event_a_id, contract_b_id, false); - contracts_and_events_graph.add_edge(event_b_id, contract_a_id, false); - contracts_and_events_graph.add_edge(event_c_id, contract_c_id, true); - - let filter = EthereumLogFilter { - contracts_and_events_graph, - wildcard_events, - events_with_topic_filters, - }; + let mut filter = EthereumLogFilter::default(); + filter + .contracts_and_events_graph + .or_add_edge(event_a_node, contract_a_node, true); + filter + .contracts_and_events_graph + .or_add_edge(event_b_node, contract_b_node, true); + filter + .contracts_and_events_graph + .or_add_edge(event_a_node, contract_b_node, false); + filter + .contracts_and_events_graph + .or_add_edge(event_b_node, contract_a_node, false); + filter + .contracts_and_events_graph + .or_add_edge(event_c_node, contract_c_node, true); + filter + .wildcard_events + .or_insert(wildcard_event_with_receipt, true); + filter + .wildcard_events + .or_insert(wildcard_event_without_receipt, false); let empty_vec: Vec = vec![]; @@ -1974,3 +2072,268 @@ fn log_filter_require_transacion_receipt_method() { &empty_vec )); } + +// Tests that `EthereumLogFilter` OR-merges per-handler `receipt` flags across +// every insertion site (`from_data_sources`, `from_mapping`, `extend`). + +#[cfg(test)] +fn receipt_merge_test_addr(n: u64) -> Address { + Address::left_padding_from(&n.to_be_bytes()) +} + +#[cfg(test)] +fn receipt_merge_test_sig(n: u64) -> B256 { + B256::left_padding_from(&n.to_be_bytes()) +} + +#[cfg(test)] +fn receipt_merge_test_mock_abi() -> std::sync::Arc { + std::sync::Arc::new(graph::data_source::common::MappingABI { + name: "mock_abi".to_string(), + contract: abi::JsonAbi::new(), + }) +} + +#[cfg(test)] +fn receipt_merge_test_event_handler( + sig: B256, + topic1: Option>, + topic2: Option>, + topic3: Option>, + receipt: bool, +) -> crate::data_source::MappingEventHandler { + crate::data_source::MappingEventHandler { + event: "Event()".to_string(), + topic0: Some(sig), + topic1, + topic2, + topic3, + handler: "handleEvent".to_string(), + receipt, + calls: graph::data_source::common::CallDecls::default(), + } +} + +#[cfg(test)] +fn receipt_merge_test_mapping( + handlers: Vec, +) -> crate::Mapping { + crate::Mapping { + kind: "ethereum/events".to_string(), + api_version: semver::Version::new(0, 0, 7), + language: "wasm/assemblyscript".to_string(), + entities: vec![], + abis: vec![receipt_merge_test_mock_abi()], + block_handlers: vec![], + call_handlers: vec![], + event_handlers: handlers, + runtime: std::sync::Arc::new(vec![]), + link: graph::prelude::Link { + link: "test".to_string(), + }, + } +} + +#[cfg(test)] +fn receipt_merge_test_data_source( + address: Option
, + handlers: Vec, +) -> crate::data_source::DataSource { + crate::data_source::DataSource { + kind: "ethereum/contract".to_string(), + network: Some("test".to_string()), + name: "Test".to_string(), + manifest_idx: 0, + address, + start_block: 0, + end_block: None, + mapping: receipt_merge_test_mapping(handlers), + context: std::sync::Arc::new(None), + creation_block: None, + contract_abi: receipt_merge_test_mock_abi(), + } +} + +/// Run two data sources with `receipt: true` and `receipt: false` at the +/// same effective filter key through `from_data_sources` and assert the +/// merged filter still requires a transaction receipt. Runs both +/// declaration orders so order-independence is verified per variant. +#[cfg(test)] +fn assert_from_data_sources_or_merges( + label: &str, + address: Option
, + topic1: Option>, + log_topics: &[B256], +) { + let event_sig = receipt_merge_test_sig(100); + let ds_yes = receipt_merge_test_data_source( + address, + vec![receipt_merge_test_event_handler( + event_sig, + topic1.clone(), + None, + None, + true, + )], + ); + let ds_no = receipt_merge_test_data_source( + address, + vec![receipt_merge_test_event_handler( + event_sig, + topic1.clone(), + None, + None, + false, + )], + ); + + for (order, dss) in [("yes,no", [&ds_yes, &ds_no]), ("no,yes", [&ds_no, &ds_yes])] { + let filter = EthereumLogFilter::from_data_sources(dss); + assert!( + filter.requires_transaction_receipt(&event_sig, address.as_ref(), log_topics), + "{label} ({order}): receipt:true must survive a later receipt:false", + ); + } +} + +/// Build two filters via `from_data_sources`, each with one handler at the +/// same effective key but with opposite receipt flags, then merge them via +/// `extend` and assert the merged filter still requires a transaction +/// receipt. Runs both extend directions so order-independence is verified +/// per variant. +#[cfg(test)] +fn assert_extend_or_merges( + label: &str, + address: Option
, + topic1: Option>, + log_topics: &[B256], +) { + let event_sig = receipt_merge_test_sig(105); + let ds_yes = receipt_merge_test_data_source( + address, + vec![receipt_merge_test_event_handler( + event_sig, + topic1.clone(), + None, + None, + true, + )], + ); + let ds_no = receipt_merge_test_data_source( + address, + vec![receipt_merge_test_event_handler( + event_sig, + topic1.clone(), + None, + None, + false, + )], + ); + + for (order, base, ext) in [ + ("yes.extend(no)", &ds_yes, &ds_no), + ("no.extend(yes)", &ds_no, &ds_yes), + ] { + let mut filter = EthereumLogFilter::from_data_sources([base]); + filter.extend(EthereumLogFilter::from_data_sources([ext])); + assert!( + filter.requires_transaction_receipt(&event_sig, address.as_ref(), log_topics), + "{label} ({order}): extend must OR-merge", + ); + } +} + +#[test] +fn from_data_sources_or_merges_at_every_insertion_site() { + let contract = receipt_merge_test_addr(1); + let event_sig = receipt_merge_test_sig(100); + let topic = receipt_merge_test_sig(200); + let with_topic = vec![event_sig, topic]; + + // Each case maps to one arm of the `match ds.address` in `from_data_sources`. + assert_from_data_sources_or_merges("graph edge", Some(contract), None, &[]); + assert_from_data_sources_or_merges( + "addressed topics", + Some(contract), + Some(vec![topic]), + &with_topic, + ); + assert_from_data_sources_or_merges("wildcard", None, None, &[]); + assert_from_data_sources_or_merges("wildcard topics", None, Some(vec![topic]), &with_topic); +} + +#[test] +fn from_mapping_or_merges_via_extend() { + // Two templates handling the same event signature with different receipt + // flags merged via `from_mapping` + `extend`. + let event_sig = receipt_merge_test_sig(104); + + let mapping_yes = receipt_merge_test_mapping(vec![receipt_merge_test_event_handler( + event_sig, None, None, None, true, + )]); + let mapping_no = receipt_merge_test_mapping(vec![receipt_merge_test_event_handler( + event_sig, None, None, None, false, + )]); + + let mut filter = EthereumLogFilter::from_mapping(&mapping_yes); + filter.extend(EthereumLogFilter::from_mapping(&mapping_no)); + + assert!(filter.requires_transaction_receipt(&event_sig, None, &[])); +} + +#[test] +fn extend_or_merges_at_every_collection() { + let contract = receipt_merge_test_addr(5); + let event_sig = receipt_merge_test_sig(105); + let topic = receipt_merge_test_sig(203); + let with_topic = vec![event_sig, topic]; + + assert_extend_or_merges("graph edge", Some(contract), None, &[]); + assert_extend_or_merges( + "addressed topics", + Some(contract), + Some(vec![topic]), + &with_topic, + ); + assert_extend_or_merges("wildcard", None, None, &[]); + assert_extend_or_merges("wildcard topics", None, Some(vec![topic]), &with_topic); +} + +#[test] +fn requires_transaction_receipt_has_or_semantics_across_handlers() { + // `requires_transaction_receipt` must return true iff at least one handler + // at the key declared `receipt: true`, regardless of declaration order. + let event_sig = receipt_merge_test_sig(108); + + for flags in [ + vec![false], + vec![true], + vec![true, false], + vec![false, true], + vec![false, false, true], + vec![true, false, false], + vec![false, true, false], + vec![true, true, false], + vec![false, false, false], + ] { + let dss: Vec<_> = flags + .iter() + .map(|&r| { + receipt_merge_test_data_source( + None, + vec![receipt_merge_test_event_handler( + event_sig, None, None, None, r, + )], + ) + }) + .collect(); + let filter = EthereumLogFilter::from_data_sources(dss.iter()); + + let any_requires_receipt = flags.iter().any(|&r| r); + assert_eq!( + filter.requires_transaction_receipt(&event_sig, None, &[]), + any_requires_receipt, + "flag sequence {flags:?}", + ); + } +} diff --git a/chain/ethereum/src/buffered_call_cache.rs b/chain/ethereum/src/buffered_call_cache.rs index 2ec0bfa40e7..29aa84e90ab 100644 --- a/chain/ethereum/src/buffered_call_cache.rs +++ b/chain/ethereum/src/buffered_call_cache.rs @@ -9,7 +9,7 @@ use graph::{ components::store::EthereumCallCache, data::store::ethereum::call, prelude::{BlockPtr, CachedEthereumCall}, - slog::{error, Logger}, + slog::{Logger, error}, }; /// A wrapper around an Ethereum call cache that buffers call results in diff --git a/chain/ethereum/src/call_helper.rs b/chain/ethereum/src/call_helper.rs index 5fc6ed80fd0..8a1216b58c5 100644 --- a/chain/ethereum/src/call_helper.rs +++ b/chain/ethereum/src/call_helper.rs @@ -3,8 +3,9 @@ use graph::{ abi, data::store::ethereum::call, prelude::{ + Logger, alloy::transports::{RpcError, TransportErrorKind}, - serde_json, Logger, + serde_json, }, slog::info, }; @@ -36,10 +37,10 @@ const PARITY_REVERT_PREFIX: &str = "revert"; const XDAI_REVERT: &str = "revert"; -// Deterministic Geth execution errors. We might need to expand this as +// Deterministic RPC execution errors. We might need to expand this as // subgraphs come across other errors. See // https://github.com/ethereum/go-ethereum/blob/cd57d5cd38ef692de8fbedaa56598b4e9fbfbabc/core/vm/errors.go -const GETH_EXECUTION_ERRORS: &[&str] = &[ +const RPC_EXECUTION_ERRORS: &[&str] = &[ // The "revert" substring covers a few known error messages, including: // Hardhat: "error: transaction reverted", // Ganache and Moonbeam: "vm exception while processing transaction: revert", @@ -53,21 +54,36 @@ const GETH_EXECUTION_ERRORS: &[&str] = &[ // See f0af4ab0-6b7c-4b68-9141-5b79346a5f61 for why the gas limit is considered deterministic. "out of gas", "stack underflow", + "vm execution error", + "invalidjump", + "notactivated", + "invalidfeopcode", + // Reth surfaces EVM halts via `EvmHalt(HaltReason)`, formatted with the + // reason's `Debug` repr (`"EVM error: {0:?}"`). revm's `HaltReason` + // variants are CamelCase with no spaces, so e.g. a stack underflow arrives + // as "EVM error: StackUnderflow", which the space-separated "stack + // underflow" above does not match. "invalidjump"/"invalidfeopcode" already + // cover the matching variants; these add the rest. Reth's OutOfGas is + // handled before EvmHalt and rendered as "out of gas: ...", so it is + // already covered above. See https://github.com/streamingfast/eth-go/pull/10. + "stackunderflow", + "stackoverflow", + "opcodenotfound", ]; -/// Helper that checks if a geth style RPC error message corresponds to a revert. -fn is_geth_revert_message(message: &str) -> bool { - let env_geth_call_errors = ENV_VARS.geth_eth_call_errors.iter(); - let mut execution_errors = GETH_EXECUTION_ERRORS +/// Helper that checks if a RPC error message corresponds to a revert. +fn is_rpc_revert_message(message: &str) -> bool { + let env_rpc_call_errors = ENV_VARS.rpc_eth_call_errors.iter(); + let mut execution_errors = RPC_EXECUTION_ERRORS .iter() .copied() - .chain(env_geth_call_errors.map(|s| s.as_str())); + .chain(env_rpc_call_errors.map(|s| s.as_str())); execution_errors.any(|e| message.to_lowercase().contains(e)) } /// Decode a Solidity revert(reason) payload, returning the reason string when possible. fn as_solidity_revert_reason(bytes: &[u8]) -> Option { - let selector = &tiny_keccak::keccak256(b"Error(string)")[..4]; + let selector = &graph::prelude::alloy::primitives::keccak256(b"Error(string)")[..4]; if bytes.len() >= 4 && &bytes[..4] == selector { abi::DynSolType::String .abi_decode(&bytes[4..]) @@ -89,10 +105,10 @@ pub fn interpret_eth_call_error( Ok(call::Retval::Null) } - if let RpcError::ErrorResp(rpc_error) = &err { - if is_geth_revert_message(&rpc_error.message) { - return reverted(logger, &rpc_error.message); - } + if let RpcError::ErrorResp(rpc_error) = &err + && is_rpc_revert_message(&rpc_error.message) + { + return reverted(logger, &rpc_error.message); } if let RpcError::ErrorResp(rpc_error) = &err { @@ -102,12 +118,11 @@ pub fn interpret_eth_call_error( .as_ref() .and_then(|d| serde_json::from_str(d.get()).ok()); - if code == PARITY_VM_EXECUTION_ERROR { - if let Some(data) = data { - if is_parity_revert(&data) { - return reverted(logger, &parity_revert_reason(&data)); - } - } + if code == PARITY_VM_EXECUTION_ERROR + && let Some(data) = data + && is_parity_revert(&data) + { + return reverted(logger, &parity_revert_reason(&data)); } } diff --git a/chain/ethereum/src/chain.rs b/chain/ethereum/src/chain.rs index 85103cc5498..405c3123bdc 100644 --- a/chain/ethereum/src/chain.rs +++ b/chain/ethereum/src/chain.rs @@ -1,5 +1,5 @@ -use anyhow::{anyhow, bail, Result}; use anyhow::{Context, Error}; +use anyhow::{Result, anyhow, bail}; use async_trait::async_trait; use graph::blockchain::client::ChainClient; use graph::blockchain::firehose_block_ingestor::{FirehoseBlockIngestor, Transforms}; @@ -13,26 +13,27 @@ use graph::data::subgraph::UnifiedMappingApiVersion; use graph::firehose::{FirehoseEndpoint, FirehoseEndpoints, ForkStep}; use graph::futures03::TryStreamExt; use graph::prelude::{ - retry, BlockHash, ComponentLoggerConfig, ElasticComponentLoggerConfig, EthereumBlock, + BlockHash, ComponentLoggerConfig, ElasticComponentLoggerConfig, EthereumBlock, EthereumCallCache, LightEthereumBlock, LightEthereumBlockExt, MetricsRegistry, StoreError, + retry, }; use graph::slog::{debug, error, trace, warn}; use graph::{ blockchain::{ + Block, BlockPtr, Blockchain, ChainHeadUpdateListener, IngestorError, + RuntimeAdapter as RuntimeAdapterTrait, TriggerFilter as _, block_stream::{ BlockRefetcher, BlockStreamEvent, BlockWithTriggers, FirehoseError, FirehoseMapper as FirehoseMapperTrait, TriggersAdapter as TriggersAdapterTrait, }, firehose_block_stream::FirehoseBlockStream, - Block, BlockPtr, Blockchain, ChainHeadUpdateListener, IngestorError, - RuntimeAdapter as RuntimeAdapterTrait, TriggerFilter as _, }, cheap_clone::CheapClone, components::store::DeploymentLocator, firehose, prelude::{ - o, serde_json as json, BlockNumber, ChainStore, EthereumBlockWithCalls, Logger, - LoggerFactory, + BlockNumber, ChainStore, EthereumBlockWithCalls, Logger, LoggerFactory, o, + serde_json as json, }, }; use prost::Message; @@ -49,7 +50,9 @@ use crate::ingestor::PollingBlockIngestor; use crate::network::EthereumNetworkAdapters; use crate::polling_block_stream::PollingBlockStream; use crate::runtime::runtime_adapter::eth_call_gas; +use crate::{BufferedCallCache, NodeCapabilities}; use crate::{ + ENV_VARS, SubgraphEthRpcMetrics, TriggerFilter, adapter::EthereumAdapter as _, codec, data_source::{DataSource, UnresolvedDataSource}, @@ -57,18 +60,53 @@ use crate::{ blocks_with_triggers, get_calls, parse_block_triggers, parse_call_triggers, parse_log_triggers, }, - SubgraphEthRpcMetrics, TriggerFilter, ENV_VARS, }; -use crate::{BufferedCallCache, NodeCapabilities}; use crate::{EthereumAdapter, RuntimeAdapter}; use graph::blockchain::block_stream::{ BlockStream, BlockStreamBuilder, BlockStreamError, BlockStreamMapper, FirehoseCursor, TriggersAdapterWrapper, }; - /// Celo Mainnet: 42220, Testnet Alfajores: 44787, Testnet Baklava: 62320 const CELO_CHAIN_IDS: [u64; 3] = [42220, 44787, 62320]; +/// Resolved per-chain settings. Populated at chain initialisation from the config file (with +/// ENV_VAR fallbacks) and stored on [`Chain`] and [`crate::EthereumAdapter`]. +#[derive(Clone, Debug)] +pub struct ChainSettings { + pub polling_interval: Duration, + pub json_rpc_timeout: Duration, + pub request_retries: usize, + pub max_block_range_size: BlockNumber, + pub block_batch_size: usize, + pub block_ptr_batch_size: usize, + pub max_event_only_range: BlockNumber, + pub target_triggers_per_block_range: u64, + pub get_logs_max_contracts: usize, + pub block_ingestor_max_concurrent_json_rpc_calls: usize, + pub genesis_block_number: u64, +} + +impl ChainSettings { + /// Constructs a [`ChainSettings`] from environment variable defaults. + /// Used in tests and for firehose-only chains that have no RPC config. + pub fn from_env_defaults() -> Self { + ChainSettings { + polling_interval: graph::env::ENV_VARS.ingestor_polling_interval, + json_rpc_timeout: ENV_VARS.json_rpc_timeout, + request_retries: ENV_VARS.request_retries, + max_block_range_size: ENV_VARS.max_block_range_size, + block_batch_size: ENV_VARS.block_batch_size, + block_ptr_batch_size: ENV_VARS.block_ptr_batch_size, + max_event_only_range: ENV_VARS.max_event_only_range, + target_triggers_per_block_range: ENV_VARS.target_triggers_per_block_range, + get_logs_max_contracts: ENV_VARS.get_logs_max_contracts, + block_ingestor_max_concurrent_json_rpc_calls: ENV_VARS + .block_ingestor_max_concurrent_json_rpc_calls, + genesis_block_number: ENV_VARS.genesis_block_number, + } + } +} + pub struct EthereumStreamBuilder {} #[async_trait] @@ -193,9 +231,9 @@ impl BlockStreamBuilder for EthereumStreamBuilder { }; let max_block_range_size = if is_using_subgraph_composition { - ENV_VARS.max_block_range_size * 10 + chain.settings.max_block_range_size * 10 } else { - ENV_VARS.max_block_range_size + chain.settings.max_block_range_size }; Ok(Box::new(PollingBlockStream::new( @@ -207,7 +245,7 @@ impl BlockStreamBuilder for EthereumStreamBuilder { reorg_threshold, logger, max_block_range_size, - ENV_VARS.target_triggers_per_block_range, + chain.settings.target_triggers_per_block_range, unified_api_version, subgraph_current_block, ))) @@ -326,13 +364,13 @@ pub struct Chain { call_cache: Arc, chain_head_update_listener: Arc, reorg_threshold: BlockNumber, - polling_ingestor_interval: Duration, pub is_ingestible: bool, block_stream_builder: Arc>, block_refetcher: Arc>, adapter_selector: Arc>, runtime_adapter_builder: Arc, eth_adapters: Arc, + pub settings: Arc, } impl std::fmt::Debug for Chain { @@ -359,10 +397,10 @@ where for _ in 0..offset { match parent_getter(current_ptr.clone()).await? { Some(parent) => { - if let Some(root_hash) = &root { - if parent.hash == *root_hash { - break; - } + if let Some(root_hash) = &root + && parent.hash == *root_hash + { + break; } current_ptr = parent; } @@ -389,8 +427,8 @@ impl Chain { runtime_adapter_builder: Arc, eth_adapters: Arc, reorg_threshold: BlockNumber, - polling_ingestor_interval: Duration, is_ingestible: bool, + settings: Arc, ) -> Self { Chain { logger_factory, @@ -407,7 +445,7 @@ impl Chain { eth_adapters, reorg_threshold, is_ingestible, - polling_ingestor_interval, + settings, } } @@ -544,6 +582,18 @@ impl Blockchain for Chain { .await .map_err(IngestorError::Unknown), ChainClient::Rpc(adapters) => { + let cached = self + .chain_store + .cheap_clone() + .block_ptrs_by_numbers(vec![number]) + .await + .unwrap_or_default(); + if let Some(ptrs) = cached.get(&number) + && ptrs.len() == 1 + { + return Ok(BlockPtr::new(ptrs[0].hash.clone(), ptrs[0].number)); + } + let adapter = adapters .cheapest() .await @@ -638,7 +688,7 @@ impl Blockchain for Chain { graph::env::ENV_VARS.reorg_threshold(), self.chain_client(), self.chain_store.cheap_clone(), - self.polling_ingestor_interval, + self.settings.polling_interval, self.name.clone(), )?) } @@ -778,8 +828,8 @@ async fn fetch_unique_blocks_from_cache( // Collect blocks and filter out ones with multiple entries let blocks: Vec> = blocks_map - .into_iter() - .filter_map(|(_, values)| { + .into_values() + .filter_map(|values| { if values.len() == 1 { Some(Arc::new(values[0].clone())) } else { @@ -1025,6 +1075,12 @@ impl TriggersAdapterTrait for TriggersAdapter { } async fn is_on_main_chain(&self, ptr: BlockPtr) -> Result { + // It is tempting to use the block cache here; but that can go wrong + // when graph-node gets shut down and some of its nonfinal blocks + // then are reorged; when graph-node gets started again when those + // block numbers have become final, it might consider a reorged + // block as canonical; allowing the use of the block cache here + // would require us to also track the finality of blocks. match &*self.chain_client { ChainClient::Firehose(endpoints) => { let endpoint = endpoints.endpoint().await?; @@ -1075,45 +1131,26 @@ impl TriggersAdapterTrait for TriggersAdapter { .ancestor_block(ptr.clone(), offset, root.clone()) .await?; - // First check if we have the ancestor in cache and can deserialize it. - // recent_blocks_cache can have full format {"block": {...}, "transaction_receipts": [...]} - // or light format (just block fields). We need full format with receipts for - // ancestor_block since it's used for trigger processing. + // Use full blocks (with receipts) directly from cache. + // Light blocks (no receipts) need to be fetched from Firehose/RPC. let block_ptr = match cached { - Some((json, ptr)) => { - if json.get("block").is_none() { - warn!( + Some((cached_block, ptr)) => match cached_block.into_full_block() { + Some(block) => { + return Ok(Some(BlockFinality::NonFinal(EthereumBlockWithCalls { + ethereum_block: block, + calls: None, + }))); + } + None => { + trace!( self.logger, - "Cached ancestor block #{} {} has light format without receipts. \ - Falling back to Firehose/RPC.", + "Cached block #{} {} is light (no receipts). Falling back to Firehose/RPC.", ptr.number, ptr.hash_hex(), ); ptr - } else { - match json::from_value::(json.clone()) { - Ok(block) => { - return Ok(Some(BlockFinality::NonFinal(EthereumBlockWithCalls { - ethereum_block: block, - calls: None, - }))); - } - Err(e) => { - warn!( - self.logger, - "Failed to deserialize cached ancestor block #{} {} (offset {} from #{}): {}. \ - Falling back to Firehose/RPC.", - ptr.number, - ptr.hash_hex(), - offset, - ptr_for_log.number, - e - ); - ptr - } - } } - } + }, None => { // Cache miss - fall back to walking the chain via parent_ptr() calls. // This provides resilience when the block cache is empty (e.g., after truncation). @@ -1168,28 +1205,9 @@ impl TriggersAdapterTrait for TriggersAdapter { let block = match self.chain_client.as_ref() { ChainClient::Firehose(endpoints) => { let chain_store = self.chain_store.cheap_clone(); - // First try to get the block from the store - if let Ok(blocks) = chain_store.blocks(vec![block.hash.clone()]).await { - if let Some(cached_json) = blocks.first() { - // recent_blocks_cache can contain full format {"block": {...}, "transaction_receipts": [...]} - // or light format (just block fields). Extract block data for deserialization. - let inner = cached_json.get("block").unwrap_or(cached_json); - match json::from_value::(inner.clone()) { - Ok(light_block) => { - return Ok(light_block.parent_ptr()); - } - Err(e) => { - warn!( - self.logger, - "Failed to deserialize cached block #{} {}: {}. \ - Falling back to Firehose.", - block.number, - block.hash_hex(), - e - ); - } - } - } + // First try to get the parent pointer from the store header columns + if let Ok(Some(parent)) = chain_store.block_parent_ptr(&block.hash).await { + return Ok(Some(parent)); } // If not in store, fetch from Firehose @@ -1261,16 +1279,14 @@ impl TriggersAdapter { adapters: &EthereumNetworkAdapters, block_ptr: &BlockPtr, ) -> Result, Error> { - let adapter = adapters.cheapest_with(&self.capabilities).await?; - - let block = adapter - .block_by_hash(&self.logger, block_ptr.hash.as_b256()) - .await?; - - match block { - Some(block) => { + // Use the cache-aware light block fetch first; it checks recent_blocks_cache + // and the DB before falling back to eth_getBlockByHash. + let light_block = self.fetch_light_block_with_rpc(adapters, block_ptr).await?; + match light_block { + Some(light_block) => { + let adapter = adapters.cheapest_with(&self.capabilities).await?; let ethereum_block = adapter - .load_full_block(&self.logger, block) + .load_full_block(&self.logger, light_block.inner().clone()) .await .map_err(|e| anyhow!("Failed to load full block: {}", e))?; Ok(Some(ethereum_block)) @@ -1373,7 +1389,9 @@ impl FirehoseMapperTrait for FirehoseMapper { } StepFinal => { - unreachable!("irreversible step is not handled and should not be requested in the Firehose request") + unreachable!( + "irreversible step is not handled and should not be requested in the Firehose request" + ) } StepUnset => { diff --git a/chain/ethereum/src/codec.rs b/chain/ethereum/src/codec.rs index a3bfb75ad49..b40bcd11910 100644 --- a/chain/ethereum/src/codec.rs +++ b/chain/ethereum/src/codec.rs @@ -8,18 +8,19 @@ use graph::{ blockchain::{ self, Block as BlockchainBlock, BlockPtr, BlockTime, ChainStoreBlock, ChainStoreData, }, - components::ethereum::{AnyBlock, AnyHeader, AnyRpcHeader, AnyRpcTransaction, AnyTxEnvelope}, + components::ethereum::{ + AnyBlock, AnyHeader, AnyRpcHeader, AnyTransactionReceiptBare, AnyTxEnvelope, + }, prelude::{ + BlockNumber, Error, EthereumBlock, EthereumBlockWithCalls, EthereumCall, + LightEthereumBlock, alloy::{ self, consensus::{ReceiptWithBloom, TxEnvelope, TxType}, network::AnyReceiptEnvelope, - primitives::{aliases::B2048, Address, Bloom, Bytes, LogData, B256, U256}, + primitives::{Address, B256, Bloom, Bytes, LogData, U256, aliases::B2048}, rpc::types::{self as alloy_rpc_types, AccessList, AccessListItem, Transaction}, - serde::WithOtherFields, }, - BlockNumber, Error, EthereumBlock, EthereumBlockWithCalls, EthereumCall, - LightEthereumBlock, }, }; use std::sync::Arc; @@ -161,6 +162,11 @@ impl<'a> TryInto> for TransactionTraceAt<'a> { // Extract data from trace and block let block_hash = self.block.hash.try_decode_proto("transaction block hash")?; let block_number = self.block.number; + let block_timestamp = self + .block + .header + .as_ref() + .and_then(|h| h.timestamp.as_ref().map(|t| t.seconds as u64)); let transaction_index = Some(self.trace.index as u64); let from_address = self .trace @@ -240,6 +246,7 @@ impl<'a> TryInto> for TransactionTraceAt<'a> { inner: recovered, block_hash: Some(block_hash), block_number: Some(block_number), + block_timestamp, transaction_index, effective_gas_price: if gas_price > 0 { Some(gas_price) } else { None }, }); @@ -450,6 +457,7 @@ impl<'a> TryInto> for TransactionTraceAt<'a> { inner: recovered, block_hash: Some(block_hash), block_number: Some(block_number), + block_timestamp, transaction_index, effective_gas_price: if gas_price > 0 { Some(gas_price) } else { None }, // gas_price already contains effective gas price per protobuf spec }) @@ -527,6 +535,8 @@ impl TryInto for &Block { } else { Some(header.requests_hash.try_decode_proto("requests hash")?) }, + block_access_list_hash: None, + slot_number: None, }; let rpc_header = alloy::rpc::types::Header { @@ -556,19 +566,12 @@ impl TryInto for &Block { let any_header: AnyRpcHeader = rpc_header.map(AnyHeader::from); - let any_transactions: Vec = transactions - .into_iter() - .map(|tx| AnyRpcTransaction::new(WithOtherFields::new(tx))) - .collect(); - - let any_block = Block { + Ok(Block { header: any_header, - transactions: alloy::rpc::types::BlockTransactions::Full(any_transactions), + transactions: alloy::rpc::types::BlockTransactions::Full(transactions), uncles, withdrawals: None, - }; - - Ok(AnyBlock::new(WithOtherFields::new(any_block))) + }) } } @@ -619,7 +622,7 @@ impl TryInto for &Block { fn transaction_trace_to_alloy_txn_reciept( t: &TransactionTrace, block: &Block, -) -> Result, Error> { +) -> Result, Error> { use alloy::consensus::{Eip658Value, Receipt}; let r = t.receipt.as_ref(); @@ -656,7 +659,7 @@ fn transaction_trace_to_alloy_txn_reciept( TransactionTraceStatus::Unknown => { return Err(format_err!( "Transaction trace has UNKNOWN status; datasource is broken" - )) + )); } TransactionTraceStatus::Succeeded => true, TransactionTraceStatus::Failed | TransactionTraceStatus::Reverted => false, @@ -719,7 +722,7 @@ fn transaction_trace_to_alloy_txn_reciept( inner: any_envelope, }; - Ok(Some(WithOtherFields::new(receipt))) + Ok(Some(receipt)) } impl BlockHeader { @@ -1014,7 +1017,7 @@ mod test { let receipt = receipt_opt.unwrap(); - assert_eq!(receipt.inner.inner.r#type, 126); + assert_eq!(receipt.inner.r#type, 126); assert_eq!(receipt.gas_used, 21000); assert_eq!(receipt.transaction_index, Some(0)); } diff --git a/chain/ethereum/src/data_source.rs b/chain/ethereum/src/data_source.rs index cd945d00c62..709884496e8 100644 --- a/chain/ethereum/src/data_source.rs +++ b/chain/ethereum/src/data_source.rs @@ -1,5 +1,5 @@ -use anyhow::{anyhow, Error}; -use anyhow::{ensure, Context}; +use anyhow::{Context, ensure}; +use anyhow::{Error, anyhow}; use async_trait::async_trait; use graph::abi; use graph::abi::EventExt; @@ -18,16 +18,17 @@ use graph::data_source::common::{ }; use graph::data_source::{CausalityRegion, MappingTrigger as MappingTriggerType}; use graph::env::ENV_VARS; +use graph::futures03::TryStreamExt; use graph::futures03::future::try_join; use graph::futures03::stream::FuturesOrdered; -use graph::futures03::TryStreamExt; +use graph::prelude::alloy::primitives::keccak256; use graph::prelude::alloy::{ consensus::{TxEnvelope, TxLegacy}, network::TransactionResponse, primitives::{Address, B256, U256}, rpc::types::Log, }; -use graph::prelude::{alloy, Link, SubgraphManifestValidationError}; +use graph::prelude::{Link, SubgraphManifestValidationError, alloy}; use graph::slog::{debug, error, o, trace}; use itertools::Itertools; use serde::de::Error as ErrorD; @@ -37,26 +38,25 @@ use std::num::NonZeroU32; use std::str::FromStr; use std::sync::Arc; use std::time::{Duration, Instant}; -use tiny_keccak::{keccak256, Keccak}; use graph::{ blockchain::{self, Blockchain}, prelude::{ - serde_json, warn, BlockNumber, CheapClone, EthereumCall, LightEthereumBlock, - LightEthereumBlockExt, LinkResolver, Logger, + BlockNumber, CheapClone, EthereumCall, LightEthereumBlock, LightEthereumBlockExt, + LinkResolver, Logger, serde_json, warn, }, }; use graph::data::subgraph::{ - calls_host_fn, DataSourceContext, Source, MIN_SPEC_VERSION, SPEC_VERSION_0_0_8, - SPEC_VERSION_1_2_0, + DataSourceContext, MIN_SPEC_VERSION, SPEC_VERSION_0_0_8, SPEC_VERSION_1_2_0, Source, + calls_host_fn, }; +use crate::NodeCapabilities; use crate::adapter::EthereumAdapter as _; use crate::chain::Chain; use crate::network::EthereumNetworkAdapters; use crate::trigger::{EthereumBlockTriggerType, EthereumTrigger, MappingTrigger}; -use crate::NodeCapabilities; // The recommended kind is `ethereum`, `ethereum/contract` is accepted for backwards compatibility. const ETHEREUM_KINDS: &[&str] = &["ethereum/contract", "ethereum"]; @@ -444,10 +444,9 @@ fn create_dummy_transaction( transaction_index: Option, transaction_hash: Option, ) -> Result { - use alloy::serde::WithOtherFields; use graph::components::ethereum::AnyTxEnvelope; use graph::prelude::alloy::{ - consensus::transaction::Recovered, consensus::Signed, primitives::Signature, + consensus::Signed, consensus::transaction::Recovered, primitives::Signature, rpc::types::Transaction, }; @@ -465,15 +464,14 @@ fn create_dummy_transaction( let recovered = Recovered::new_unchecked(any_envelope, Address::ZERO); - let inner_tx = Transaction { + Ok(Transaction { inner: recovered, block_hash: Some(block_hash), block_number: Some(block_number), + block_timestamp: None, transaction_index, effective_gas_price: None, - }; - - Ok(AnyTransaction::new(WithOtherFields::new(inner_tx))) + }) } impl DataSource { @@ -812,7 +810,7 @@ impl DataSource { let logging_extras = Arc::new(o! { "signature" => event_handler.event.to_string(), "address" => format!("{}", &log.address()), - "transaction" => format!("{}", &transaction.inner.tx_hash()), + "transaction" => format!("{}", &transaction.tx_hash()), }); let handler = event_handler.handler.clone(); let calls = DeclaredCall::from_log_trigger_with_event( @@ -921,7 +919,7 @@ impl DataSource { let logging_extras = Arc::new(o! { "function" => handler.function.to_string(), "to" => format!("{}", &call.to), - "transaction" => format!("{}", &transaction.inner.tx_hash()), + "transaction" => format!("{}", &transaction.tx_hash()), }); Ok(Some(TriggerWithHandler::::new_with_logging_extras( MappingTrigger::Call { @@ -1592,13 +1590,7 @@ impl MappingEventHandler { /// Hashes a string to a B256 hash. fn string_to_b256(s: &str) -> B256 { - let mut result = [0u8; 32]; - let data = s.replace(' ', "").into_bytes(); - let mut sponge = Keccak::new_keccak256(); - sponge.update(&data); - sponge.finalize(&mut result); - - B256::from_slice(&result) + keccak256(s.replace(' ', "").as_bytes()) } #[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] diff --git a/chain/ethereum/src/env.rs b/chain/ethereum/src/env.rs index 027a26b623f..b15ee0ee018 100644 --- a/chain/ethereum/src/env.rs +++ b/chain/ethereum/src/env.rs @@ -1,6 +1,6 @@ use envconfig::Envconfig; use graph::env::EnvVarBoolean; -use graph::prelude::{envconfig, lazy_static, BlockNumber}; +use graph::prelude::{BlockNumber, envconfig, lazy_static}; use std::fmt; use std::time::Duration; @@ -13,9 +13,9 @@ lazy_static! { pub struct EnvVars { /// Additional deterministic errors that have not yet been hardcoded. /// - /// Set by the environment variable `GRAPH_GETH_ETH_CALL_ERRORS`, separated + /// Set by the environment variable `GRAPH_RPC_ETH_CALL_ERRORS`, separated /// by `;`. - pub geth_eth_call_errors: Vec, + pub rpc_eth_call_errors: Vec, /// Set by the environment variable `GRAPH_ETH_GET_LOGS_MAX_CONTRACTS`. The /// default value is 2000. pub get_logs_max_contracts: usize, @@ -114,8 +114,8 @@ impl From for EnvVars { fn from(x: Inner) -> Self { Self { get_logs_max_contracts: x.get_logs_max_contracts, - geth_eth_call_errors: x - .geth_eth_call_errors + rpc_eth_call_errors: x + .rpc_eth_call_errors .split(';') .filter(|s| !s.is_empty()) .map(str::to_string) @@ -158,8 +158,8 @@ impl Default for EnvVars { #[derive(Clone, Debug, Envconfig)] struct Inner { - #[envconfig(from = "GRAPH_GETH_ETH_CALL_ERRORS", default = "")] - geth_eth_call_errors: String, + #[envconfig(from = "GRAPH_RPC_ETH_CALL_ERRORS", default = "")] + rpc_eth_call_errors: String, #[envconfig(from = "GRAPH_ETH_GET_LOGS_MAX_CONTRACTS", default = "2000")] get_logs_max_contracts: usize, diff --git a/chain/ethereum/src/ethereum_adapter.rs b/chain/ethereum/src/ethereum_adapter.rs index 64affbeec0b..9c85f0ac551 100644 --- a/chain/ethereum/src/ethereum_adapter.rs +++ b/chain/ethereum/src/ethereum_adapter.rs @@ -3,39 +3,40 @@ use futures03::{future::BoxFuture, stream::FuturesUnordered}; use graph::abi; use graph::abi::DynSolValueExt; use graph::abi::FunctionExt; -use graph::blockchain::client::ChainClient; use graph::blockchain::BlockHash; use graph::blockchain::ChainIdentifier; use graph::blockchain::ExtendedBlockPtr; +use graph::blockchain::client::ChainClient; use graph::components::ethereum::*; use graph::components::transaction_receipt::LightTransactionReceipt; use graph::data::store::ethereum::call; use graph::data::store::scalar; -use graph::data::subgraph::UnifiedMappingApiVersion; use graph::data::subgraph::API_VERSION_0_0_7; +use graph::data::subgraph::UnifiedMappingApiVersion; use graph::data_source::common::ContractCall; -use graph::futures01::stream; +use graph::derive::CheapClone; use graph::futures01::Future; use graph::futures01::Stream; +use graph::futures01::stream; use graph::futures03::future::try_join_all; use graph::futures03::{ - self, compat::Future01CompatExt, FutureExt, StreamExt, TryFutureExt, TryStreamExt, + self, FutureExt, StreamExt, TryFutureExt, TryStreamExt, compat::Future01CompatExt, }; use graph::prelude::{ alloy::{ self, - network::{AnyNetwork, TransactionResponse}, + network::TransactionResponse, primitives::{Address, B256}, providers::{ + Identity, Provider, RootProvider, ext::TraceApi, fillers::{ BlobGasFiller, ChainIdFiller, FillProvider, GasFiller, JoinFill, NonceFiller, }, - Identity, Provider, RootProvider, }, rpc::types::{ - trace::{filter::TraceFilter as AlloyTraceFilter, parity::LocalizedTransactionTrace}, TransactionInput, TransactionRequest, + trace::{filter::TraceFilter as AlloyTraceFilter, parity::LocalizedTransactionTrace}, }, transports::{RpcError, TransportErrorKind}, }, @@ -43,11 +44,12 @@ use graph::prelude::{ }; use graph::slog::o; use graph::{ - blockchain::{block_stream::BlockWithTriggers, BlockPtr, IngestorError}, + blockchain::{BlockPtr, IngestorError, block_stream::BlockWithTriggers}, prelude::{ - anyhow::{self, anyhow, bail, ensure, Context}, - debug, error, hex, info, retry, serde_json as json, trace, warn, BlockNumber, ChainStore, - CheapClone, DynTryFuture, Error, EthereumCallCache, Logger, TimeoutError, + BlockNumber, ChainStore, CheapClone, DynTryFuture, Error, EthereumCallCache, Logger, + TimeoutError, + anyhow::{self, Context, anyhow, bail, ensure}, + debug, error, hex, info, retry, trace, warn, }, }; use itertools::Itertools; @@ -56,27 +58,28 @@ use std::convert::TryFrom; use std::iter::FromIterator; use std::pin::Pin; use std::sync::Arc; -use std::time::Instant; +use std::time::{Duration, Instant}; use tokio::sync::RwLock; use tokio::time::timeout; +use crate::Chain; +use crate::NodeCapabilities; +use crate::TriggerFilter; use crate::adapter::EthGetLogsFilter; use crate::adapter::EthereumRpcError; use crate::adapter::ProviderStatus; use crate::call_helper::interpret_eth_call_error; use crate::chain::BlockFinality; +use crate::chain::ChainSettings; use crate::trigger::{LogPosition, LogRef}; -use crate::Chain; -use crate::NodeCapabilities; -use crate::TriggerFilter; use crate::{ + ENV_VARS, adapter::{ ContractCallError, EthereumAdapter as EthereumAdapterTrait, EthereumBlockFilter, EthereumCallFilter, EthereumLogFilter, ProviderEthRpcMetrics, SubgraphEthRpcMetrics, }, transport::Transport, trigger::{EthereumBlockTriggerType, EthereumTrigger}, - ENV_VARS, }; type AlloyProvider = FillProvider< @@ -84,8 +87,8 @@ type AlloyProvider = FillProvider< Identity, JoinFill>>, >, - RootProvider, - AnyNetwork, + RootProvider, + AnyNetworkBare, >; #[derive(Clone)] @@ -97,6 +100,7 @@ pub struct EthereumAdapter { supports_eip_1898: bool, call_only: bool, supports_block_receipts: Arc>>, + pub(crate) settings: Arc, } impl std::fmt::Debug for EthereumAdapter { @@ -109,6 +113,7 @@ impl std::fmt::Debug for EthereumAdapter { .field("supports_eip_1898", &self.supports_eip_1898) .field("call_only", &self.call_only) .field("supports_block_receipts", &self.supports_block_receipts) + .field("settings", &self.settings) .finish() } } @@ -123,15 +128,39 @@ impl CheapClone for EthereumAdapter { supports_eip_1898: self.supports_eip_1898, call_only: self.call_only, supports_block_receipts: self.supports_block_receipts.cheap_clone(), + settings: self.settings.clone(), } } } +/// Logger with provider context. Internal methods should take this type +/// to ensure provider info is always included in logs. +#[derive(Clone, CheapClone)] +struct ProviderLogger(Logger); + +impl ProviderLogger { + fn new(logger: &Logger, provider: &str) -> Self { + Self(Logger::new(logger, o!("provider" => provider.to_string()))) + } +} + +impl std::ops::Deref for ProviderLogger { + type Target = Logger; + fn deref(&self) -> &Logger { + &self.0 + } +} + impl EthereumAdapter { pub fn is_call_only(&self) -> bool { self.call_only } + /// Returns a logger with provider context added + fn provider_logger(&self, logger: &Logger) -> ProviderLogger { + ProviderLogger::new(logger, &self.provider) + } + pub async fn new( logger: Logger, provider: String, @@ -139,26 +168,24 @@ impl EthereumAdapter { provider_metrics: Arc, supports_eip_1898: bool, call_only: bool, + settings: Arc, ) -> Self { let alloy = match &transport { Transport::RPC(client) => Arc::new( - alloy::providers::ProviderBuilder::<_, _, AnyNetwork>::default() - .network::() - .with_recommended_fillers() + alloy::providers::ProviderBuilder::<_, _, AnyNetworkBare>::default() + .network::() .connect_client(client.clone()), ), Transport::IPC(ipc_connect) => Arc::new( - alloy::providers::ProviderBuilder::<_, _, AnyNetwork>::default() - .network::() - .with_recommended_fillers() + alloy::providers::ProviderBuilder::<_, _, AnyNetworkBare>::default() + .network::() .connect_ipc(ipc_connect.clone()) .await .expect("Failed to connect to Ethereum IPC"), ), Transport::WS(ws_connect) => Arc::new( - alloy::providers::ProviderBuilder::<_, _, AnyNetwork>::default() - .network::() - .with_recommended_fillers() + alloy::providers::ProviderBuilder::<_, _, AnyNetworkBare>::default() + .network::() .connect_ws(ws_connect.clone()) .await .expect("Failed to connect to Ethereum WS"), @@ -173,12 +200,13 @@ impl EthereumAdapter { supports_eip_1898, call_only, supports_block_receipts: Arc::new(RwLock::new(None)), + settings, } } async fn traces( self, - logger: Logger, + logger: ProviderLogger, subgraph_metrics: Arc, from: BlockNumber, to: BlockNumber, @@ -192,8 +220,8 @@ impl EthereumAdapter { retry(retry_log_message, &logger) .redact_log_urls(true) - .limit(ENV_VARS.request_retries) - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .limit(self.settings.request_retries) + .timeout_secs(self.settings.json_rpc_timeout.as_secs()) .run(move || { let eth = eth.clone(); let logger = logger.clone(); @@ -219,7 +247,7 @@ impl EthereumAdapter { async fn execute_trace_filter_request( &self, - logger: Logger, + logger: ProviderLogger, subgraph_metrics: Arc, from: BlockNumber, to: BlockNumber, @@ -264,7 +292,7 @@ impl EthereumAdapter { fn log_trace_results( &self, - logger: &Logger, + logger: &ProviderLogger, from: BlockNumber, to: BlockNumber, trace_len: usize, @@ -288,7 +316,7 @@ impl EthereumAdapter { result: &Result, RpcError>, from: BlockNumber, to: BlockNumber, - logger: &Logger, + logger: &ProviderLogger, ) { self.metrics .observe_request(elapsed, "trace_filter", &self.provider); @@ -312,7 +340,7 @@ impl EthereumAdapter { block_hash: B256, supports_eip_1898: bool, call_only: bool, - logger: Logger, + logger: ProviderLogger, ) -> bool { // This is the lazy part. If the result is already in `supports_block_receipts`, we don't need // to check again. @@ -362,7 +390,7 @@ impl EthereumAdapter { /// Alloy-exclusive version of logs_with_sigs using alloy types and methods async fn logs_with_sigs( &self, - logger: Logger, + logger: ProviderLogger, subgraph_metrics: Arc, from: BlockNumber, to: BlockNumber, @@ -386,8 +414,8 @@ impl EthereumAdapter { .any(|f| e.to_string().contains(f)), }, ) - .limit(ENV_VARS.request_retries) - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .limit(self.settings.request_retries) + .timeout_secs(self.settings.json_rpc_timeout.as_secs()) .run(move || { let eth_adapter = eth_adapter.cheap_clone(); let subgraph_metrics = subgraph_metrics.clone(); @@ -416,7 +444,7 @@ impl EthereumAdapter { fn trace_stream( self, - logger: &Logger, + logger: ProviderLogger, subgraph_metrics: Arc, from: BlockNumber, to: BlockNumber, @@ -446,8 +474,8 @@ impl EthereumAdapter { ranges }; + let block_batch_size = self.settings.block_batch_size; let eth = self; - let logger = logger.clone(); futures03::stream::iter(ranges.into_iter().map(move |(start, end)| { let eth = eth.clone(); @@ -466,14 +494,14 @@ impl EthereumAdapter { .await } })) - .buffered(ENV_VARS.block_batch_size) + .buffered(block_batch_size) .map_ok(|traces| futures03::stream::iter(traces.into_iter().map(Ok))) .try_flatten() } fn log_stream( &self, - logger: Logger, + logger: ProviderLogger, subgraph_metrics: Arc, from: BlockNumber, to: BlockNumber, @@ -502,7 +530,7 @@ impl EthereumAdapter { let step = match filter.contracts.is_empty() { // `to - from + 1` blocks will be scanned. false => to - from, - true => (to - from).min(ENV_VARS.max_event_only_range - 1), + true => (to - from).min(self.settings.max_event_only_range - 1), }; // Typically this will loop only once and fetch the entire range in one request. But if the @@ -575,12 +603,11 @@ impl EthereumAdapter { async fn code( &self, - logger: &Logger, + logger: ProviderLogger, address: Address, block_ptr: BlockPtr, ) -> Result { let alloy = self.alloy.clone(); - let logger = Logger::new(logger, o!("provider" => self.provider.clone())); let block_id = self.block_ptr_to_id(&block_ptr); let retry_log_message = format!("eth_getCode RPC call for block {}", block_ptr); @@ -588,8 +615,8 @@ impl EthereumAdapter { retry(retry_log_message, &logger) .redact_log_urls(true) .when(|result| result.is_err()) - .limit(ENV_VARS.request_retries) - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .limit(self.settings.request_retries) + .timeout_secs(self.settings.json_rpc_timeout.as_secs()) .run(move || { let alloy = alloy.cheap_clone(); async move { @@ -606,12 +633,11 @@ impl EthereumAdapter { async fn balance( &self, - logger: &Logger, + logger: ProviderLogger, address: Address, block_ptr: BlockPtr, ) -> Result { let alloy = self.alloy.clone(); - let logger = Logger::new(logger, o!("provider" => self.provider.clone())); let block_id = self.block_ptr_to_id(&block_ptr); let retry_log_message = format!("eth_getBalance RPC call for block {}", block_ptr); @@ -619,8 +645,8 @@ impl EthereumAdapter { retry(retry_log_message, &logger) .redact_log_urls(true) .when(|result| result.is_err()) - .limit(ENV_VARS.request_retries) - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .limit(self.settings.request_retries) + .timeout_secs(self.settings.json_rpc_timeout.as_secs()) .run(move || { let alloy = alloy.cheap_clone(); async move { @@ -637,20 +663,19 @@ impl EthereumAdapter { async fn call( &self, - logger: Logger, + logger: ProviderLogger, call_data: call::Request, block_ptr: BlockPtr, gas: Option, ) -> Result { let alloy = self.alloy.clone(); - let logger = Logger::new(&logger, o!("provider" => self.provider.clone())); let alloy_block_id = self.block_ptr_to_id(&block_ptr); let retry_log_message = format!("eth_call RPC call for block {}", block_ptr); retry(retry_log_message, &logger) .redact_log_urls(true) - .limit(ENV_VARS.request_retries) - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .limit(self.settings.request_retries) + .timeout_secs(self.settings.json_rpc_timeout.as_secs()) .run(move || { let call_data = call_data.clone(); let alloy = alloy.cheap_clone(); @@ -680,7 +705,7 @@ impl EthereumAdapter { async fn call_and_cache( &self, - logger: &Logger, + logger: &ProviderLogger, call: &ContractCall, req: call::Request, cache: Arc, @@ -716,6 +741,8 @@ impl EthereumAdapter { ids: Vec, ) -> impl futures03::Stream, Error>> + Send { let alloy = self.alloy.clone(); + let request_retries = self.settings.request_retries; + let json_rpc_timeout_secs = self.settings.json_rpc_timeout.as_secs(); futures03::stream::iter(ids.into_iter().map(move |hash| { let alloy = alloy.clone(); @@ -724,8 +751,8 @@ impl EthereumAdapter { async move { retry(format!("load block {}", hash), &logger) .redact_log_urls(true) - .limit(ENV_VARS.request_retries) - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .limit(request_retries) + .timeout_secs(json_rpc_timeout_secs) .run(move || { let alloy = alloy.cheap_clone(); async move { @@ -754,7 +781,7 @@ impl EthereumAdapter { }) } })) - .buffered(ENV_VARS.block_batch_size) + .buffered(self.settings.block_batch_size) } /// Request blocks by number through JSON-RPC. @@ -764,6 +791,8 @@ impl EthereumAdapter { numbers: Vec, ) -> impl futures03::Stream, Error>> + Send { let alloy = self.alloy.clone(); + let request_retries = self.settings.request_retries; + let json_rpc_timeout_secs = self.settings.json_rpc_timeout.as_secs(); futures03::stream::iter(numbers.into_iter().map(move |number| { let alloy = alloy.clone(); @@ -772,8 +801,8 @@ impl EthereumAdapter { async move { retry(format!("load block {}", number), &logger) .redact_log_urls(true) - .limit(ENV_VARS.request_retries) - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .limit(request_retries) + .timeout_secs(json_rpc_timeout_secs) .run(move || { let alloy = alloy.cheap_clone(); @@ -814,7 +843,7 @@ impl EthereumAdapter { }) } })) - .buffered(ENV_VARS.block_ptr_batch_size) + .buffered(self.settings.block_ptr_batch_size) } /// Request blocks ptrs for numbers through JSON-RPC. @@ -826,6 +855,7 @@ impl EthereumAdapter { block_nums: Vec, ) -> impl Stream + Send { let alloy = self.alloy.clone(); + let json_rpc_timeout_secs = self.settings.json_rpc_timeout.as_secs(); stream::iter_ok::<_, Error>(block_nums.into_iter().map(move |block_num| { let alloy = alloy.clone(); @@ -833,7 +863,7 @@ impl EthereumAdapter { .redact_log_urls(true) .when(|res| !res.is_ok() && !detect_null_block(res)) .no_limit() - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .timeout_secs(json_rpc_timeout_secs) .run(move || { let alloy = alloy.cheap_clone(); async move { @@ -859,7 +889,7 @@ impl EthereumAdapter { } }) })) - .buffered(ENV_VARS.block_batch_size) + .buffered(self.settings.block_batch_size) .filter_map(|b| b) .map(|b| BlockPtr::from((b.header.hash, b.header.number))) } @@ -896,19 +926,24 @@ impl EthereumAdapter { log_filter: EthereumLogFilter, ) -> DynTryFuture<'static, Vec, Error> { let eth: Self = self.cheap_clone(); - let logger = logger.clone(); - - futures03::stream::iter(log_filter.eth_get_logs_filters().map(move |filter| { - eth.cheap_clone().log_stream( - logger.cheap_clone(), - subgraph_metrics.cheap_clone(), - from, - to, - filter, - ) - })) + let logger = self.provider_logger(logger); + + let max_contracts = eth.settings.get_logs_max_contracts; + futures03::stream::iter( + log_filter + .eth_get_logs_filters(max_contracts) + .map(move |filter| { + eth.cheap_clone().log_stream( + logger.cheap_clone(), + subgraph_metrics.cheap_clone(), + from, + to, + filter, + ) + }), + ) // Real limits on the number of parallel requests are imposed within the adapter. - .buffered(ENV_VARS.block_ingestor_max_concurrent_json_rpc_calls) + .buffered(self.settings.block_ingestor_max_concurrent_json_rpc_calls) .try_concat() .boxed() } @@ -922,6 +957,7 @@ impl EthereumAdapter { call_filter: &'a EthereumCallFilter, ) -> Box + Send + 'a> { let eth = self.clone(); + let logger = self.provider_logger(logger); let EthereumCallFilter { contract_addresses_function_signatures, @@ -976,34 +1012,20 @@ impl EthereumAdapter { ) -> Pin< Box< dyn std::future::Future, anyhow::Error>> - + std::marker::Send, + + std::marker::Send + + '_, >, > { - // Create a HashMap of block numbers to Vec + // Create a HashMap of block numbers to Vec. let matching_blocks = (from..=to) .filter_map(|block_number| { - filter - .polling_intervals - .iter() - .find_map(|(start_block, interval)| { - let has_once_trigger = (*interval == 0) && (block_number == *start_block); - let has_polling_trigger = block_number >= *start_block - && *interval > 0 - && ((block_number - start_block) % *interval) == 0; - - if has_once_trigger || has_polling_trigger { - let mut triggers = Vec::new(); - if has_once_trigger { - triggers.push(EthereumBlockTriggerType::Start); - } - if has_polling_trigger { - triggers.push(EthereumBlockTriggerType::End); - } - Some((block_number, triggers)) - } else { - None - } - }) + let triggers = + block_trigger_types_from_intervals(block_number, &filter.polling_intervals); + if triggers.is_empty() { + None + } else { + Some((block_number, triggers)) + } }) .collect::>(); @@ -1015,15 +1037,13 @@ impl EthereumAdapter { let block_futures = blocks_matching_polling_filter.map(move |ptrs| { ptrs.into_iter() .flat_map(|ptr| { - let triggers = matching_blocks + matching_blocks .get(&ptr.number) // Safe to unwrap since we are iterating over ptrs which was created from // the keys of matching_blocks .unwrap() .iter() - .map(move |trigger| EthereumTrigger::Block(ptr.clone(), trigger.clone())); - - triggers + .map(move |trigger| EthereumTrigger::Block(ptr.clone(), trigger.clone())) }) .collect::>() }); @@ -1039,6 +1059,7 @@ impl EthereumAdapter { block_hash: alloy::primitives::B256, ) -> Result, Error> { let eth = self.clone(); + let logger = self.provider_logger(logger); let addresses = Vec::new(); let traces: Vec = eth .trace_stream( @@ -1087,7 +1108,7 @@ impl EthereumAdapter { logger: Logger, from: BlockNumber, to: BlockNumber, - ) -> Box, Error = Error> + Send> { + ) -> Box, Error = Error> + Send + '_> { // Currently we can't go to the DB for this because there might be duplicate entries for // the same block number. debug!(&logger, "Requesting hashes for blocks [{}, {}]", from, to); @@ -1101,7 +1122,7 @@ impl EthereumAdapter { &self, logger: Logger, blocks: Vec, - ) -> Box, Error = Error> + Send> { + ) -> Box, Error = Error> + Send + '_> { // Currently we can't go to the DB for this because there might be duplicate entries for // the same block number. debug!(&logger, "Requesting hashes for blocks {:?}", blocks); @@ -1114,7 +1135,7 @@ impl EthereumAdapter { retry("chain_id RPC call", &logger) .redact_log_urls(true) .no_limit() - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .timeout_secs(self.settings.json_rpc_timeout.as_secs()) .run(move || { let alloy = alloy.cheap_clone(); async move { alloy.get_chain_id().await.map_err(Error::from) } @@ -1171,9 +1192,10 @@ impl EthereumAdapterTrait for EthereumAdapter { let alloy_provider = self.alloy.clone(); let metrics = self.metrics.clone(); let provider = self.provider().to_string(); + let genesis_block_number = self.settings.genesis_block_number; let retry_log_message = format!( "eth_getBlockByNumber({}, false) RPC call", - ENV_VARS.genesis_block_number + genesis_block_number ); let gen_block_hash_future = retry(retry_log_message, &logger) .redact_log_urls(true) @@ -1186,7 +1208,7 @@ impl EthereumAdapterTrait for EthereumAdapter { async move { alloy_genesis .get_block_by_number(alloy::rpc::types::BlockNumberOrTag::Number( - ENV_VARS.genesis_block_number, + genesis_block_number, )) .await .inspect_err(|_| { @@ -1224,8 +1246,8 @@ impl EthereumAdapterTrait for EthereumAdapter { let alloy = self.alloy.clone(); retry("eth_getBlockByNumber(latest) no txs RPC call", logger) .redact_log_urls(true) - .no_limit() - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .limit(ENV_VARS.request_retries) + .timeout_secs(self.settings.json_rpc_timeout.as_secs()) .run(move || { let alloy = alloy.cheap_clone(); async move { @@ -1248,6 +1270,14 @@ impl EthereumAdapterTrait for EthereumAdapter { .await } + async fn is_reachable(&self) -> bool { + let alloy = self.alloy.clone(); + tokio::time::timeout(Duration::from_secs(10), alloy.get_block_number()) + .await + .map(|r| r.is_ok()) + .unwrap_or(false) + } + async fn block_by_hash( &self, logger: &Logger, @@ -1262,8 +1292,8 @@ impl EthereumAdapterTrait for EthereumAdapter { retry(retry_log_message, &logger) .redact_log_urls(true) - .limit(ENV_VARS.request_retries) - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .limit(self.settings.request_retries) + .timeout_secs(self.settings.json_rpc_timeout.as_secs()) .run(move || { let alloy = alloy.cheap_clone(); async move { @@ -1296,7 +1326,7 @@ impl EthereumAdapterTrait for EthereumAdapter { retry(retry_log_message, &logger) .redact_log_urls(true) .no_limit() - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .timeout_secs(self.settings.json_rpc_timeout.as_secs()) .run(move || { let alloy = alloy.clone(); async move { @@ -1326,7 +1356,7 @@ impl EthereumAdapterTrait for EthereumAdapter { block: AnyBlock, ) -> Result { let alloy = self.alloy.clone(); - let logger = logger.clone(); + let logger = self.provider_logger(logger); let block_hash = block.header.hash; // The early return is necessary for correctness, otherwise we'll @@ -1350,12 +1380,19 @@ impl EthereumAdapterTrait for EthereumAdapter { ) .await; - fetch_receipts_with_retry(alloy, hashes, block_hash, logger, supports_block_receipts) - .await - .map(|transaction_receipts| EthereumBlock { - block: Arc::new(LightEthereumBlock::new(block)), - transaction_receipts, - }) + fetch_receipts_with_retry( + alloy, + hashes, + block_hash, + logger, + supports_block_receipts, + &self.settings, + ) + .await + .map(|transaction_receipts| EthereumBlock { + block: Arc::new(LightEthereumBlock::new(block)), + transaction_receipts, + }) } async fn get_balance( @@ -1364,6 +1401,7 @@ impl EthereumAdapterTrait for EthereumAdapter { address: Address, block_ptr: BlockPtr, ) -> Result { + let logger = self.provider_logger(logger); debug!( logger, "eth_getBalance"; "address" => format!("{}", address), @@ -1378,6 +1416,7 @@ impl EthereumAdapterTrait for EthereumAdapter { address: Address, block_ptr: BlockPtr, ) -> Result { + let logger = self.provider_logger(logger); debug!( logger, "eth_getCode"; "address" => format!("{}", address), @@ -1403,7 +1442,7 @@ impl EthereumAdapterTrait for EthereumAdapter { .redact_log_urls(true) .when(|res| !res.is_ok() && !detect_null_block(res)) .no_limit() - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .timeout_secs(self.settings.json_rpc_timeout.as_secs()) .run(move || { let alloy = alloy.cheap_clone(); async move { @@ -1457,7 +1496,7 @@ impl EthereumAdapterTrait for EthereumAdapter { cache: Arc, ) -> Result>, call::Source)>, ContractCallError> { fn as_req( - logger: &Logger, + logger: &ProviderLogger, call: &ContractCall, index: u32, ) -> Result { @@ -1498,7 +1537,7 @@ impl EthereumAdapterTrait for EthereumAdapter { } fn decode( - logger: &Logger, + logger: &ProviderLogger, resp: call::Response, call: &ContractCall, ) -> (Option>, call::Source) { @@ -1528,20 +1567,33 @@ impl EthereumAdapterTrait for EthereumAdapter { } } - fn log_call_error(logger: &Logger, e: &ContractCallError, call: &ContractCall) { + fn log_call_error(logger: &ProviderLogger, e: &ContractCallError, call: &ContractCall) { match e { - ContractCallError::AlloyError(e) => error!(logger, + ContractCallError::AlloyError(e) => error!( + logger, "Ethereum node returned an error when calling function \"{}\" of contract \"{}\": {}", - call.function.name, call.contract_name, e), - ContractCallError::Timeout => error!(logger, + call.function.name, + call.contract_name, + e + ), + ContractCallError::Timeout => error!( + logger, "Ethereum node did not respond when calling function \"{}\" of contract \"{}\"", - call.function.name, call.contract_name), - _ => error!(logger, + call.function.name, + call.contract_name + ), + _ => error!( + logger, "Failed to call function \"{}\" of contract \"{}\": {}", - call.function.name, call.contract_name, e), + call.function.name, + call.contract_name, + e + ), } } + let logger = self.provider_logger(logger); + if calls.is_empty() { return Ok(Vec::new()); } @@ -1556,7 +1608,7 @@ impl EthereumAdapterTrait for EthereumAdapter { let reqs: Vec<_> = calls .iter() .enumerate() - .map(|(index, call)| as_req(logger, call, index as u32)) + .map(|(index, call)| as_req(&logger, call, index as u32)) .collect::>()?; let (mut resps, missing) = cache @@ -1567,12 +1619,13 @@ impl EthereumAdapterTrait for EthereumAdapter { let futs = missing.into_iter().map(|req| { let cache = cache.clone(); + let logger = logger.clone(); async move { let call = calls[req.index as usize]; - match self.call_and_cache(logger, call, req, cache.clone()).await { + match self.call_and_cache(&logger, call, req, cache.clone()).await { Ok(resp) => Ok(resp), Err(e) => { - log_call_error(logger, &e, call); + log_call_error(&logger, &e, call); Err(e) } } @@ -1590,7 +1643,7 @@ impl EthereumAdapterTrait for EthereumAdapter { .into_iter() .map(|res| { let call = &calls[res.req.index as usize]; - decode(logger, res, call) + decode(&logger, res, call) }) .collect(); @@ -1612,27 +1665,8 @@ impl EthereumAdapterTrait for EthereumAdapter { .await .map_err(|e| error!(&logger, "Error accessing block cache {}", e)) .unwrap_or_default() - .into_iter() - .filter_map(|value| { - // recent_blocks_cache can contain full format {"block": {...}, "transaction_receipts": [...]} - // or light format (just block fields). Extract block data for deserialization. - let inner = value.get("block").unwrap_or(&value); - json::from_value(inner.clone()) - .map_err(|e| { - let block_num = inner.get("number").and_then(|n| n.as_str()); - let block_hash = inner.get("hash").and_then(|h| h.as_str()); - warn!( - &logger, - "Failed to deserialize cached block #{:?} {:?}: {}. \ - Block will be re-fetched from RPC.", - block_num, - block_hash, - e - ); - }) - .ok() - }) - .map(|b| Arc::new(LightEthereumBlock::new(b))) + .iter() + .map(|b| b.to_light_block()) .collect(); let missing_blocks = Vec::from_iter( @@ -1693,6 +1727,7 @@ pub(crate) async fn blocks_with_triggers( // while searching for a trigger type, the entire operation fails. let eth = adapter.clone(); let call_filter = EthereumCallFilter::from(&filter.block); + let logger = ProviderLogger::new(&logger, eth.provider()); // Scan the block range to find relevant triggers let trigger_futs: FuturesUnordered, anyhow::Error>>> = @@ -1708,8 +1743,9 @@ pub(crate) async fn blocks_with_triggers( // This is for `start` triggers which can be initialization handlers which needs to be run // before all other triggers if filter.block.trigger_every_block { + let logger = logger.clone(); let block_future = eth - .block_range_to_ptrs(logger.clone(), from, to) + .block_range_to_ptrs((*logger).clone(), from, to) .map(move |ptrs| { ptrs.into_iter() .flat_map(|ptr| { @@ -1725,7 +1761,7 @@ pub(crate) async fn blocks_with_triggers( trigger_futs.push(block_future) } else if !filter.block.polling_intervals.is_empty() { let block_futures_matching_once_filter = - eth.blocks_matching_polling_intervals(logger.clone(), from, to, &filter.block); + eth.blocks_matching_polling_intervals((*logger).clone(), from, to, &filter.block); trigger_futs.push(block_futures_matching_once_filter); } @@ -1733,7 +1769,7 @@ pub(crate) async fn blocks_with_triggers( if !filter.log.is_empty() { let logs_future = get_logs_and_transactions( ð, - &logger, + logger.clone(), subgraph_metrics.clone(), from, to, @@ -1798,7 +1834,7 @@ pub(crate) async fn blocks_with_triggers( let logger2 = logger.cheap_clone(); let blocks: Vec<_> = eth - .load_blocks(logger.cheap_clone(), chain_store.clone(), block_hashes) + .load_blocks((*logger).cheap_clone(), chain_store.clone(), block_hashes) .await? .into_iter() .map( @@ -1901,31 +1937,36 @@ pub(crate) fn parse_log_triggers( return vec![]; } - block + let total_logs: usize = block .transaction_receipts .iter() - .flat_map(move |receipt| { - receipt.logs().iter().enumerate().map(move |(index, log)| { - let requires_transaction_receipt = log - .topics() - .first() - .map(|signature| { - log_filter.requires_transaction_receipt( - signature, - Some(&log.address()), - log.topics(), - ) - }) - .unwrap_or(false); + .map(|r| r.logs().len()) + .sum(); + let mut triggers = Vec::with_capacity(total_logs); + + for receipt in &block.transaction_receipts { + for (index, log) in receipt.logs().iter().enumerate() { + let requires_transaction_receipt = log + .topics() + .first() + .map(|signature| { + log_filter.requires_transaction_receipt( + signature, + Some(&log.address()), + log.topics(), + ) + }) + .unwrap_or(false); - EthereumTrigger::Log(LogRef::LogPosition(LogPosition { - index, - receipt: receipt.cheap_clone(), - requires_transaction_receipt, - })) - }) - }) - .collect() + triggers.push(EthereumTrigger::Log(LogRef::LogPosition(LogPosition { + index, + receipt: receipt.cheap_clone(), + requires_transaction_receipt, + }))); + } + } + + triggers } pub(crate) fn parse_call_triggers( @@ -1953,6 +1994,36 @@ pub(crate) fn parse_call_triggers( } } +/// For a given `block_number`, return the block trigger types that fire +/// based on the rules in `polling_intervals`. Each entry is `(start_block, +/// interval)` where `interval == 0` encodes a `once` rule and `interval > 0` +/// encodes a `polling every interval` rule. Both rule kinds can fire at the +/// same block (e.g. a once and polling rule sharing a `start_block`), so the +/// returned Vec may contain `Start`, `End`, both, or neither. +pub(crate) fn block_trigger_types_from_intervals( + block_number: i32, + polling_intervals: &HashSet<(i32, i32)>, +) -> Vec { + let has_once_trigger = polling_intervals + .iter() + .any(|(start_block, interval)| *interval == 0 && block_number == *start_block); + + let has_polling_trigger = polling_intervals.iter().any(|(start_block, interval)| { + *interval > 0 + && block_number >= *start_block + && (block_number - start_block) % *interval == 0 + }); + + let mut triggers = Vec::new(); + if has_once_trigger { + triggers.push(EthereumBlockTriggerType::Start); + } + if has_polling_trigger { + triggers.push(EthereumBlockTriggerType::End); + } + triggers +} + /// This method does not parse block triggers with `once` filters. /// This is because it is to be run before any other triggers are run. /// So we have `parse_initialization_triggers` for that. @@ -1994,38 +2065,12 @@ pub(crate) fn parse_block_triggers( EthereumBlockTriggerType::End, )); } else if !block_filter.polling_intervals.is_empty() { - let has_polling_trigger = - &block_filter - .polling_intervals - .iter() - .any(|(start_block, interval)| match interval { - 0 => false, - _ => { - block_number >= *start_block - && (block_number - *start_block) % *interval == 0 - } - }); - - let has_once_trigger = - &block_filter - .polling_intervals - .iter() - .any(|(start_block, interval)| match interval { - 0 => block_number == *start_block, - _ => false, - }); - - if *has_once_trigger { - triggers.push(EthereumTrigger::Block( - block_ptr3.clone(), - EthereumBlockTriggerType::Start, - )); - } - - if *has_polling_trigger { + for trigger_type in + block_trigger_types_from_intervals(block_number, &block_filter.polling_intervals) + { triggers.push(EthereumTrigger::Block( - block_ptr3, - EthereumBlockTriggerType::End, + block_ptr3.cheap_clone(), + trigger_type, )); } } @@ -2035,7 +2080,7 @@ pub(crate) fn parse_block_triggers( async fn fetch_receipt_from_ethereum_client( eth: &EthereumAdapter, transaction_hash: B256, -) -> anyhow::Result { +) -> anyhow::Result { match eth.alloy.get_transaction_receipt(transaction_hash).await { Ok(Some(receipt)) => Ok(receipt), Ok(None) => bail!("Could not find transaction receipt"), @@ -2047,7 +2092,7 @@ async fn filter_call_triggers_from_unsuccessful_transactions( mut block: BlockWithTriggers, eth: &EthereumAdapter, chain_store: &Arc, - logger: &Logger, + logger: &ProviderLogger, ) -> anyhow::Result> { // Return early if there is no trigger data if block.trigger_data.is_empty() { @@ -2077,7 +2122,7 @@ async fn filter_call_triggers_from_unsuccessful_transactions( // And obtain all Transaction values for the calls in this block. let transactions: Vec<&AnyTransaction> = { match &block.block { - BlockFinality::Final(ref block) => block + BlockFinality::Final(block) => block .transactions() .ok_or_else(|| anyhow!("Block transactions not available"))? .iter() @@ -2187,17 +2232,18 @@ async fn fetch_transaction_receipts_in_batch_with_retry( alloy: Arc, hashes: Vec, block_hash: B256, - logger: Logger, -) -> Result>, IngestorError> { + logger: ProviderLogger, + settings: &ChainSettings, +) -> Result>, IngestorError> { let retry_log_message = format!( "batch eth_getTransactionReceipt RPC call for block {:?}", block_hash ); retry(retry_log_message, &logger) .redact_log_urls(true) - .limit(ENV_VARS.request_retries) + .limit(settings.request_retries) .no_logging() - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .timeout_secs(settings.json_rpc_timeout.as_secs()) .run(move || { let alloy = alloy.cheap_clone(); let hashes = hashes.clone(); @@ -2213,8 +2259,8 @@ async fn fetch_transaction_receipts_in_batch( alloy: Arc, hashes: Vec, block_hash: B256, - logger: Logger, -) -> Result>, IngestorError> { + logger: ProviderLogger, +) -> Result>, IngestorError> { // Use the batch method to get all receipts at once let receipts = batch_get_transaction_receipts(alloy, hashes.clone()) .await @@ -2243,17 +2289,16 @@ async fn fetch_transaction_receipts_in_batch( async fn batch_get_transaction_receipts( provider: Arc, tx_hashes: Vec, -) -> Result>, Box> { +) -> Result>, Box> { let mut batch = alloy::rpc::client::BatchRequest::new(provider.client()); let mut receipt_futures = Vec::new(); // Add all receipt requests to batch for tx_hash in &tx_hashes { - let receipt_future = batch - .add_call::<(B256,), Option>( - "eth_getTransactionReceipt", - &(*tx_hash,), - )?; + let receipt_future = batch.add_call::<(B256,), Option>( + "eth_getTransactionReceipt", + &(*tx_hash,), + )?; receipt_futures.push(receipt_future); } @@ -2303,13 +2348,14 @@ async fn fetch_receipts_with_retry( alloy: Arc, hashes: Vec, block_hash: B256, - logger: Logger, + logger: ProviderLogger, supports_block_receipts: bool, -) -> Result>, IngestorError> { + settings: &ChainSettings, +) -> Result>, IngestorError> { if supports_block_receipts { - return fetch_block_receipts_with_retry(alloy, hashes, block_hash, logger).await; + return fetch_block_receipts_with_retry(alloy, hashes, block_hash, logger, settings).await; } - fetch_individual_receipts_with_retry(alloy, hashes, block_hash, logger).await + fetch_individual_receipts_with_retry(alloy, hashes, block_hash, logger, settings).await } // Fetches receipts for each transaction in the block individually. @@ -2317,13 +2363,20 @@ async fn fetch_individual_receipts_with_retry( alloy: Arc, hashes: Vec, block_hash: B256, - logger: Logger, -) -> Result>, IngestorError> { + logger: ProviderLogger, + settings: &ChainSettings, +) -> Result>, IngestorError> { if ENV_VARS.fetch_receipts_in_batches { - return fetch_transaction_receipts_in_batch_with_retry(alloy, hashes, block_hash, logger) - .await; + return fetch_transaction_receipts_in_batch_with_retry( + alloy, hashes, block_hash, logger, settings, + ) + .await; } + let request_retries = settings.request_retries; + let json_rpc_timeout = settings.json_rpc_timeout; + let concurrent_requests = settings.block_ingestor_max_concurrent_json_rpc_calls; + // Use a stream to fetch receipts individually let hash_stream = tokio_stream::iter(hashes); let receipt_stream = hash_stream @@ -2333,13 +2386,15 @@ async fn fetch_individual_receipts_with_retry( tx_hash, block_hash, logger.cheap_clone(), + request_retries, + json_rpc_timeout, ) }) - .buffered(ENV_VARS.block_ingestor_max_concurrent_json_rpc_calls); + .buffered(concurrent_requests); - tokio_stream::StreamExt::collect::< - Result>, IngestorError>, - >(receipt_stream) + tokio_stream::StreamExt::collect::>, IngestorError>>( + receipt_stream, + ) .await } @@ -2348,17 +2403,17 @@ async fn fetch_block_receipts_with_retry( alloy: Arc, hashes: Vec, block_hash: B256, - logger: Logger, -) -> Result>, IngestorError> { + logger: ProviderLogger, + settings: &ChainSettings, +) -> Result>, IngestorError> { use graph::prelude::alloy::rpc::types::BlockId; - let logger = logger.cheap_clone(); let retry_log_message = format!("eth_getBlockReceipts RPC call for block {:?}", block_hash); // Perform the retry operation let receipts_option = retry(retry_log_message, &logger) .redact_log_urls(true) - .limit(ENV_VARS.request_retries) - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .limit(settings.request_retries) + .timeout_secs(settings.json_rpc_timeout.as_secs()) .run(move || alloy.get_block_receipts(BlockId::from(block_hash)).boxed()) .await .map_err(|_timeout| -> IngestorError { anyhow!(block_hash).into() })?; @@ -2393,9 +2448,10 @@ async fn fetch_transaction_receipt_with_retry( alloy: Arc, transaction_hash: B256, block_hash: B256, - logger: Logger, -) -> Result, IngestorError> { - let logger = logger.cheap_clone(); + logger: ProviderLogger, + request_retries: usize, + json_rpc_timeout: Duration, +) -> Result, IngestorError> { let retry_log_message = format!( "eth_getTransactionReceipt RPC call for transaction {:?}", transaction_hash @@ -2403,8 +2459,8 @@ async fn fetch_transaction_receipt_with_retry( retry(retry_log_message, &logger) .redact_log_urls(true) - .limit(ENV_VARS.request_retries) - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .limit(request_retries) + .timeout_secs(json_rpc_timeout.as_secs()) .run(move || { let alloy_clone = alloy.clone(); async move { alloy_clone.get_transaction_receipt(transaction_hash).await }.boxed() @@ -2418,11 +2474,11 @@ async fn fetch_transaction_receipt_with_retry( } fn resolve_transaction_receipt( - transaction_receipt: Option, + transaction_receipt: Option, transaction_hash: B256, block_hash: B256, - logger: Logger, -) -> Result { + logger: ProviderLogger, +) -> Result { match transaction_receipt { // A receipt might be missing because the block was uncled, and the transaction never // made it back into the main chain. @@ -2475,7 +2531,7 @@ fn resolve_transaction_receipt( /// Retrieves logs and the associated transaction receipts, if required by the [`EthereumLogFilter`]. async fn get_logs_and_transactions( adapter: &Arc, - logger: &Logger, + logger: ProviderLogger, subgraph_metrics: Arc, from: BlockNumber, to: BlockNumber, @@ -2485,7 +2541,7 @@ async fn get_logs_and_transactions( // Obtain logs externally let logs = adapter .logs_in_block_range( - logger, + &logger, subgraph_metrics.cheap_clone(), from, to, @@ -2554,12 +2610,11 @@ async fn get_transaction_receipts_for_transaction_hashes( adapter: &EthereumAdapter, transaction_hashes_by_block: &HashMap>, subgraph_metrics: Arc, - logger: Logger, -) -> Result>, anyhow::Error> { + logger: ProviderLogger, +) -> Result>, anyhow::Error> { use std::collections::hash_map::Entry::Vacant; - let mut receipts_by_hash: HashMap> = - HashMap::new(); + let mut receipts_by_hash: HashMap> = HashMap::new(); // Return early if input set is empty if transaction_hashes_by_block.is_empty() { @@ -2582,6 +2637,8 @@ async fn get_transaction_receipts_for_transaction_hashes( *transaction_hash, *block_hash, logger.cheap_clone(), + adapter.settings.request_retries, + adapter.settings.json_rpc_timeout, ); receipt_futures.push(receipt_future) } @@ -2636,16 +2693,15 @@ mod tests { use crate::trigger::{EthereumBlockTriggerType, EthereumTrigger}; use super::{ - check_block_receipt_support, parse_block_triggers, EthereumBlock, EthereumBlockFilter, - EthereumBlockWithCalls, + EthereumBlock, EthereumBlockFilter, EthereumBlockWithCalls, + block_trigger_types_from_intervals, check_block_receipt_support, parse_block_triggers, }; use graph::blockchain::BlockPtr; - use graph::components::ethereum::AnyBlock; - use graph::prelude::alloy::network::AnyNetwork; - use graph::prelude::alloy::primitives::{Address, Bytes, B256}; - use graph::prelude::alloy::providers::mock::Asserter; + use graph::components::ethereum::AnyNetworkBare; + use graph::prelude::alloy::primitives::{Address, B256, Bytes}; use graph::prelude::alloy::providers::ProviderBuilder; - use graph::prelude::{create_minimal_block_for_test, EthereumCall, LightEthereumBlock}; + use graph::prelude::alloy::providers::mock::Asserter; + use graph::prelude::{EthereumCall, LightEthereumBlock, create_minimal_block_for_test}; use jsonrpc_core::serde_json::{self, Value}; use std::collections::HashSet; use std::iter::FromIterator; @@ -2657,7 +2713,7 @@ mod tests { let block = EthereumBlockWithCalls { ethereum_block: EthereumBlock { - block: Arc::new(LightEthereumBlock::new(AnyBlock::from(block))), + block: Arc::new(LightEthereumBlock::new(block)), ..Default::default() }, calls: Some(vec![EthereumCall { @@ -2718,9 +2774,8 @@ mod tests { let json_value: Value = serde_json::from_str(json_response).unwrap(); let asserter = Asserter::new(); - let provider = ProviderBuilder::<_, _, AnyNetwork>::default() - .network::() - .with_recommended_fillers() + let provider = ProviderBuilder::<_, _, AnyNetworkBare>::default() + .network::() .connect_mocked_client(asserter.clone()); asserter.push_success(&json_value); @@ -2802,7 +2857,7 @@ mod tests { #[allow(unreachable_code)] let block = EthereumBlockWithCalls { ethereum_block: EthereumBlock { - block: Arc::new(LightEthereumBlock::new(AnyBlock::from(block))), + block: Arc::new(LightEthereumBlock::new(block)), ..Default::default() }, calls: Some(vec![EthereumCall { @@ -2833,7 +2888,7 @@ mod tests { #[allow(unreachable_code)] let block = EthereumBlockWithCalls { ethereum_block: EthereumBlock { - block: Arc::new(LightEthereumBlock::new(AnyBlock::from(block))), + block: Arc::new(LightEthereumBlock::new(block)), ..Default::default() }, calls: Some(vec![EthereumCall { @@ -2860,6 +2915,115 @@ mod tests { ); } + #[test] + fn block_trigger_types_once_only() { + let intervals = HashSet::from_iter(vec![(100, 0)]); + + assert_eq!( + vec![EthereumBlockTriggerType::Start], + block_trigger_types_from_intervals(100, &intervals), + "once rule fires Start at start_block" + ); + assert_eq!( + Vec::::new(), + block_trigger_types_from_intervals(99, &intervals), + "once rule does not fire before start_block" + ); + assert_eq!( + Vec::::new(), + block_trigger_types_from_intervals(101, &intervals), + "once rule does not fire after start_block" + ); + } + + #[test] + fn block_trigger_types_polling_only() { + let intervals = HashSet::from_iter(vec![(100, 10)]); + + assert_eq!( + vec![EthereumBlockTriggerType::End], + block_trigger_types_from_intervals(100, &intervals), + "polling rule fires End at start_block" + ); + assert_eq!( + vec![EthereumBlockTriggerType::End], + block_trigger_types_from_intervals(110, &intervals), + "polling rule fires End at start_block + interval" + ); + assert_eq!( + Vec::::new(), + block_trigger_types_from_intervals(105, &intervals), + "polling rule does not fire off-interval" + ); + assert_eq!( + Vec::::new(), + block_trigger_types_from_intervals(90, &intervals), + "polling rule does not fire before start_block" + ); + } + + #[test] + fn block_trigger_types_once_and_polling_same_start_block() { + // A single data source with both a `once` handler and a `polling` handler + // contributes both entries at its start_block. Both must fire at start_block. + let intervals = HashSet::from_iter(vec![(100, 0), (100, 10)]); + + assert_eq!( + vec![ + EthereumBlockTriggerType::Start, + EthereumBlockTriggerType::End, + ], + block_trigger_types_from_intervals(100, &intervals), + "both Start and End should fire when once and polling rules share start_block" + ); + assert_eq!( + vec![EthereumBlockTriggerType::End], + block_trigger_types_from_intervals(110, &intervals), + "only polling fires at later interval matches" + ); + } + + #[test] + fn block_trigger_types_cross_datasource_collision() { + // Two data sources: DS-A with once at 100, DS-B with polling every 10 from 50. + // Block 100 satisfies DS-A's once rule and also (100-50) % 10 == 0, so both fire. + let intervals = HashSet::from_iter(vec![(100, 0), (50, 10)]); + + assert_eq!( + vec![ + EthereumBlockTriggerType::Start, + EthereumBlockTriggerType::End, + ], + block_trigger_types_from_intervals(100, &intervals), + "both triggers fire when a once rule and an unrelated polling rule collide" + ); + assert_eq!( + vec![EthereumBlockTriggerType::End], + block_trigger_types_from_intervals(60, &intervals), + "only polling fires at a block where only the polling rule matches" + ); + } + + #[test] + fn block_trigger_types_no_match() { + let intervals = HashSet::from_iter(vec![(100, 0), (100, 10)]); + assert_eq!( + Vec::::new(), + block_trigger_types_from_intervals(99, &intervals), + "no triggers when block matches neither rule" + ); + } + + #[test] + fn block_trigger_types_empty_intervals() { + let intervals: HashSet<(i32, i32)> = HashSet::new(); + assert_eq!( + Vec::::new(), + block_trigger_types_from_intervals(100, &intervals), + "empty intervals yields no triggers" + ); + } + fn address(id: u64) -> Address { Address::left_padding_from(&id.to_be_bytes()) } diff --git a/chain/ethereum/src/ingestor.rs b/chain/ethereum/src/ingestor.rs index 9e7a8c01725..dcbcc42c6f8 100644 --- a/chain/ethereum/src/ingestor.rs +++ b/chain/ethereum/src/ingestor.rs @@ -1,8 +1,9 @@ -use crate::{chain::BlockFinality, ENV_VARS}; +use crate::{ENV_VARS, chain::BlockFinality}; use crate::{EthereumAdapter, EthereumAdapterTrait as _}; use async_trait::async_trait; -use graph::blockchain::client::ChainClient; +use futures::future::select_ok; use graph::blockchain::BlockchainKind; +use graph::blockchain::client::ChainClient; use graph::components::network_provider::ChainName; use graph::prelude::alloy::primitives::B256; use graph::slog::o; @@ -11,7 +12,8 @@ use graph::{ blockchain::{BlockHash, BlockIngestor, BlockPtr, IngestorError}, cheap_clone::CheapClone, prelude::{ - error, info, tokio, trace, warn, ChainStore, Error, EthereumBlockWithCalls, LogCode, Logger, + ChainStore, Error, EthereumBlockWithCalls, LogCode, Logger, debug, error, info, tokio, + trace, warn, }, }; use std::{sync::Arc, time::Duration}; @@ -213,12 +215,111 @@ impl PollingBlockIngestor { eth_adapter.latest_block_ptr(logger).await } - async fn eth_adapter(&self) -> anyhow::Result> { - self.chain_client - .rpc()? - .cheapest() - .await - .ok_or_else(|| graph::anyhow::anyhow!("unable to get eth adapter")) + /// Executes one polling iteration. On failure delegates to `on_poll_failure` for + /// probe+switch logic. + async fn poll_once( + &self, + providers: &[Arc], + current_provider: &mut Option, + ) { + // Resolve by name; resets to first provider if the tracked one left the list. + let eth_adapter = resolve_provider(providers, current_provider, &self.logger).clone(); + // Pin the name so the next iteration knows which provider is active. + let provider_name = eth_adapter.provider().to_string(); + let logger = self.logger.new(o!("provider" => provider_name.clone())); + *current_provider = Some(provider_name); + + if let Err(err) = self.do_poll(&logger, eth_adapter).await { + error!(logger, "Trying again after block polling failed: {}", err); + on_poll_failure(providers, current_provider, &self.logger).await; + } + } +} + +/// Returns the currently-tracked provider from `providers`. +/// +/// If the tracked provider is no longer in the list (it became invalid and was removed by +/// `ProviderManager`), logs a warning, resets the state, and returns the first available +/// provider. +fn resolve_provider<'a, A: crate::EthereumAdapterTrait>( + providers: &'a [Arc], + current_provider: &mut Option, + logger: &Logger, +) -> &'a Arc { + if let Some(name) = current_provider.as_ref() { + if let Some(found) = providers.iter().find(|p| p.provider() == name) { + return found; + } + warn!( + logger, + "Current RPC provider is no longer available, resetting to first provider"; + "provider" => name, + ); + *current_provider = None; + } + &providers[0] +} + +async fn on_poll_failure( + providers: &[Arc], + current_provider: &mut Option, + logger: &Logger, +) { + if providers.len() <= 1 { + return; + } + + let current_name = match current_provider.as_ref() { + Some(name) => name.clone(), + None => return, + }; + + // Probe the current provider before trying alternatives. do_poll() can fail for + // reasons unrelated to RPC availability (e.g. DB errors from chain_head_ptr or + // attempt_chain_head_update, or a BlockUnavailable from a chain reorg). All of + // these surface as IngestorError::Unknown, indistinguishable from an RPC failure + // at the match level. If the current provider responds to eth_blockNumber, the + // failure was not caused by provider unavailability — switching cannot help. + let current = providers.iter().find(|p| p.provider() == current_name); + if let Some(current) = current + && current.is_reachable().await + { + return; + } + + // Probe all alternatives in parallel; switch to the first that responds. + let futs = providers + .iter() + .filter(|p| p.provider() != current_name) + .map(|p| { + let name = p.provider().to_string(); + debug!(logger, "Trying RPC provider"; "provider" => &name); + Box::pin(async move { + if p.is_reachable().await { + Ok(name) + } else { + Err(()) + } + }) + }); + + match select_ok(futs).await { + Ok((next_name, _)) => { + warn!( + logger, + "Switching RPC provider for block ingestor"; + "from" => ¤t_name, + "to" => &next_name, + ); + *current_provider = Some(next_name); + } + Err(_) => { + warn!( + logger, + "All RPC providers unreachable, continuing on current provider"; + "provider" => ¤t_name, + ); + } } } @@ -227,33 +328,28 @@ impl BlockIngestor for PollingBlockIngestor { async fn run(self: Box) { let mut backoff = ExponentialBackoff::new(Duration::from_millis(250), Duration::from_secs(30)); + // Name of the provider currently in use. `None` until the first poll. + let mut current_provider: Option = None; loop { - let eth_adapter = match self.eth_adapter().await { - Ok(adapter) => { - backoff.reset(); - adapter - } - Err(err) => { - error!( - &self.logger, - "unable to get ethereum adapter, backing off... error: {}", - err.to_string() - ); - backoff.sleep_async().await; - continue; - } - }; - let logger = self - .logger - .new(o!("provider" => eth_adapter.provider().to_string())); + let providers = self + .chain_client + .rpc() + .expect("PollingBlockIngestor is only created for RPC chains") + .all_cheapest() + .await; - if let Err(err) = self.do_poll(&logger, eth_adapter).await { - error!(logger, "Trying again after block polling failed: {}", err); + if providers.is_empty() { + error!(self.logger, "No RPC providers available for block ingestor"); + backoff.sleep_async().await; + continue; } + backoff.reset(); + + self.poll_once(&providers, &mut current_provider).await; if ENV_VARS.cleanup_blocks { - self.cleanup_cached_blocks().await + self.cleanup_cached_blocks().await; } tokio::time::sleep(self.polling_interval).await; @@ -268,3 +364,266 @@ impl BlockIngestor for PollingBlockIngestor { BlockchainKind::Ethereum } } + +#[cfg(test)] +mod tests { + use super::on_poll_failure; + use super::*; + use crate::adapter::{ + ContractCallError, EthereumAdapter as EthereumAdapterTrait, EthereumRpcError, + }; + use async_trait::async_trait; + use graph::blockchain::{BlockPtr, ChainIdentifier}; + use graph::components::ethereum::AnyBlock; + use graph::components::ethereum::LightEthereumBlock; + use graph::data::store::ethereum::call; + use graph::data_source::common::ContractCall; + use graph::prelude::alloy::primitives::{Address, B256, Bytes, U256}; + use graph::prelude::{BlockNumber, Error, EthereumCallCache, Logger}; + use graph::slog::Discard; + use std::collections::HashSet; + use std::sync::Arc; + + struct MockEthAdapter { + provider_name: String, + reachable: bool, + } + + impl MockEthAdapter { + fn new(name: &str, reachable: bool) -> Arc { + Arc::new(Self { + provider_name: name.to_string(), + reachable, + }) + } + } + + #[async_trait] + impl EthereumAdapterTrait for MockEthAdapter { + fn provider(&self) -> &str { + &self.provider_name + } + + async fn is_reachable(&self) -> bool { + self.reachable + } + + async fn net_identifiers(&self) -> Result { + unimplemented!() + } + async fn latest_block_ptr( + &self, + _: &Logger, + ) -> Result { + unimplemented!() + } + async fn load_blocks( + &self, + _: Logger, + _: Arc, + _: HashSet, + ) -> Result>, Error> { + unimplemented!() + } + async fn block_by_hash(&self, _: &Logger, _: B256) -> Result, Error> { + unimplemented!() + } + async fn block_by_number( + &self, + _: &Logger, + _: BlockNumber, + ) -> Result, Error> { + unimplemented!() + } + async fn load_full_block( + &self, + _: &Logger, + _: AnyBlock, + ) -> Result { + unimplemented!() + } + async fn next_existing_ptr_to_number( + &self, + _: &Logger, + _: BlockNumber, + ) -> Result { + unimplemented!() + } + async fn contract_call( + &self, + _: &Logger, + _: &ContractCall, + _: Arc, + ) -> Result<(Option>, call::Source), ContractCallError> + { + unimplemented!() + } + async fn contract_calls( + &self, + _: &Logger, + _: &[&ContractCall], + _: Arc, + ) -> Result>, call::Source)>, ContractCallError> + { + unimplemented!() + } + async fn get_balance( + &self, + _: &Logger, + _: Address, + _: BlockPtr, + ) -> Result { + unimplemented!() + } + async fn get_code( + &self, + _: &Logger, + _: Address, + _: BlockPtr, + ) -> Result { + unimplemented!() + } + } + + fn discard_logger() -> Logger { + Logger::root(Discard, o!()) + } + + #[test] + fn test_current_provider_unavailable_resets_to_first() { + // p0 left the validated list; only p1 and p2 remain (p1 is now at index 0). + let providers: Vec> = vec![ + MockEthAdapter::new("p1", true), + MockEthAdapter::new("p2", true), + ]; + let mut current_provider = Some("p0".to_string()); + let resolved = resolve_provider(&providers, &mut current_provider, &discard_logger()); + assert_eq!(resolved.provider(), "p1"); + assert_eq!(current_provider, None); + } + + #[tokio::test] + async fn test_current_reachable_no_switch() { + // Current provider is reachable: on_poll_failure should return early without + // switching, even if alternatives are also reachable. + let providers: Vec> = vec![ + MockEthAdapter::new("p0", true), + MockEthAdapter::new("p1", true), + ]; + let mut current_provider = Some("p0".to_string()); + on_poll_failure(&providers, &mut current_provider, &discard_logger()).await; + assert_eq!(current_provider, Some("p0".to_string())); + } + + #[tokio::test] + async fn test_successful_probe_switches_provider() { + let providers: Vec> = vec![ + MockEthAdapter::new("p0", false), + MockEthAdapter::new("p1", true), + ]; + let mut current_provider = Some("p0".to_string()); + on_poll_failure(&providers, &mut current_provider, &discard_logger()).await; + assert_eq!(current_provider, Some("p1".to_string())); + } + + #[tokio::test] + async fn test_all_unreachable_stays_on_current() { + let providers: Vec> = vec![ + MockEthAdapter::new("p0", false), + MockEthAdapter::new("p1", false), + ]; + let mut current_provider = Some("p0".to_string()); + on_poll_failure(&providers, &mut current_provider, &discard_logger()).await; + assert_eq!(current_provider, Some("p0".to_string())); + } + + #[tokio::test] + async fn test_single_provider_no_switch() { + let providers: Vec> = vec![MockEthAdapter::new("p0", true)]; + let mut current_provider = Some("p0".to_string()); + on_poll_failure(&providers, &mut current_provider, &discard_logger()).await; + assert_eq!(current_provider, Some("p0".to_string())); + } + + // --- Provider-list churn tests --- + // + // These tests simulate the state carried across loop iterations where + // all_cheapest() returns a different subset each time. + + /// Current provider survives a list shrink; its index shifts but identity is preserved. + /// + /// Before: [p0, p1, p2], active = p1 (index 1) + /// After: [p1, p2], active = p1 (index 0) + /// + /// With index-based tracking this would silently move to p2. Name-based tracking + /// must keep us on p1. + #[test] + fn test_list_shrinks_current_provider_index_remaps() { + let providers: Vec> = vec![ + MockEthAdapter::new("p1", true), + MockEthAdapter::new("p2", true), + ]; + let mut current_provider = Some("p1".to_string()); + let resolved = resolve_provider(&providers, &mut current_provider, &discard_logger()); + // p1 is now at index 0 — must not drift to p2 + assert_eq!(resolved.provider(), "p1"); + assert_eq!(current_provider, Some("p1".to_string())); + } + + /// After failing over to p1, p1 itself disappears from the validated list in the + /// next iteration. The ingestor must fall back to the first available provider (p0). + /// + /// Iteration 1: [p0(unreachable), p1(reachable)] — p0 fails, switch to p1 + /// Iteration 2: [p0(reachable), p2(reachable)] — p1 gone, reset to p0 + #[tokio::test] + async fn test_failover_target_then_disappears_resets_to_first() { + let logger = discard_logger(); + + // Iteration 1: p0 fails, switch to p1. + let providers_iter1: Vec> = vec![ + MockEthAdapter::new("p0", false), + MockEthAdapter::new("p1", true), + ]; + let mut current_provider = Some("p0".to_string()); + on_poll_failure(&providers_iter1, &mut current_provider, &logger).await; + assert_eq!(current_provider, Some("p1".to_string())); + + // Iteration 2: p1 has dropped from the validated list. + let providers_iter2: Vec> = vec![ + MockEthAdapter::new("p0", true), + MockEthAdapter::new("p2", true), + ]; + let resolved = resolve_provider(&providers_iter2, &mut current_provider, &logger); + assert_eq!(resolved.provider(), "p0"); // reset to first + assert_eq!(current_provider, None); + } + + /// After failing over to p1, the original provider p0 reappears in the list. + /// The ingestor must stay on p1 — there is no proactive return to p0. + /// + /// Iteration 1: [p0(unreachable), p1(reachable)] — switch to p1 + /// Iteration 2: [p0(reachable), p1(reachable)] — p0 is back; must stay on p1 + #[tokio::test] + async fn test_original_provider_reappears_no_involuntary_return() { + let logger = discard_logger(); + + // Iteration 1: switch to p1. + let providers_iter1: Vec> = vec![ + MockEthAdapter::new("p0", false), + MockEthAdapter::new("p1", true), + ]; + let mut current_provider = Some("p0".to_string()); + on_poll_failure(&providers_iter1, &mut current_provider, &logger).await; + assert_eq!(current_provider, Some("p1".to_string())); + + // Iteration 2: p0 is back in the validated list alongside p1. + let providers_iter2: Vec> = vec![ + MockEthAdapter::new("p0", true), + MockEthAdapter::new("p1", true), + ]; + let resolved = resolve_provider(&providers_iter2, &mut current_provider, &logger); + // Must resolve to p1, not drift back to p0. + assert_eq!(resolved.provider(), "p1"); + assert_eq!(current_provider, Some("p1".to_string())); + } +} diff --git a/chain/ethereum/src/lib.rs b/chain/ethereum/src/lib.rs index 8850764d63b..b21eb4717ae 100644 --- a/chain/ethereum/src/lib.rs +++ b/chain/ethereum/src/lib.rs @@ -14,14 +14,16 @@ mod transport; pub use self::capabilities::NodeCapabilities; pub use self::ethereum_adapter::EthereumAdapter; pub use self::runtime::RuntimeAdapter; -pub use self::transport::Transport; +pub use self::transport::{Compression, Transport}; pub use env::ENV_VARS; pub use buffered_call_cache::BufferedCallCache; // ETHDEP: These concrete types should probably not be exposed. pub use data_source::{ - BlockHandlerFilter, DataSource, DataSourceTemplate, Mapping, TemplateSource, + BlockHandlerFilter, DataSource, DataSourceTemplate, Mapping, MappingBlockHandler, + MappingCallHandler, TemplateSource, UnresolvedDataSource, UnresolvedDataSourceTemplate, + UnresolvedMapping, UnresolvedMappingEventHandler, }; pub mod chain; diff --git a/chain/ethereum/src/network.rs b/chain/ethereum/src/network.rs index 536f7a8a54d..ad9cd0a83c0 100644 --- a/chain/ethereum/src/network.rs +++ b/chain/ethereum/src/network.rs @@ -15,9 +15,9 @@ use std::sync::Arc; pub use graph::impl_slog_value; use graph::prelude::Error; +use crate::EthereumAdapter; use crate::adapter::EthereumAdapter as _; use crate::capabilities::NodeCapabilities; -use crate::EthereumAdapter; pub const DEFAULT_ADAPTER_ERROR_RETEST_PERCENT: f64 = 0.2; @@ -107,7 +107,7 @@ impl EthereumNetworkAdapters { use std::cmp::Ordering; use graph::components::network_provider::ProviderCheckStrategy; - use graph::slog::{o, Discard, Logger}; + use graph::slog::{Discard, Logger, o}; let chain_id: ChainName = "testing".into(); adapters.sort_by(|a, b| { @@ -118,7 +118,7 @@ impl EthereumNetworkAdapters { let provider = ProviderManager::new( Logger::root(Discard, o!()), - vec![(chain_id.clone(), adapters)].into_iter(), + vec![(chain_id.clone(), adapters)], ProviderCheckStrategy::MarkAsValid, ); @@ -249,6 +249,15 @@ impl EthereumNetworkAdapters { Self::cheapest_from(cheapest, required_capabilities, self.retest_percent) } + /// Returns all validated providers. Unvalidated providers are excluded. + pub async fn all_cheapest(&self) -> Vec> { + self.manager + .providers(&self.chain_id) + .await + .map(|adapters| adapters.map(|a| a.adapter.clone()).collect()) + .unwrap_or_default() + } + pub async fn cheapest(&self) -> Option> { // EthereumAdapters are sorted by their NodeCapabilities when the EthereumNetworks // struct is instantiated so they do not need to be sorted here @@ -314,17 +323,21 @@ mod tests { use graph::components::network_provider::ProviderManager; use graph::components::network_provider::ProviderName; use graph::data::value::Word; + use graph::http::HeaderMap; use graph::{ endpoint::EndpointMetrics, firehose::SubgraphLimit, prelude::MetricsRegistry, - slog::{o, Discard, Logger}, + slog::{Discard, Logger, o}, url::Url, }; use std::sync::Arc; - use crate::{EthereumAdapter, EthereumAdapterTrait, ProviderEthRpcMetrics, Transport}; + use crate::{ + Compression, EthereumAdapter, EthereumAdapterTrait, ProviderEthRpcMetrics, Transport, + chain::ChainSettings, + }; use super::{EthereumNetworkAdapter, EthereumNetworkAdapters, NodeCapabilities}; @@ -394,6 +407,8 @@ mod tests { HeaderMap::new(), metrics.clone(), "", + false, + Compression::None, ); let provider_metrics = Arc::new(ProviderEthRpcMetrics::new(mock_registry.clone())); @@ -405,6 +420,7 @@ mod tests { provider_metrics.clone(), true, true, + Arc::new(ChainSettings::from_env_defaults()), ) .await, ); @@ -417,6 +433,7 @@ mod tests { provider_metrics.clone(), true, false, + Arc::new(ChainSettings::from_env_defaults()), ) .await, ); @@ -448,13 +465,15 @@ mod tests { { // Not Found - assert!(adapters - .cheapest_with(&NodeCapabilities { - archive: false, - traces: true, - }) - .await - .is_err()); + assert!( + adapters + .cheapest_with(&NodeCapabilities { + archive: false, + traces: true, + }) + .await + .is_err() + ); // Check cheapest is not call only let adapter = adapters @@ -497,6 +516,8 @@ mod tests { HeaderMap::new(), metrics.clone(), "", + false, + Compression::None, ); let provider_metrics = Arc::new(ProviderEthRpcMetrics::new(mock_registry.clone())); @@ -508,6 +529,7 @@ mod tests { provider_metrics.clone(), true, true, + Arc::new(ChainSettings::from_env_defaults()), ) .await, ); @@ -520,6 +542,7 @@ mod tests { provider_metrics.clone(), true, false, + Arc::new(ChainSettings::from_env_defaults()), ) .await, ); @@ -568,6 +591,8 @@ mod tests { HeaderMap::new(), metrics.clone(), "", + false, + Compression::None, ); let provider_metrics = Arc::new(ProviderEthRpcMetrics::new(mock_registry.clone())); @@ -579,6 +604,7 @@ mod tests { provider_metrics.clone(), true, true, + Arc::new(ChainSettings::from_env_defaults()), ) .await, ); @@ -591,6 +617,7 @@ mod tests { provider_metrics.clone(), true, false, + Arc::new(ChainSettings::from_env_defaults()), ) .await, ); @@ -632,6 +659,8 @@ mod tests { HeaderMap::new(), metrics.clone(), "", + false, + Compression::None, ); let provider_metrics = Arc::new(ProviderEthRpcMetrics::new(mock_registry.clone())); @@ -643,6 +672,7 @@ mod tests { provider_metrics.clone(), true, false, + Arc::new(ChainSettings::from_env_defaults()), ) .await, ); @@ -919,6 +949,8 @@ mod tests { HeaderMap::new(), endpoint_metrics.clone(), "", + false, + Compression::None, ); Arc::new( @@ -929,6 +961,7 @@ mod tests { provider_metrics.clone(), true, call_only, + Arc::new(ChainSettings::from_env_defaults()), ) .await, ) diff --git a/chain/ethereum/src/polling_block_stream.rs b/chain/ethereum/src/polling_block_stream.rs index 705fdaaaf7f..339386207f1 100644 --- a/chain/ethereum/src/polling_block_stream.rs +++ b/chain/ethereum/src/polling_block_stream.rs @@ -1,4 +1,4 @@ -use anyhow::{anyhow, Error}; +use anyhow::{Error, anyhow}; use std::cmp; use std::collections::VecDeque; use std::pin::Pin; @@ -7,13 +7,13 @@ use std::task::{Context, Poll}; use std::time::Duration; use graph::blockchain::block_stream::{ - BlockStream, BlockStreamError, BlockStreamEvent, BlockWithTriggers, ChainHeadUpdateStream, - FirehoseCursor, TriggersAdapterWrapper, BUFFERED_BLOCK_STREAM_SIZE, + BUFFERED_BLOCK_STREAM_SIZE, BlockStream, BlockStreamError, BlockStreamEvent, BlockWithTriggers, + ChainHeadUpdateStream, FirehoseCursor, TriggersAdapterWrapper, }; use graph::blockchain::{Block, BlockPtr, TriggerFilterWrapper}; -use graph::futures03::{stream::Stream, Future, FutureExt}; -use graph::prelude::{DeploymentHash, BLOCK_NUMBER_MAX}; -use graph::slog::{info, trace, warn, Logger}; +use graph::futures03::{Future, FutureExt, stream::Stream}; +use graph::prelude::{BLOCK_NUMBER_MAX, DeploymentHash}; +use graph::slog::{Logger, info, trace, warn}; use graph::components::store::BlockNumber; use graph::data::subgraph::UnifiedMappingApiVersion; @@ -182,7 +182,7 @@ impl PollingBlockStreamContext { return Ok(NextBlocks::Done); } ReconciliationStep::Revert(parent_ptr) => { - return Ok(NextBlocks::Revert(parent_ptr)) + return Ok(NextBlocks::Revert(parent_ptr)); } } } @@ -224,10 +224,10 @@ impl PollingBlockStreamContext { // Only continue if the subgraph block ptr is behind the head block ptr. // subgraph_ptr > head_ptr shouldn't happen, but if it does, it's safest to just stop. - if let Some(ptr) = &subgraph_ptr { - if ptr.number >= head_ptr.number { - return Ok(ReconciliationStep::Done); - } + if let Some(ptr) = &subgraph_ptr + && ptr.number >= head_ptr.number + { + return Ok(ReconciliationStep::Done); } // Subgraph ptr is behind head ptr. @@ -370,10 +370,6 @@ impl PollingBlockStreamContext { return Ok(ReconciliationStep::Retry); } - if to > head_ptr.number - reorg_threshold { - return Ok(ReconciliationStep::Retry); - } - info!( ctx.logger, "Scanned blocks [{}, {}]", from, to; @@ -573,7 +569,7 @@ impl Stream for PollingBlockStream { } // Yielding blocks from reconciliation process - BlockStreamState::YieldingBlocks(ref mut next_blocks) => { + BlockStreamState::YieldingBlocks(next_blocks) => { match next_blocks.pop_front() { // Yield one block Some(next_block) => { @@ -593,7 +589,7 @@ impl Stream for PollingBlockStream { } // Pausing after an error, before looking for more blocks - BlockStreamState::RetryAfterDelay(ref mut delay) => match delay.as_mut().poll(cx) { + BlockStreamState::RetryAfterDelay(delay) => match delay.as_mut().poll(cx) { Poll::Ready(Ok(..)) | Poll::Ready(Err(_)) => { self.state = BlockStreamState::BeginReconciliation; } diff --git a/chain/ethereum/src/runtime/abi.rs b/chain/ethereum/src/runtime/abi.rs index 7ac13752671..4311fed3c48 100644 --- a/chain/ethereum/src/runtime/abi.rs +++ b/chain/ethereum/src/runtime/abi.rs @@ -9,12 +9,11 @@ use graph::prelude::alloy; use graph::prelude::alloy::consensus::TxReceipt; use graph::prelude::alloy::network::ReceiptResponse; use graph::prelude::alloy::rpc::types::{Log, TransactionReceipt}; -use graph::prelude::alloy::serde::WithOtherFields; use graph::{ prelude::BigInt, runtime::{ - asc_get, asc_new, asc_new_or_null, gas::GasCounter, AscHeap, AscIndexId, AscPtr, AscType, - DeterministicHostError, FromAscObj, HostExportError, IndexForAscTypeId, ToAscObj, + AscHeap, AscIndexId, AscPtr, AscType, DeterministicHostError, FromAscObj, HostExportError, + IndexForAscTypeId, ToAscObj, asc_get, asc_new, asc_new_or_null, gas::GasCounter, }, }; use graph_runtime_derive::AscType; @@ -587,10 +586,7 @@ where #[async_trait] impl<'a, T, B, Inner> ToAscObj> - for ( - EthereumEventData<'a>, - Option<&WithOtherFields>>, - ) + for (EthereumEventData<'a>, Option<&TransactionReceipt>) where T: AscType + AscIndexId + Send, B: AscType + AscIndexId + Send, @@ -615,7 +611,7 @@ where params, } = event_data.to_asc_obj(heap, gas).await?; let receipt = if let Some(receipt_data) = optional_receipt { - asc_new(heap, &receipt_data.inner(), gas).await? + asc_new(heap, receipt_data, gas).await? } else { AscPtr::null() }; diff --git a/chain/ethereum/src/runtime/runtime_adapter.rs b/chain/ethereum/src/runtime/runtime_adapter.rs index a5597efcd4d..a5d68a761a7 100644 --- a/chain/ethereum/src/runtime/runtime_adapter.rs +++ b/chain/ethereum/src/runtime/runtime_adapter.rs @@ -2,10 +2,10 @@ use std::{sync::Arc, time::Instant}; use crate::adapter::EthereumRpcError; use crate::{ - capabilities::NodeCapabilities, network::EthereumNetworkAdapters, Chain, ContractCallError, - EthereumAdapter, EthereumAdapterTrait, ENV_VARS, + Chain, ContractCallError, ENV_VARS, EthereumAdapter, EthereumAdapterTrait, + capabilities::NodeCapabilities, network::EthereumNetworkAdapters, }; -use anyhow::{anyhow, Context, Error}; +use anyhow::{Context, Error, anyhow}; use blockchain::HostFn; use graph::abi; use graph::abi::DynSolValueExt; @@ -23,8 +23,8 @@ use graph::{ blockchain::{self, BlockPtr, HostFnCtx}, cheap_clone::CheapClone, futures03::FutureExt, - prelude::{alloy::primitives::Address, EthereumCallCache}, - runtime::{asc_get, asc_new, AscPtr, HostExportError}, + prelude::{EthereumCallCache, alloy::primitives::Address}, + runtime::{AscPtr, HostExportError, asc_get, asc_new}, slog::Logger, }; use graph_runtime_wasm::asc_abi::class::{AscBigInt, AscEnumArray, AscWrapped, EthereumValueKind}; @@ -346,32 +346,34 @@ async fn eth_call( Err(e) => (Err(e), call::Source::Rpc), }; let result = match result { - Ok(res) => Ok(res), + Ok(res) => Ok(res), - // Any error reported by the Ethereum node could be due to the block no longer being on - // the main chain. This is very unespecific but we don't want to risk failing a - // subgraph due to a transient error such as a reorg. - Err(ContractCallError::AlloyError(e)) => Err(HostExportError::PossibleReorg(anyhow::anyhow!( + // Any error reported by the Ethereum node could be due to the block no longer being on + // the main chain. This is very unespecific but we don't want to risk failing a + // subgraph due to a transient error such as a reorg. + Err(ContractCallError::AlloyError(e)) => { + Err(HostExportError::PossibleReorg(anyhow::anyhow!( "Ethereum node returned an error when calling function \"{}\" of contract \"{}\": {}", unresolved_call.function_name, unresolved_call.contract_name, e - ))), - - // Also retry on timeouts. - Err(ContractCallError::Timeout) => Err(HostExportError::PossibleReorg(anyhow::anyhow!( - "Ethereum node did not respond when calling function \"{}\" of contract \"{}\"", - unresolved_call.function_name, - unresolved_call.contract_name, - ))), + ))) + } - Err(e) => Err(HostExportError::Unknown(anyhow::anyhow!( - "Failed to call function \"{}\" of contract \"{}\": {}", - unresolved_call.function_name, - unresolved_call.contract_name, - e - ))), - }; + // Also retry on timeouts. + Err(ContractCallError::Timeout) => Err(HostExportError::PossibleReorg(anyhow::anyhow!( + "Ethereum node did not respond when calling function \"{}\" of contract \"{}\"", + unresolved_call.function_name, + unresolved_call.contract_name, + ))), + + Err(e) => Err(HostExportError::Unknown(anyhow::anyhow!( + "Failed to call function \"{}\" of contract \"{}\": {}", + unresolved_call.function_name, + unresolved_call.contract_name, + e + ))), + }; let elapsed = start_time.elapsed(); diff --git a/chain/ethereum/src/tests.rs b/chain/ethereum/src/tests.rs index 5fa236323a4..a5f0a17bff0 100644 --- a/chain/ethereum/src/tests.rs +++ b/chain/ethereum/src/tests.rs @@ -1,17 +1,17 @@ use std::sync::Arc; use graph::{ - blockchain::{block_stream::BlockWithTriggers, BlockPtr, Trigger}, + blockchain::{BlockPtr, Trigger, block_stream::BlockWithTriggers}, prelude::{ + EthereumCall, LightEthereumBlock, alloy::{ self, - primitives::{Address, Bytes, LogData, B256}, + primitives::{Address, B256, Bytes, LogData}, rpc::types::{Block, Log}, }, rand::{self, Rng}, - EthereumCall, LightEthereumBlock, }, - slog::{self, o, Logger}, + slog::{self, Logger, o}, }; use crate::{ diff --git a/chain/ethereum/src/transport.rs b/chain/ethereum/src/transport.rs index d5fac8523bb..5b5414c4a5d 100644 --- a/chain/ethereum/src/transport.rs +++ b/chain/ethereum/src/transport.rs @@ -1,15 +1,36 @@ +use alloy::transports::{TransportError, TransportErrorKind, TransportFut}; +use graph::components::ethereum::json_patch; use graph::components::network_provider::ProviderName; use graph::endpoint::{ConnectionType, EndpointMetrics, RequestLabels}; use graph::prelude::alloy::rpc::json_rpc::{RequestPacket, ResponsePacket}; +use graph::prelude::alloy::transports::{ipc::IpcConnect, ws::WsConnect}; use graph::prelude::*; use graph::url::Url; +use serde_json::Value; use std::sync::Arc; use std::task::{Context, Poll}; use tower::Service; -use alloy::transports::{TransportError, TransportFut}; +/// Compression method for RPC requests. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Compression { + #[default] + None, + Gzip, + Brotli, + Deflate, +} -use graph::prelude::alloy::transports::{http::Http, ipc::IpcConnect, ws::WsConnect}; +impl std::fmt::Display for Compression { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Compression::None => write!(f, "none"), + Compression::Gzip => write!(f, "gzip"), + Compression::Brotli => write!(f, "brotli"), + Compression::Deflate => write!(f, "deflate"), + } + } +} /// Abstraction over different transport types for Alloy providers. #[derive(Clone, Debug)] @@ -21,9 +42,17 @@ pub enum Transport { impl Transport { /// Creates an IPC transport. + /// + /// Accepts both a raw file path (`/tmp/geth.ipc`) and an `ipc://` URL + /// (`ipc:///tmp/geth.ipc`). When a URL is provided the file path is + /// extracted so that the underlying IPC connector receives a plain path. #[cfg(unix)] pub async fn new_ipc(ipc: &str) -> Self { - let transport = IpcConnect::new(ipc.to_string()); + let path = Url::parse(ipc) + .ok() + .map(|u| u.path().to_string()) + .unwrap_or_else(|| ipc.to_string()); + let transport = IpcConnect::new(path); Transport::IPC(transport) } @@ -41,19 +70,37 @@ impl Transport { } /// Creates a JSON-RPC over HTTP transport. + /// + /// Set `no_eip2718` to true for chains that don't return the `type` field + /// in transaction receipts (pre-EIP-2718 chains). Use provider feature `no_eip2718`. pub fn new_rpc( rpc: Url, headers: graph::http::HeaderMap, metrics: Arc, provider: impl AsRef, + no_eip2718: bool, + compression: Compression, ) -> Self { - let client = reqwest::Client::builder() - .default_headers(headers) - .build() - .expect("Failed to build HTTP client"); + let mut client_builder = reqwest::Client::builder().default_headers(headers); - let http_transport = Http::with_client(client, rpc); - let metrics_transport = MetricsHttp::new(http_transport, metrics, provider.as_ref().into()); + match compression { + Compression::None => {} + Compression::Gzip => { + client_builder = client_builder.gzip(true); + } + Compression::Brotli => { + client_builder = client_builder.brotli(true); + } + Compression::Deflate => { + client_builder = client_builder.deflate(true); + } + } + + let client = client_builder.build().expect("Failed to build HTTP client"); + + let patching_transport = PatchingHttp::new(client, rpc, no_eip2718); + let metrics_transport = + MetricsHttp::new(patching_transport, metrics, provider.as_ref().into()); let rpc_client = alloy::rpc::client::RpcClient::new(metrics_transport, false); Transport::RPC(rpc_client) @@ -63,17 +110,13 @@ impl Transport { /// Custom HTTP transport wrapper that collects metrics #[derive(Clone)] pub struct MetricsHttp { - inner: Http, + inner: PatchingHttp, metrics: Arc, provider: ProviderName, } impl MetricsHttp { - pub fn new( - inner: Http, - metrics: Arc, - provider: ProviderName, - ) -> Self { + pub fn new(inner: PatchingHttp, metrics: Arc, provider: ProviderName) -> Self { Self { inner, metrics, @@ -125,3 +168,138 @@ impl Service for MetricsHttp { }) } } + +/// HTTP transport that patches receipts for chains that don't support EIP-2718 (typed transactions). +/// When `no_eip2718` is set, adds missing `type` field to receipts. +#[derive(Clone)] +pub struct PatchingHttp { + client: reqwest::Client, + url: Url, + no_eip2718: bool, +} + +impl PatchingHttp { + pub fn new(client: reqwest::Client, url: Url, no_eip2718: bool) -> Self { + Self { + client, + url, + no_eip2718, + } + } + + fn is_receipt_method(method: &str) -> bool { + method == "eth_getTransactionReceipt" || method == "eth_getBlockReceipts" + } + + fn patch_rpc_response(response: &mut Value) -> bool { + response + .get_mut("result") + .map(json_patch::patch_receipts) + .unwrap_or(false) + } + + fn patch_response(body: &[u8]) -> Option> { + let mut json: Value = serde_json::from_slice(body).ok()?; + + let patched = match &mut json { + Value::Object(_) => Self::patch_rpc_response(&mut json), + Value::Array(batch) => { + let mut patched = false; + for r in batch { + patched |= Self::patch_rpc_response(r); + } + patched + } + _ => false, + }; + + if patched { + serde_json::to_vec(&json).ok() + } else { + None + } + } +} + +impl Service for PatchingHttp { + type Response = ResponsePacket; + type Error = TransportError; + type Future = TransportFut<'static>; + + fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, request: RequestPacket) -> Self::Future { + let client = self.client.clone(); + let url = self.url.clone(); + let no_eip2718 = self.no_eip2718; + + let should_patch = if no_eip2718 { + match &request { + RequestPacket::Single(req) => Self::is_receipt_method(req.method()), + RequestPacket::Batch(reqs) => { + reqs.iter().any(|r| Self::is_receipt_method(r.method())) + } + } + } else { + false + }; + + Box::pin(async move { + let resp = client + .post(url) + .json(&request) + .headers(request.headers()) + .send() + .await + .map_err(TransportErrorKind::custom)?; + + let status = resp.status(); + let body = resp.bytes().await.map_err(TransportErrorKind::custom)?; + + if !status.is_success() { + return Err(TransportErrorKind::http_error( + status.as_u16(), + String::from_utf8_lossy(&body).into_owned(), + )); + } + + if should_patch && let Some(patched) = Self::patch_response(&body) { + return serde_json::from_slice(&patched).map_err(|err| { + TransportError::deser_err(err, String::from_utf8_lossy(&patched)) + }); + } + serde_json::from_slice(&body) + .map_err(|err| TransportError::deser_err(err, String::from_utf8_lossy(&body))) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn patch_response_single() { + let body = br#"{"jsonrpc":"2.0","id":1,"result":{"status":"0x1"}}"#; + let patched = PatchingHttp::patch_response(body).unwrap(); + let json: Value = serde_json::from_slice(&patched).unwrap(); + assert_eq!(json["result"]["type"], "0x0"); + } + + #[test] + fn patch_response_returns_none_when_type_exists() { + let body = br#"{"jsonrpc":"2.0","id":1,"result":{"status":"0x1","type":"0x2"}}"#; + assert!(PatchingHttp::patch_response(body).is_none()); + } + + #[test] + fn patch_response_batch() { + let body = br#"[{"jsonrpc":"2.0","id":1,"result":{"status":"0x1"}},{"jsonrpc":"2.0","id":2,"result":{"status":"0x1"}}]"#; + let patched = PatchingHttp::patch_response(body).unwrap(); + let json: Value = serde_json::from_slice(&patched).unwrap(); + assert_eq!(json[0]["result"]["type"], "0x0"); + assert_eq!(json[1]["result"]["type"], "0x0"); + } +} diff --git a/chain/ethereum/src/trigger.rs b/chain/ethereum/src/trigger.rs index f969fa7063c..b5d51d9a379 100644 --- a/chain/ethereum/src/trigger.rs +++ b/chain/ethereum/src/trigger.rs @@ -3,24 +3,24 @@ use graph::abi; use graph::blockchain::MappingTriggerTrait; use graph::blockchain::TriggerData; use graph::components::ethereum::AnyTransaction; +use graph::components::ethereum::AnyTransactionReceiptBare as AlloyTransactionReceipt; use graph::data::subgraph::API_VERSION_0_0_2; use graph::data::subgraph::API_VERSION_0_0_6; use graph::data::subgraph::API_VERSION_0_0_7; use graph::data_source::common::DeclaredCall; +use graph::prelude::BlockNumber; +use graph::prelude::BlockPtr; +use graph::prelude::LightEthereumBlock; use graph::prelude::alloy::consensus::Transaction as TransactionTrait; -use graph::prelude::alloy::network::AnyTransactionReceipt as AlloyTransactionReceipt; use graph::prelude::alloy::network::TransactionResponse; use graph::prelude::alloy::primitives::{Address, B256, U256}; use graph::prelude::alloy::rpc::types::Log; -use graph::prelude::BlockNumber; -use graph::prelude::BlockPtr; -use graph::prelude::LightEthereumBlock; use graph::prelude::{CheapClone, EthereumCall}; -use graph::runtime::asc_new; -use graph::runtime::gas::GasCounter; use graph::runtime::AscHeap; use graph::runtime::AscPtr; use graph::runtime::HostExportError; +use graph::runtime::asc_new; +use graph::runtime::gas::GasCounter; use graph::semver::Version; use graph_runtime_wasm::module::ToAscPtr; use std::{cmp::Ordering, sync::Arc}; diff --git a/chain/near/src/adapter.rs b/chain/near/src/adapter.rs index 8a9de408bad..31c7dca7c97 100644 --- a/chain/near/src/adapter.rs +++ b/chain/near/src/adapter.rs @@ -1,7 +1,7 @@ use std::collections::HashSet; use crate::data_source::PartialAccounts; -use crate::{data_source::DataSource, Chain}; +use crate::{Chain, data_source::DataSource}; use graph::blockchain as bc; use graph::firehose::{BasicReceiptFilter, PrefixSuffixPair}; use graph::itertools::Itertools; @@ -251,7 +251,7 @@ mod test { use std::collections::HashSet; use super::NearBlockFilter; - use crate::adapter::{TriggerFilter, BASIC_RECEIPT_FILTER_TYPE_URL}; + use crate::adapter::{BASIC_RECEIPT_FILTER_TYPE_URL, TriggerFilter}; use graph::{ blockchain::TriggerFilter as _, firehose::{BasicReceiptFilter, PrefixSuffixPair}, diff --git a/chain/near/src/chain.rs b/chain/near/src/chain.rs index 6dae3c3a1f0..5698301b5e8 100644 --- a/chain/near/src/chain.rs +++ b/chain/near/src/chain.rs @@ -14,17 +14,17 @@ use graph::prelude::MetricsRegistry; use graph::{ anyhow::Result, blockchain::{ + BlockHash, BlockPtr, Blockchain, EmptyNodeCapabilities, IngestorError, + RuntimeAdapter as RuntimeAdapterTrait, block_stream::{ BlockStreamEvent, BlockWithTriggers, FirehoseError, FirehoseMapper as FirehoseMapperTrait, TriggersAdapter as TriggersAdapterTrait, }, firehose_block_stream::FirehoseBlockStream, - BlockHash, BlockPtr, Blockchain, EmptyNodeCapabilities, IngestorError, - RuntimeAdapter as RuntimeAdapterTrait, }, components::store::DeploymentLocator, firehose::{self as firehose, ForkStep}, - prelude::{o, BlockNumber, Error, Logger, LoggerFactory}, + prelude::{BlockNumber, Error, Logger, LoggerFactory, o}, }; use prost::Message; use std::collections::BTreeSet; @@ -197,7 +197,9 @@ impl Blockchain for Chain { _logger: &Logger, _cursor: FirehoseCursor, ) -> Result { - unimplemented!("This chain does not support Dynamic Data Sources. is_refetch_block_required always returns false, this shouldn't be called.") + unimplemented!( + "This chain does not support Dynamic Data Sources. is_refetch_block_required always returns false, this shouldn't be called." + ) } async fn chain_head_ptr(&self) -> Result, Error> { @@ -367,7 +369,7 @@ impl BlockStreamMapper for FirehoseMapper { return Err(anyhow::anyhow!( "near mapper is expected to always have a block" )) - .map_err(BlockStreamError::from) + .map_err(BlockStreamError::from); } }; @@ -439,7 +441,9 @@ impl FirehoseMapperTrait for FirehoseMapper { } StepFinal => { - panic!("irreversible step is not handled and should not be requested in the Firehose request") + panic!( + "irreversible step is not handled and should not be requested in the Firehose request" + ) } StepUnset => { @@ -477,23 +481,23 @@ mod test { use std::{collections::HashSet, sync::Arc, vec}; use graph::{ - blockchain::{block_stream::BlockWithTriggers, DataSource as _, TriggersAdapter as _}, + blockchain::{DataSource as _, TriggersAdapter as _, block_stream::BlockWithTriggers}, data::subgraph::LATEST_VERSION, prelude::Link, semver::Version, - slog::{self, o, Logger}, + slog::{self, Logger, o}, }; use crate::{ + Chain, adapter::{NearReceiptFilter, TriggerFilter}, codec::{ - self, execution_outcome, receipt, Block, BlockHeader, DataReceiver, ExecutionOutcome, - ExecutionOutcomeWithId, IndexerExecutionOutcomeWithReceipt, IndexerShard, - ReceiptAction, SuccessValueExecutionStatus, + self, Block, BlockHeader, DataReceiver, ExecutionOutcome, ExecutionOutcomeWithId, + IndexerExecutionOutcomeWithReceipt, IndexerShard, ReceiptAction, + SuccessValueExecutionStatus, execution_outcome, receipt, }, - data_source::{DataSource, Mapping, PartialAccounts, ReceiptHandler, NEAR_KIND}, + data_source::{DataSource, Mapping, NEAR_KIND, PartialAccounts, ReceiptHandler}, trigger::{NearTrigger, ReceiptWithOutcome}, - Chain, }; use super::TriggersAdapter; diff --git a/chain/near/src/codec.rs b/chain/near/src/codec.rs index fca0a4f4c9e..569e8cf5243 100644 --- a/chain/near/src/codec.rs +++ b/chain/near/src/codec.rs @@ -4,7 +4,7 @@ pub mod pbcodec; use graph::{ blockchain::{Block as BlockchainBlock, BlockPtr, BlockTime}, - prelude::{alloy::primitives::B256, hex, BlockNumber}, + prelude::{BlockNumber, alloy::primitives::B256, hex}, }; use std::convert::TryFrom; use std::fmt::LowerHex; diff --git a/chain/near/src/data_source.rs b/chain/near/src/data_source.rs index e64197eeb0d..01484905a58 100644 --- a/chain/near/src/data_source.rs +++ b/chain/near/src/data_source.rs @@ -7,7 +7,7 @@ use graph::components::subgraph::InstanceDSTemplateInfo; use graph::data::subgraph::{DataSourceContext, DeploymentHash}; use graph::prelude::SubgraphManifestValidationError; use graph::{ - anyhow::{anyhow, Error}, + anyhow::{Error, anyhow}, blockchain::{self, Blockchain}, prelude::{BlockNumber, CheapClone, Deserialize, Link, LinkResolver, Logger}, semver, diff --git a/chain/near/src/runtime/abi.rs b/chain/near/src/runtime/abi.rs index 87e224dc4d0..b74d8f5f7ff 100644 --- a/chain/near/src/runtime/abi.rs +++ b/chain/near/src/runtime/abi.rs @@ -3,7 +3,7 @@ use crate::trigger::ReceiptWithOutcome; use async_trait::async_trait; use graph::anyhow::anyhow; use graph::runtime::gas::GasCounter; -use graph::runtime::{asc_new, AscHeap, AscPtr, DeterministicHostError, HostExportError, ToAscObj}; +use graph::runtime::{AscHeap, AscPtr, DeterministicHostError, HostExportError, ToAscObj, asc_new}; use graph_runtime_wasm::asc_abi::class::{Array, AscEnum, EnumPayload, Uint8Array}; pub(crate) use super::generated::*; @@ -517,7 +517,7 @@ impl ToAscObj for codec::MerklePathItem { "Invalid direction value {}", x )) - .into()) + .into()); } }, }) @@ -557,7 +557,7 @@ impl ToAscObj for codec::Signature { "Invalid signature type {}", value, )) - .into()) + .into()); } }, bytes: asc_new(heap, self.bytes.as_slice(), gas).await?, @@ -596,7 +596,7 @@ impl ToAscObj for codec::PublicKey { "Invalid public key type {}", value, )) - .into()) + .into()); } }, bytes: asc_new(heap, self.bytes.as_slice(), gas).await?, diff --git a/chain/near/src/trigger.rs b/chain/near/src/trigger.rs index ab68ab71beb..bf9794d2b20 100644 --- a/chain/near/src/trigger.rs +++ b/chain/near/src/trigger.rs @@ -3,11 +3,11 @@ use graph::blockchain::Block; use graph::blockchain::MappingTriggerTrait; use graph::blockchain::TriggerData; use graph::derive::CheapClone; +use graph::prelude::BlockNumber; use graph::prelude::alloy::primitives::B256; use graph::prelude::hex; -use graph::prelude::BlockNumber; use graph::runtime::HostExportError; -use graph::runtime::{asc_new, gas::GasCounter, AscHeap, AscPtr}; +use graph::runtime::{AscHeap, AscPtr, asc_new, gas::GasCounter}; use graph_runtime_wasm::module::ToAscPtr; use std::{cmp::Ordering, sync::Arc}; @@ -158,8 +158,8 @@ mod tests { anyhow::anyhow, components::metrics::gas::GasMetrics, data::subgraph::API_VERSION_0_0_5, - prelude::{hex, BigInt}, - runtime::{gas::GasCounter, DeterministicHostError, HostExportError}, + prelude::{BigInt, hex}, + runtime::{DeterministicHostError, HostExportError, gas::GasCounter}, util::mem::init_slice, }; diff --git a/core/graphman/src/commands/deployment/info.rs b/core/graphman/src/commands/deployment/info.rs index 55e00e917ca..155a821d715 100644 --- a/core/graphman/src/commands/deployment/info.rs +++ b/core/graphman/src/commands/deployment/info.rs @@ -11,10 +11,10 @@ use graph_store_postgres::ConnectionPool; use graph_store_postgres::Store; use itertools::Itertools; +use crate::GraphmanError; use crate::deployment::Deployment; use crate::deployment::DeploymentSelector; use crate::deployment::DeploymentVersionSelector; -use crate::GraphmanError; #[derive(Clone, Debug)] pub struct DeploymentStatus { diff --git a/core/graphman/src/commands/deployment/pause.rs b/core/graphman/src/commands/deployment/pause.rs index 0257768a03b..e2405b5f68b 100644 --- a/core/graphman/src/commands/deployment/pause.rs +++ b/core/graphman/src/commands/deployment/pause.rs @@ -3,15 +3,15 @@ use std::sync::Arc; use anyhow::anyhow; use graph::components::store::DeploymentLocator; use graph::components::store::StoreEvent; -use graph_store_postgres::command_support::catalog; -use graph_store_postgres::command_support::catalog::Site; use graph_store_postgres::ConnectionPool; use graph_store_postgres::NotificationSender; +use graph_store_postgres::command_support::catalog; +use graph_store_postgres::command_support::catalog::Site; use thiserror::Error; +use crate::GraphmanError; use crate::deployment::DeploymentSelector; use crate::deployment::DeploymentVersionSelector; -use crate::GraphmanError; pub struct ActiveDeployment { locator: DeploymentLocator, diff --git a/core/graphman/src/commands/deployment/reassign.rs b/core/graphman/src/commands/deployment/reassign.rs index b1ead37cd12..f4279946917 100644 --- a/core/graphman/src/commands/deployment/reassign.rs +++ b/core/graphman/src/commands/deployment/reassign.rs @@ -5,15 +5,15 @@ use graph::components::store::DeploymentLocator; use graph::components::store::StoreEvent; use graph::prelude::AssignmentChange; use graph::prelude::NodeId; -use graph_store_postgres::command_support::catalog; -use graph_store_postgres::command_support::catalog::Site; use graph_store_postgres::ConnectionPool; use graph_store_postgres::NotificationSender; +use graph_store_postgres::command_support::catalog; +use graph_store_postgres::command_support::catalog::Site; use thiserror::Error; +use crate::GraphmanError; use crate::deployment::DeploymentSelector; use crate::deployment::DeploymentVersionSelector; -use crate::GraphmanError; pub struct Deployment { locator: DeploymentLocator, @@ -134,7 +134,10 @@ pub async fn reassign_deployment( .map_err(GraphmanError::from)? .len(); if count == 1 { - let warning_msg = format!("This is the only deployment assigned to '{}'. Please make sure that the node ID is spelled correctly.",node.as_str()); + let warning_msg = format!( + "This is the only deployment assigned to '{}'. Please make sure that the node ID is spelled correctly.", + node.as_str() + ); Ok(ReassignResult::CompletedWithWarnings(vec![warning_msg])) } else { Ok(ReassignResult::Ok) diff --git a/core/graphman/src/commands/deployment/resume.rs b/core/graphman/src/commands/deployment/resume.rs index a8f9f4769e7..4fa0162f83e 100644 --- a/core/graphman/src/commands/deployment/resume.rs +++ b/core/graphman/src/commands/deployment/resume.rs @@ -3,15 +3,15 @@ use std::sync::Arc; use anyhow::anyhow; use graph::components::store::DeploymentLocator; use graph::prelude::StoreEvent; -use graph_store_postgres::command_support::catalog; -use graph_store_postgres::command_support::catalog::Site; use graph_store_postgres::ConnectionPool; use graph_store_postgres::NotificationSender; +use graph_store_postgres::command_support::catalog; +use graph_store_postgres::command_support::catalog::Site; use thiserror::Error; +use crate::GraphmanError; use crate::deployment::DeploymentSelector; use crate::deployment::DeploymentVersionSelector; -use crate::GraphmanError; pub struct PausedDeployment { locator: DeploymentLocator, diff --git a/core/graphman/src/commands/deployment/unassign.rs b/core/graphman/src/commands/deployment/unassign.rs index 474358c297e..cef72e947f1 100644 --- a/core/graphman/src/commands/deployment/unassign.rs +++ b/core/graphman/src/commands/deployment/unassign.rs @@ -3,15 +3,15 @@ use std::sync::Arc; use anyhow::anyhow; use graph::components::store::DeploymentLocator; use graph::components::store::StoreEvent; -use graph_store_postgres::command_support::catalog; -use graph_store_postgres::command_support::catalog::Site; use graph_store_postgres::ConnectionPool; use graph_store_postgres::NotificationSender; +use graph_store_postgres::command_support::catalog; +use graph_store_postgres::command_support::catalog::Site; use thiserror::Error; +use crate::GraphmanError; use crate::deployment::DeploymentSelector; use crate::deployment::DeploymentVersionSelector; -use crate::GraphmanError; pub struct AssignedDeployment { locator: DeploymentLocator, diff --git a/core/graphman/src/deployment.rs b/core/graphman/src/deployment.rs index 0a38f175586..6538017edaa 100644 --- a/core/graphman/src/deployment.rs +++ b/core/graphman/src/deployment.rs @@ -1,6 +1,4 @@ use anyhow::anyhow; -use diesel::dsl::sql; -use diesel::sql_types::Text; use diesel::BoolExpressionMethods; use diesel::ExpressionMethods; use diesel::JoinOnDsl; @@ -8,12 +6,14 @@ use diesel::NullableExpressionMethods; use diesel::PgTextExpressionMethods; use diesel::QueryDsl; use diesel::Queryable; +use diesel::dsl::sql; +use diesel::sql_types::Text; use diesel_async::RunQueryDsl; use graph::components::store::DeploymentId; use graph::components::store::DeploymentLocator; use graph::data::subgraph::DeploymentHash; -use graph_store_postgres::command_support::catalog; use graph_store_postgres::AsyncPgConnection; +use graph_store_postgres::command_support::catalog; use itertools::Itertools; use crate::GraphmanError; diff --git a/core/graphman_store/src/lib.rs b/core/graphman_store/src/lib.rs index f986d6484c0..8b1965df7cd 100644 --- a/core/graphman_store/src/lib.rs +++ b/core/graphman_store/src/lib.rs @@ -7,6 +7,9 @@ use anyhow::Result; use async_trait::async_trait; use chrono::DateTime; use chrono::Utc; +use diesel::AsExpression; +use diesel::FromSqlRow; +use diesel::Queryable; use diesel::deserialize::FromSql; use diesel::pg::Pg; use diesel::pg::PgValue; @@ -14,9 +17,6 @@ use diesel::serialize::Output; use diesel::serialize::ToSql; use diesel::sql_types::BigSerial; use diesel::sql_types::Varchar; -use diesel::AsExpression; -use diesel::FromSqlRow; -use diesel::Queryable; use strum::Display; use strum::EnumString; use strum::IntoStaticStr; diff --git a/core/src/amp_subgraph/manager.rs b/core/src/amp_subgraph/manager.rs index ae272830880..61ec421ca17 100644 --- a/core/src/amp_subgraph/manager.rs +++ b/core/src/amp_subgraph/manager.rs @@ -18,7 +18,7 @@ use graph::{ use slog::{debug, error}; use tokio_util::sync::CancellationToken; -use super::{runner, Metrics, Monitor}; +use super::{Metrics, Monitor, runner}; /// Manages Amp subgraph runner futures. /// diff --git a/core/src/amp_subgraph/metrics.rs b/core/src/amp_subgraph/metrics.rs index e54b07c1a6a..7088cf9a48f 100644 --- a/core/src/amp_subgraph/metrics.rs +++ b/core/src/amp_subgraph/metrics.rs @@ -4,7 +4,7 @@ use alloy::primitives::BlockNumber; use graph::{ cheap_clone::CheapClone, components::{ - metrics::{stopwatch::StopwatchMetrics, MetricsRegistry}, + metrics::{MetricsRegistry, stopwatch::StopwatchMetrics}, store::WritableStore, }, prelude::DeploymentHash, diff --git a/core/src/amp_subgraph/monitor.rs b/core/src/amp_subgraph/monitor.rs index 2bdd32ab504..314f464c0d1 100644 --- a/core/src/amp_subgraph/monitor.rs +++ b/core/src/amp_subgraph/monitor.rs @@ -7,11 +7,11 @@ //! `subgraph instance` - A background task that executes the subgraph runner future. use std::{ - collections::{hash_map::Entry, HashMap}, + collections::{HashMap, hash_map::Entry}, fmt, sync::{ - atomic::{AtomicU32, Ordering::SeqCst}, Arc, + atomic::{AtomicU32, Ordering::SeqCst}, }, time::Duration, }; @@ -21,8 +21,12 @@ use futures::future::BoxFuture; use graph::{ cheap_clone::CheapClone, components::store::DeploymentLocator, log::factory::LoggerFactory, }; -use slog::{debug, error, info, warn, Logger}; -use tokio::{sync::mpsc, task::JoinHandle, time::timeout}; +use slog::{Logger, debug, error, info, warn}; +use tokio::{ + sync::mpsc::{self, error::SendError}, + task::JoinHandle, + time::timeout, +}; use tokio_util::sync::CancellationToken; /// Represents the maximum amount of time a subgraph instance is allowed to run @@ -137,7 +141,7 @@ impl Monitor { .new(slog::o!("method" => "start")); info!(logger, "Starting subgraph"); - handle_send_result( + log_send_error( &logger, self.command_tx.send(Command::Start { id: self.subgraph_instance_id.fetch_add(1, SeqCst), @@ -164,7 +168,7 @@ impl Monitor { .new(slog::o!("method" => "stop")); info!(logger, "Stopping subgraph"); - handle_send_result(&logger, self.command_tx.send(Command::Stop { deployment })); + log_send_error(&logger, self.command_tx.send(Command::Stop { deployment })); } /// Processes commands sent through the command channel. @@ -184,10 +188,6 @@ impl Monitor { loop { tokio::select! { Some(command) = command_rx.recv() => { - debug!(logger, "Processing a new command"; - "command" => ?command - ); - match &command { Command::Start { .. } => { Self::process_start_command( @@ -390,7 +390,7 @@ impl Monitor { if let Some(pending_start_command) = pending_start_commands.remove(&deployment) { debug!(logger, "Resending a pending start command"); - handle_send_result(&logger, command_tx.send(pending_start_command)); + log_send_error(&logger, command_tx.send(pending_start_command)); } } @@ -476,7 +476,7 @@ impl Monitor { } debug!(logger, "Sending clear command"); - handle_send_result(&logger, command_tx.send(Command::Clear { id, deployment })); + log_send_error(&logger, command_tx.send(Command::Clear { id, deployment })); } }); @@ -546,13 +546,10 @@ impl fmt::Debug for Command { } } -fn handle_send_result( - logger: &Logger, - result: Result<(), tokio::sync::mpsc::error::SendError>, -) { +fn log_send_error(logger: &Logger, result: Result<(), SendError>) { match result { Ok(()) => { - debug!(logger, "Command was sent successfully"); + // No need to log anything } // This should only happen if the parent cancel token of the subgraph monitor was cancelled diff --git a/core/src/amp_subgraph/runner/context.rs b/core/src/amp_subgraph/runner/context.rs index 32e96148acf..d8b5b7b638e 100644 --- a/core/src/amp_subgraph/runner/context.rs +++ b/core/src/amp_subgraph/runner/context.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use alloy::primitives::{BlockHash, BlockNumber}; use graph::{ - amp::{log::Logger as _, Codec, Manifest}, + amp::{Codec, Manifest, log::Logger as _}, cheap_clone::CheapClone, components::store::WritableStore, data::subgraph::DeploymentHash, @@ -18,8 +18,8 @@ pub(in super::super) struct Context { pub(super) logger: Logger, pub(super) client: Arc, pub(super) store: Arc, - pub(super) max_buffer_size: usize, - pub(super) max_block_range: usize, + pub(super) buffer_size: usize, + pub(super) block_range: usize, pub(super) backoff: ExponentialBackoff, pub(super) deployment: DeploymentHash, pub(super) manifest: Manifest, @@ -45,8 +45,8 @@ impl Context { logger, client, store, - max_buffer_size: env.max_buffer_size, - max_block_range: env.max_block_range, + buffer_size: env.buffer_size, + block_range: env.block_range, backoff, deployment, manifest, @@ -77,15 +77,7 @@ impl Context { .map(|block_ptr| (block_ptr.number.compat(), block_ptr.hash.compat())) } - pub(super) fn total_queries(&self) -> usize { - self.manifest - .data_sources - .iter() - .map(|data_source| data_source.transformer.tables.len()) - .sum() - } - - pub(super) fn min_start_block(&self) -> BlockNumber { + pub(super) fn start_block(&self) -> BlockNumber { self.manifest .data_sources .iter() @@ -94,7 +86,7 @@ impl Context { .unwrap() } - pub(super) fn max_end_block(&self) -> BlockNumber { + pub(super) fn end_block(&self) -> BlockNumber { self.manifest .data_sources .iter() diff --git a/core/src/amp_subgraph/runner/data_processing.rs b/core/src/amp_subgraph/runner/data_processing.rs index 8c403de2b7f..d5bf991d505 100644 --- a/core/src/amp_subgraph/runner/data_processing.rs +++ b/core/src/amp_subgraph/runner/data_processing.rs @@ -6,27 +6,27 @@ use arrow::array::RecordBatch; use chrono::{DateTime, Utc}; use graph::{ amp::{ - codec::{utils::auto_block_timestamp_decoder, DecodeOutput, DecodedEntity, Decoder}, + codec::{DecodeOutput, DecodedEntity, Decoder, utils::auto_block_timestamp_decoder}, stream_aggregator::{RecordBatchGroup, RecordBatchGroups, StreamRecordBatch}, }, blockchain::block_stream::FirehoseCursor, cheap_clone::CheapClone, - components::store::{EntityCache, ModificationsAndCache}, + components::store::{EntityCache, EntityLfuCache, ModificationsAndCache, SeqGenerator}, }; use slog::{debug, trace}; -use super::{data_stream::TablePtr, Compat, Context, Error}; +use super::{Compat, Context, Error, data_stream::TablePtr}; pub(super) async fn process_record_batch_groups( cx: &mut Context, - mut entity_cache: EntityCache, + mut entity_lfu_cache: EntityLfuCache, record_batch_groups: RecordBatchGroups, stream_table_ptr: Arc<[TablePtr]>, latest_block: BlockNumber, -) -> Result { +) -> Result { if record_batch_groups.is_empty() { debug!(cx.logger, "Received no record batch groups"); - return Ok(entity_cache); + return Ok(entity_lfu_cache); } let from_block = record_batch_groups @@ -50,9 +50,9 @@ pub(super) async fn process_record_batch_groups( "record_batches_count" => record_batch_group.record_batches.len() ); - entity_cache = process_record_batch_group( + entity_lfu_cache = process_record_batch_group( cx, - entity_cache, + entity_lfu_cache, block_number, block_hash, record_batch_group, @@ -79,18 +79,18 @@ pub(super) async fn process_record_batch_groups( "to_block" => to_block ); - Ok(entity_cache) + Ok(entity_lfu_cache) } async fn process_record_batch_group( cx: &mut Context, - mut entity_cache: EntityCache, + entity_lfu_cache: EntityLfuCache, block_number: BlockNumber, block_hash: BlockHash, record_batch_group: RecordBatchGroup, stream_table_ptr: &[TablePtr], latest_block: BlockNumber, -) -> Result { +) -> Result { let _section = cx .metrics .stopwatch @@ -100,9 +100,15 @@ async fn process_record_batch_group( if record_batches.is_empty() { debug!(cx.logger, "Record batch group is empty"); - return Ok(entity_cache); + return Ok(entity_lfu_cache); } + let mut entity_cache = EntityCache::with_current( + cx.store.cheap_clone(), + entity_lfu_cache, + SeqGenerator::new(block_number.compat()), + ); + let block_timestamp = if cx.manifest.schema.has_aggregations() { decode_block_timestamp(&record_batches) .map_err(|e| e.context("failed to decode block timestamp"))? @@ -121,7 +127,6 @@ async fn process_record_batch_group( process_record_batch( cx, &mut entity_cache, - block_number, record_batch, stream_table_ptr[stream_index], ) @@ -139,7 +144,7 @@ async fn process_record_batch_group( entity_lfu_cache, evict_stats: _, } = entity_cache - .as_modifications(block_number.compat()) + .as_modifications(block_number.compat(), &cx.metrics.stopwatch) .await .map_err(Error::from) .map_err(|e| e.context("failed to extract entity modifications from the state"))?; @@ -169,16 +174,12 @@ async fn process_record_batch_group( cx.metrics.deployment_synced.record(true); } - Ok(EntityCache::with_current( - cx.store.cheap_clone(), - entity_lfu_cache, - )) + Ok(entity_lfu_cache) } async fn process_record_batch( cx: &mut Context, entity_cache: &mut EntityCache, - block_number: BlockNumber, record_batch: RecordBatch, (i, j): TablePtr, ) -> Result<(), Error> { @@ -209,13 +210,11 @@ async fn process_record_batch( let key = match key { Some(key) => key, None => { - let entity_id = entity_cache - .generate_id(id_type, block_number.compat()) - .map_err(|e| { - Error::Deterministic(e.context(format!( - "failed to generate a new id for an entity of type '{entity_name}'" - ))) - })?; + let entity_id = entity_cache.seq_gen.id(id_type).map_err(|e| { + Error::Deterministic(e.context(format!( + "failed to generate a new id for an entity of type '{entity_name}'" + ))) + })?; entity_data.push(("id".into(), entity_id.clone().into())); entity_type.key(entity_id) @@ -229,14 +228,11 @@ async fn process_record_batch( ))) })?; - entity_cache - .set(key, entity, block_number.compat(), None) - .await - .map_err(|e| { - Error::Deterministic(e.context(format!( - "failed to store a new entity of type '{entity_name}' with id '{entity_id}'" - ))) - })?; + entity_cache.set(key, entity, None).await.map_err(|e| { + Error::Deterministic(e.context(format!( + "failed to store a new entity of type '{entity_name}' with id '{entity_id}'" + ))) + })?; } Ok(()) diff --git a/core/src/amp_subgraph/runner/data_stream.rs b/core/src/amp_subgraph/runner/data_stream.rs index ad6d6d471f8..693ae9384a7 100644 --- a/core/src/amp_subgraph/runner/data_stream.rs +++ b/core/src/amp_subgraph/runner/data_stream.rs @@ -3,16 +3,19 @@ use std::{collections::HashMap, ops::RangeInclusive, sync::Arc}; use alloy::primitives::BlockNumber; use anyhow::anyhow; use futures::{ - stream::{empty, BoxStream}, StreamExt, TryStreamExt, + stream::{self, BoxStream}, }; use graph::{ amp::{ + Client, + client::ResponseBatch, + error::IsDeterministic, manifest::DataSource, stream_aggregator::{RecordBatchGroups, StreamAggregator}, - Client, }, cheap_clone::CheapClone, + prelude::StopwatchMetrics, }; use slog::{debug, warn}; @@ -25,135 +28,158 @@ pub(super) fn new_data_stream( latest_block: BlockNumber, ) -> BoxStream<'static, Result<(RecordBatchGroups, Arc<[TablePtr]>), Error>> where - AC: Client, + AC: Client + Send + Sync + 'static, { let logger = cx.logger.new(slog::o!("process" => "new_data_stream")); - - let total_queries = cx.total_queries(); - let mut total_queries_to_execute = 0; - let mut data_streams = Vec::new(); - let mut latest_queried_block = cx.latest_synced_block(); - let mut max_end_block = BlockNumber::MIN; + let client = cx.client.cheap_clone(); + let manifest = cx.manifest.clone(); + let buffer_size = cx.buffer_size; + let block_range = cx.block_range; + let stopwatch = cx.metrics.stopwatch.cheap_clone(); debug!(logger, "Creating data stream"; - "from_block" => latest_queried_block.unwrap_or(BlockNumber::MIN), + "from_block" => cx.latest_synced_block().unwrap_or(BlockNumber::MIN), "to_block" => latest_block, - "min_start_block" => cx.min_start_block(), - "max_block_range" => cx.max_block_range, + "start_block" => cx.start_block(), + "block_range" => block_range, ); - loop { - let next_block_ranges = next_block_ranges(cx, latest_queried_block, latest_block); - - if next_block_ranges.is_empty() { - if data_streams.is_empty() { - warn!(logger, "There are no unprocessed block ranges"); - } - break; - } - - let mut query_streams = Vec::with_capacity(total_queries); - let mut query_streams_table_ptr = Vec::with_capacity(total_queries); - let mut min_start_block = BlockNumber::MAX; - - for (i, data_source) in cx.manifest.data_sources.iter().enumerate() { - let Some(block_range) = next_block_ranges.get(&i) else { - continue; - }; - - if *block_range.start() < min_start_block { - min_start_block = *block_range.start(); + // State: (latest_queried_block, end_block, is_first) + let initial_state = (cx.latest_synced_block(), BlockNumber::MIN, true); + + stream::unfold( + initial_state, + move |(latest_queried_block, mut end_block, is_first)| { + let block_ranges = next_block_ranges( + &manifest.data_sources, + block_range, + latest_queried_block, + latest_block, + ); + + if block_ranges.is_empty() { + if is_first { + warn!(logger, "There are no unprocessed block ranges"); + } + return futures::future::ready(None); } - if *block_range.end() > max_end_block { - max_end_block = *block_range.end(); - } + let start_block = block_ranges.values().map(|r| *r.start()).min().unwrap(); + end_block = end_block.max(block_ranges.values().map(|r| *r.end()).max().unwrap()); + + let (query_streams, table_ptrs) = + build_query_streams(&*client, &logger, &manifest.data_sources, &block_ranges); + + let data_stream = build_data_stream( + &logger, + query_streams, + table_ptrs, + buffer_size, + &stopwatch, + start_block, + ); + + debug!(logger, "Created a new data stream"; + "latest_queried_block" => latest_queried_block, + "start_block" => start_block, + "end_block" => end_block, + ); + futures::future::ready(Some((data_stream, (Some(end_block), end_block, false)))) + }, + ) + .flatten() + .boxed() +} - for (j, table) in data_source.transformer.tables.iter().enumerate() { - let query = table.query.build_with_block_range(block_range); - let stream = cx.client.query(&cx.logger, query, None); - let stream_name = format!("{}.{}", data_source.name, table.name); +fn build_query_streams( + client: &AC, + logger: &slog::Logger, + data_sources: &[DataSource], + block_ranges: &HashMap>, +) -> ( + Vec<(String, BoxStream<'static, Result>)>, + Arc<[TablePtr]>, +) { + let total_queries: usize = data_sources + .iter() + .map(|ds| ds.transformer.tables.len()) + .sum(); - query_streams.push((stream_name, stream)); - query_streams_table_ptr.push((i, j)); - } - } + let mut query_streams = Vec::with_capacity(total_queries); + let mut table_ptrs = Vec::with_capacity(total_queries); - let query_streams_table_ptr: Arc<[TablePtr]> = query_streams_table_ptr.into(); - total_queries_to_execute += query_streams.len(); - - let mut min_start_block_checked = false; - let mut load_first_record_batch_group_section = Some( - cx.metrics - .stopwatch - .start_section("load_first_record_batch_group"), - ); - - data_streams.push( - StreamAggregator::new(&cx.logger, query_streams, cx.max_buffer_size) - .map_ok(move |response| (response, query_streams_table_ptr.cheap_clone())) - .map_err(Error::from) - .map(move |result| { - if load_first_record_batch_group_section.is_some() { - let _section = load_first_record_batch_group_section.take(); - } + for (i, data_source) in data_sources.iter().enumerate() { + let Some(block_range) = block_ranges.get(&i) else { + continue; + }; - match result { - Ok(response) => { - if !min_start_block_checked { - if let Some(((first_block, _), _)) = response.0.first_key_value() { - if *first_block < min_start_block { - return Err(Error::NonDeterministic(anyhow!( - "chain reorg" - ))); - } - } - - min_start_block_checked = true; - } - - Ok(response) - } - Err(e) => Err(e), - } - }) - .boxed(), - ); + for (j, table) in data_source.transformer.tables.iter().enumerate() { + let query = table.query.build_with_block_range(block_range); + let stream = client.query(logger, query, None); + let stream_name = format!("{}.{}", data_source.name, table.name); - if max_end_block >= latest_block { - break; + query_streams.push((stream_name, stream)); + table_ptrs.push((i, j)); } - - latest_queried_block = Some(max_end_block); } - debug!(logger, "Created aggregated data streams"; - "total_data_streams" => data_streams.len(), - "total_queries_to_execute" => total_queries_to_execute - ); + (query_streams, table_ptrs.into()) +} + +fn build_data_stream( + logger: &slog::Logger, + query_streams: Vec<(String, BoxStream<'static, Result>)>, + table_ptrs: Arc<[TablePtr]>, + buffer_size: usize, + stopwatch: &StopwatchMetrics, + start_block: BlockNumber, +) -> BoxStream<'static, Result<(RecordBatchGroups, Arc<[TablePtr]>), Error>> +where + E: std::error::Error + IsDeterministic + Send + Sync + 'static, +{ + let mut start_block_checked = false; + let mut load_first_record_batch_group_section = + Some(stopwatch.start_section("load_first_record_batch_group")); + + StreamAggregator::new(logger, query_streams, buffer_size) + .map_ok(move |response| (response, table_ptrs.cheap_clone())) + .map_err(Error::from) + .map(move |result| { + if load_first_record_batch_group_section.is_some() { + let _section = load_first_record_batch_group_section.take(); + } - let mut iter = data_streams.into_iter(); - let mut merged_data_stream = iter.next().unwrap_or_else(|| empty().boxed()); + match result { + Ok(response) => { + if !start_block_checked { + if let Some(((first_block, _), _)) = response.0.first_key_value() + && *first_block < start_block + { + return Err(Error::NonDeterministic(anyhow!("chain reorg"))); + } - for data_stream in iter { - merged_data_stream = merged_data_stream.chain(data_stream).boxed(); - } + start_block_checked = true; + } - merged_data_stream + Ok(response) + } + Err(e) => Err(e), + } + }) + .boxed() } -fn next_block_ranges( - cx: &Context, +fn next_block_ranges( + data_sources: &[DataSource], + block_range: usize, latest_queried_block: Option, latest_block: BlockNumber, ) -> HashMap> { - let block_ranges = cx - .manifest - .data_sources + let block_ranges = data_sources .iter() .enumerate() .filter_map(|(i, data_source)| { - next_block_range(cx, data_source, latest_queried_block, latest_block) + next_block_range(block_range, data_source, latest_queried_block, latest_block) .map(|block_range| (i, block_range)) }) .collect::>(); @@ -172,8 +198,8 @@ fn next_block_ranges( .collect() } -fn next_block_range( - cx: &Context, +fn next_block_range( + block_range: usize, data_source: &DataSource, latest_queried_block: Option, latest_block: BlockNumber, @@ -190,7 +216,7 @@ fn next_block_range( }; let end_block = [ - start_block.saturating_add(cx.max_block_range as BlockNumber), + start_block.saturating_add(block_range as BlockNumber), data_source.source.end_block, latest_block, ] diff --git a/core/src/amp_subgraph/runner/latest_blocks.rs b/core/src/amp_subgraph/runner/latest_blocks.rs index cb62f2e3a42..17bdc23dd29 100644 --- a/core/src/amp_subgraph/runner/latest_blocks.rs +++ b/core/src/amp_subgraph/runner/latest_blocks.rs @@ -1,13 +1,13 @@ use alloy::primitives::BlockNumber; use anyhow::anyhow; use arrow::array::RecordBatch; -use futures::{future::try_join_all, stream::BoxStream, StreamExt, TryFutureExt}; +use futures::{StreamExt, TryFutureExt, future::try_join_all, stream::BoxStream}; use graph::amp::{ + Client, client::ResponseBatch, - codec::{utils::block_number_decoder, Decoder}, + codec::{Decoder, utils::block_number_decoder}, error::IsDeterministic, manifest::DataSource, - Client, }; use itertools::Itertools; use slog::debug; @@ -152,7 +152,9 @@ async fn latest_block_changed( where AC: Client, { - let query = format!("SELECT _block_num FROM {dataset}.{table} WHERE _block_num > {latest_block} SETTINGS stream = true"); + let query = format!( + "SELECT _block_num FROM {dataset}.{table} WHERE _block_num > {latest_block} SETTINGS stream = true" + ); let stream = cx.client.query(&cx.logger, query, None); let _record_batch = read_once(stream).await?; diff --git a/core/src/amp_subgraph/runner/mod.rs b/core/src/amp_subgraph/runner/mod.rs index a74cf3871de..4e0e85294b6 100644 --- a/core/src/amp_subgraph/runner/mod.rs +++ b/core/src/amp_subgraph/runner/mod.rs @@ -11,8 +11,8 @@ use std::time::{Duration, Instant}; use anyhow::Result; use futures::StreamExt; use graph::{ - amp::Client, cheap_clone::CheapClone, components::store::EntityCache, - data::subgraph::schema::SubgraphError, + amp::Client, cheap_clone::CheapClone, components::store::EntityLfuCache, + data::subgraph::schema::SubgraphError, util::lfu_cache::LfuCache, }; use slog::{debug, error, warn}; use tokio_util::sync::CancellationToken; @@ -67,7 +67,7 @@ where async fn run_indexing(cx: &mut Context) -> Result<(), Error> where - AC: Client, + AC: Client + Send + Sync + 'static, { cx.metrics.deployment_status.starting(); @@ -101,32 +101,44 @@ where cx.metrics .deployment_target - .update(latest_block.min(cx.max_end_block())); + .update(latest_block.min(cx.end_block())); let mut deployment_is_failed = cx.store.health().await?.is_failed(); - let mut entity_cache = EntityCache::new(cx.store.cheap_clone()); + let mut entity_lfu_cache: EntityLfuCache = LfuCache::new(); let mut stream = new_data_stream(cx, latest_block); while let Some(result) = stream.next().await { let (record_batch_groups, stream_table_ptr) = result?; - entity_cache = process_record_batch_groups( + entity_lfu_cache = process_record_batch_groups( cx, - entity_cache, + entity_lfu_cache, record_batch_groups, stream_table_ptr, latest_block, ) .await?; - if deployment_is_failed { - if let Some(block_ptr) = cx.store.block_ptr() { - cx.store.unfail_non_deterministic_error(&block_ptr).await?; - deployment_is_failed = false; - } + if deployment_is_failed && let Some(block_ptr) = cx.store.block_ptr() { + cx.store.unfail_non_deterministic_error(&block_ptr).await?; + deployment_is_failed = false; } } + // Check if the Amp Flight server has data covering through every data + // source's endBlock. This handles the case where endBlock has no entity + // data — the persisted block pointer never advances to endBlock, but the + // server's latest block confirms all queries have been served. + if latest_block >= cx.end_block() { + cx.metrics.deployment_synced.record(true); + + debug!(cx.logger, "Indexing completed; endBlock reached via server latest block"; + "latest_block" => latest_block, + "end_block" => cx.end_block() + ); + return Ok(()); + } + debug!(cx.logger, "Completed indexing iteration"; "latest_synced_block_ptr" => ?cx.latest_synced_block_ptr() ); @@ -138,7 +150,7 @@ where async fn run_indexing_with_retries(cx: &mut Context) -> Result<()> where - AC: Client, + AC: Client + Send + Sync + 'static, { loop { match run_indexing(cx).await { diff --git a/core/src/amp_subgraph/runner/reorg_handler.rs b/core/src/amp_subgraph/runner/reorg_handler.rs index 03130b19625..8f59169304b 100644 --- a/core/src/amp_subgraph/runner/reorg_handler.rs +++ b/core/src/amp_subgraph/runner/reorg_handler.rs @@ -1,10 +1,10 @@ use alloy::primitives::{BlockHash, BlockNumber}; use anyhow::anyhow; -use futures::{future::try_join_all, StreamExt, TryFutureExt}; +use futures::{StreamExt, TryFutureExt, future::try_join_all}; use graph::{ amp::{ - client::{LatestBlockBeforeReorg, RequestMetadata, ResponseBatch, ResumeStreamingQuery}, Client, + client::{LatestBlockBeforeReorg, RequestMetadata, ResponseBatch, ResumeStreamingQuery}, }, blockchain::block_stream::FirehoseCursor, }; diff --git a/core/src/polling_monitor/arweave_service.rs b/core/src/polling_monitor/arweave_service.rs index 51249324df7..cc42f7af5ea 100644 --- a/core/src/polling_monitor/arweave_service.rs +++ b/core/src/polling_monitor/arweave_service.rs @@ -2,18 +2,18 @@ use anyhow::Error; use bytes::Bytes; use graph::futures03::future::BoxFuture; use graph::{ - components::link_resolver::{ArweaveClient, ArweaveResolver, FileSizeLimit}, + components::link_resolver::{ArweaveResolver, FileSizeLimit}, data_source::offchain::Base64, derive::CheapClone, prelude::CheapClone, }; use std::{sync::Arc, time::Duration}; -use tower::{buffer::Buffer, ServiceBuilder, ServiceExt}; +use tower::{ServiceBuilder, ServiceExt, buffer::Buffer}; pub type ArweaveService = Buffer, Error>>>; pub fn arweave_service( - client: Arc, + client: Arc, rate_limit: u16, max_file_size: FileSizeLimit, ) -> ArweaveService { @@ -34,7 +34,7 @@ pub fn arweave_service( #[derive(Clone, CheapClone)] struct ArweaveServiceInner { - client: Arc, + client: Arc, max_file_size: FileSizeLimit, } diff --git a/core/src/polling_monitor/ipfs_service.rs b/core/src/polling_monitor/ipfs_service.rs index e35fcf30d30..6847bb82acd 100644 --- a/core/src/polling_monitor/ipfs_service.rs +++ b/core/src/polling_monitor/ipfs_service.rs @@ -1,13 +1,13 @@ use std::sync::Arc; use std::time::Duration; -use anyhow::anyhow; use anyhow::Error; +use anyhow::anyhow; use bytes::Bytes; use graph::futures03::future::BoxFuture; use graph::ipfs::{ContentPath, IpfsClient, IpfsContext, RetryPolicy}; use graph::{derive::CheapClone, prelude::CheapClone}; -use tower::{buffer::Buffer, ServiceBuilder, ServiceExt}; +use tower::{ServiceBuilder, ServiceExt, buffer::Buffer}; pub type IpfsService = Buffer, Error>>>; @@ -104,16 +104,15 @@ mod test { use std::time::Duration; use graph::components::link_resolver::ArweaveClient; - use graph::components::link_resolver::ArweaveResolver; use graph::data::value::Word; use graph::ipfs::test_utils::add_files_to_local_ipfs_node_for_testing; use graph::ipfs::{IpfsContext, IpfsMetrics, IpfsRpcClient, ServerAddress}; use graph::log::discard; use tower::ServiceExt; - use wiremock::matchers as m; use wiremock::Mock; use wiremock::MockServer; use wiremock::ResponseTemplate; + use wiremock::matchers as m; use super::*; @@ -133,7 +132,7 @@ mod test { let dir_cid = add_resp.into_iter().find(|x| x.name == "dir").unwrap().hash; let client = IpfsRpcClient::new_unchecked( - ServerAddress::local_rpc_api(), + ServerAddress::test_rpc_api(), IpfsMetrics::test(), &graph::log::discard(), ) @@ -154,12 +153,15 @@ mod test { assert_eq!(content.to_vec(), random_bytes); } + // Ignored because arweave.net is unreliable right now #[graph::test] + #[ignore] async fn arweave_get() { const ID: &str = "8APeQ5lW0-csTcBaGdPBDLAL2ci2AT9pTn2tppGPU_8"; - let cl = ArweaveClient::default(); - let body = cl.get(&Word::from(ID)).await.unwrap(); + let Some(body) = ArweaveClient::get_test(&Word::from(ID)).await else { + return; + }; let body = String::from_utf8(body).unwrap(); let expected = r#" diff --git a/core/src/polling_monitor/mod.rs b/core/src/polling_monitor/mod.rs index fa0b433f4de..f8a24db97d7 100644 --- a/core/src/polling_monitor/mod.rs +++ b/core/src/polling_monitor/mod.rs @@ -14,11 +14,11 @@ use graph::cheap_clone::CheapClone; use graph::env::ENV_VARS; use graph::futures03::future::BoxFuture; use graph::futures03::stream::StreamExt; -use graph::futures03::{stream, Future, FutureExt, TryFutureExt}; +use graph::futures03::{Future, FutureExt, TryFutureExt, stream}; use graph::parking_lot::Mutex; use graph::prelude::tokio; use graph::prometheus::{Counter, Gauge}; -use graph::slog::{debug, Logger}; +use graph::slog::{Logger, debug}; use graph::util::monitored::MonitoredVecDeque as VecDeque; use tokio::sync::{mpsc, watch}; use tower::retry::backoff::{Backoff, ExponentialBackoff, ExponentialBackoffMaker, MakeBackoff}; @@ -28,8 +28,8 @@ use tower::{Service, ServiceExt}; use self::request::RequestId; pub use self::metrics::PollingMonitorMetrics; -pub use arweave_service::{arweave_service, ArweaveService}; -pub use ipfs_service::{ipfs_service, IpfsRequest, IpfsService}; +pub use arweave_service::{ArweaveService, arweave_service}; +pub use ipfs_service::{IpfsRequest, IpfsService, ipfs_service}; const MIN_BACKOFF: Duration = Duration::from_secs(5); @@ -53,7 +53,7 @@ impl Backoffs { } } - fn next_backoff(&mut self, id: ID) -> impl Future { + fn next_backoff(&mut self, id: ID) -> impl Future + Send + use { self.backoffs .entry(id) .or_insert_with(|| self.backoff_maker.make_backoff()) diff --git a/core/src/subgraph/context/instance/mod.rs b/core/src/subgraph/context/instance/mod.rs index e9c311420e6..494ae227e97 100644 --- a/core/src/subgraph/context/instance/mod.rs +++ b/core/src/subgraph/context/instance/mod.rs @@ -5,8 +5,8 @@ use graph::futures01::sync::mpsc::Sender; use graph::{ blockchain::{Blockchain, TriggerData as _}, data_source::{ - causality_region::CausalityRegionSeq, offchain, CausalityRegion, DataSource, - DataSourceTemplate, TriggerData, + CausalityRegion, DataSource, DataSourceTemplate, TriggerData, + causality_region::CausalityRegionSeq, offchain, }, prelude::*, }; @@ -100,11 +100,11 @@ where ) -> Result>, Error> { let module_bytes = match &data_source.runtime() { None => return Ok(None), - Some(ref module_bytes) => module_bytes.cheap_clone(), + Some(module_bytes) => module_bytes.cheap_clone(), }; let mapping_request_sender = { - let module_hash = tiny_keccak::keccak256(module_bytes.as_ref()); + let module_hash = alloy::primitives::keccak256(module_bytes.as_ref()).0; if let Some(sender) = self.module_cache.get(&module_hash) { sender.clone() } else { diff --git a/core/src/subgraph/context/mod.rs b/core/src/subgraph/context/mod.rs index 2b7d560dfc1..affbc09aa0f 100644 --- a/core/src/subgraph/context/mod.rs +++ b/core/src/subgraph/context/mod.rs @@ -1,18 +1,18 @@ mod instance; use crate::polling_monitor::{ - spawn_monitor, ArweaveService, IpfsRequest, IpfsService, PollingMonitor, PollingMonitorMetrics, + ArweaveService, IpfsRequest, IpfsService, PollingMonitor, PollingMonitorMetrics, spawn_monitor, }; use anyhow::{self, Error}; use bytes::Bytes; use graph::{ - blockchain::{Blockchain, TriggerFilterWrapper}, + blockchain::Blockchain, components::{store::DeploymentId, subgraph::HostMetrics}, data::subgraph::SubgraphManifest, data_source::{ + CausalityRegion, DataSource, DataSourceTemplate, causality_region::CausalityRegionSeq, offchain::{self, Base64}, - CausalityRegion, DataSource, DataSourceTemplate, }, derive::CheapClone, ipfs::IpfsContext, @@ -74,7 +74,6 @@ where pub(crate) instance: SubgraphInstance, pub instances: SubgraphKeepAlive, pub offchain_monitor: OffchainMonitor, - pub filter: Option>, pub(crate) trigger_processor: Box>, pub(crate) decoder: Box>, } @@ -101,7 +100,6 @@ impl> IndexingContext { instance, instances, offchain_monitor, - filter: None, trigger_processor, decoder, } @@ -116,12 +114,12 @@ impl> IndexingContext { /// `process_trigger`. /// /// File data sources that have been marked not done during this process will get re-queued - pub fn revert_data_sources(&mut self, reverted_block: BlockNumber) -> Result<(), Error> { + pub fn revert_data_sources(&mut self, reverted_block: BlockNumber) { let removed = self.instance.revert_data_sources(reverted_block); removed .into_iter() - .try_for_each(|source| self.offchain_monitor.add_source(source)) + .for_each(|source| self.offchain_monitor.add_source(source)) } pub fn add_dynamic_data_source( @@ -134,12 +132,12 @@ impl> IndexingContext { .map(|ds| (ds.source.clone(), ds.is_processed())); let host = self.instance.add_dynamic_data_source(logger, data_source)?; - if host.is_some() { - if let Some((source, is_processed)) = offchain_fields { - // monitor data source only if it has not yet been processed. - if !is_processed { - self.offchain_monitor.add_source(source)?; - } + if host.is_some() + && let Some((source, is_processed)) = offchain_fields + { + // monitor data source only if it has not yet been processed. + if !is_processed { + self.offchain_monitor.add_source(source); } } @@ -214,7 +212,7 @@ impl OffchainMonitor { } } - fn add_source(&mut self, source: offchain::Source) -> Result<(), Error> { + fn add_source(&mut self, source: offchain::Source) { match source { offchain::Source::Ipfs(path) => self.ipfs_monitor.monitor(IpfsRequest { ctx: IpfsContext::new(&self.deployment_hash, &self.logger), @@ -222,7 +220,6 @@ impl OffchainMonitor { }), offchain::Source::Arweave(base64) => self.arweave_monitor.monitor(base64), }; - Ok(()) } pub fn ready_offchain_events(&mut self) -> Result, Error> { diff --git a/core/src/subgraph/error.rs b/core/src/subgraph/error.rs index 502a28dbc66..00413acb258 100644 --- a/core/src/subgraph/error.rs +++ b/core/src/subgraph/error.rs @@ -1,6 +1,6 @@ use graph::data::subgraph::schema::SubgraphError; use graph::env::ENV_VARS; -use graph::prelude::{anyhow, thiserror, Error, StoreError}; +use graph::prelude::{Error, StoreError, anyhow, thiserror}; pub trait DeterministicError: std::fmt::Debug + std::fmt::Display + Send + Sync + 'static {} @@ -10,6 +10,37 @@ impl DeterministicError for StoreError {} impl DeterministicError for anyhow::Error {} +/// Classification of processing errors for unified error handling. +/// +/// This enum provides a consistent way to categorize errors and determine +/// the appropriate response. The error handling invariants are: +/// +/// - **Deterministic**: Stop processing the current block, persist PoI only. +/// The subgraph will be marked as failed. These errors are reproducible +/// and indicate a bug in the subgraph or a permanent data issue. +/// +/// - **NonDeterministic**: Retry with exponential backoff. These errors are +/// transient (network issues, temporary database problems) and may succeed +/// on retry. +/// +/// - **PossibleReorg**: Restart the block stream cleanly without persisting. +/// The block stream needs to be restarted to detect and handle a potential +/// blockchain reorganization. +/// +/// - **Canceled**: The subgraph was canceled (unassigned or shut down). +/// No error should be recorded; this is a clean shutdown. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProcessingErrorKind { + /// Error is deterministic - stop processing, persist PoI only + Deterministic, + /// Error is non-deterministic - retry with backoff + NonDeterministic, + /// Possible blockchain reorg detected - restart block stream cleanly + PossibleReorg, + /// Processing was canceled - clean shutdown + Canceled, +} + /// An error happened during processing and we need to classify errors into /// deterministic and non-deterministic errors. This struct holds the result /// of that classification @@ -23,13 +54,72 @@ pub enum ProcessingError { #[error("{0}")] Deterministic(Box), + /// A possible blockchain reorganization was detected. + /// The block stream should be restarted to detect and handle the reorg. + #[error("possible reorg detected: {0:#}")] + PossibleReorg(Error), + #[error("subgraph stopped while processing triggers")] Canceled, } impl ProcessingError { + /// Classify the error into one of the defined error kinds. + /// + /// This method provides a unified way to determine how to handle an error: + /// - `Deterministic`: Stop processing, persist PoI only + /// - `NonDeterministic`: Retry with backoff + /// - `PossibleReorg`: Restart block stream cleanly + /// - `Canceled`: Clean shutdown, no error recording + pub fn kind(&self) -> ProcessingErrorKind { + match self { + ProcessingError::Unknown(_) => ProcessingErrorKind::NonDeterministic, + ProcessingError::Deterministic(_) => ProcessingErrorKind::Deterministic, + ProcessingError::PossibleReorg(_) => ProcessingErrorKind::PossibleReorg, + ProcessingError::Canceled => ProcessingErrorKind::Canceled, + } + } + + #[allow(dead_code)] pub fn is_deterministic(&self) -> bool { - matches!(self, ProcessingError::Deterministic(_)) + matches!(self.kind(), ProcessingErrorKind::Deterministic) + } + + /// Returns true if this error should stop processing the current block. + /// + /// Deterministic errors stop processing because continuing would produce + /// incorrect results. The PoI is still persisted for debugging purposes. + #[allow(dead_code)] + pub fn should_stop_processing(&self) -> bool { + matches!(self.kind(), ProcessingErrorKind::Deterministic) + } + + /// Returns true if this error requires a clean restart of the block stream. + /// + /// Possible reorgs require restarting to allow the block stream to detect + /// and properly handle the reorganization. No state should be persisted + /// in this case. + #[allow(dead_code)] + pub fn should_restart(&self) -> bool { + matches!(self.kind(), ProcessingErrorKind::PossibleReorg) + } + + /// Returns true if this error is retryable with exponential backoff. + /// + /// Non-deterministic errors (network issues, temporary failures) may + /// succeed on retry and should not immediately fail the subgraph. + #[allow(dead_code)] + pub fn is_retryable(&self) -> bool { + matches!(self.kind(), ProcessingErrorKind::NonDeterministic) + } + + /// Returns true if processing was canceled (clean shutdown). + /// + /// Canceled errors indicate the subgraph was unassigned or shut down + /// intentionally and should not be treated as failures. + #[allow(dead_code)] + pub fn is_canceled(&self) -> bool { + matches!(self.kind(), ProcessingErrorKind::Canceled) } pub fn detail(self, ctx: &str) -> ProcessingError { @@ -41,6 +131,9 @@ impl ProcessingError { ProcessingError::Deterministic(e) => { ProcessingError::Deterministic(Box::new(anyhow!("{e}").context(ctx.to_string()))) } + ProcessingError::PossibleReorg(e) => { + ProcessingError::PossibleReorg(e.context(ctx.to_string())) + } ProcessingError::Canceled => ProcessingError::Canceled, } } diff --git a/core/src/subgraph/inputs.rs b/core/src/subgraph/inputs.rs index 88e89de4ff4..03a8cec29bc 100644 --- a/core/src/subgraph/inputs.rs +++ b/core/src/subgraph/inputs.rs @@ -1,5 +1,5 @@ use graph::{ - blockchain::{block_stream::TriggersAdapterWrapper, Blockchain}, + blockchain::{Blockchain, block_stream::TriggersAdapterWrapper}, components::{ store::{DeploymentLocator, SourceableStore, SubgraphFork, WritableStore}, subgraph::ProofOfIndexingVersion, diff --git a/core/src/subgraph/instance_manager.rs b/core/src/subgraph/instance_manager.rs index 5d0c89ae171..ea301ffc723 100644 --- a/core/src/subgraph/instance_manager.rs +++ b/core/src/subgraph/instance_manager.rs @@ -2,10 +2,10 @@ use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; use crate::polling_monitor::{ArweaveService, IpfsService}; +use crate::subgraph::Decoder; use crate::subgraph::context::{IndexingContext, SubgraphKeepAlive}; use crate::subgraph::inputs::IndexingInputs; use crate::subgraph::loader::load_dynamic_data_sources; -use crate::subgraph::Decoder; use std::collections::BTreeSet; use crate::subgraph::runner::SubgraphRunner; @@ -17,18 +17,18 @@ use graph::components::metrics::gas::GasMetrics; use graph::components::metrics::subgraph::DeploymentStatusMetric; use graph::components::store::SourceableStore; use graph::components::subgraph::ProofOfIndexingVersion; -use graph::data::subgraph::{UnresolvedSubgraphManifest, SPEC_VERSION_0_0_6}; +use graph::data::subgraph::{SPEC_VERSION_0_0_6, UnresolvedSubgraphManifest}; use graph::data::value::Word; use graph::data_source::causality_region::CausalityRegionSeq; use graph::env::EnvVars; use graph::prelude::{SubgraphInstanceManager as SubgraphInstanceManagerTrait, *}; use graph::{blockchain::BlockchainMap, components::store::DeploymentLocator}; -use graph_runtime_wasm::module::ToAscPtr; use graph_runtime_wasm::RuntimeHostBuilder; +use graph_runtime_wasm::module::ToAscPtr; use tokio::task; -use super::context::OffchainMonitor; use super::SubgraphTriggerProcessor; +use super::context::OffchainMonitor; use crate::{subgraph::runner::SubgraphRunnerError, subgraph_manifest}; #[derive(Clone)] @@ -278,17 +278,17 @@ impl SubgraphInstanceManager { .with_retries(), ); - if let Some(graft) = &manifest.graft { - if self.subgraph_store.is_deployed(&graft.base).await? { - // Makes sure the raw manifest is cached in the subgraph store - let _raw_manifest = subgraph_manifest::load_raw_subgraph_manifest( - &logger, - &*self.subgraph_store, - &*self.link_resolver, - &graft.base, - ) - .await?; - } + if let Some(graft) = &manifest.graft + && self.subgraph_store.is_deployed(&graft.base).await? + { + // Makes sure the raw manifest is cached in the subgraph store + let _raw_manifest = subgraph_manifest::load_raw_subgraph_manifest( + &logger, + &*self.subgraph_store, + &*self.link_resolver, + &graft.base, + ) + .await?; } info!(logger, "Resolve subgraph files using IPFS"; @@ -399,7 +399,15 @@ impl SubgraphInstanceManager { // when to stop processing them. // - Offchain data sources might require processing beyond the end block of // onchain data sources, so the subgraph needs to continue. - let max_end_block: Option = if data_sources.len() == end_blocks.len() { + // + // Note: we explicitly check each data source rather than comparing lengths, because + // `end_blocks` is a BTreeSet and deduplicates equal values, which would cause the + // length comparison to fail when multiple data sources share the same `end_block`. + let max_end_block: Option = if data_sources.iter().all(|d| { + d.as_onchain() + .and_then(|d: &C::DataSource| d.end_block()) + .is_some() + }) { end_blocks.iter().max().cloned() } else { None diff --git a/core/src/subgraph/registrar.rs b/core/src/subgraph/registrar.rs index a60f2b01b03..e9422671e25 100644 --- a/core/src/subgraph/registrar.rs +++ b/core/src/subgraph/registrar.rs @@ -5,16 +5,17 @@ use graph::amp; use graph::blockchain::{Blockchain, BlockchainKind, BlockchainMap}; use graph::components::{ link_resolver::LinkResolverContext, + network_provider::AmpChainNames, store::{DeploymentId, DeploymentLocator, SubscriptionManager}, subgraph::Settings, }; use graph::data::{ - subgraph::{schema::DeploymentCreate, Graft}, + subgraph::{Graft, schema::DeploymentCreate}, value::Word, }; -use graph::futures03::{self, future::TryFutureExt, Stream, StreamExt}; +use graph::futures03::{self, Stream, StreamExt, future::TryFutureExt}; use graph::prelude::{CreateSubgraphResult, SubgraphRegistrar as SubgraphRegistrarTrait, *}; -use graph::util::futures::{retry_strategy, RETRY_DEFAULT_LIMIT}; +use graph::util::futures::{RETRY_DEFAULT_LIMIT, retry_strategy}; use tokio_retry::Retry; pub struct SubgraphRegistrar { @@ -30,6 +31,7 @@ pub struct SubgraphRegistrar { version_switching_mode: SubgraphVersionSwitchingMode, assignment_event_stream_cancel_guard: CancelGuard, // cancels on drop settings: Arc, + amp_chain_names: Arc, } impl SubgraphRegistrar @@ -50,6 +52,7 @@ where node_id: NodeId, version_switching_mode: SubgraphVersionSwitchingMode, settings: Arc, + amp_chain_names: Arc, ) -> Self { let logger = logger_factory.component_logger("SubgraphRegistrar", None); let logger_factory = logger_factory.with_parent(logger.clone()); @@ -69,6 +72,7 @@ where version_switching_mode, assignment_event_stream_cancel_guard: CancelGuard::new(), settings, + amp_chain_names, } } @@ -314,6 +318,7 @@ where &resolver, self.amp_client.cheap_clone(), history_blocks, + &self.amp_chain_names, ) .await? } @@ -333,6 +338,7 @@ where &resolver, self.amp_client.cheap_clone(), history_blocks, + &self.amp_chain_names, ) .await? } @@ -349,9 +355,12 @@ where } async fn remove_subgraph(&self, name: SubgraphName) -> Result<(), SubgraphRegistrarError> { - self.store.clone().remove_subgraph(name.clone()).await?; + use itertools::Itertools; - debug!(self.logger, "Removed subgraph"; "subgraph_name" => name.to_string()); + let hashes = self.store.clone().remove_subgraph(name.clone()).await?; + let hashes = hashes.into_iter().join(", "); + + debug!(self.logger, "Removed subgraph"; "subgraph_name" => name.to_string(), "deployments" => format!("[{}]", hashes)); Ok(()) } @@ -411,7 +420,7 @@ async fn resolve_start_block( .expect("cannot identify minimum start block because there are no data sources") { 0 => Ok(None), - min_start_block => Retry::spawn(retry_strategy(Some(2), RETRY_DEFAULT_LIMIT), move || { + min_start_block => Retry::start(retry_strategy(Some(2), RETRY_DEFAULT_LIMIT), move || { chain .block_pointer_from_number(logger, min_start_block - 1) .inspect_err(move |e| warn!(&logger, "Failed to get block number: {}", e)) @@ -460,6 +469,7 @@ async fn create_subgraph_version, amp_client: Option>, history_blocks_override: Option, + amp_chain_names: &AmpChainNames, ) -> Result { let raw_string = serde_yaml::to_string(&raw).unwrap(); @@ -488,9 +498,10 @@ async fn create_subgraph_version(network_name.clone()) + .get::(resolved_name.clone()) .map_err(SubgraphRegistrarError::NetworkNotSupported)? .cheap_clone(); @@ -570,7 +581,7 @@ async fn create_subgraph_version, + /// The current state in the runner's state machine. + /// This field drives the main loop of the runner. + runner_state: RunnerState, } #[derive(Debug, thiserror::Error)] @@ -97,10 +109,12 @@ where ), entity_lfu_cache: LfuCache::new(), cached_head_ptr: None, + postponed_indexes_created: false, }, logger, metrics, cancel_handle: None, + runner_state: RunnerState::Initializing, } } @@ -113,14 +127,13 @@ where /// be removed. The same thing also applies to the block cache. /// This function must be called before continuing to process in order to avoid /// duplicated host insertion and POI issues with dirty entity changes. - fn revert_state_to(&mut self, block_number: BlockNumber) -> Result<(), Error> { + fn revert_state_to(&mut self, block_number: BlockNumber) { self.state.entity_lfu_cache = LfuCache::new(); // 1. Revert all hosts(created by DDS) at a block higher than `block_number`. // 2. Unmark any offchain data sources that were marked done on the blocks being removed. // When no offchain datasources are present, 2. should be a noop. - self.ctx.revert_data_sources(block_number + 1)?; - Ok(()) + self.ctx.revert_data_sources(block_number + 1); } #[cfg(debug_assertions)] @@ -212,15 +225,11 @@ where let block_stream_canceler = CancelGuard::new(); let block_stream_cancel_handle = block_stream_canceler.handle(); // TriggerFilter needs to be rebuilt eveytime the blockstream is restarted - self.ctx.filter = Some(self.build_filter()); + let filter = self.build_filter(); - let block_stream = new_block_stream( - &self.inputs, - self.ctx.filter.clone().unwrap(), // Safe to unwrap as we just called `build_filter` in the previous line - &self.metrics.subgraph, - ) - .await? - .cancelable(&block_stream_canceler); + let block_stream = new_block_stream(&self.inputs, filter, &self.metrics.subgraph) + .await? + .cancelable(&block_stream_canceler); self.cancel_handle = Some(block_stream_cancel_handle); @@ -241,11 +250,17 @@ where } } - pub async fn run(self) -> Result<(), SubgraphRunnerError> { - self.run_inner(false).await.map(|_| ()) - } - - async fn run_inner(mut self, break_on_restart: bool) -> Result { + /// Initialize the runner by performing pre-loop setup. + /// + /// This method handles: + /// - Updating the deployment synced metric + /// - Attempting to unfail deterministic errors from the previous run + /// - Checking if the subgraph has already reached its max end block + /// + /// Returns the next state to transition to: + /// - `Restarting` to start the block stream (normal case) + /// - `Stopped` if the max end block was already reached + async fn initialize(&self) -> Result, SubgraphRunnerError> { self.update_deployment_synced_metric(); // If a subgraph failed for deterministic reasons, before start indexing, we first @@ -276,93 +291,345 @@ where } // Stop subgraph when we reach maximum endblock. - if let Some(max_end_block) = self.inputs.max_end_block { - if max_end_block <= current_ptr.block_number() { - info!(self.logger, "Stopping subgraph as we reached maximum endBlock"; + if let Some(max_end_block) = self.inputs.max_end_block + && max_end_block <= current_ptr.block_number() + { + info!(self.logger, "Stopping subgraph as we reached maximum endBlock"; "max_end_block" => max_end_block, "current_block" => current_ptr.block_number()); - self.inputs.store.flush().await?; - return Ok(self); - } + self.inputs.store.flush().await?; + return Ok(RunnerState::Stopped { + reason: StopReason::MaxEndBlockReached, + }); } } - loop { - debug!(self.logger, "Starting or restarting subgraph"); + // Normal case: proceed to start the block stream + Ok(RunnerState::Restarting { + reason: RestartReason::StoreError, // Initial start uses the same path as restart + }) + } - let mut block_stream = self.start_block_stream().await?; + /// Await the next block stream event and transition to the appropriate state. + /// + /// This method waits for the next event from the block stream and determines + /// which state the runner should transition to: + /// - `ProcessingBlock` for new blocks to process + /// - `Reverting` for revert events + /// - `Stopped` when the stream ends or is canceled + /// - Returns back to `AwaitingBlock` for non-fatal errors that allow continuation + async fn await_block( + &mut self, + mut block_stream: Cancelable>>, + ) -> Result, SubgraphRunnerError> { + let event = { + let _section = self.metrics.stream.stopwatch.start_section("scan_blocks"); + block_stream.next().await + }; - debug!(self.logger, "Started block stream"); + if self.is_canceled() { + return self.cancel(); + } - self.metrics.subgraph.deployment_status.running(); + match event { + Some(Ok(BlockStreamEvent::ProcessBlock(block, cursor))) => { + Ok(RunnerState::ProcessingBlock { + block_stream, + block, + cursor, + }) + } + Some(Ok(BlockStreamEvent::Revert(to_ptr, cursor))) => Ok(RunnerState::Reverting { + block_stream, + to_ptr, + cursor, + }), + // Log and drop the errors from the block_stream + // The block stream will continue attempting to produce blocks + Some(Err(e)) => { + // Log error and continue waiting for blocks + debug!( + &self.logger, + "Block stream produced a non-fatal error"; + "error" => format!("{}", e), + ); + Ok(RunnerState::AwaitingBlock { block_stream }) + } + // If the block stream ends, that means that there is no more indexing to do. + None => Ok(RunnerState::Stopped { + reason: StopReason::StreamEnded, + }), + } + } - // Process events from the stream as long as no restart is needed - loop { - let event = { - let _section = self.metrics.stream.stopwatch.start_section("scan_blocks"); + fn cancel(&mut self) -> Result, SubgraphRunnerError> { + if self.ctx.instances.contains(&self.inputs.deployment.id) { + warn!( + self.logger, + "Terminating the subgraph runner because a newer one is active. \ + Possible reassignment detected while the runner was in a non-cancellable pending state", + ); + return Err(SubgraphRunnerError::Duplicate); + } + warn!( + self.logger, + "Terminating the subgraph runner because subgraph was unassigned", + ); + Ok(RunnerState::Stopped { + reason: StopReason::Unassigned, + }) + } - block_stream.next().await - }; + /// Construct a SubgraphError and mark the subgraph as failed in the store. + async fn fail_subgraph( + &mut self, + message: String, + block_ptr: Option, + deterministic: bool, + ) -> Result<(), SubgraphRunnerError> { + let error = SubgraphError { + subgraph_id: self.inputs.deployment.hash.clone(), + message, + block_ptr, + handler: None, + deterministic, + }; + self.inputs + .store + .fail_subgraph(error) + .await + .context("Failed to set subgraph status to `failed`")?; + Ok(()) + } - // TODO: move cancel handle to the Context - // This will require some code refactor in how the BlockStream is created - let block_start = Instant::now(); - - let action = self.handle_stream_event(event).await.inspect(|res| { - self.metrics - .subgraph - .observe_block_processed(block_start.elapsed(), res.block_finished()); - })?; - - self.update_deployment_synced_metric(); - - // It is possible that the subgraph was unassigned, but the runner was in - // a retry delay state and did not observe the cancel signal. - if self.is_canceled() { - // It is also possible that the runner was in a retry delay state while - // the subgraph was reassigned and a new runner was started. - if self.ctx.instances.contains(&self.inputs.deployment.id) { - warn!( - self.logger, - "Terminating the subgraph runner because a newer one is active. \ - Possible reassignment detected while the runner was in a non-cancellable pending state", - ); - return Err(SubgraphRunnerError::Duplicate); - } + /// Handle a restart by potentially restarting the store and starting a new block stream. + /// + /// This method handles: + /// - Restarting the store if there were errors (to clear error state) + /// - Reverting state to the last good block if the store was restarted + /// - Starting a new block stream with updated filters + /// + /// Returns the next state to transition to: + /// - `AwaitingBlock` with the new block stream (normal case) + async fn restart( + &mut self, + reason: RestartReason, + ) -> Result, SubgraphRunnerError> { + debug!(self.logger, "Starting or restarting subgraph"; "reason" => ?reason); + + // If restarting due to store error, try to restart the store + if matches!(reason, RestartReason::StoreError) { + let store = self.inputs.store.cheap_clone(); + if let Some(store) = store.restart().await? { + let last_good_block = store.block_ptr().map(|ptr| ptr.number).unwrap_or(0); + self.revert_state_to(last_good_block); + self.inputs = Arc::new(self.inputs.with_store(store)); + } + } - warn!( - self.logger, - "Terminating the subgraph runner because subgraph was unassigned", - ); - return Ok(self); - } + let block_stream = self.start_block_stream().await?; - match action { - Action::Continue => continue, - Action::Stop => { - info!(self.logger, "Stopping subgraph"); - self.inputs.store.flush().await?; - return Ok(self); - } - Action::Restart if break_on_restart => { + debug!(self.logger, "Started block stream"); + self.metrics.subgraph.deployment_status.running(); + self.update_deployment_synced_metric(); + + Ok(RunnerState::AwaitingBlock { block_stream }) + } + + /// Finalize the runner when it reaches a terminal state. + /// + /// This method handles cleanup tasks when the runner stops: + /// - Flushing the store to ensure all changes are persisted + /// - Logging the stop reason + async fn finalize(self, reason: StopReason) -> Result { + match reason { + StopReason::MaxEndBlockReached => { + info!(self.logger, "Stopping subgraph - max end block reached"); + } + StopReason::Canceled => { + info!(self.logger, "Stopping subgraph - canceled"); + } + StopReason::Unassigned => { + info!(self.logger, "Stopping subgraph - unassigned"); + } + StopReason::StreamEnded => { + info!(self.logger, "Stopping subgraph - stream ended"); + } + } + + self.inputs.store.flush().await?; + Ok(self) + } + + pub async fn run(self) -> Result<(), SubgraphRunnerError> { + self.run_inner(false).await.map(|_| ()) + } + + /// Main state machine loop for the subgraph runner. + /// + /// This method drives the runner through its state machine, transitioning + /// between states based on events and actions. The state machine replaces + /// the previous nested loop structure with explicit state transitions. + /// + /// ## State Machine + /// + /// The runner starts in `Initializing` and transitions through states: + /// - `Initializing` → `Restarting` (or `Stopped` if max end block reached) + /// - `Restarting` → `AwaitingBlock` + /// - `AwaitingBlock` → `ProcessingBlock`, `Reverting`, or `Stopped` + /// - `ProcessingBlock` → `AwaitingBlock` or `Restarting` + /// - `Reverting` → `AwaitingBlock` or `Restarting` + /// - `Stopped` → terminal (returns) + async fn run_inner(mut self, break_on_restart: bool) -> Result { + // Start in Initializing state + self.runner_state = RunnerState::Initializing; + + // Track whether we've started processing blocks (not just initialized). + // This is used for break_on_restart logic - we should only stop on restart + // after we've actually started processing, not on the initial "restart" + // which is really the first start of the block stream. + let mut has_processed_blocks = false; + + loop { + self.runner_state = match std::mem::take(&mut self.runner_state) { + RunnerState::Initializing => self.initialize().await?, + + RunnerState::Restarting { reason } => { + if break_on_restart && has_processed_blocks { + // In test mode, stop on restart after first block processing info!(self.logger, "Stopping subgraph on break"); - self.inputs.store.flush().await?; - return Ok(self); - } - Action::Restart => { - // Restart the store to clear any errors that it - // might have encountered and use that from now on - let store = self.inputs.store.cheap_clone(); - if let Some(store) = store.restart().await? { - let last_good_block = - store.block_ptr().map(|ptr| ptr.number).unwrap_or(0); - self.revert_state_to(last_good_block)?; - self.inputs = Arc::new(self.inputs.with_store(store)); + RunnerState::Stopped { + reason: StopReason::Canceled, } - break; + } else { + self.restart(reason).await? } - }; - } + } + + RunnerState::AwaitingBlock { block_stream } => { + self.await_block(block_stream).await? + } + + RunnerState::ProcessingBlock { + block_stream, + block, + cursor, + } => { + has_processed_blocks = true; + self.process_block_state(block_stream, block, cursor) + .await? + } + + RunnerState::Reverting { + block_stream, + to_ptr, + cursor, + } => self.handle_revert_state(block_stream, to_ptr, cursor).await, + + RunnerState::Stopped { reason } => { + return self.finalize(reason).await; + } + }; + } + } + + /// Process a block and determine the next state. + /// + /// This is the state machine wrapper around `process_block` that handles + /// the block processing action and determines state transitions. + async fn process_block_state( + &mut self, + block_stream: Cancelable>>, + block: BlockWithTriggers, + cursor: FirehoseCursor, + ) -> Result, SubgraphRunnerError> { + let block_ptr = block.ptr(); + self.metrics + .stream + .deployment_head + .set(block_ptr.number as f64); + + if block.trigger_count() > 0 { + self.metrics + .subgraph + .block_trigger_count + .observe(block.trigger_count() as f64); + } + + // Check if we should skip this block (optimization for blocks without triggers). + // Do not skip if max_end_block has been reached — fall through to process_block so the + // block pointer is persisted and the existing max_end_block check in handle_action fires. + let max_end_block_reached = self + .inputs + .max_end_block + .is_some_and(|max| block_ptr.number >= max); + if block.trigger_count() == 0 + && self.state.skip_ptr_updates_timer.elapsed() <= SKIP_PTR_UPDATES_THRESHOLD + && !self.inputs.store.is_deployment_synced() + && !close_to_chain_head(&block_ptr, &self.inputs.chain.chain_head_ptr().await?, 1000) + && !max_end_block_reached + { + // Skip this block and continue with the same stream + return Ok(RunnerState::AwaitingBlock { block_stream }); + } else { + self.state.skip_ptr_updates_timer = Instant::now(); + } + + let block_start = Instant::now(); + + let action = { + let stopwatch = &self.metrics.stream.stopwatch; + let _section = stopwatch.start_section(PROCESS_BLOCK_SECTION_NAME); + self.process_block(block, cursor).await + }; + + let action = self.handle_action(block_start, block_ptr, action).await?; + + self.update_deployment_synced_metric(); + + if self.is_canceled() { + return self.cancel(); + } + + self.metrics + .subgraph + .observe_block_processed(block_start.elapsed(), action.block_finished()); + + // Convert Action to RunnerState + match action { + Action::Continue => Ok(RunnerState::AwaitingBlock { block_stream }), + Action::Restart => Ok(RunnerState::Restarting { + reason: RestartReason::DynamicDataSourceCreated, + }), + Action::Stop => Ok(RunnerState::Stopped { + reason: StopReason::MaxEndBlockReached, + }), + } + } + + /// Handle a revert event and determine the next state. + /// + /// This is the state machine wrapper around `handle_revert` that handles + /// the revert action and determines state transitions. + async fn handle_revert_state( + &mut self, + block_stream: Cancelable>>, + revert_to_ptr: BlockPtr, + cursor: FirehoseCursor, + ) -> RunnerState { + let stopwatch = &self.metrics.stream.stopwatch; + let _section = stopwatch.start_section(HANDLE_REVERT_SECTION_NAME); + + let action = self.handle_revert(revert_to_ptr, cursor).await; + + match action { + Action::Continue => RunnerState::AwaitingBlock { block_stream }, + Action::Restart => RunnerState::Restarting { + reason: RestartReason::StoreError, + }, + Action::Stop => RunnerState::Stopped { + reason: StopReason::Canceled, + }, } } @@ -425,7 +692,7 @@ where entity_lfu_cache: cache, evict_stats, } = entity_cache - .as_modifications(block_ptr.number) + .as_modifications(block_ptr.number, &self.metrics.host.stopwatch) .await .classify()?; section.end(); @@ -470,6 +737,21 @@ where let is_caught_up = self.is_caught_up(&block_ptr).await.non_deterministic()?; + if !self.state.postponed_indexes_created + && close_to_chain_head( + &block_ptr, + &self.state.cached_head_ptr, + ENV_VARS.postpone_indexes_creation_threshold, + ) + { + self.state.postponed_indexes_created = true; + self.inputs + .store + .create_postponed_indexes() + .await + .non_deterministic()?; + } + self.inputs .store .transact_block_operations( @@ -569,13 +851,85 @@ where .await } - /// Processes a block and returns the updated context and a boolean flag indicating - /// whether new dynamic data sources have been added to the subgraph. - async fn process_block( + // ========================================================================= + // Pipeline Stage Methods + // ========================================================================= + // + // The following methods implement the block processing pipeline stages. + // Each stage handles a specific phase of block processing: + // + // 1. match_triggers: Match and decode triggers against hosts + // 2. execute_triggers: Execute the matched triggers + // 3. process_dynamic_data_sources: Handle dynamically created data sources + // 4. (process_offchain_triggers): Existing handle_offchain_triggers method + // 5. (persist_block_state): Existing transact_block_state method + // + // ========================================================================= + + /// Pipeline Stage 1: Match triggers to hosts and decode them. + /// + /// Takes raw triggers from a block and matches them against all registered + /// hosts, returning runnable triggers ready for execution. + async fn match_triggers<'a>( + &'a self, + logger: &Logger, + block: &Arc, + triggers: Vec>, + ) -> Result>, MappingError> { + let hosts_filter = |trigger: &TriggerData| self.ctx.instance.hosts_for_trigger(trigger); + self.match_and_decode_many(logger, block, triggers, hosts_filter) + .await + } + + /// Pipeline Stage 2: Execute matched triggers. + /// + /// Takes runnable triggers and executes them using the TriggerRunner, + /// accumulating state changes in the block state. + async fn execute_triggers( + &self, + block: &Arc, + runnables: Vec>, + block_state: BlockState, + proof_of_indexing: &SharedProofOfIndexing, + causality_region: &str, + ) -> Result { + let trigger_runner = TriggerRunner::new( + self.ctx.trigger_processor.as_ref(), + &self.logger, + &self.metrics.subgraph, + &self.inputs.debug_fork, + self.inputs.instrument, + ); + trigger_runner + .execute( + block, + runnables, + block_state, + proof_of_indexing, + causality_region, + ) + .await + } + + /// Pipeline Stage 3: Process dynamically created data sources. + /// + /// This loop processes data sources created during trigger execution: + /// 1. Instantiate the created data sources + /// 2. Reprocess triggers from this block that match the new data sources + /// 3. Repeat until no more data sources are created + /// + /// Note: This algorithm processes data sources spawned on the same block + /// _breadth first_ on the tree implied by the parent-child relationship + /// between data sources. + async fn process_dynamic_data_sources( &mut self, - block: BlockWithTriggers, - firehose_cursor: FirehoseCursor, - ) -> Result { + logger: &Logger, + block: &Arc, + firehose_cursor: &FirehoseCursor, + mut block_state: BlockState, + proof_of_indexing: &SharedProofOfIndexing, + causality_region: &str, + ) -> Result { fn log_triggers_found(logger: &Logger, triggers: &[Trigger]) { if triggers.len() == 1 { info!(logger, "1 trigger found in this block"); @@ -584,6 +938,141 @@ where } } + let _section = self + .metrics + .stream + .stopwatch + .start_section(HANDLE_CREATED_DS_SECTION_NAME); + + while block_state.has_created_data_sources() { + // Instantiate dynamic data sources, removing them from the block state. + let (data_sources, runtime_hosts) = + self.create_dynamic_data_sources(block_state.drain_created_data_sources())?; + + let filter = &Arc::new(TriggerFilterWrapper::new( + C::TriggerFilter::from_data_sources( + data_sources.iter().filter_map(DataSource::as_onchain), + ), + vec![], + )); + + // TODO: We have to pass a reference to `block` to + // `refetch_block`, otherwise the call to + // handle_offchain_triggers below gets an error that `block` + // has moved. That is extremely fishy since it means that + // `handle_offchain_triggers` uses the non-refetched block + // + // It's also not clear why refetching needs to happen inside + // the loop; will firehose really return something diffrent + // each time even though the cursor doesn't change? + let block = self.refetch_block(logger, block, firehose_cursor).await?; + + // Reprocess the triggers from this block that match the new data sources + let block_with_triggers = self + .inputs + .triggers_adapter + .triggers_in_block(logger, block.as_ref().clone(), filter) + .await + .non_deterministic()?; + + let triggers = block_with_triggers.trigger_data; + log_triggers_found::(logger, &triggers); + + // Add entity operations for the new data sources to the block state + // and add runtimes for the data sources to the subgraph instance. + self.persist_dynamic_data_sources(&mut block_state, data_sources); + + // Process the triggers in each host in the same order the + // corresponding data sources have been created. + let hosts_filter = |_: &'_ TriggerData| -> Box + Send> { + Box::new(runtime_hosts.iter().map(Arc::as_ref)) + }; + let runnables = self + .match_and_decode_many(logger, &block, triggers, hosts_filter) + .await; + + let trigger_runner = TriggerRunner::new( + self.ctx.trigger_processor.as_ref(), + &self.logger, + &self.metrics.subgraph, + &self.inputs.debug_fork, + self.inputs.instrument, + ); + let res = match runnables { + Ok(runnables) => { + trigger_runner + .execute( + &block, + runnables, + block_state, + proof_of_indexing, + causality_region, + ) + .await + } + Err(e) => Err(e), + }; + + block_state = res.map_err(|e| { + // This treats a `PossibleReorg` as an ordinary error which will fail the subgraph. + // This can cause an unnecessary subgraph failure, to fix it we need to figure out a + // way to revert the effect of `create_dynamic_data_sources` so we may return a + // clean context as in b21fa73b-6453-4340-99fb-1a78ec62efb1. + match e { + MappingError::PossibleReorg(e) | MappingError::Unknown(e) => { + ProcessingError::Unknown(e) + } + } + })?; + } + + Ok(block_state) + } + + /// Pipeline Stage 4: Process offchain triggers. + /// + /// Retrieves ready offchain events and processes them, returning entity + /// modifications and processed data sources to be included in the transaction. + async fn process_offchain_triggers( + &mut self, + block: &Arc, + block_state: &mut BlockState, + ) -> Result<(Vec, Vec), ProcessingError> { + let offchain_events = self + .ctx + .offchain_monitor + .ready_offchain_events() + .non_deterministic()?; + + let vid_gen = block_state.seq_gen(); + let (offchain_mods, processed_offchain_data_sources, persisted_off_chain_data_sources) = + self.handle_offchain_triggers(offchain_events, block, vid_gen) + .await + .non_deterministic()?; + + block_state + .persisted_data_sources + .extend(persisted_off_chain_data_sources); + + Ok((offchain_mods, processed_offchain_data_sources)) + } + + /// Processes a block and returns the updated context and a boolean flag indicating + /// whether new dynamic data sources have been added to the subgraph. + /// + /// ## Pipeline Stages + /// + /// Block processing follows a pipeline of stages: + /// 1. **match_triggers**: Match and decode triggers against hosts + /// 2. **execute_triggers**: Execute the matched triggers via TriggerRunner + /// 3. **process_dynamic_data_sources**: Handle dynamically created data sources + /// 4. **process_offchain_triggers**: Process offchain events + /// 5. **persist_block**: Persist block state to the store + async fn process_block( + &mut self, + block: BlockWithTriggers, + firehose_cursor: FirehoseCursor, + ) -> Result { let triggers = block.trigger_data; let block = Arc::new(block.block); let block_ptr = block.ptr(); @@ -602,9 +1091,11 @@ where // Causality region for onchain triggers. let causality_region = PoICausalityRegion::from_network(&self.inputs.network); + let vid_gen = SeqGenerator::new(block_ptr.number); let mut block_state = BlockState::new( self.inputs.store.clone(), std::mem::take(&mut self.state.entity_lfu_cache), + vid_gen, ); let _section = self @@ -613,46 +1104,22 @@ where .stopwatch .start_section(PROCESS_TRIGGERS_SECTION_NAME); - // Match and decode all triggers in the block - let hosts_filter = |trigger: &TriggerData| self.ctx.instance.hosts_for_trigger(trigger); - let match_res = self - .match_and_decode_many(&logger, &block, triggers, hosts_filter) - .await; - - // Process events one after the other, passing in entity operations - // collected previously to every new event being processed - let mut res = Ok(block_state); - match match_res { + // Stage 1: Match triggers to hosts and decode + let runnables = self.match_triggers(&logger, &block, triggers).await; + + // Stage 2: Execute triggers + let res = match runnables { Ok(runnables) => { - for runnable in runnables { - let process_res = self - .ctx - .trigger_processor - .process_trigger( - &self.logger, - runnable.hosted_triggers, - &block, - res.unwrap(), - &proof_of_indexing, - &causality_region, - &self.inputs.debug_fork, - &self.metrics.subgraph, - self.inputs.instrument, - ) - .await - .map_err(|e| e.add_trigger_context(&runnable.trigger)); - match process_res { - Ok(state) => res = Ok(state), - Err(e) => { - res = Err(e); - break; - } - } - } - } - Err(e) => { - res = Err(e); + self.execute_triggers( + &block, + runnables, + block_state, + &proof_of_indexing, + &causality_region, + ) + .await } + Err(e) => Err(e), }; match res { @@ -661,21 +1128,10 @@ where // Some form of unknown or non-deterministic error ocurred. Err(MappingError::Unknown(e)) => return Err(ProcessingError::Unknown(e)), - Err(MappingError::PossibleReorg(e)) => { - info!(logger, - "Possible reorg detected, retrying"; - "error" => format!("{:#}", e), - ); - // In case of a possible reorg, we want this function to do nothing and restart the - // block stream so it has a chance to detect the reorg. - // - // The state is unchanged at this point, except for having cleared the entity cache. - // Losing the cache is a bit annoying but not an issue for correctness. - // - // See also b21fa73b-6453-4340-99fb-1a78ec62efb1. - return Ok(Action::Restart); - } + // Possible blockchain reorg detected - signal restart via ProcessingError::PossibleReorg. + // See also b21fa73b-6453-4340-99fb-1a78ec62efb1. + Err(MappingError::PossibleReorg(e)) => return Err(ProcessingError::PossibleReorg(e)), } // Check if there are any datasources that have expired in this block. ie: the end_block @@ -691,137 +1147,33 @@ where // or data sources that have reached their end block. let needs_restart = created_data_sources_needs_restart || has_expired_data_sources; - { - let _section = self - .metrics - .stream - .stopwatch - .start_section(HANDLE_CREATED_DS_SECTION_NAME); - - // This loop will: - // 1. Instantiate created data sources. - // 2. Process those data sources for the current block. - // Until no data sources are created or MAX_DATA_SOURCES is hit. - - // Note that this algorithm processes data sources spawned on the same block _breadth - // first_ on the tree implied by the parent-child relationship between data sources. Only a - // very contrived subgraph would be able to observe this. - while block_state.has_created_data_sources() { - // Instantiate dynamic data sources, removing them from the block state. - let (data_sources, runtime_hosts) = - self.create_dynamic_data_sources(block_state.drain_created_data_sources())?; - - let filter = &Arc::new(TriggerFilterWrapper::new( - C::TriggerFilter::from_data_sources( - data_sources.iter().filter_map(DataSource::as_onchain), - ), - vec![], - )); - - // TODO: We have to pass a reference to `block` to - // `refetch_block`, otherwise the call to - // handle_offchain_triggers below gets an error that `block` - // has moved. That is extremely fishy since it means that - // `handle_offchain_triggers` uses the non-refetched block - // - // It's also not clear why refetching needs to happen inside - // the loop; will firehose really return something diffrent - // each time even though the cursor doesn't change? - let block = self - .refetch_block(&logger, &block, &firehose_cursor) - .await?; - - // Reprocess the triggers from this block that match the new data sources - let block_with_triggers = self - .inputs - .triggers_adapter - .triggers_in_block(&logger, block.as_ref().clone(), filter) - .await - .non_deterministic()?; - - let triggers = block_with_triggers.trigger_data; - log_triggers_found(&logger, &triggers); - - // Add entity operations for the new data sources to the block state - // and add runtimes for the data sources to the subgraph instance. - self.persist_dynamic_data_sources(&mut block_state, data_sources); - - // Process the triggers in each host in the same order the - // corresponding data sources have been created. - let hosts_filter = |_: &'_ TriggerData| -> Box + Send> { - Box::new(runtime_hosts.iter().map(Arc::as_ref)) - }; - let match_res: Result, _> = self - .match_and_decode_many(&logger, &block, triggers, hosts_filter) - .await; - - let mut res = Ok(block_state); - match match_res { - Ok(runnables) => { - for runnable in runnables { - let process_res = self - .ctx - .trigger_processor - .process_trigger( - &self.logger, - runnable.hosted_triggers, - &block, - res.unwrap(), - &proof_of_indexing, - &causality_region, - &self.inputs.debug_fork, - &self.metrics.subgraph, - self.inputs.instrument, - ) - .await - .map_err(|e| e.add_trigger_context(&runnable.trigger)); - match process_res { - Ok(state) => res = Ok(state), - Err(e) => { - res = Err(e); - break; - } - } - } - } - Err(e) => { - res = Err(e); - } - } - - block_state = res.map_err(|e| { - // This treats a `PossibleReorg` as an ordinary error which will fail the subgraph. - // This can cause an unnecessary subgraph failure, to fix it we need to figure out a - // way to revert the effect of `create_dynamic_data_sources` so we may return a - // clean context as in b21fa73b-6453-4340-99fb-1a78ec62efb1. - match e { - MappingError::PossibleReorg(e) | MappingError::Unknown(e) => { - ProcessingError::Unknown(e) - } - } - })?; - } - } + // Checkpoint before dynamic DS processing for potential rollback scenarios. + // This captures the current state so it can be restored if dynamic data source + // processing fails in a way that requires partial rollback. + let _checkpoint = block_state.checkpoint(); + + // Stage 3: Process dynamic data sources + block_state = self + .process_dynamic_data_sources( + &logger, + &block, + &firehose_cursor, + block_state, + &proof_of_indexing, + &causality_region, + ) + .await?; - // Check for offchain events and process them, including their entity modifications in the - // set to be transacted. - let offchain_events = self - .ctx - .offchain_monitor - .ready_offchain_events() - .non_deterministic()?; - let (offchain_mods, processed_offchain_data_sources, persisted_off_chain_data_sources) = - self.handle_offchain_triggers(offchain_events, &block) - .await - .non_deterministic()?; - block_state - .persisted_data_sources - .extend(persisted_off_chain_data_sources); + // Stage 4: Process offchain triggers + let (offchain_mods, processed_offchain_data_sources) = self + .process_offchain_triggers(&block, &mut block_state) + .await?; + // Stage 5: Persist block state self.transact_block_state( &logger, - block_ptr.clone(), - firehose_cursor.clone(), + block_ptr, + firehose_cursor, block.timestamp(), block_state, proof_of_indexing, @@ -967,114 +1319,126 @@ where .unfail_non_deterministic_error(&block_ptr) .await?; - // Stop trying to unfail. - self.state.should_try_unfail_non_deterministic = false; + // Stop trying unless we're still behind the error block. + if outcome != UnfailOutcome::BehindErrorBlock { + self.state.should_try_unfail_non_deterministic = false; - if let UnfailOutcome::Unfailed = outcome { - self.metrics.subgraph.deployment_status.running(); - self.state.backoff.reset(); + if outcome == UnfailOutcome::Unfailed { + self.metrics.subgraph.deployment_status.running(); + self.state.backoff.reset(); + } } } - if let Some(stop_block) = self.inputs.stop_block { - if block_ptr.number >= stop_block { - info!(self.logger, "Stop block reached for subgraph"); - return Ok(Action::Stop); - } + if let Some(stop_block) = self.inputs.stop_block + && block_ptr.number >= stop_block + { + info!(self.logger, "Stop block reached for subgraph"); + return Ok(Action::Stop); } - if let Some(max_end_block) = self.inputs.max_end_block { - if block_ptr.number >= max_end_block { - info!( - self.logger, - "Stopping subgraph as maximum endBlock reached"; - "max_end_block" => max_end_block, - "current_block" => block_ptr.number - ); - return Ok(Action::Stop); - } + if let Some(max_end_block) = self.inputs.max_end_block + && block_ptr.number >= max_end_block + { + info!( + self.logger, + "Stopping subgraph as maximum endBlock reached"; + "max_end_block" => max_end_block, + "current_block" => block_ptr.number + ); + return Ok(Action::Stop); } Ok(action) } - Err(ProcessingError::Canceled) => { - debug!(self.logger, "Subgraph block stream shut down cleanly"); - Ok(Action::Stop) - } + // Handle errors based on their kind using the unified error classification. + // + // Error handling invariants: + // - Deterministic: Stop processing, persist PoI only, fail subgraph + // - NonDeterministic: Retry with backoff, may succeed on retry + // - PossibleReorg: Restart cleanly without persisting (don't fail subgraph) + // - Canceled: Clean shutdown, no error recording + Err(e) => match e.kind() { + ProcessingErrorKind::Canceled => { + debug!(self.logger, "Subgraph block stream shut down cleanly"); + Ok(Action::Stop) + } - // Handle unexpected stream errors by marking the subgraph as failed. - Err(e) => { - self.metrics.subgraph.deployment_status.failed(); - let last_good_block = self - .inputs - .store - .block_ptr() - .map(|ptr| ptr.number) - .unwrap_or(0); - self.revert_state_to(last_good_block)?; - - let message = format!("{:#}", e).replace('\n', "\t"); - let err = anyhow!("{}, code: {}", message, LogCode::SubgraphSyncingFailure); - let deterministic = e.is_deterministic(); - - let error = SubgraphError { - subgraph_id: self.inputs.deployment.hash.clone(), - message, - block_ptr: Some(block_ptr), - handler: None, - deterministic, - }; + ProcessingErrorKind::PossibleReorg => { + // Possible reorg detected - restart the block stream cleanly. + // Don't persist anything and don't mark subgraph as failed. + // The block stream restart will allow detection of the actual reorg. + info!(self.logger, + "Possible reorg detected, restarting block stream"; + "error" => format!("{:#}", e), + ); - match deterministic { - true => { - // Fail subgraph: - // - Change status/health. - // - Save the error to the database. - self.inputs - .store - .fail_subgraph(error) - .await - .context("Failed to set subgraph status to `failed`")?; + // Revert in-memory state to last good block but don't touch the store + let last_good_block = self + .inputs + .store + .block_ptr() + .map(|ptr| ptr.number) + .unwrap_or(0); + self.revert_state_to(last_good_block); - Err(err) - } - false => { - // Shouldn't fail subgraph if it's already failed for non-deterministic - // reasons. - // - // If we don't do this check we would keep adding the same error to the - // database. - let should_fail_subgraph = - self.inputs.store.health().await? != SubgraphHealth::Failed; - - if should_fail_subgraph { - // Fail subgraph: - // - Change status/health. - // - Save the error to the database. - self.inputs - .store - .fail_subgraph(error) - .await - .context("Failed to set subgraph status to `failed`")?; - } + Ok(Action::Restart) + } - // Retry logic below: + ProcessingErrorKind::Deterministic => { + // Deterministic error - fail the subgraph permanently. + self.metrics.subgraph.deployment_status.failed(); + let last_good_block = self + .inputs + .store + .block_ptr() + .map(|ptr| ptr.number) + .unwrap_or(0); + self.revert_state_to(last_good_block); - let message = format!("{:#}", e).replace('\n', "\t"); - error!(self.logger, "Subgraph failed with non-deterministic error: {}", message; - "attempt" => self.state.backoff.attempt, - "retry_delay_s" => self.state.backoff.delay().as_secs()); + let message = format!("{:#}", e).replace('\n', "\t"); + let err = anyhow!("{}, code: {}", message, LogCode::SubgraphSyncingFailure); - // Sleep before restarting. - self.state.backoff.sleep_async().await; + self.fail_subgraph(message, Some(block_ptr), true).await?; - self.state.should_try_unfail_non_deterministic = true; + Err(err) + } - // And restart the subgraph. - Ok(Action::Restart) + ProcessingErrorKind::NonDeterministic => { + // Non-deterministic error - retry with backoff. + self.metrics.subgraph.deployment_status.failed(); + let last_good_block = self + .inputs + .store + .block_ptr() + .map(|ptr| ptr.number) + .unwrap_or(0); + self.revert_state_to(last_good_block); + + let message = format!("{:#}", e).replace('\n', "\t"); + + error!(self.logger, "Subgraph failed with non-deterministic error: {}", message; + "attempt" => self.state.backoff.attempt, + "retry_delay_s" => self.state.backoff.delay().as_secs()); + + // Shouldn't fail subgraph if it's already failed for non-deterministic + // reasons. + // + // If we don't do this check we would keep adding the same error to the + // database. + if !self.inputs.store.health().await?.is_failed() { + self.fail_subgraph(message, Some(block_ptr), false).await?; } + + // Sleep before restarting. + self.state.backoff.sleep_async().await; + + self.state.should_try_unfail_non_deterministic = true; + + // And restart the subgraph. + Ok(Action::Restart) } - } + }, } } @@ -1132,35 +1496,11 @@ where C: Blockchain, T: RuntimeHostBuilder, { - async fn handle_stream_event( - &mut self, - event: Option, CancelableError>>, - ) -> Result { - let stopwatch = &self.metrics.stream.stopwatch; - let action = match event { - Some(Ok(BlockStreamEvent::ProcessBlock(block, cursor))) => { - let _section = stopwatch.start_section(PROCESS_BLOCK_SECTION_NAME); - self.handle_process_block(block, cursor).await? - } - Some(Ok(BlockStreamEvent::Revert(revert_to_ptr, cursor))) => { - let _section = stopwatch.start_section(HANDLE_REVERT_SECTION_NAME); - self.handle_revert(revert_to_ptr, cursor).await? - } - // Log and drop the errors from the block_stream - // The block stream will continue attempting to produce blocks - Some(Err(e)) => self.handle_err(e).await?, - // If the block stream ends, that means that there is no more indexing to do. - // Typically block streams produce indefinitely, but tests are an example of finite block streams. - None => Action::Stop, - }; - - Ok(action) - } - async fn handle_offchain_triggers( &mut self, triggers: Vec, block: &Arc, + vid_gen: SeqGenerator, ) -> Result< ( Vec, @@ -1177,7 +1517,11 @@ where // Using an `EmptyStore` and clearing the cache for each trigger is a makeshift way to // get causality region isolation. let schema = ReadStore::input_schema(&self.inputs.store); - let mut block_state = BlockState::new(EmptyStore::new(schema), LfuCache::new()); + let mut block_state = BlockState::new( + EmptyStore::new(schema), + LfuCache::new(), + vid_gen.cheap_clone(), + ); // PoI ignores offchain events. // See also: poi-ignores-offchain @@ -1247,7 +1591,7 @@ where mods.extend( block_state .entity_cache - .as_modifications(block.number()) + .as_modifications(block.number(), &self.metrics.subgraph.stopwatch) .await? .modifications, ); @@ -1288,52 +1632,7 @@ where C: Blockchain, T: RuntimeHostBuilder, { - async fn handle_process_block( - &mut self, - block: BlockWithTriggers, - cursor: FirehoseCursor, - ) -> Result { - let block_ptr = block.ptr(); - self.metrics - .stream - .deployment_head - .set(block_ptr.number as f64); - - if block.trigger_count() > 0 { - self.metrics - .subgraph - .block_trigger_count - .observe(block.trigger_count() as f64); - } - - if block.trigger_count() == 0 - && self.state.skip_ptr_updates_timer.elapsed() <= SKIP_PTR_UPDATES_THRESHOLD - && !self.inputs.store.is_deployment_synced() - && !close_to_chain_head( - &block_ptr, - &self.inputs.chain.chain_head_ptr().await?, - // The "skip ptr updates timer" is ignored when a subgraph is at most 1000 blocks - // behind the chain head. - 1000, - ) - { - return Ok(Action::Continue); - } else { - self.state.skip_ptr_updates_timer = Instant::now(); - } - - let start = Instant::now(); - - let res = self.process_block(block, cursor).await; - - self.handle_action(start, block_ptr, res).await - } - - async fn handle_revert( - &mut self, - revert_to_ptr: BlockPtr, - cursor: FirehoseCursor, - ) -> Result { + async fn handle_revert(&mut self, revert_to_ptr: BlockPtr, cursor: FirehoseCursor) -> Action { // Current deployment head in the database / WritableAgent Mutex cache. // // Safe unwrap because in a Revert event we're sure the subgraph has @@ -1341,7 +1640,7 @@ where let subgraph_ptr = self.inputs.store.block_ptr().unwrap(); if revert_to_ptr.number >= subgraph_ptr.number { info!(&self.logger, "Block to revert is higher than subgraph pointer, nothing to do"; "subgraph_ptr" => &subgraph_ptr, "revert_to_ptr" => &revert_to_ptr); - return Ok(Action::Continue); + return Action::Continue; } info!(&self.logger, "Reverting block to get back to main chain"; "subgraph_ptr" => &subgraph_ptr, "revert_to_ptr" => &revert_to_ptr); @@ -1355,7 +1654,7 @@ where error!(&self.logger, "Could not revert block. Retrying"; "error" => %e); // Exit inner block stream consumption loop and go up to loop that restarts subgraph - return Ok(Action::Restart); + return Action::Restart; } self.metrics @@ -1367,62 +1666,15 @@ where .deployment_head .set(subgraph_ptr.number as f64); - self.revert_state_to(revert_to_ptr.number)?; + self.revert_state_to(revert_to_ptr.number); let needs_restart: bool = self.needs_restart(revert_to_ptr, subgraph_ptr); - let action = if needs_restart { + if needs_restart { Action::Restart } else { Action::Continue - }; - - Ok(action) - } - - async fn handle_err( - &mut self, - err: CancelableError, - ) -> Result { - if self.is_canceled() { - debug!(&self.logger, "Subgraph block stream shut down cleanly"); - return Ok(Action::Stop); } - - let err = match err { - CancelableError::Error(BlockStreamError::Fatal(msg)) => { - error!( - &self.logger, - "The block stream encountered a substreams fatal error and will not retry: {}", - msg - ); - - // If substreams returns a deterministic error we may not necessarily have a specific block - // but we should not retry since it will keep failing. - self.inputs - .store - .fail_subgraph(SubgraphError { - subgraph_id: self.inputs.deployment.hash.clone(), - message: msg, - block_ptr: None, - handler: None, - deterministic: true, - }) - .await - .context("Failed to set subgraph status to `failed`")?; - - return Ok(Action::Stop); - } - e => e, - }; - - debug!( - &self.logger, - "Block stream produced a non-fatal error"; - "error" => format!("{}", err), - ); - - Ok(Action::Continue) } /// Determines if the subgraph needs to be restarted. @@ -1457,7 +1709,6 @@ async fn update_proof_of_indexing( key: EntityKey, digest: Bytes, block_time: BlockTime, - block: BlockNumber, ) -> Result<(), Error> { let digest_name = entity_cache.schema.poi_digest(); let mut data = vec![ @@ -1472,12 +1723,11 @@ async fn update_proof_of_indexing( data.push((entity_cache.schema.poi_block_time(), block_time)); } let poi = entity_cache.make_entity(data)?; - entity_cache.set(key, poi, block, None).await + entity_cache.set(key, poi, None).await } let _section_guard = stopwatch.start_section("update_proof_of_indexing"); - let block_number = proof_of_indexing.get_block(); let mut proof_of_indexing = proof_of_indexing.take(); for (causality_region, stream) in proof_of_indexing.drain() { @@ -1514,7 +1764,6 @@ async fn update_proof_of_indexing( entity_key, updated_proof_of_indexing, block_time, - block_number, ) .await?; } diff --git a/core/src/subgraph/runner/state.rs b/core/src/subgraph/runner/state.rs new file mode 100644 index 00000000000..d6401842522 --- /dev/null +++ b/core/src/subgraph/runner/state.rs @@ -0,0 +1,96 @@ +//! State machine types for SubgraphRunner. +//! +//! This module defines the explicit state machine that controls the runner's lifecycle, +//! replacing the previous nested loop structure with clear state transitions. + +use graph::blockchain::Blockchain; +use graph::blockchain::block_stream::{BlockStream, BlockWithTriggers, FirehoseCursor}; +use graph::ext::futures::Cancelable; +use graph::prelude::BlockPtr; + +/// The current state of the SubgraphRunner's lifecycle. +/// +/// The runner transitions through these states as it processes blocks, +/// handles reverts, and responds to errors or cancellation signals. +/// +/// ## State Transitions +/// +/// ```text +/// Initializing ───────────────────────────────────┐ +/// │ │ +/// v │ +/// AwaitingBlock ◄────────────────────────────────┤ +/// │ │ +/// ├── ProcessBlock event ──► ProcessingBlock │ +/// │ │ │ +/// │ ├── success ┼──► AwaitingBlock +/// │ │ │ +/// │ └── restart ┼──► Restarting +/// │ │ +/// ├── Revert event ──────────► Reverting ────┤ +/// │ │ +/// ├── Error ─────────────────► Restarting ───┤ +/// │ │ +/// └── Cancel/MaxBlock ───────► Stopped │ +/// │ +/// Restarting ─────────────────────────────────────┘ +/// ``` +#[derive(Default)] +pub enum RunnerState { + /// Initial state, ready to start block stream. + #[default] + Initializing, + + /// Block stream active, waiting for next event. + AwaitingBlock { + block_stream: Cancelable>>, + }, + + /// Processing a block through the pipeline. + /// The block stream is kept alive to continue processing after this block. + ProcessingBlock { + block_stream: Cancelable>>, + block: BlockWithTriggers, + cursor: FirehoseCursor, + }, + + /// Handling a revert event. + /// The block stream is kept alive to continue processing after the revert. + Reverting { + block_stream: Cancelable>>, + to_ptr: BlockPtr, + cursor: FirehoseCursor, + }, + + /// Restarting block stream (new filters, store restart, etc.). + Restarting { reason: RestartReason }, + + /// Terminal state. + Stopped { reason: StopReason }, +} + +/// Reasons for restarting the block stream. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RestartReason { + /// New dynamic data source was created that requires filter updates. + DynamicDataSourceCreated, + /// Store error occurred and store needs to be restarted. + StoreError, + /// Possible reorg detected, need to restart to detect it. + /// NOTE: Currently unused but reserved for future error handling consolidation (Phase 5). + #[allow(dead_code)] + PossibleReorg, +} + +/// Reasons for stopping the runner. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StopReason { + /// The maximum end block was reached. + MaxEndBlockReached, + /// The runner was canceled (unassigned or shutdown). + Canceled, + /// The subgraph was unassigned while this runner was active. + Unassigned, + /// The block stream ended (typically in tests). + StreamEnded, +} diff --git a/core/src/subgraph/runner/trigger_runner.rs b/core/src/subgraph/runner/trigger_runner.rs new file mode 100644 index 00000000000..6c89052e5dc --- /dev/null +++ b/core/src/subgraph/runner/trigger_runner.rs @@ -0,0 +1,79 @@ +use std::sync::Arc; + +use graph::blockchain::Blockchain; +use graph::components::store::SubgraphFork; +use graph::components::subgraph::{MappingError, SharedProofOfIndexing}; +use graph::components::trigger_processor::RunnableTriggers; +use graph::prelude::{BlockState, RuntimeHostBuilder, SubgraphInstanceMetrics, TriggerProcessor}; +use graph::slog::Logger; + +/// Handles the execution of triggers against runtime hosts, accumulating state. +/// +/// This component unifies the trigger processing loop that was previously duplicated +/// for initial triggers and dynamically created data source triggers. +pub struct TriggerRunner<'a, C: Blockchain, T: RuntimeHostBuilder> { + processor: &'a dyn TriggerProcessor, + logger: &'a Logger, + metrics: &'a Arc, + debug_fork: &'a Option>, + instrument: bool, +} + +impl<'a, C, T> TriggerRunner<'a, C, T> +where + C: Blockchain, + T: RuntimeHostBuilder, +{ + /// Create a new TriggerRunner with the given dependencies. + pub fn new( + processor: &'a dyn TriggerProcessor, + logger: &'a Logger, + metrics: &'a Arc, + debug_fork: &'a Option>, + instrument: bool, + ) -> Self { + Self { + processor, + logger, + metrics, + debug_fork, + instrument, + } + } + + /// Execute a sequence of runnable triggers, accumulating state changes. + /// + /// Processes each trigger in order. If any trigger fails with a non-deterministic + /// error, processing stops and the error is returned. Deterministic errors are + /// accumulated in the block state. + pub async fn execute( + &self, + block: &Arc, + runnables: Vec>, + block_state: BlockState, + proof_of_indexing: &SharedProofOfIndexing, + causality_region: &str, + ) -> Result { + let mut state = block_state; + + for runnable in runnables { + state = self + .processor + .process_trigger( + self.logger, + runnable.hosted_triggers, + block, + state, + proof_of_indexing, + causality_region, + self.debug_fork, + self.metrics, + self.instrument, + ) + .await + .map_err(|e| e.add_trigger_context(&runnable.trigger))?; + } + + Ok(state) + } +} diff --git a/core/src/subgraph/state.rs b/core/src/subgraph/state.rs index 0ce6ab48b15..c1da4b43199 100644 --- a/core/src/subgraph/state.rs +++ b/core/src/subgraph/state.rs @@ -16,4 +16,7 @@ pub struct IndexingState { pub skip_ptr_updates_timer: Instant, pub entity_lfu_cache: EntityLfuCache, pub cached_head_ptr: Option, + /// Set to `true` once postponed indexes have been created. This + /// ensures we only trigger index creation once per subgraph run. + pub postponed_indexes_created: bool, } diff --git a/core/src/subgraph_manifest.rs b/core/src/subgraph_manifest.rs index c02497c10c5..ddc01e0bc69 100644 --- a/core/src/subgraph_manifest.rs +++ b/core/src/subgraph_manifest.rs @@ -6,7 +6,7 @@ use graph::{ }, data::subgraph::DeploymentHash, }; -use slog::{debug, Logger}; +use slog::{Logger, debug}; pub(super) async fn load_raw_subgraph_manifest( logger: &Logger, diff --git a/docker/Dockerfile b/docker/Dockerfile index 7ecbe905d54..29d8e142db0 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,20 +22,18 @@ ARG TAG_NAME=unknown ADD . /graph-node -RUN apt-get update \ - && apt-get install -y cmake protobuf-compiler && \ - cd /graph-node && \ - RUSTFLAGS="-g" cargo build --release --package graph-node \ +RUN apt-get update && apt-get install -y cmake protobuf-compiler + +RUN cd /graph-node \ + && RUSTFLAGS="-g -C force-frame-pointers=yes" cargo build --release --package graph-node \ && cp target/release/graph-node /usr/local/bin/graph-node \ && cp target/release/graphman /usr/local/bin/graphman \ - # Reduce the size of the layer by removing unnecessary files. && cargo clean \ - && objcopy --only-keep-debug /usr/local/bin/graph-node /usr/local/bin/graph-node.debug \ - && strip -g /usr/local/bin/graph-node \ - && strip -g /usr/local/bin/graphman \ - && cd /usr/local/bin \ - && objcopy --add-gnu-debuglink=graph-node.debug graph-node \ - && echo "REPO_NAME='$REPO_NAME'" > /etc/image-info \ + && cp /usr/local/bin/graph-node /usr/local/bin/graph-node-debug \ + && cp /usr/local/bin/graphman /usr/local/bin/graphman-debug \ + && strip -g /usr/local/bin/graph-node /usr/local/bin/graphman + +RUN echo "REPO_NAME='$REPO_NAME'" > /etc/image-info \ && echo "TAG_NAME='$TAG_NAME'" >> /etc/image-info \ && echo "BRANCH_NAME='$BRANCH_NAME'" >> /etc/image-info \ && echo "COMMIT_SHA='$COMMIT_SHA'" >> /etc/image-info \ @@ -43,14 +41,6 @@ RUN apt-get update \ && echo "RUST_VERSION='$(rustc --version)'" >> /etc/image-info \ && echo "CARGO_DEV_BUILD='$CARGO_DEV_BUILD'" >> /etc/image-info -# Debug image to access core dumps -FROM graph-node-build AS graph-node-debug -RUN apt-get update \ - && apt-get install -y curl gdb postgresql-client - -COPY docker/Dockerfile /Dockerfile -COPY docker/bin/* /usr/local/bin/ - # The graph-node runtime image with only the executable FROM debian:bookworm-20241111-slim AS graph-node ENV RUST_LOG="" @@ -97,8 +87,7 @@ EXPOSE 8020 EXPOSE 8030 RUN apt-get update \ - && apt-get install -y libpq-dev ca-certificates \ - netcat-openbsd + && apt-get install -y libpq-dev ca-certificates netcat-openbsd ADD docker/wait_for docker/start /usr/local/bin/ COPY --from=graph-node-build /usr/local/bin/graph-node /usr/local/bin/graphman /usr/local/bin/ @@ -107,3 +96,12 @@ COPY --from=envsubst /go/bin/envsubst /usr/local/bin/ COPY docker/Dockerfile /Dockerfile CMD ["start"] +# Debug and profiling image +# Build with: docker build --target graph-node-debug -f docker/Dockerfile . +FROM graph-node AS graph-node-debug +COPY --from=graph-node-build /usr/local/bin/graph-node-debug /usr/local/bin/graph-node +COPY --from=graph-node-build /usr/local/bin/graphman-debug /usr/local/bin/graphman +RUN apt-get update \ + && apt-get install -y curl gdb postgresql-client linux-perf procps binutils +COPY docker/Dockerfile /Dockerfile +COPY docker/bin/* /usr/local/bin/ diff --git a/docs/aggregations.md b/docs/aggregations.md index fafbd4d3305..3e90d7ee62f 100644 --- a/docs/aggregations.md +++ b/docs/aggregations.md @@ -178,8 +178,8 @@ accepts the following arguments: dimension - A mandatory `interval` - An optional `current` to indicate whether to include the current, - partially filled bucket in the response. Can be either `ignore` (the - default) or `include` (still **TODO** and not implemented) + partially filled bucket in the response. Can be either `exclude` (the + default) or `include` - Optional `timestamp_{gte|gt|lt|lte|eq|in}` filters to restrict the range of timestamps to return. The timestamp to filter by must be a string containing microseconds since the epoch. The value `"1704164640000000"` @@ -189,7 +189,7 @@ accepts the following arguments: ```graphql token_stats(interval: "hour", - current: ignore, + current: exclude, where: { token: "0x1234", timestamp_gte: 1234567890, @@ -201,3 +201,53 @@ token_stats(interval: "hour", avgVolume } ``` + +### Current Bucket + +By default, aggregation queries return only completed, rolled-up buckets +(`current: exclude`). These are buckets whose time interval has ended and +whose data has been fully aggregated by `graph-node`'s rollup process. + +Setting `current: include` adds an additional, partially filled bucket that +is computed on-the-fly from the unrolled timeseries data that has been +inserted since the last rollup. This current bucket aggregates raw data +points from the source timeseries table that have not yet been rolled up +into the aggregation table. It covers the time period from the end of the +last completed bucket up to the most recent data point. + +- `current: exclude` (default) — return only completed, rolled-up buckets +- `current: include` — also return the in-progress bucket computed from + unrolled source data + +The current bucket is useful when you need near-real-time aggregation data +without waiting for the next rollup cycle to complete. + +#### Nested Aggregation Queries + +The `current` argument also works on nested aggregation fields accessed +through a parent entity. For example, if a `Token` entity has a derived +aggregation field `tokenStats`, you can query the current bucket for each +token: + +```graphql +{ + tokens { + id + name + tokenStats(interval: "hour", current: include) { + timestamp + totalVolume + } + } +} +``` + +This returns both the completed rolled-up hourly buckets and the current +in-progress bucket for each token's stats. + +#### Limitations + +Current bucket support for nested aggregation fields is only available when +the field references a single aggregation type. It is not supported when the +aggregation field is accessed through an interface with multiple +implementations. diff --git a/docs/amp-powered-subgraphs.md b/docs/amp-powered-subgraphs.md index abec0123e88..60bccdde449 100644 --- a/docs/amp-powered-subgraphs.md +++ b/docs/amp-powered-subgraphs.md @@ -1,7 +1,10 @@ # Amp-powered subgraphs +> [!WARNING] +> This feature is experimental and may change in future releases. + > [!NOTE] -> This features is available starting from spec version `1.5.0` +> This feature is available starting from spec version `1.5.0` Amp-powered subgraphs are a new kind of subgraphs with SQL data sources that query and index data from the Amp servers. They are significantly more efficient than the standard subgraphs, and the indexing time can be reduced from days and weeks, @@ -41,6 +44,7 @@ The minimum spec version for Amp-powered subgraphs is `1.5.0`. - name: Transfer file: ``` + ### Data source structure @@ -70,6 +74,7 @@ This is used to assign the subgraph to the appropriate indexing process. - name: Transfer file: ``` + ### `name` @@ -97,6 +102,7 @@ This name is used for observability purposes and to identify progress and potent - name: Transfer file: ``` + ### `network` @@ -127,6 +133,7 @@ This is used to validate that the SQL queries for this data source produce resul - name: Transfer file: ``` + ### `source` @@ -158,6 +165,7 @@ This is used to validate that the SQL queries for this data source only query th - name: Transfer file: ``` + ### `source.tables` @@ -185,6 +193,7 @@ This is used to validate that the SQL queries for this data source only query th - name: Transfer file: ``` + ### Optional `source.address` @@ -193,7 +202,7 @@ Contains the contract address with which SQL queries in the data source interact Enables SQL query reuse through `sg_source_address()` calls instead of hard-coding the contract address. SQL queries resolve `sg_source_address()` calls to this contract address. - +
Example YAML: @@ -215,6 +224,7 @@ SQL queries resolve `sg_source_address()` calls to this contract address. - name: Transfer file: ``` +
### Optional `source.startBlock` @@ -245,6 +255,7 @@ _When not provided, defaults to block number `0`._ - name: Transfer file: ``` + ### Optional `source.endBlock` @@ -275,6 +286,7 @@ _When not provided, defaults to the maximum possible block number._ - name: Transfer file: ``` + ### `transformer` @@ -309,6 +321,7 @@ Represents the version of this transformer. Each version may contain a different - name: Transfers file: ``` + ### Optional `transformer.abis` @@ -344,6 +357,7 @@ _When not provided, defaults to an empty list._ - name: Transfer file: ``` + ### `transformer.tables` @@ -368,6 +382,7 @@ type Block @entity(immutable: true) { ``` **YAML manifest:** + ```diff specVersion: 1.5.0 + dataSources: @@ -386,6 +401,7 @@ type Block @entity(immutable: true) { + - name: Block file: ``` + ### `transformer.tables[i].query` @@ -419,6 +435,7 @@ _When not provided, the `file` field is used instead._ - name: Block + query: SELECT * FROM "edgeandnode/ethereum_mainnet".blocks; ``` + ### `transformer.tables[i].file` @@ -451,6 +468,7 @@ _When not provided, the `query` field is used instead._ - name: Block + file: ``` + ### Amp-powered subgraph examples @@ -460,6 +478,11 @@ https://github.com/edgeandnode/amp-subgraph-examples ## SQL query requirements +### Names + +The names of tables, columns, and aliases must not start with `amp_` as this +prefix is reserved for internal use. + ### Block numbers Every SQL query in Amp-powered subgraphs must return the block number for every row. @@ -543,8 +566,8 @@ Amp-powered subgraphs feature introduces the following new ENV variables: - `GRAPH_AMP_FLIGHT_SERVICE_ADDRESS` – The address of the Amp Flight gRPC service. _Defaults to `None`, which disables support for Amp-powered subgraphs._ - `GRAPH_AMP_FLIGHT_SERVICE_TOKEN` – Token used to authenticate Amp Flight gRPC service requests. _Defaults to `None`, which disables authentication._ -- `GRAPH_AMP_MAX_BUFFER_SIZE` – Maximum number of response batches to buffer in memory per stream for each SQL query. _Defaults to `1,000`._ -- `GRAPH_AMP_MAX_BLOCK_RANGE` – Maximum number of blocks to request per stream for each SQL query. _Defaults to `2,000,000`._ +- `GRAPH_AMP_BUFFER_SIZE` – Maximum number of response batches to buffer in memory per stream for each SQL query. _Defaults to `1,000`._ +- `GRAPH_AMP_BLOCK_RANGE` – Maximum number of blocks to request per stream for each SQL query. _Defaults to `100,000`._ - `GRAPH_AMP_QUERY_RETRY_MIN_DELAY_SECONDS` – Minimum time to wait before retrying a failed SQL query to the Amp server. _Defaults to `1` second._ - `GRAPH_AMP_QUERY_RETRY_MAX_DELAY_SECONDS` – Maximum time to wait before retrying a failed SQL query to the Amp server. _Defaults to `600` seconds._ diff --git a/docs/config.md b/docs/config.md index 8641398867c..db5fc083a5f 100644 --- a/docs/config.md +++ b/docs/config.md @@ -5,12 +5,11 @@ CLI. The location of the file is passed with the `--config` command line switch. configuration file, it is not possible to use the options `--postgres-url`, `--postgres-secondary-hosts`, and `--postgres-host-weights`. -The TOML file consists of four sections: +The TOML file consists of three sections: -- `[chains]` sets the endpoints to blockchain clients. -- `[store]` describes the available databases. -- `[ingestor]` sets the name of the node responsible for block ingestion. -- `[deployment]` describes how to place newly deployed subgraphs. +- [`[chains]`](#configuring-chains) lists the available chains and how to access them. +- [`[store]`](#configuring-multiple-databases) describes the available databases. +- [`[deployment]`](#controlling-deployment) describes how to place newly deployed subgraphs. Some of these sections support environment variable expansion out of the box, most notably Postgres connection strings. The official `graph-node` Docker image @@ -104,8 +103,19 @@ names that will use this configuration file. The `[chains]` section controls the providers that `graph-node` connects to, and where blocks and other metadata for each chain are -stored. The section consists of the name of the node doing block ingestion -(currently not used), and a list of chains. +stored. The section consists of the name of the node responsible for block +ingestion and a list of chains. Block ingestion only runs on the node +whose `--node-id` matches the `ingestor` value. The +`--disable-block-ingestor` flag (or `DISABLE_BLOCK_INGESTOR` env var) +acts as a hard override that always prevents ingestion regardless of +the config. + +The section-level setting `cache_size` controls the default number of +blocks from the chain head for which block data is kept cached. Individual +chains can override this value. The default is 500. When the environment +variable `GRAPH_STORE_IGNORE_BLOCK_CACHE` is set, blocks older than +`cache_size` are treated as if they have no data. The value must be greater +than the reorg threshold. The configuration for a chain `name` is specified in the section `[chains.]`, with the following: @@ -113,9 +123,43 @@ The configuration for a chain `name` is specified in the section - `shard`: where chain data is stored - `protocol`: the protocol type being indexed, default `ethereum` (alternatively `near`, `cosmos`,`arweave`,`starknet`) -- `polling_interval`: the polling interval for the block ingestor (default 500ms) +- `amp`: the network name used by AMP for this chain; defaults to the chain name. + Set this when AMP uses a different name than graph-node (e.g., `amp = "ethereum-mainnet"` + on a chain named `mainnet`). +- `cache_size`: number of blocks from the chain head for which to keep + block data cached. Defaults to the section-level `cache_size`. - `provider`: a list of providers for that chain +Additionally, Ethereum chains support per-chain RPC tuning settings. When +omitted, each setting falls back to its corresponding environment variable +default (see [environment-variables.md](environment-variables.md) for +details): + +- `polling_interval`: block ingestor polling interval in milliseconds. + Default: `ETHEREUM_POLLING_INTERVAL` (1000ms). +- `json_rpc_timeout`: timeout for JSON-RPC requests in seconds. + Default: `GRAPH_ETHEREUM_JSON_RPC_TIMEOUT` (180s). +- `request_retries`: number of times to retry failed JSON-RPC requests. + Default: `GRAPH_ETHEREUM_REQUEST_RETRIES` (10). +- `max_block_range_size`: maximum number of blocks to scan for triggers per + request. Default: `GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE` (1000). +- `block_batch_size`: number of blocks to request in parallel. + Default: `ETHEREUM_BLOCK_BATCH_SIZE` (10). +- `block_ptr_batch_size`: number of block pointers to request in parallel. + Default: `ETHEREUM_BLOCK_PTR_BATCH_SIZE` (100). +- `max_event_only_range`: maximum range for `eth_getLogs` requests that + don't filter on contract address. + Default: `GRAPH_ETHEREUM_MAX_EVENT_ONLY_RANGE` (500). +- `target_triggers_per_block_range`: ideal number of triggers per batch. + Default: `GRAPH_ETHEREUM_TARGET_TRIGGERS_PER_BLOCK_RANGE` (100). +- `get_logs_max_contracts`: maximum contracts per `eth_getLogs` call. + Default: `GRAPH_ETH_GET_LOGS_MAX_CONTRACTS` (2000). +- `block_ingestor_max_concurrent_json_rpc_calls`: maximum concurrent + JSON-RPC calls for transaction receipts during block ingestion. + Default: `GRAPH_ETHEREUM_BLOCK_INGESTOR_MAX_CONCURRENT_JSON_RPC_CALLS_FOR_TXN_RECEIPTS` (1000). +- `genesis_block_number`: genesis block number for this chain. + Default: `GRAPH_ETHEREUM_GENESIS_BLOCK_NUMBER` (0). + A `provider` is an object with the following characteristics: - `label`: the name of the provider, which will appear in logs @@ -127,8 +171,19 @@ A `provider` is an object with the following characteristics: - `transport`: one of `rpc`, `ws`, and `ipc`. Defaults to `rpc`. - `url`: the URL for the provider - `features`: an array of features that the provider supports, either empty - or any combination of `traces` and `archive` for Web3 providers, or - `compression` and `filters` for Firehose providers + or any combination of the following for Web3 providers: + - `traces`: provider supports `debug_traceBlockByNumber` for call tracing + - `archive`: provider is an archive node with full historical state + - `no_eip1898`: provider doesn't support EIP-1898 (block parameter by hash/number object) + - `no_eip2718`: provider doesn't return the `type` field in transaction receipts + (pre-EIP-2718 chains). When set, receipts are patched to add + `"type": "0x0"` for legacy transaction compatibility. + - `compression/`: provider supports compression for RPC requests, + where `` is the compression method supported by the provider, + one of `gzip`, `brotli`, or `deflate`. The default is no compression. + + For Firehose providers: `compression` and `filters` + - `headers`: HTTP headers to be added on every request. Defaults to none. - `limit`: the maximum number of subgraphs that can use this provider. Defaults to unlimited. At least one provider should be unlimited, @@ -151,8 +206,14 @@ optimisations. ```toml [chains] ingestor = "block_ingestor_node" +cache_size = 500 [chains.mainnet] shard = "vip" +amp = "ethereum-mainnet" +# Per-chain RPC tuning (all optional — omitted fields use env var defaults) +json_rpc_timeout = 300 +request_retries = 15 +max_block_range_size = 2000 provider = [ { label = "mainnet1", url = "http://..", features = [], headers = { Authorization = "Bearer foo" } }, { label = "mainnet2", url = "http://..", features = [ "archive", "traces" ] } @@ -167,6 +228,33 @@ protocol = "near" provider = [ { label = "near", details = { type = "firehose", url = "https://..", key = "", features = ["compression", "filters"] } } ] ``` +### Block ingestor failover + +When the block ingestor's `do_poll()` call fails (after all internal per-request retries are +exhausted), `graph-node` automatically attempts to switch to a healthier provider. The logic +is: + +1. **Probe the current provider first.** `do_poll()` can fail for reasons unrelated to RPC + availability (e.g. a database error or a chain reorg). If the current provider still + responds to `eth_blockNumber`, the failure was not caused by the provider — no switch + occurs. +2. **Probe all alternatives in parallel.** If the current provider is unreachable, all other + validated providers are probed simultaneously via `eth_blockNumber` to minimise wait time + when providers are timing out. +3. **Switch to the first reachable provider.** The first provider to respond successfully + to the probe is selected as the new provider for the ingestor. + The remaining probes are cancelled at this point. +4. **If all providers are unreachable**, the ingestor stays on the current provider and + re-probes on the next `do_poll()` failure. + +There is no automatic return to the original provider. Once the ingestor switches, it stays on +the new provider until that provider fails, at which point the same probe-and-switch logic +applies. + +Only validated providers are eligible as failover candidates. A provider in a temporary failure +state (e.g. unreachable at startup, pending re-validation) is excluded until it passes +validation again. + ### Controlling the number of subgraphs using a provider **This feature is experimental and might be removed in a future release** diff --git a/docs/dump.md b/docs/dump.md new file mode 100644 index 00000000000..c8a3d32372b --- /dev/null +++ b/docs/dump.md @@ -0,0 +1,311 @@ +## Dump Format + +The `graphman dump` command exports all entity data and metadata for a +single subgraph deployment into a self-contained directory of Parquet files +and JSON metadata. The resulting dump can be used to restore the deployment +into a different `graph-node` instance via `graphman restore`. Dumps are +consistent snapshots of the deployment's state at a specific point in time. + +**WARNING**: The dump and restore commands are experimental and can not +replace proper database backups at this point. In particular, there is no +guarantee that a dump will be restorable. Having said that, we encourage +users to try out the dump and restore commands in non-production +environments and report any issues they encounter. + +**WARNING**: Dumping happens in a single transaction and can put significant +load on the database for large subgraphs. Use with caution on production +instances. + +**WARNING**: Restoring a dump will currently create all the default indexes +that a new deployment gets, and ignores the indexes that might have been +carefully curated for the original deployment, even though they are recorded +in the dump. This can lead to very long restore times for large subgraphs. +The restore process will be optimized in the future to only create indexes +that are present in the dump's metadata. + +### Usage + +#### Dumping a deployment + +```bash +graphman dump +``` + +`` identifies the subgraph deployment to dump. It can be a +subgraph name, a deployment hash (`Qm...`), or a database namespace +(`sgdNNN`). `` is the path where the dump will be written; it +will be created if it does not exist. + +Running `graphman dump` against an existing dump directory performs an +**incremental dump**: only rows added since the last dump are exported, and +new chunk files are appended rather than rewriting existing ones. + +```bash +# Full dump +graphman dump my-subgraph /backups/my-subgraph + +# Incremental update of the same dump +graphman dump my-subgraph /backups/my-subgraph +``` + +#### Restoring a deployment + +```bash +graphman restore [options] +``` + +`` is the path to a dump previously created with `graphman +dump`. + +| Option | Description | +| ----------- | --------------------------------------------------------------------------------------------------- | +| `--shard` | Target database shard. Uses deployment rules (or primary shard) when omitted. Required with `--add` | +| `--name` | Subgraph name for deployment rule matching and node assignment. Falls back to an existing name | +| `--replace` | Drop and recreate if the deployment already exists in the target shard | +| `--add` | Create a copy in a shard that doesn't already have this deployment (requires `--shard`) | +| `--force` | Replace if the deployment exists in the target shard, add if it doesn't | + +`--replace`, `--add`, and `--force` are mutually exclusive. When none is +given, restore fails if the deployment already exists in the target shard. + +```bash +# Restore into the default shard +graphman restore /backups/my-subgraph + +# Restore into a specific shard, replacing if it already exists +graphman restore /backups/my-subgraph --shard shard1 --replace + +# Force-restore (replace or add as needed) +graphman restore /backups/my-subgraph --force +``` + +### Directory layout + +A dump directory has the following structure: + +``` +/ + metadata.json -- deployment metadata + per-table state + schema.graphql -- raw GraphQL schema text + subgraph.yaml -- raw subgraph manifest YAML (optional) + / + chunk_000000.parquet -- rows ordered by vid + chunk_000001.parquet -- incremental append (future chunks) + ... + data_sources$/ + chunk_000000.parquet -- dynamic data sources +``` + +Each entity type defined in the GraphQL schema gets its own subdirectory, +named after the entity type exactly as it appears in the schema (e.g. +`Token/`, `Pool/`). The Proof of Indexing appears as a regular entity +directory name `Poi$`. The special `data_sources$` directory holds dynamic +data sources created at runtime. + +Within each directory, data is stored in numbered chunk files +(`chunk_000000.parquet`, `chunk_000001.parquet`, ...). A fresh dump +produces a single `chunk_000000.parquet` per table. Incremental dumps +append new chunks rather than rewriting existing ones. + +The GraphQL schema and subgraph manifest are stored as separate plain-text +files `schema.graphql` and `subgraph.yaml`. + +### metadata.json + +The top-level `metadata.json` contains everything needed to reconstruct the +deployment's table structure, plus diagnostic information captured at dump +time. Its structure is: + +```json +{ + "version": 1, + "deployment": "Qm...", + "network": "mainnet", + + "manifest": { + "spec_version": "1.0.0", + "description": "Optional subgraph description", + "repository": "https://github.com/...", + "features": ["..."], + "entities_with_causality_region": ["EntityType1"], + "history_blocks": 2147483647 + }, + + "earliest_block_number": 12345, + "start_block": { "number": 12345, "hash": "0xabc..." }, + "head_block": { "number": 99999, "hash": "0xdef..." }, + "entity_count": 150000, + + "graft_base": null, + "graft_block": null, + "debug_fork": null, + + "health": { + "failed": false, + "health": "healthy", + "fatal_error": null, + "non_fatal_errors": [] + }, + + "indexes": { + "token": [ + "CREATE INDEX CONCURRENTLY IF NOT EXISTS attr_0_0_id ON sgd.token USING btree (id)" + ] + }, + + "tables": { + "Token": { + "immutable": true, + "has_causality_region": false, + "chunks": [ + { + "file": "Token/chunk_000000.parquet", + "min_vid": 0, + "max_vid": 50000, + "row_count": 50000 + } + ], + "max_vid": 50000 + }, + "data_sources$": { + "immutable": false, + "has_causality_region": true, + "chunks": [ + { + "file": "data_sources$/chunk_000000.parquet", + "min_vid": 0, + "max_vid": 100, + "row_count": 100 + } + ], + "max_vid": 100 + } + } +} +``` + +**Field descriptions:** + +| Field | Description | +| ----------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | +| `version` | Format version. Must be `1`. | +| `deployment` | Deployment hash (`Qm...`). | +| `network` | The blockchain network (e.g. `mainnet`, `goerli`). | +| `manifest` | Manifest metadata extracted from `subgraphs.subgraph_manifest`. | +| `manifest.spec_version` | Subgraph API version. Required to parse `schema.graphql`. | +| `manifest.entities_with_causality_region` | Entity types that have a `causality_region` column. | +| `manifest.history_blocks` | How many blocks of entity version history are retained. | +| `earliest_block_number` | Earliest block for which data exists (accounts for pruning). | +| `start_block` | The block where indexing started. Null if not set. | +| `head_block` | The latest indexed block at dump time. | +| `entity_count` | Total entity count across all tables. | +| `graft_base` | Deployment hash of the graft base, if any. | +| `graft_block` | Block pointer of the graft point, if any. | +| `debug_fork` | Debug fork deployment hash, if any. | +| `health` | Point-in-time health snapshot. Not used during restore. | +| `indexes` | Point-in-time index definitions as SQL. Not used during restore (indexes are auto-created by `Layout::create_relational_schema()`). | +| `tables` | Per-table metadata keyed by entity type name (or `data_sources$`). | + +Each entry in `tables` contains: + +| Field | Description | +| ---------------------- | ------------------------------------------------------------------------------ | +| `immutable` | Whether the entity type is immutable (uses `block$` instead of `block_range`). | +| `has_causality_region` | Whether rows have a `causality_region` column. | +| `chunks` | Ordered list of Parquet chunk files for this table. | +| `chunks[].file` | Relative path from the dump directory. | +| `chunks[].min_vid` | Minimum `vid` value in this chunk. | +| `chunks[].max_vid` | Maximum `vid` value in this chunk. | +| `chunks[].row_count` | Number of rows in this chunk. | +| `max_vid` | Maximum `vid` across all chunks. `-1` if the table is empty. | + +### Parquet schema: entity tables + +Each entity table's Parquet file uses an Arrow schema derived from the +entity's GraphQL definition. Columns are ordered as follows: + +1. **System columns** (always present, in this order): + - `vid` (Int64, non-nullable) -- row version ID + - Block tracking (one of): + - Immutable entities: `block$` (Int32, non-nullable) + - Mutable entities: `block_range_start` (Int32, non-nullable), + `block_range_end` (Int32, nullable -- null means unbounded/current) + - `causality_region` (Int32, non-nullable) -- only if the entity has one + +2. **Data columns** in GraphQL declaration order, skipping fulltext + (`TSVector`) columns which are generated and rebuilt on restore. + +The PostgreSQL `int4range` type used for `block_range` is decomposed into +two scalar columns (`block_range_start`, `block_range_end`) in the Parquet +representation. This avoids the need for a custom range type in Arrow. + +#### Type mapping + +GraphQL/PostgreSQL column types map to Arrow data types as follows: + +| ColumnType | Arrow DataType | Notes | +| --------------- | ------------------------------ | -------------------------------------------------------------- | +| `Boolean` | `Boolean` | | +| `Int` | `Int32` | | +| `Int8` | `Int64` | | +| `Bytes` | `Binary` | Raw bytes, no hex encoding | +| `BigInt` | `Utf8` | Stored as decimal string for arbitrary precision | +| `BigDecimal` | `Utf8` | Stored as decimal string for arbitrary precision | +| `Timestamp` | `Timestamp(Microsecond, None)` | Microseconds since epoch, no timezone | +| `String` | `Utf8` | | +| `Enum(...)` | `Utf8` | Enum variant as string (cast from PG enum to text during dump) | +| `TSVector(...)` | _skipped_ | Fulltext index columns are generated; rebuilt on restore | + +**Array columns:** A GraphQL list field (e.g. `tags: [String!]!`) is +stored as `List` where `T` is the base Arrow type from the table +above. Whether a column is a list is determined by the GraphQL field type, +not by `ColumnType`. For example, `[String!]!` becomes `List` and +`[Int!]` becomes `List`. + +**Nullability** follows the GraphQL schema: non-null fields produce +non-nullable Arrow columns; optional fields produce nullable columns. List +elements within list columns are always marked nullable in the Arrow schema. + +### Parquet schema: data_sources$ + +The `data_sources$` table has a fixed schema independent of the GraphQL +definition: + +| Column | Arrow DataType | Nullable | Description | +| ------------------- | -------------- | -------- | -------------------------------------------------- | +| `vid` | `Int64` | no | Row version ID | +| `block_range_start` | `Int32` | no | Lower bound of `block_range` | +| `block_range_end` | `Int32` | yes | Upper bound (null = unbounded) | +| `causality_region` | `Int32` | no | Causality region | +| `manifest_idx` | `Int32` | no | Index into the manifest's data source list | +| `parent` | `Int32` | yes | Self-referencing parent data source | +| `id` | `Binary` | yes | Data source identifier | +| `param` | `Binary` | yes | Data source parameter | +| `context` | `Utf8` | yes | JSON context | +| `done_at` | `Int32` | yes | Block number where the data source was marked done | + +### Compression + +All Parquet files use ZSTD compression (default level). + +### Row ordering + +Within each Parquet chunk file, rows are ordered by `vid` (ascending). +This matches the primary key ordering in PostgreSQL and enables efficient +sequential reads during restore. + +### Incremental dumps + +An incremental dump reads the existing `metadata.json`, determines the +`max_vid` for each table, and queries only rows with `vid > max_vid`. New +rows are written to new chunk files (e.g. `chunk_000001.parquet`) and the +metadata is updated atomically (write to a temp file, then rename). + +### Atomicity + +The `metadata.json` file is always written atomically: the dump writes to +`metadata.json.tmp` first, then renames it to `metadata.json`. This +ensures that a reader never sees a partially-written metadata file. If the +dump process crashes mid-write, the previous `metadata.json` remains +intact. The Parquet chunk files are written before `metadata.json` is +updated, so chunk files referenced by `metadata.json` are always complete. diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 9c4fc5dc8b4..7a601e32e64 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -10,10 +10,15 @@ those. ## JSON-RPC configuration for EVM chains +> **Note**: Many of these settings can now be overridden per-chain in the TOML +> configuration file under `[chains.]`. When a per-chain value is set in +> the config file it takes precedence over the environment variable. See +> [config.md](config.md#configuring-chains) for details. + - `ETHEREUM_REORG_THRESHOLD`: Maximum expected reorg size, if a larger reorg happens, subgraphs might process inconsistent data. Defaults to 250. - `ETHEREUM_POLLING_INTERVAL`: how often to poll Ethereum for new blocks (in ms, - defaults to 500ms) + defaults to 1000ms) - `GRAPH_ETHEREUM_TARGET_TRIGGERS_PER_BLOCK_RANGE`: The ideal amount of triggers to be processed in a batch. If this is too small it may cause too many requests to the ethereum node, if it is too large it may cause unreasonably expensive @@ -216,6 +221,9 @@ those. decisions. Set to `true` to turn simulation on, defaults to `false` - `GRAPH_STORE_CONNECTION_TIMEOUT`: How long to wait to connect to a database before assuming the database is down in ms. Defaults to 5000ms. +- `GRAPH_STORE_SETUP_TIMEOUT`: Timeout for database setup operations + (migrations, schema creation) in milliseconds. Defaults to 30000ms (30s). + Setup operations can legitimately take longer than normal runtime operations. - `GRAPH_STORE_CONNECTION_UNAVAILABLE_RETRY`: When a database shard is marked unavailable due to connection timeouts, this controls how often to allow a single probe request through to check if the database has recovered. Only one @@ -307,3 +315,55 @@ those. Disabling the store call cache may significantly impact performance; the actual impact depends on the average execution time of an `eth_call` compared to the cost of a database lookup for a cached result. (default: false) + +## Log Store Configuration + +`graph-node` supports storing and querying subgraph logs through multiple backends: Elasticsearch, Loki, local files, or disabled. + +**For complete log store documentation**, including detailed configuration, querying examples, and choosing the right backend, see the **[Log Store Guide](log-store.md)**. + +### Quick Reference + +**Backend selection:** +- `GRAPH_LOG_STORE_BACKEND`: `disabled` (default), `elasticsearch`, `loki`, or `file` + +**Elasticsearch:** +- `GRAPH_LOG_STORE_ELASTICSEARCH_URL`: Elasticsearch endpoint URL (required) +- `GRAPH_LOG_STORE_ELASTICSEARCH_USER`: Username (optional) +- `GRAPH_LOG_STORE_ELASTICSEARCH_PASSWORD`: Password (optional) +- `GRAPH_LOG_STORE_ELASTICSEARCH_INDEX`: Index name (default: `subgraph`) + +**Loki:** +- `GRAPH_LOG_STORE_LOKI_URL`: Loki endpoint URL (required) +- `GRAPH_LOG_STORE_LOKI_TENANT_ID`: Tenant ID (optional) + +**File-based:** +- `GRAPH_LOG_STORE_FILE_DIR`: Log directory (required) +- `GRAPH_LOG_STORE_FILE_MAX_SIZE`: Max file size in bytes (default: 104857600 = 100MB) +- `GRAPH_LOG_STORE_FILE_RETENTION_DAYS`: Retention period (default: 30) + +**Deprecated variables** (will be removed in future versions): +- `GRAPH_ELASTICSEARCH_URL` → use `GRAPH_LOG_STORE_ELASTICSEARCH_URL` +- `GRAPH_ELASTICSEARCH_USER` → use `GRAPH_LOG_STORE_ELASTICSEARCH_USER` +- `GRAPH_ELASTICSEARCH_PASSWORD` → use `GRAPH_LOG_STORE_ELASTICSEARCH_PASSWORD` +- `GRAPH_ELASTIC_SEARCH_INDEX` → use `GRAPH_LOG_STORE_ELASTICSEARCH_INDEX` + +### Example: File-based Logs for Local Development + +```bash +mkdir -p ./graph-logs +export GRAPH_LOG_STORE_BACKEND=file +export GRAPH_LOG_STORE_FILE_DIR=./graph-logs + +graph-node \ + --postgres-url postgresql://graph:pass@localhost/graph-node \ + --ethereum-rpc mainnet:https://... \ + --ipfs 127.0.0.1:5001 +``` + +See the **[Log Store Guide](log-store.md)** for: +- Detailed configuration for all backends +- How log stores work internally +- GraphQL query examples +- Choosing the right backend for your use case +- Best practices and troubleshooting diff --git a/docs/implementation/README.md b/docs/implementation/README.md index 31d4eb694a6..d54a39babbe 100644 --- a/docs/implementation/README.md +++ b/docs/implementation/README.md @@ -10,3 +10,4 @@ the code should go into comments. * [SQL Query Generation](./sql-query-generation.md) * [Adding support for a new chain](./add-chain.md) * [Pruning](./pruning.md) +* [Dump Format](./dump.md) diff --git a/docs/log-store.md b/docs/log-store.md new file mode 100644 index 00000000000..8be1cddecd8 --- /dev/null +++ b/docs/log-store.md @@ -0,0 +1,853 @@ +# Log Store Configuration and Usage + +This guide explains how to configure subgraph indexing logs storage in graph-node. + +## Table of Contents + +- [Overview](#overview) +- [How Log Stores Work](#how-log-stores-work) +- [Log Store Types](#log-store-types) + - [File-based Logs](#file-based-logs) + - [Elasticsearch](#elasticsearch) + - [Loki](#loki) + - [Disabled](#disabled) +- [Configuration](#configuration) + - [Environment Variables](#environment-variables) + - [CLI Arguments](#cli-arguments) + - [Configuration Precedence](#configuration-precedence) +- [Querying Logs](#querying-logs) +- [Migrating from Deprecated Configuration](#migrating-from-deprecated-configuration) +- [Choosing the Right Backend](#choosing-the-right-backend) +- [Best Practices](#best-practices) +- [Troubleshooting](#troubleshooting) + +## Overview + +Graph Node supports multiple logs storage backends for subgraph indexing logs. Subgraph indexing logs include: +- **User-generated logs**: Explicit logging from subgraph mapping code (`log.info()`, `log.error()`, etc.) +- **Runtime logs**: Handler execution, event processing, data source activity +- **System logs**: Warnings, errors, and diagnostics from the indexing system + +**Available backends:** +- **File**: JSON Lines files on local filesystem (for local development) +- **Elasticsearch**: Enterprise-grade search and analytics (for production) +- **Loki**: Grafana's lightweight log aggregation system (for production) +- **Disabled**: No log storage (default) + +All backends share the same query interface through GraphQL, making it easy to switch between them. + +**Important Note:** When log storage is disabled (the default), subgraph logs still appear in stdout/stderr as they always have. The "disabled" setting simply means logs are not stored separately in a queryable format. You can still see logs in your terminal or container logs - they just won't be available via the `_logs` GraphQL query. + +## How Log Stores Work + +### Architecture + +``` +┌─────────────────┐ +│ Subgraph Code │ +│ (mappings) │ +└────────┬────────┘ + │ log.info(), log.error(), etc. + ▼ +┌─────────────────┐ +│ Graph Runtime │ +│ (WebAssembly) │ +└────────┬────────┘ + │ Log events + ▼ +┌─────────────────┐ +│ Log Drain │ ◄─── slog-based logging system +└────────┬────────┘ + │ Write + ▼ +┌─────────────────┐ +│ Log Store │ ◄─── Configurable backend +│ (ES/Loki/File) │ +└────────┬────────┘ + │ + ▼ +┌─────────────────┐ +│ GraphQL API │ ◄─── Unified query interface +│ (port 8000) │ +└─────────────────┘ +``` + +### Log Flow + +1. **Log sources** generate logs from: + - User mapping code (explicit `log.info()`, `log.error()`, etc. calls) + - Subgraph runtime (handler execution, event processing, data source triggers) + - System warnings and errors (indexing issues, constraint violations, etc.) +2. **Graph runtime** captures these logs with metadata (timestamp, level, source location) +3. **Log drain** formats logs and writes to configured backend +4. **Log store** persists logs and handles queries +5. **GraphQL API** exposes logs through the `_logs` query + +### Log Entry Structure + +Each log entry contains: +- **`id`**: Unique identifier +- **`subgraphId`**: Deployment hash (QmXxx...) +- **`timestamp`**: ISO 8601 timestamp (e.g., `2024-01-15T10:30:00.123456789Z`) +- **`level`**: CRITICAL, ERROR, WARNING, INFO, or DEBUG +- **`text`**: Log message +- **`arguments`**: Key-value pairs from structured logging +- **`meta`**: Source location (module, line, column) + +## Log Store Types + +### File-based Logs + +**Best for:** Local development, testing + +#### How It Works + +File-based logs store each subgraph's logs in a separate JSON Lines (`.jsonl`) file: + +``` +graph-logs/ +├── QmSubgraph1Hash.jsonl +├── QmSubgraph2Hash.jsonl +└── QmSubgraph3Hash.jsonl +``` + +Each line in the file is a complete JSON object representing one log entry. + +#### Storage Format + +```json +{"id":"QmTest-2024-01-15T10:30:00.123456789Z","subgraphId":"QmTest","timestamp":"2024-01-15T10:30:00.123456789Z","level":"error","text":"Handler execution failed, retries: 3","arguments":[{"key":"retries","value":"3"}],"meta":{"module":"mapping.ts","line":42,"column":10}} +``` + +#### Query Performance + +File-based logs stream through files line-by-line with bounded memory usage. + +**Performance characteristics:** +- Query time: O(n) where n = number of log entries +- Memory usage: O(skip + first) - only matching entries kept in memory +- Suitable for: Development and testing + +#### Configuration + +**Minimum configuration (CLI):** +```bash +graph-node \ + --postgres-url postgresql://graph:pass@localhost/graph-node \ + --ethereum-rpc mainnet:https://... \ + --ipfs 127.0.0.1:5001 \ + --log-store-backend file \ + --log-store-file-dir ./graph-logs +``` + +**Full configuration (environment variables):** +```bash +export GRAPH_LOG_STORE_BACKEND=file +export GRAPH_LOG_STORE_FILE_DIR=/var/log/graph-node +export GRAPH_LOG_STORE_FILE_MAX_SIZE=104857600 # 100MB +export GRAPH_LOG_STORE_FILE_RETENTION_DAYS=30 +``` + +#### Features + +**Advantages:** +- No external dependencies +- Simple setup (just specify a directory) +- Human-readable format (JSON Lines) +- Easy to inspect with standard tools (`jq`, `grep`, etc.) +- Good for debugging during development + +**Limitations:** +- Not suitable for production with high log volume +- No indexing (O(n) query time scales with file size) +- No automatic log rotation or retention management +- Single file per subgraph (no sharding) + +#### When to Use + +Use file-based logs when: +- Developing subgraphs locally +- Testing on a development machine +- Running low-traffic subgraphs (< 1000 total logs/day including system logs) +- You want simple log access without external services + +### Elasticsearch + +**Best for:** Production deployments, high log volume, advanced search + +#### How It Works + +Elasticsearch stores logs in indices with full-text search capabilities, making it ideal for production deployments with high log volume. + +**Architecture:** +``` +graph-node → Elasticsearch HTTP API → Elasticsearch cluster + → Index: subgraph-logs-* + → Query DSL for filtering +``` + +#### Features + +**Advantages:** +- **Indexed searching**: Fast queries even with millions of logs +- **Full-text search**: Powerful text search across log messages +- **Scalability**: Handles billions of log entries +- **High availability**: Supports clustering and replication +- **Kibana integration**: Rich visualization and dashboards for operators +- **Time-based indices**: Efficient retention management + +**Considerations:** +- Requires Elasticsearch cluster (infrastructure overhead) +- Resource-intensive (CPU, memory, disk) + +#### Configuration + +**Minimum configuration (CLI):** +```bash +graph-node \ + --postgres-url postgresql://graph:pass@localhost/graph-node \ + --ethereum-rpc mainnet:https://... \ + --ipfs 127.0.0.1:5001 \ + --log-store-backend elasticsearch \ + --log-store-elasticsearch-url http://localhost:9200 +``` + +**Full configuration with authentication:** +```bash +graph-node \ + --postgres-url postgresql://graph:pass@localhost/graph-node \ + --ethereum-rpc mainnet:https://... \ + --ipfs 127.0.0.1:5001 \ + --log-store-backend elasticsearch \ + --log-store-elasticsearch-url https://es.example.com:9200 \ + --log-store-elasticsearch-user elastic \ + --log-store-elasticsearch-password secret \ + --log-store-elasticsearch-index subgraph-logs +``` + +**Environment variables:** +```bash +export GRAPH_LOG_STORE_BACKEND=elasticsearch +export GRAPH_LOG_STORE_ELASTICSEARCH_URL=http://localhost:9200 +export GRAPH_LOG_STORE_ELASTICSEARCH_USER=elastic +export GRAPH_LOG_STORE_ELASTICSEARCH_PASSWORD=secret +export GRAPH_LOG_STORE_ELASTICSEARCH_INDEX=subgraph-logs +``` + +#### Index Configuration + +Logs are stored in the configured index (default: `subgraph`). The index mapping is automatically created. + +**Recommended index settings for production:** +```json +{ + "settings": { + "number_of_shards": 3, + "number_of_replicas": 1, + "refresh_interval": "5s" + } +} +``` + +#### Query Performance + +**Performance characteristics:** +- Query time: O(log n) with indexing +- Memory usage: Minimal (server-side filtering) +- Suitable for: Millions to billions of log entries + +#### When to Use + +Use Elasticsearch when: +- Running production deployments +- High log volume +- Need advanced search and filtering +- Want to build dashboards with Kibana +- Need high availability and scalability +- Have DevOps resources to manage Elasticsearch or can set up a managed ElasticSearch deployment + +### Loki + +**Best for:** Production deployments, Grafana users, cost-effective at scale + +#### How It Works + +Loki is Grafana's log aggregation system, designed to be cost-effective and easy to operate. Unlike Elasticsearch, Loki only indexes metadata (not full-text), making it more efficient for time-series log data. + +**Architecture:** +``` +graph-node → Loki HTTP API → Loki + → Stores compressed chunks + → Indexes labels only +``` + +#### Features + +**Advantages:** +- **Cost-effective**: Lower storage costs than Elasticsearch +- **Grafana integration**: Native integration with Grafana +- **Horizontal scalability**: Designed for cloud-native deployments +- **Multi-tenancy**: Built-in tenant isolation +- **Efficient compression**: Optimized for log data +- **LogQL**: Powerful query language similar to PromQL +- **Lower resource usage**: Less CPU/memory than Elasticsearch + +**Considerations:** +- No full-text indexing (slower text searches) +- Best used with Grafana (less tooling than Elasticsearch) +- Younger ecosystem than Elasticsearch +- Query performance depends on label cardinality + +#### Configuration + +**Minimum configuration (CLI):** +```bash +graph-node \ + --postgres-url postgresql://graph:pass@localhost/graph-node \ + --ethereum-rpc mainnet:https://... \ + --ipfs 127.0.0.1:5001 \ + --log-store-backend loki \ + --log-store-loki-url http://localhost:3100 +``` + +**With multi-tenancy:** +```bash +graph-node \ + --postgres-url postgresql://graph:pass@localhost/graph-node \ + --ethereum-rpc mainnet:https://... \ + --ipfs 127.0.0.1:5001 \ + --log-store-backend loki \ + --log-store-loki-url http://localhost:3100 \ + --log-store-loki-tenant-id my-graph-node +``` + +**Environment variables:** +```bash +export GRAPH_LOG_STORE_BACKEND=loki +export GRAPH_LOG_STORE_LOKI_URL=http://localhost:3100 +export GRAPH_LOG_STORE_LOKI_TENANT_ID=my-graph-node +``` + +#### Labels + +Loki uses labels for indexing. Graph Node automatically creates labels: +- `subgraph_id`: Deployment hash +- `level`: Log level +- `job`: "graph-node" + +#### Query Performance + +**Performance characteristics:** +- Query time: O(n) for text searches, O(log n) for label queries +- Memory usage: Minimal (server-side processing) +- Suitable for: Millions to billions of log entries +- Best performance with label-based filtering + +#### When to Use + +Use Loki when: +- Already using Grafana for monitoring +- Need cost-effective log storage at scale +- Want simpler operations than Elasticsearch +- Multi-tenancy is required +- Log volume is very high (> 1M logs/day) +- Full-text search is not critical + +### Disabled + +**Best for:** Minimalist deployments, reduced overhead + +#### How It Works + +When log storage is disabled (the default), subgraph logs are **still written to stdout/stderr** along with all other graph-node logs. They are just **not stored separately** in a queryable format. + +**Important:** "Disabled" does NOT mean logs are discarded. It means: +- Logs appear in stdout/stderr (traditional behavior) +- Logs are not stored in a separate queryable backend +- The `_logs` GraphQL query returns empty results + +This is the default behavior - logs continue to work exactly as they did before this feature was added. + +#### Configuration + +**Explicitly disable:** +```bash +export GRAPH_LOG_STORE_BACKEND=disabled +``` + +**Or simply don't configure a backend** (defaults to disabled): +```bash +# No log store configuration = disabled +graph-node \ + --postgres-url postgresql://graph:pass@localhost/graph-node \ + --ethereum-rpc mainnet:https://... \ + --ipfs 127.0.0.1:5001 +``` + +#### Features + +**Advantages:** +- Zero additional overhead +- No external dependencies +- Minimal configuration +- Logs still appear in stdout/stderr for debugging + +**Limitations:** +- Cannot query logs via GraphQL (`_logs` returns empty results) +- No separation of subgraph logs from other graph-node logs in stdout +- Logs mixed with system logs (harder to filter programmatically) +- No structured querying or filtering capabilities + +#### When to Use + +Use disabled log storage when: +- Running minimal test deployments with less dependencies +- Exposing logs to users is not required for your use case +- You'd like subgraph logs sent to external log collection (e.g., container logs) + +## Configuration + +### Environment Variables + +Environment variables are the recommended way to configure log stores, especially in containerized deployments. + +#### Backend Selection + +```bash +GRAPH_LOG_STORE_BACKEND= +``` +Valid values: `disabled`, `elasticsearch`, `loki`, `file` + +#### Elasticsearch + +```bash +GRAPH_LOG_STORE_ELASTICSEARCH_URL=http://localhost:9200 +GRAPH_LOG_STORE_ELASTICSEARCH_USER=elastic # Optional +GRAPH_LOG_STORE_ELASTICSEARCH_PASSWORD=secret # Optional +GRAPH_LOG_STORE_ELASTICSEARCH_INDEX=subgraph # Default: "subgraph" +``` + +#### Loki + +```bash +GRAPH_LOG_STORE_LOKI_URL=http://localhost:3100 +GRAPH_LOG_STORE_LOKI_TENANT_ID=my-tenant # Optional +``` + +#### File + +```bash +GRAPH_LOG_STORE_FILE_DIR=/var/log/graph-node +GRAPH_LOG_STORE_FILE_MAX_SIZE=104857600 # Default: 100MB +GRAPH_LOG_STORE_FILE_RETENTION_DAYS=30 # Default: 30 +``` + +### CLI Arguments + +CLI arguments provide the same functionality as environment variables and the two can be mixed together. + +#### Backend Selection + +```bash +--log-store-backend +``` + +#### Elasticsearch + +```bash +--log-store-elasticsearch-url +--log-store-elasticsearch-user +--log-store-elasticsearch-password +--log-store-elasticsearch-index +``` + +#### Loki + +```bash +--log-store-loki-url +--log-store-loki-tenant-id +``` + +#### File + +```bash +--log-store-file-dir +--log-store-file-max-size +--log-store-file-retention-days +``` + +### Configuration Precedence + +When multiple configuration methods are used: + +1. **CLI arguments** take highest precedence +2. **Environment variables** are used if no CLI args provided +3. **Defaults** are used if neither is set + +## Querying Logs + +All log backends share the same GraphQL query interface. Logs are queried through the subgraph-specific GraphQL endpoint: + +- **Subgraph by deployment**: `http://localhost:8000/subgraphs/id/` +- **Subgraph by name**: `http://localhost:8000/subgraphs/name/` + +The `_logs` query is automatically scoped to the subgraph in the URL, so you don't need to pass a `subgraphId` parameter. + +**Note**: Queries return all log types - both user-generated logs from mapping code and system-generated runtime logs (handler execution, events, warnings, etc.). Use the `search` filter to search for specific messages, or `level` to filter by severity. + +### Basic Query + +Query the `_logs` field at your subgraph's GraphQL endpoint: + +```graphql +query { + _logs( + first: 100 + ) { + id + timestamp + level + text + } +} +``` + +**Example endpoint**: `http://localhost:8000/subgraphs/id/QmYourDeploymentHash` + +### Query with Filters + +```graphql +query { + _logs( + level: ERROR + from: "2024-01-01T00:00:00Z" + to: "2024-01-31T23:59:59Z" + search: "timeout" + first: 50 + skip: 0 + ) { + id + timestamp + level + text + arguments { + key + value + } + meta { + module + line + column + } + } +} +``` + +### Available Filters + +| Filter | Type | Description | +|--------|------|-------------| +| `level` | LogLevel | Filter by level: CRITICAL, ERROR, WARNING, INFO, DEBUG | +| `from` | String | Start timestamp (ISO 8601) | +| `to` | String | End timestamp (ISO 8601) | +| `search` | String | Case-insensitive substring search in log messages | +| `first` | Int | Number of results to return (default: 100, max: 1000) | +| `skip` | Int | Number of results to skip for pagination (max: 10000) | + +### Response Fields + +| Field | Type | Description | +|-------|------|-------------| +| `id` | String | Unique log entry ID | +| `timestamp` | String | ISO 8601 timestamp with nanosecond precision | +| `level` | LogLevel | Log level (CRITICAL, ERROR, WARNING, INFO, DEBUG) | +| `text` | String | Complete log message with arguments | +| `arguments` | [(String, String)] | Structured key-value pairs | +| `meta.module` | String | Source file name | +| `meta.line` | Int | Line number | +| `meta.column` | Int | Column number | + +### Query Examples + +#### Recent Errors + +```graphql +query RecentErrors { + _logs( + level: ERROR + first: 20 + ) { + timestamp + text + meta { + module + line + } + } +} +``` + +#### Search for Specific Text + +```graphql +query SearchTimeout { + _logs( + search: "timeout" + first: 50 + ) { + timestamp + level + text + } +} +``` + +#### Handler Execution Logs + +```graphql +query HandlerLogs { + _logs( + search: "handler" + first: 50 + ) { + timestamp + level + text + } +} +``` + +#### Time Range Query + +```graphql +query LogsInRange { + _logs( + from: "2024-01-15T00:00:00Z" + to: "2024-01-15T23:59:59Z" + first: 1000 + ) { + timestamp + level + text + } +} +``` + +#### Pagination + +```graphql +# First page +query Page1 { + _logs( + first: 100 + skip: 0 + ) { + id + text + } +} + +# Second page +query Page2 { + _logs( + first: 100 + skip: 100 + ) { + id + text + } +} +``` + +### Querying the logs store using cURL + +```bash +curl -X POST http://localhost:8000/subgraphs/id/ \ + -H "Content-Type: application/json" \ + -d '{ + "query": "{ _logs(level: ERROR, first: 10) { timestamp level text } }" + }' +``` + +### Performance Considerations + +**File-based:** _for development only_ +- Streams through files line-by-line (bounded memory usage) +- Memory usage limited to O(skip + first) entries +- Query time is O(n) where n = total log entries in file + +**Elasticsearch:** +- Indexed queries are fast regardless of size +- Text searches are optimized with full-text indexing +- Can handle billions of log entries +- Best for production with high query volume + +**Loki:** +- Label-based queries are fast (indexed) +- Text searches scan compressed chunks (slower than Elasticsearch) +- Good performance with proper label filtering +- Best for production with Grafana integration + +## Choosing the Right Backend + +### Decision Matrix + +| Scenario | Recommended Backend | Reason | +|----------|-------------------|-----------------------------------------------------------------------------------| +| Local development | **File** | Simple, no dependencies, easy to inspect | +| Testing/staging | **File** or **Elasticsearch** | File for simplicity, ES if testing production config | +| Production | **Elasticsearch** or **Loki** | Both handle scale well | +| Using Grafana | **Loki** | Native integration | +| Cost-sensitive at scale | **Loki** | Lower storage costs | +| Want rich ecosystem | **Elasticsearch** | More tools and plugins | +| Minimal deployment | **Disabled** | No overhead | + +### Resource Requirements + +#### File-based +- **Disk**: Minimal (log files only) +- **Memory**: Depends on file size during queries +- **CPU**: Minimal +- **Network**: None +- **External services**: None + +#### Elasticsearch +- **Disk**: High (indices + replicas) +- **Memory**: 4-8GB minimum for small deployments +- **CPU**: Medium to high +- **Network**: HTTP API calls +- **External services**: Elasticsearch cluster + +#### Loki +- **Disk**: Medium (compressed chunks) +- **Memory**: 2-4GB minimum +- **CPU**: Low to medium +- **Network**: HTTP API calls +- **External services**: Loki server + +## Best Practices + +### General + +1. **Start with file-based for development** - Simplest setup, easy debugging +2. **Use Elasticsearch or Loki for production** - Better performance and features +3. **Monitor log volume** - Set up alerts if log volume grows unexpectedly (includes both user logs and system-generated runtime logs) +4. **Set retention policies** - Don't keep logs forever (disk space and cost) +5. **Use structured logging** - Pass key-value pairs to log functions for better filtering + +### File-based Logs + +1. **Monitor file size** - While queries use bounded memory, larger files take longer to scan (O(n) query time) +2. **Archive old logs** - Manually archive/delete old files or implement external rotation +3. **Monitor disk usage** - Files can grow quickly with verbose logging +4. **Use JSON tools** - `jq` is excellent for inspecting .jsonl files locally + +**Example local inspection:** +```bash +# Count logs by level +cat graph-logs/QmExample.jsonl | jq -r '.level' | sort | uniq -c + +# Find errors in last 1000 lines +tail -n 1000 graph-logs/QmExample.jsonl | jq 'select(.level == "error")' + +# Search for specific text +cat graph-logs/QmExample.jsonl | jq 'select(.text | contains("timeout"))' +``` + +### Elasticsearch + +1. **Use index patterns** - Time-based indices for easier management +2. **Configure retention** - Use Index Lifecycle Management (ILM) +3. **Monitor cluster health** - Set up Elasticsearch monitoring +4. **Tune for your workload** - Adjust shards/replicas based on log volume +5. **Use Kibana** - Visualize and explore logs effectively + +**Example Elasticsearch retention policy:** +```json +{ + "policy": "graph-logs-policy", + "phases": { + "hot": { "min_age": "0ms", "actions": {} }, + "warm": { "min_age": "7d", "actions": {} }, + "delete": { "min_age": "30d", "actions": { "delete": {} } } + } +} +``` + +### Loki + +1. **Use proper labels** - Don't over-index, keep label cardinality low +2. **Configure retention** - Set retention period in Loki config +3. **Use Grafana** - Native integration provides best experience +4. **Compress efficiently** - Loki's compression works best with batch writes +5. **Multi-tenancy** - Use tenant IDs if running multiple environments + +**Example Grafana query:** +```logql +{subgraph_id="QmExample", level="error"} |= "timeout" +``` + +## Troubleshooting + +### File-based Logs + +**Problem: Log file doesn't exist** +- Check `GRAPH_LOG_STORE_FILE_DIR` is set correctly +- Verify directory is writable by graph-node + +**Problem: Queries are slow** +- Subgraph logs file may be very large +- Consider archiving old logs or implementing retention +- For high-volume production use, switch to Elasticsearch or Loki + +**Problem: Disk filling up** +- Implement log rotation +- Reduce log verbosity in subgraph code +- Set up monitoring for disk usage + +### Elasticsearch + +**Problem: Cannot connect to Elasticsearch** +- Verify `GRAPH_LOG_STORE_ELASTICSEARCH_URL` is correct +- Check Elasticsearch is running: `curl http://localhost:9200` +- Verify authentication credentials if using security features +- Check network connectivity and firewall rules + +**Problem: No logs appearing in Elasticsearch** +- Check Elasticsearch cluster health +- Verify index exists: `curl http://localhost:9200/_cat/indices` +- Check graph-node logs for write errors +- Verify Elasticsearch has disk space + +**Problem: Queries are slow** +- Check Elasticsearch cluster health and resources +- Verify indices are not over-sharded +- Consider adding replicas for query performance +- Review query patterns and add appropriate indices + +### Loki + +**Problem: Cannot connect to Loki** +- Verify `GRAPH_LOG_STORE_LOKI_URL` is correct +- Check Loki is running: `curl http://localhost:3100/ready` +- Verify tenant ID if using multi-tenancy +- Check network connectivity + +**Problem: No logs appearing in Loki** +- Check Loki service health +- Verify Loki has disk space for chunks +- Check graph-node logs for write errors +- Verify Loki retention settings aren't deleting logs immediately + +**Problem: Queries return no results in Grafana** +- Check label selectors match what graph-node is sending +- Verify time range includes when logs were written +- Check Loki retention period +- Verify tenant ID matches if using multi-tenancy + +## Further Reading + +- [Environment Variables Reference](environment-variables.md) +- [Graph Node Configuration](config.md) +- [Elasticsearch Documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html) +- [Grafana Loki Documentation](https://grafana.com/docs/loki/latest/) diff --git a/docs/plans/gnd-cli-expansion.md b/docs/plans/gnd-cli-expansion.md new file mode 100644 index 00000000000..b1bef8bc108 --- /dev/null +++ b/docs/plans/gnd-cli-expansion.md @@ -0,0 +1,418 @@ +# Plan: Extend gnd with graph-cli functionality + +## Overview + +Extend the existing `gnd` (Graph Node Dev) CLI to be a **drop-in replacement** for the TypeScript-based `graph-cli`. The goal is identical CLI behavior, flags, output format, and byte-for-byte identical AssemblyScript code generation. + +**Spec document**: `docs/specs/gnd-cli-expansion.md` + +**Scope**: + +- Ethereum protocol only (other protocols are future work) +- All graph-cli commands except `local` and `node` subcommand +- Match latest graph-cli version (0.98.x) + +**Current state of gnd** (`~/code/graph-node/gnd/`): + +- ~700 lines of Rust across 3 files +- Single command with flags (no subcommands) +- Already uses clap, imports graph crates +- Runs graph-node in dev mode with file watching + +## Status Summary (Updated 2026-01-19) + +**Overall Progress**: ~99% complete (implementation + docs + automated tests done, only publish/test commands need manual verification) + +| Phase | Status | Notes | +|-------|--------|-------| +| Phase 1: CLI Restructure | ✅ Complete | | +| Phase 2: Infrastructure | ✅ Complete | | +| Phase 3: Simple Commands | ✅ Complete | | +| Phase 4: Code Generation | ✅ Complete | 11 verification fixtures passing | +| Phase 5: Migrations | ✅ Complete | | +| Phase 6: Build Command | ✅ Complete | Tested via `just test-gnd-commands` | +| Phase 7: Deploy Command | ✅ Complete | Tested via `just test-gnd-cli` | +| Phase 8: Init Command | ✅ Complete | Tested via `just test-gnd-commands` | +| Phase 9: Add Command | ✅ Complete | | +| Phase 10: Publish Command | 🟡 Needs Testing | Manual end-to-end verification | +| Phase 11: Test Command | 🟡 Needs Testing | Manual Matchstick test | +| Phase 12: Testing & Polish | ✅ Complete | Documentation done, test porting analyzed | +| Phase 13: CLI Integration Tests | ✅ Complete | Test infrastructure created, ready for manual verification | + +**Total LOC**: ~15,000 lines of Rust (158 unit tests, 11 verification tests passing) + +**Documentation**: +- `gnd/README.md` - CLI command reference with examples +- `gnd/docs/MIGRATING_FROM_GRAPH_CLI.md` - Migration guide from graph-cli + +## Git Workflow + +**Branch**: All work should be committed to the `gnd-cli` branch. + +**Commit discipline**: + +- Commit work in small, reviewable chunks +- Each commit should be self-contained and pass all checks +- Prefer many small commits over few large ones +- Each commit message should clearly describe what it does + +**Before each commit**: + +```bash +just format +just lint +just test-unit # only run the tests in gnd/ +``` + +- MANDATORY: Work must be committed, a task is only done when work is committed +- MANDATORY: Make sure to follow the commit discipline above + +## Commands + +| Command | Description | Complexity | +| ------------- | ----------------------------------- | ----------------------- | +| `gnd dev` | Existing gnd functionality | Done (restructure only) | +| `gnd codegen` | Generate AssemblyScript types | High | +| `gnd build` | Compile to WASM | Medium | +| `gnd deploy` | Deploy to Graph Node | Medium | +| `gnd init` | Scaffold new subgraph | High | +| `gnd add` | Add datasource to existing subgraph | Medium | +| `gnd create` | Register subgraph name | Low | +| `gnd remove` | Unregister subgraph name | Low | +| `gnd auth` | Set deploy key | Low | +| `gnd publish` | Publish to decentralized network | Medium | +| `gnd test` | Run Matchstick tests | Low (shell out) | +| `gnd clean` | Remove build artifacts | Low | + +**Not implemented** (documented differences): + +- `gnd local` - Use existing test infrastructure +- `gnd node` - Use graphman for node management +- `--uncrashable` flag - Float Capital third-party feature + +## Reusable from graph-node + +| Feature | Location | Status | +| ------------------- | ------------------------------------- | ---------------------------- | +| Manifest parsing | `graph/src/data/subgraph/mod.rs` | Ready | +| Manifest validation | `graph/src/data/subgraph/mod.rs` | Ready (may need refactoring) | +| GraphQL schema | `graph/src/schema/input/` | Ready | +| ABI parsing | `ethabi` + chain/ethereum wrappers | Ready | +| IPFS client | `graph/src/ipfs/` | Ready | +| Link resolution | `graph/src/components/link_resolver/` | Ready | +| File watching | `gnd/src/watcher.rs` | Ready | +| CLI framework | `clap` (already in gnd) | Ready | + +## Module Structure + +``` +gnd/src/ +├── main.rs # Entry point, clap setup +├── lib.rs +├── commands/ +│ ├── mod.rs +│ ├── dev.rs # Existing gnd functionality +│ ├── codegen.rs +│ ├── build.rs +│ ├── deploy.rs +│ ├── init.rs +│ ├── add.rs +│ ├── create.rs +│ ├── remove.rs +│ ├── auth.rs +│ ├── publish.rs +│ ├── test.rs +│ └── clean.rs +├── codegen/ +│ ├── mod.rs +│ ├── schema.rs # Entity class generation +│ ├── abi.rs # ABI binding generation +│ └── template.rs # Template binding generation +├── scaffold/ +│ ├── mod.rs +│ ├── manifest.rs # Generate subgraph.yaml +│ ├── schema.rs # Generate schema.graphql +│ └── mapping.rs # Generate mapping.ts +├── migrations/ +│ ├── mod.rs +│ └── ... # One module per migration version +├── compiler/ +│ ├── mod.rs +│ └── asc.rs # Shell out to asc +├── services/ +│ ├── mod.rs +│ ├── etherscan.rs # Etherscan API client +│ ├── sourcify.rs # Sourcify API client +│ ├── registry.rs # Network registry client +│ └── graph_node.rs # Graph Node JSON-RPC client +├── config/ +│ ├── mod.rs +│ ├── auth.rs # ~/.graphprotocol/ management +│ └── networks.rs # networks.json handling +├── output/ +│ ├── mod.rs +│ └── spinner.rs # Progress/spinner output (match TS CLI exactly) +└── watcher.rs # Existing file watcher +``` + +## Dependencies to Add + +| Crate | Purpose | +| ----------- | ------------------------------------------- | +| `inquire` | Interactive prompts (init) | +| `minijinja` | Template rendering (scaffold) | +| `indicatif` | Progress bars and spinners | +| `reqwest` | HTTP client (Etherscan, Sourcify, registry) | + +## TODO List + +### Phase 1: CLI Restructure + +- [x] Create `gnd-cli` branch +- [x] Restructure main.rs with clap subcommands +- [x] Move existing gnd logic to `commands/dev.rs` +- [x] Add empty command stubs for all commands +- [x] Verify `gnd dev` works exactly as before +- [x] Add `--version` output showing both gnd and graph-cli versions + +### Phase 2: Infrastructure + +- [x] Implement `output/spinner.rs` matching TS CLI output format exactly +- [x] Implement `config/auth.rs` for `~/.graph-cli.json` management (in commands/auth.rs) +- [x] Implement `config/networks.rs` for networks.json parsing +- [x] Add reqwest dependency and basic HTTP client setup (in services/graph_node.rs) +- [x] Implement `services/registry.rs` for @pinax/graph-networks-registry (implemented as `NetworksRegistry` in services/contract.rs) + +### Phase 3: Simple Commands + +- [x] Implement `gnd clean` command +- [x] Implement `gnd auth` command +- [x] Implement `gnd create` command +- [x] Implement `gnd remove` command +- [x] Add tests for simple commands + +### Phase 4: Code Generation (High Priority, High Risk) + +- [x] Study TS CLI codegen output in detail +- [x] Implement `codegen/schema.rs` - entity classes from GraphQL +- [x] Implement `codegen/typescript.rs` - AST builders for TypeScript/AssemblyScript +- [x] Implement `codegen/types.rs` - type conversion utilities +- [x] Implement `codegen/abi.rs` - ABI bindings +- [x] Implement `codegen/template.rs` - template bindings +- [x] Integrate prettier formatting (shell out) +- [x] Implement `commands/codegen.rs` with all flags +- [x] Implement `--watch` mode using existing file watcher +- [x] Create snapshot tests comparing output to TS CLI (11 fixtures from graph-cli validation tests) +- [x] Verify byte-for-byte identical output - tests pass with documented known differences: + - Int8 import always included (gnd simplicity) + - Trailing commas in multi-line constructs (gnd style) + - 2D array accessors use correct `toStringMatrix()` (gnd fixes graph-cli bug) + +### Phase 5: Migrations + +- [x] Study TS CLI migrations in `/packages/cli/src/migrations/` +- [x] Implement migration framework in `migrations/mod.rs` +- [x] Implement each migration version (0.0.1 → current) +- [x] Add `--skip-migrations` flag support +- [x] Test migrations with old manifest versions + +### Phase 6: Build Command + +- [x] Implement `compiler/asc.rs` - shell out to asc +- [x] Implement `commands/build.rs` with all flags (~1084 lines, full build pipeline) +- [x] Handle network file resolution +- [x] Implement `--watch` mode +- [x] Implement optional IPFS upload with deduplication +- [x] Test build output matches TS CLI - Covered by `just test-gnd-commands` (test_build_after_codegen) + +### Phase 7: Deploy Command + +- [x] Implement `services/graph_node.rs` - JSON-RPC client (~258 lines, create/remove/deploy) +- [x] Implement `commands/deploy.rs` with all flags (~239 lines) +- [x] Support all deploy targets (local, studio, hosted service) - defaults to Subgraph Studio +- [x] Handle access token / deploy key authentication +- [x] Test deployment to local Graph Node - Covered by `just test-gnd-cli` (block-handlers, value-roundtrip, int8) + +### Phase 8: Init Command + +- [x] Add `inquire` dependency for interactive prompts +- [x] Implement `services/contract.rs` - ABI fetching (Etherscan, Blockscout, Sourcify) (~730 lines) +- [x] Implement networks registry loading +- [x] Implement `scaffold/manifest.rs` (~271 lines) +- [x] Implement `scaffold/schema.rs` (~206 lines) +- [x] Implement `scaffold/mapping.rs` (~205 lines) +- [x] Implement `commands/init.rs` with all flags (~1060 lines including --from-subgraph) +- [x] Implement --from-example mode +- [x] Implement --from-contract mode (uses ContractService for ABI fetching) +- [x] Add interactive prompts when required options are missing +- [x] Implement --from-subgraph mode (fetches manifest from IPFS, extracts immutable entities) +- [x] Test scaffold output matches TS CLI - Covered by `just test-gnd-commands` (test_init_from_example, test_init_from_contract_with_abi) + +### Phase 9: Add Command + +- [x] Implement `commands/add.rs` (~773 lines, fully functional) +- [x] Reuse scaffold components for actual implementation +- [x] ABI validation, entity generation, mapping creation, manifest updates +- [x] Test adding datasource to existing subgraph (unit tests for helper functions) +- [x] Comprehensive error handling + +### Phase 10: Publish Command + +- [x] Implement `commands/publish.rs` (~230 lines, fully functional) +- [x] Build subgraph and upload to IPFS (reuses build command) +- [x] Open browser to webapp URL with query params (same approach as graph-cli) +- [x] Support --ipfs-hash flag to skip build +- [x] Support --subgraph-id, --api-key for updating existing subgraphs +- [x] Support --protocol-network (arbitrum-one, arbitrum-sepolia) +- [x] Interactive prompt before opening browser +- [ ] Test publish workflow (NEEDS HUMAN: manual end-to-end verification) + +### Phase 11: Test Command + +- [x] Implement Matchstick binary download/detection +- [x] Implement `commands/test.rs` (~254 lines, shell out to Matchstick) +- [x] Handle Docker mode (Dockerfile generation on demand) +- [x] Recompilation flag support +- [ ] Test with actual Matchstick tests (NEEDS HUMAN: install matchstick + run end-to-end) + +### Phase 12: Testing & Polish + +- [x] Port all tests from TS CLI test suite not already covered by gnd tests + - Location: `/home/lutter/code/subgraphs/graph-cli/packages/cli/tests/` + - Review existing coverage and identify gaps + - Analysis: gnd has 11/12 success fixtures from graph-cli validation tests (missing `near-is-valid` which is out of scope for Ethereum-only support). Error/validation fixtures test manifest parsing which is handled by graph-node's validation layer. +- [x] Add snapshot tests for code generation scenarios (11 fixtures from graph-cli validation tests) +- [x] Add tests for overloaded events/functions in ABI codegen (disambiguation) +- [x] Add tests for simple array fields in schema codegen +- [x] Add tests for nested array types (`[[String]]` etc.) - schema codegen refactored to track list depth +- [x] Add tests for tuple/struct types in ABI codegen (functions, events, nested, arrays) +- [x] Add tests for array types in ABI codegen (array params in events, 2D matrices) +- [x] Codegen verification test framework (gnd/tests/codegen_verification.rs) +- [x] Add comprehensive tests for prompt module (network completer, source type) +- [x] Add comprehensive tests for init command (interactive mode detection) +- [ ] Ensure all edge cases are covered (ongoing) +- [x] Documentation: + - [x] CLI usage docs (README with command reference, examples, common workflows) + - [x] Migration guide from graph-cli to gnd (differences, compatibility notes) +- [x] Shell completions (bash, elvish, fish, powershell, zsh via clap_complete) + +### Phase 13: CLI Integration Tests + +Verify gnd works as a drop-in replacement for graph-cli by running integration tests with `GRAPH_CLI=../target/debug/gnd`. + +**Key Insight**: The existing integration test infrastructure uses `CONFIG.graph_cli` which can be set via the `GRAPH_CLI` environment variable. By setting this to the gnd binary, existing integration tests will use gnd for the deployment flow (codegen, create, deploy). + +#### Integration Deployment Tests (`tests/tests/gnd_cli_tests.rs`) + +- [x] Create test file that sets `GRAPH_CLI` env var to gnd binary +- [x] Verify gnd binary exists (fail fast with clear message) +- [x] Run subset of integration tests: `block-handlers`, `value-roundtrip`, `int8` +- [x] Reuse `Contract::deploy_all()`, `CONFIG.reset_database()`, `CONFIG.spawn_graph_node()` +- [x] Use `Subgraph::deploy()` which calls: `gnd codegen`, `gnd create`, `gnd deploy` +- [x] Verify subgraphs sync and queries return expected data + +**Commands Tested via Subgraph::deploy()**: +- `gnd codegen` - via `Subgraph::deploy()` +- `gnd create` - via `Subgraph::deploy()` +- `gnd deploy` - via `Subgraph::deploy()` + +#### Standalone Command Tests (`gnd/tests/cli_commands.rs`) + +Tests for commands that don't require running graph-node: + +**`gnd init` Tests**: +- [x] `test_init_from_example` - Run `gnd init --from-example ethereum-gravatar`, verify scaffold created +- [x] `test_init_from_contract_with_abi` - Run with `--from-contract` + `--abi`, verify manifest/schema/mapping created +- [ ] `test_init_from_subgraph` - Run with `--from-subgraph`, verify immutable entities extracted (requires IPFS) - **Skipped**: Requires IPFS with subgraph data + +**`gnd add` Tests**: +- [x] `test_add_datasource` - Create base subgraph with init, then add contract, verify manifest updated + +**`gnd build` Tests**: +- [x] `test_build_after_codegen` - Copy fixture, run codegen + build, verify `build/` directory created with WASM + +**`gnd codegen` Tests**: +- [x] `test_codegen_generates_types` - Run codegen on scaffolded subgraph, verify `generated/schema.ts` exists + +**`gnd clean` Tests**: +- [x] `test_clean_removes_artifacts` - Verify clean removes `generated/` and `build/` directories + +**ABIs Available**: `tests/contracts/abis/*.json` + +#### Justfile Targets + +- [x] Add `test-gnd-cli` - Run gnd CLI integration tests (deployment with gnd as CLI) +- [x] Add `test-gnd-commands` - Run gnd standalone command tests (init, add, build) + +#### Service Requirements + +Tests use the same services as `just test-integration`: + +| Service | Port | Start Command | +|---------|------|---------------| +| PostgreSQL | 3011 | `nix run .#integration` | +| IPFS | 3001 | (included above) | +| Anvil | 3021 | (included above) | + +#### Verification + +1. `just build` - Build gnd binary +2. `just test-gnd-cli` - Run integration deployment tests with gnd +3. `just test-gnd-commands` - Run standalone command tests + +Expected output: All tests pass, showing gnd works as drop-in replacement. + +## Key Decisions + +| Decision | Choice | Rationale | +| ---------------- | ---------------------------- | ------------------------------------------- | +| Binary name | `gnd` | Existing name, avoid confusion with `graph` | +| Protocol support | Ethereum only | Simplify initial scope | +| Compatibility | Drop-in replacement | Same flags, output, exit codes | +| Code generation | Byte-for-byte identical | Snapshot testable, no surprises | +| Formatting | Shell out to prettier | Guarantees identical output | +| Compilation | Shell out to asc | Same as TS CLI | +| Testing | Shell out to Matchstick | Same as TS CLI | +| Debug logging | RUST_LOG | Rust standard, not DEBUG env var | +| Validation | Use graph-node's | Refactor if needed, single source of truth | +| Network registry | Fetch at runtime | Registry contents change over time | +| Error messages | Same info, format may differ | Convey same information | +| TS CLI bugs | Fix them | Don't replicate bugs | + +## Risk Assessment + +| Risk | Severity | Mitigation | +| --------------------------- | -------- | ------------------------------------ | +| Code generation fidelity | High | Snapshot tests against TS CLI output | +| Migration correctness | High | Test with real old manifests | +| Breaking existing gnd users | Medium | Keep `gnd dev` behavior identical | +| `asc` version compat | Medium | Version detection, clear errors | +| Network API changes | Low | Error handling, clear messages | + +## TS CLI References + +Key files to study in `/home/lutter/code/subgraphs/graph-cli/packages/cli/src/`: + +| Feature | TS CLI Location | +| ---------------- | ----------------------------------- | +| Commands | `commands/*.ts` | +| Type generator | `type-generator.ts` | +| Schema codegen | `codegen/schema.ts` | +| ABI codegen | `protocols/ethereum/codegen/abi.ts` | +| Template codegen | `codegen/template.ts` | +| Compiler | `compiler/index.ts` | +| Migrations | `migrations/` | +| Scaffold | `scaffold/` | +| Spinner/output | `command-helpers/spinner.ts` | +| Auth | `command-helpers/auth.ts` | +| Network config | `command-helpers/network.ts` | +| Tests | `../tests/cli/` | + +## Effort Estimate + +**~3-4 person-months** for full implementation because: + +- Code generation must be byte-for-byte identical (requires careful study) +- All migrations must be implemented +- Full test coverage needed +- But we reuse: manifest parsing, validation, IPFS client, file watching, CLI framework diff --git a/docs/plans/runner-refactor.md b/docs/plans/runner-refactor.md new file mode 100644 index 00000000000..c5ce612f8db --- /dev/null +++ b/docs/plans/runner-refactor.md @@ -0,0 +1,281 @@ +# Runner Refactor Implementation Plan + +This document outlines the implementation plan for the runner refactor described in [the spec](../specs/runner-refactor.md). + +## Overview + +The refactor transforms `core/src/subgraph/runner.rs` from a complex nested-loop structure into a cleaner state machine with explicit pipeline stages. + +## Git Workflow + +**Branch**: All work should be committed to the `runner-refactor` branch. + +**Commit discipline**: + +- Commit work in small, reviewable chunks +- Each commit should be self-contained and pass all checks +- Prefer many small commits over few large ones +- Each commit message should clearly describe what it does +- Each step in the implementation phases should correspond to one or more commits + +**Before each commit**: + +```bash +just format +just lint +just test-unit +just test-runner +``` + +- MANDATORY: Work must be committed, a task is only done when work is committed +- MANDATORY: Make sure to follow the commit discipline above +- IMPORTANT: The runner tests produce output in `tests/runner-tests.log`. Use that to investigate failures. + +## Implementation Phases + +### Phase 1: Extract TriggerRunner Component + +**Goal:** Eliminate duplicated trigger processing code (lines 616-656 vs 754-790). + +**Files to modify:** + +- `core/src/subgraph/runner.rs` - Extract logic +- Create `core/src/subgraph/runner/trigger_runner.rs` + +**Steps:** + +1. Create `TriggerRunner` struct with execute method +2. Replace first trigger loop (lines 616-656) with `TriggerRunner::execute()` +3. Replace second trigger loop (lines 754-790) with same call +4. Verify tests pass + +**Verification:** + +- `just test-unit` passes +- `just test-runner` passes +- No behavioral changes + +### Phase 2: Define RunnerState Enum + +**Goal:** Introduce explicit state machine types without changing control flow yet. + +**Files to modify:** + +- Create `core/src/subgraph/runner/state.rs` +- `core/src/subgraph/runner.rs` - Add state field + +**Steps:** + +1. Define `RunnerState` enum with all variants +2. Define `RestartReason` and `StopReason` enums +3. Add `state: RunnerState` field to `SubgraphRunner` +4. Initialize state in constructor +5. Verify tests pass (no behavioral changes yet) + +**Verification:** + +- Code compiles +- Tests pass unchanged + +### Phase 3: Refactor run_inner to State Machine + +**Goal:** Replace nested loops with explicit state transitions. + +**Files to modify:** + +- `core/src/subgraph/runner.rs` - Rewrite `run_inner` + +**Steps:** + +1. Extract `initialize()` method for pre-loop setup +2. Extract `await_block()` method for stream event handling +3. Extract `restart()` method for restart logic +4. Extract `finalize()` method for cleanup +5. Rewrite `run_inner` as state machine loop +6. Remove nested loop structure +7. Verify tests pass + +**Verification:** + +- `just test-unit` passes +- `just test-runner` passes +- Same behavior, cleaner structure + +### Phase 4: Define Pipeline Stages + +**Goal:** Break `process_block` into explicit stages. + +**Files to modify:** + +- Create `core/src/subgraph/runner/pipeline.rs` +- `core/src/subgraph/runner.rs` - Refactor `process_block` + +**Steps:** + +1. Extract `match_triggers()` stage method +2. Extract `execute_triggers()` stage method (uses `TriggerRunner`) +3. Extract `process_dynamic_data_sources()` stage method +4. Extract `process_offchain_triggers()` stage method +5. Extract `persist_block_state()` stage method +6. Rewrite `process_block` to call stages in sequence +7. Verify tests pass + +**Verification:** + +- `just test-unit` passes +- `just test-runner` passes +- Same behavior, cleaner structure + +### Phase 5: Consolidate Error Handling + +**Goal:** Unify scattered error handling into explicit classification. + +**Files to modify:** + +- `graph/src/components/subgraph/error.rs` (or wherever `ProcessingError` lives) +- `core/src/subgraph/runner.rs` - Use new error methods + +**Steps:** + +1. Add `ProcessingErrorKind` enum with Deterministic/NonDeterministic/PossibleReorg variants +2. Add `kind()` method to `ProcessingError` +3. Add helper methods: `should_stop_processing()`, `should_restart()`, `is_retryable()` +4. Replace scattered error checks in `process_block` with unified logic +5. Replace scattered error checks in dynamic DS handling +6. Replace scattered error checks in `handle_offchain_triggers` +7. Document error handling invariants in code comments +8. Verify tests pass + +**Verification:** + +- `just test-unit` passes +- `just test-runner` passes +- Error behavior unchanged (same semantics, cleaner code) + +### Phase 6: Add BlockState Checkpoints + +**Goal:** Enable rollback capability with minimal overhead. + +**Files to modify:** + +- `graph/src/prelude.rs` or wherever `BlockState` is defined +- `core/src/subgraph/runner.rs` - Use checkpoints + +**Steps:** + +1. Add `checkpoint()` method to `BlockState` +2. Add `BlockStateCheckpoint` struct +3. Add `restore()` method to `BlockState` +4. Use checkpoint before dynamic DS processing +5. Verify tests pass + +**Verification:** + +- `just test-unit` passes +- No performance regression (checkpoints are lightweight) + +### Phase 7: Module Organization + +**Goal:** Organize code into proper module structure. + +**Files to create/modify:** + +- `core/src/subgraph/runner/mod.rs` +- Move/organize existing extracted modules + +**Steps:** + +1. Create `runner/` directory +2. Move `state.rs`, `pipeline.rs`, `trigger_runner.rs` into it +3. Update `runner.rs` to re-export from module +4. Update imports in dependent files +5. Verify tests pass + +**Verification:** + +- `just test-unit` passes +- `just test-runner` passes +- `just lint` shows no warnings + +## Completion Criteria + +Each phase is complete when: + +1. `just format` - Code is formatted +2. `just lint` - Zero warnings +3. `just check --release` - Builds in release mode +4. `just test-unit` - Unit tests pass +5. `just test-runner` - Runner tests pass + +## Progress Checklist + +### Phase 1: Extract TriggerRunner Component + +- [x] Create `TriggerRunner` struct with execute method +- [x] Replace first trigger loop (lines 616-656) +- [x] Replace second trigger loop (lines 754-790) +- [x] Verify tests pass + +### Phase 2: Define RunnerState Enum + +- [x] Define `RunnerState` enum with all variants +- [x] Define `RestartReason` and `StopReason` enums +- [x] Add `state: RunnerState` field to `SubgraphRunner` +- [x] Initialize state in constructor +- [x] Verify tests pass + +### Phase 3: Refactor run_inner to State Machine + +- [x] Extract `initialize()` method +- [x] Extract `await_block()` method +- [x] Extract `restart()` method +- [x] Extract `finalize()` method +- [x] Rewrite `run_inner` as state machine loop +- [x] Remove nested loop structure +- [x] Verify tests pass + +### Phase 4: Define Pipeline Stages + +- [x] Extract `match_triggers()` stage method +- [x] Extract `execute_triggers()` stage method +- [x] Extract `process_dynamic_data_sources()` stage method +- [x] Extract `process_offchain_triggers()` stage method +- [x] Extract `persist_block_state()` stage method +- [x] Rewrite `process_block` to call stages in sequence +- [x] Verify tests pass + +### Phase 5: Consolidate Error Handling + +- [x] Add `ProcessingErrorKind` enum +- [x] Add `kind()` method to `ProcessingError` +- [x] Add helper methods (`should_stop_processing()`, `should_restart()`, `is_retryable()`) +- [x] Replace scattered error checks in `process_block` +- [x] Replace scattered error checks in dynamic DS handling (preserved existing behavior per spec) +- [x] Replace scattered error checks in `handle_offchain_triggers` (preserved existing behavior per spec) +- [x] Document error handling invariants +- [x] Verify tests pass + +### Phase 6: Add BlockState Checkpoints + +- [x] Add `BlockStateCheckpoint` struct +- [x] Add `checkpoint()` method to `BlockState` +- [x] Add `restore()` method to `BlockState` +- [x] Use checkpoint before dynamic DS processing +- [x] Verify tests pass + +### Phase 7: Module Organization + +- [x] Create `runner_components/` directory (named to avoid conflict with `runner.rs`) +- [x] Move `state.rs`, `trigger_runner.rs` into it (pipeline.rs was not created as stages are methods) +- [x] Create `runner_components/mod.rs` with re-exports +- [x] Update imports in `runner.rs` to use the new module path +- [x] Verify tests pass +- [x] `just lint` shows zero warnings + +## Notes + +- Each phase should be a separate, reviewable PR +- Phases 1-4 can potentially be combined if changes are small +- Phase 3 (FSM refactor of run_inner) is the most invasive and should be reviewed carefully +- Phase 5 (error handling) can be done earlier if it helps simplify other phases +- Preserve all existing behavior - this is a refactor, not a feature change diff --git a/docs/specs/dump-restore.md b/docs/specs/dump-restore.md new file mode 100644 index 00000000000..6d7e117b78d --- /dev/null +++ b/docs/specs/dump-restore.md @@ -0,0 +1,269 @@ +# Spec: Parquet Dump/Restore for Subgraph Data + +## Problem + +Subgraph entity data lives exclusively in PostgreSQL. There's no way to export a subgraph's data for backup, migration between environments, or sharing. We need a file-based dump/restore mechanism. + +## Goals + +1. **Dump** a subgraph's entity data to parquet files +2. **Restore** a subgraph from parquet files +3. **Incremental append** -- add newly arrived data to an existing dump without rewriting it +4. Long-term: make dumping an **ongoing process** (not just a one-off CLI operation) + +## Non-goals (for now) + +- S3/GCS output +- Schema evolution / migration between different schema versions + +## Data Model Recap + +Each subgraph deployment has: +- A PostgreSQL schema (e.g., `sgd123`) +- One table per entity type, with columns: + - `vid` (bigserial) -- row version ID, primary key + - `block_range` (int4range) for mutable entities OR `block$` (int) for immutable + - `causality_region` (int, optional) -- for offchain data sources + - Data columns matching GraphQL fields (text, int, bigint, numeric, bytea, bool, timestamptz, arrays, enums) +- `data_sources$` table -- dynamic data sources created at runtime (defined in `dynds/private.rs`, separate from Layout): + - `vid` (int, identity PK), `block_range` (int4range), `causality_region` (int) + - `manifest_idx` (int), `parent` (int, self-ref FK), `id` (bytea) + - `param` (bytea, nullable), `context` (jsonb, nullable), `done_at` (int, nullable) + - Note: `parent` and `id` exist in the DDL but are currently unused by `insert()`, `load()`, and `copy_to()`. The dump should include them for completeness (they may contain data in older deployments). +- `poi2$` table -- Proof of Indexing data (a **mutable** entity table in the Layout, i.e. it uses `block_range` not `block$`; has `digest` (bytea), `id` (text), optionally `block_time$` (int8); `has_causality_region: false`). Conditionally created when `catalog.use_poi` is true. Excluded from some Layout operations like `find_changes()`. +- Metadata in `subgraphs.subgraph_manifest`, `subgraphs.deployment`, and `subgraphs.head` tables + +## Dump Format + +### Directory layout + +``` +/ + metadata.json -- deployment metadata + per-table state + schema.graphql -- raw GraphQL schema text + subgraph.yaml -- raw subgraph manifest YAML (optional) + / + chunk_000000.parquet -- rows ordered by vid + chunk_000001.parquet -- incremental append + ... + data_sources$/ + chunk_000000.parquet -- dynamic data sources +``` + +One parquet file per entity type (each has a different columnar schema). The `data_sources$` table is dumped alongside entity tables. The `poi2$` table, when present, is a regular entity table in the `Layout` (conditionally created when `catalog.use_poi` is true) and appears as any other entity type directory. Incremental dumps produce new chunk files rather than rewriting existing ones. + +The GraphQL schema and subgraph manifest YAML are stored as separate files rather than embedded in `metadata.json`. This matches the existing dump code in `dump.rs` and keeps the files human-readable and diffable. + +### Parquet schema per entity type + +System columns (always present): +- `vid` -> Int64 +- Immutable entities: `block$` -> Int32 +- Mutable entities: `block_range_start` -> Int32, `block_range_end` -> Int32 (nullable; null = unbounded/current) +- `causality_region` -> Int32 (only if table has it) + +Data columns mapped from `ColumnType` (all 10 variants defined in `relational.rs:1342`): + +| ColumnType | Arrow DataType | Notes | +|--------------------|---------------------------|-------------------------------| +| Boolean | Boolean | | +| Int | Int32 | | +| Int8 | Int64 | | +| Bytes | Binary | Raw bytes | +| BigInt | Utf8 | Arbitrary precision as string | +| BigDecimal | Utf8 | Arbitrary precision as string | +| Timestamp | TimestampMicrosecond(None) | Matches Value::Timestamp | +| String | Utf8 | | +| Enum(EnumType) | Utf8 | String value of enum variant | +| TSVector(FulltextConfig) | **Skip** | Generated; rebuild on restore | + +**List/array columns:** `List(T)` is not a `ColumnType` variant. Whether a column is an array is determined by `Column.is_list()` (delegates to the GraphQL `field_type`). A `[String]` field has `column_type: ColumnType::String` with a list-typed `field_type`. For Arrow mapping, check `column.is_list()` and wrap the base Arrow type in `List`. In `OidValue`, arrays have separate variants (`StringArray`, `BytesArray`, `BoolArray`, `Ints`, `Int8Array`, `BigDecimalArray`, `TimestampArray`). + +Nullability follows the GraphQL schema (non-null fields -> non-nullable Arrow columns). + +### metadata.json + +Contains everything needed to reconstruct the deployment's table structure, plus diagnostic information (health, indexes) captured at dump time. The GraphQL schema and manifest YAML are stored in separate files (`schema.graphql`, `subgraph.yaml`), not embedded here. + +The struct backing this file is `Metadata` (evolved from the existing `Control` struct in `dump.rs`). + +```json +{ + "version": 1, + "deployment": "Qm...", + "network": "mainnet", + + "manifest": { + "spec_version": "1.0.0", + "description": "Optional subgraph description", + "repository": "https://github.com/...", + "features": ["..."], + "entities_with_causality_region": ["EntityType1"], + "history_blocks": 2147483647 + }, + + "earliest_block_number": 12345, + "start_block": { "number": 12345, "hash": "0xabc..." }, + "head_block": { "number": 99999, "hash": "0xdef..." }, + "entity_count": 150000, + + "graft_base": null, + "graft_block": null, + "debug_fork": null, + + "health": { + "failed": false, + "health": "healthy", + "fatal_error": null, + "non_fatal_errors": [] + }, + + "indexes": { + "token": [ + "CREATE INDEX CONCURRENTLY IF NOT EXISTS attr_0_0_id ON sgd.token USING btree (id)" + ] + }, + + "tables": { + "Token": { + "immutable": true, + "has_causality_region": false, + "chunks": [ + { "file": "Token/chunk_000000.parquet", "min_vid": 0, "max_vid": 50000, "row_count": 50000 } + ], + "max_vid": 50000 + }, + "data_sources$": { + "immutable": false, + "has_causality_region": true, + "chunks": [ + { "file": "data_sources$/chunk_000000.parquet", "min_vid": 0, "max_vid": 100, "row_count": 100 } + ], + "max_vid": 100 + } + } +} +``` + +**Field sources:** + +| Field | Source | Code path | +|-------|--------|-----------| +| `manifest.*` | `subgraphs.subgraph_manifest` | `SubgraphManifestEntity` via `deployment_entity()` in `detail.rs` | +| `start_block` | `subgraphs.subgraph_manifest` | `start_block_number`, `start_block_hash` columns; available via `StoredSubgraphManifest` in `detail.rs:542-543`, assembled into `SubgraphDeploymentEntity.start_block` | +| `earliest_block_number` | `subgraphs.deployment` | `SubgraphDeploymentEntity.earliest_block_number` | +| `graft_base`, `graft_block` | `subgraphs.deployment` | `SubgraphDeploymentEntity.graft_base`, `.graft_block` | +| `debug_fork` | `subgraphs.deployment` | `SubgraphDeploymentEntity.debug_fork` | +| `head_block` | `subgraphs.head` | `SubgraphDeploymentEntity.latest_block` | +| `entity_count` | `subgraphs.head` | `DeploymentDetail.entity_count` (i64 in DB, usize in Rust) | +| `health.*` | `subgraphs.deployment` + `subgraph_error` | `SubgraphDeploymentEntity.{failed, health, fatal_error, non_fatal_errors}` | +| `indexes` | `pg_indexes` catalog | `IndexList::load()` → `CreateIndex::to_sql()` (existing code in `dump.rs:163-179`) | +| `network` | `deployment_schemas` | `Site.network` | +| `tables.*` | `Layout.tables` | `Table.{immutable, has_causality_region}` | + +**Notes:** +- `use_bytea_prefix` is not stored in the dump. It is hardcoded to `true` in `create_deployment` (deployment.rs:1302) and will always be set to `true` on restore. +- `health` and `indexes` are point-in-time diagnostic snapshots. They are not used during restore (a restored deployment starts healthy; indexes are auto-created by `Layout::create_relational_schema()`). They are included for inspection and debugging. +- `indexes` are serialized as SQL strings using `CreateIndex::with_nsp("sgd")` + `to_sql(true, true)`, producing `CREATE INDEX CONCURRENTLY IF NOT EXISTS` statements with a normalized `sgd` namespace. +- The `manifest` fields mirror the existing `Manifest` struct in `dump.rs` (derived from `SubgraphManifestEntity`). The `schema` and `raw_yaml` fields of `SubgraphManifestEntity` are written to separate files instead. +- The `poi2$` table, when present, is a regular mutable entity table in `Layout.tables` and appears in the `tables` map like any other entity. It does not need special handling. + +The raw GraphQL schema (in `schema.graphql`) is sufficient to reconstruct the full relational layout via `InputSchema::parse(spec_version, schema, deployment_hash)` → `Layout::new()`. The `InputSchema::parse()` call requires `manifest.spec_version` for version-specific parsing logic. + +## Dump Process + +**Existing code:** There is already a metadata-only dump in `store/postgres/src/relational/dump.rs` (`Layout::dump()`) that writes `control.json`, `schema.graphql`, and `subgraph.yaml`. It is called via `DeploymentStore::dump()` (deployment_store.rs:901) which loads `Layout` + `IndexList` and passes both to `Layout::dump()`. The connection is `AsyncPgConnection` via `pool.get_permitted()`. The new parquet dump extends this to include entity data. + +The existing `Control` struct is renamed to `Metadata` and extended with the fields described above. The existing `Manifest`, `BlockPtr`, `Health`, and `Error` structs in `dump.rs` are reused and extended. + +1. Resolve the deployment (by name, hash, or sgdN) +2. Read deployment metadata from `subgraph_manifest` + `deployment` + `head` tables (via `deployment_entity()` in `detail.rs`) +3. Write `schema.graphql` and `subgraph.yaml` (existing behavior) +4. For each entity type table in `Layout.tables` (sorted by name for determinism; includes `poi2$` when present): + a. Query rows in vid order, batched (adaptive sizing like `VidBatcher`) + b. Convert PG rows directly to Arrow `RecordBatch` (no JSON intermediate) + c. Write batches to parquet file + d. Record chunk info (file path, min_vid, max_vid, row_count) +5. Dump `data_sources$` table (fixed schema, same batch approach; include all DDL columns: `vid`, `block_range`, `causality_region`, `manifest_idx`, `parent`, `id`, `param`, `context`, `done_at`). Note: `parent` and `id` are in the DDL (`private.rs:68-69`) but not in the `DataSourcesTable` struct — dumping them requires raw SQL or extending the struct. +6. Write `metadata.json` atomically (write to tmp file, rename) + +### Incremental append + +- Read existing `metadata.json` to get `max_vid` per entity type +- Query rows with `vid > max_vid` +- Write as new chunk files (`chunk_000001.parquet`, etc.) +- Update metadata atomically + +## Restore Process + +1. Read `metadata.json` and `schema.graphql` +2. Parse schema via `InputSchema::parse(manifest.spec_version, schema_text, deployment_hash)` +3. Create a `Site` entry in `deployment_schemas` (needed for the deployment to be discoverable) +4. Create deployment via `create_deployment(conn, site, DeploymentCreate { .. })` -- this populates three tables: + - `subgraphs.head` (block pointers, entity count -- initially null/0) + - `subgraphs.deployment` (deployment hash, earliest_block, graft info, health) + - `subgraphs.subgraph_manifest` (schema from file, features/spec_version/etc. from metadata) +5. Create tables via `Layout::create_relational_schema()` -- this generates DDL from the parsed schema and creates all entity tables with default indexes +6. Restore `data_sources$` table via DDL from `DataSourcesTable::new().as_ddl()` + batch-insert +7. For each entity type (including `poi2$` if present), read all parquet chunks in order, batch-insert into PG +8. Reset vid sequences to `max_vid + 1` for all entity tables and data_sources$ +9. Update `subgraphs.head` with `head_block.number`, `head_block.hash`, and `entity_count` from dump metadata + +## PG Read Strategy: OidValue-based Dynamic Columns + +Use the existing `dsl::Table::select_cols()` + `DynamicRow` pattern (see `store/postgres/src/relational/dsl.rs` and `store/postgres/src/relational/value.rs`). This already solves dynamic-schema typed extraction through the connection pool: + +1. `select_cols()` builds a typed SELECT using `DynamicSelectClause` for any set of columns +2. Results are `DynamicRow` where `OidValue` dispatches on PG OID at runtime +3. `OidValue` captures all needed types: String, Bytes, Bool, Int, Int8, BigDecimal, Timestamp, plus array variants +4. Convert `OidValue` -> Arrow `ArrayBuilder` (analogous to existing `OidValue` -> `Entity` in `FromOidRow`) + +No JSON, no separate connection. The existing `DeploymentStore::dump()` already uses `AsyncPgConnection` via `pool.get_permitted()`. + +**Block range handling:** Add `OidValue::Int4Range(Bound, Bound)` variant (OID 3904). Diesel already has `FromSql, Pg>` for `(Bound, Bound)` which parses the binary format. ~15 lines of code in `value.rs` + fix the `BLOCK_RANGE_COL` placeholder in `dsl.rs:46-49` (currently `ColumnType::Bytes`, with comment "we can't deserialize in4range"). This resolves the existing TODO at dsl.rs line 294. + +**Key existing code:** +- `dsl::Table::select_cols()` (`store/postgres/src/relational/dsl.rs:305`) +- `OidValue` enum and `FromSql` impl (`store/postgres/src/relational/value.rs:33`) +- `FromOidRow` trait for result deserialization (`value.rs:206`) +- `selected_columns()` for building column list with system columns (`dsl.rs:246`) + +### Vid continuity on restore + +Preserve original vid values. Needed for incremental consistency and simpler to implement. Reset vid sequence to `max_vid + 1` after restore. + +## Where Code Lives + +- `store/postgres/src/relational/dump.rs` -- **Existing** metadata-only dump. Contains `Manifest`, `BlockPtr`, `Error`, `Health`, `Control` structs. `Control` will be renamed to `Metadata` and extended. The existing helper structs (`Manifest`, `BlockPtr`, `Health`, `Error`) are reused. Called via `DeploymentStore::dump()` → `Layout::dump()`. +- `store/postgres/src/parquet/` -- New module for parquet read/write/schema mapping +- `node/src/manager/commands/dump.rs` -- **Existing** CLI command skeleton (resolves deployment, calls `SubgraphStore::dump()`) +- `node/src/manager/commands/restore.rs` -- New CLI command for restore +- Expose via `command_support` in `store/postgres/src/lib.rs` + +## Dependencies to Add + +- `parquet = "=57.3.0"` (same version as existing `arrow`) to workspace and `store/postgres` +- `arrow` workspace dep to `store/postgres` + +## Existing Code to Reuse + +- `Layout`, `Table`, `Column`, `ColumnType` (`store/postgres/src/relational.rs`) -- schema introspection; `poi2$` is in Layout as a mutable entity table (conditionally, when `catalog.use_poi` is true) +- `Column.is_list()` (`relational.rs:1574`) -- determines if a column is an array type (delegates to GraphQL `field_type.is_list()`) +- `DataSourcesTable` (`store/postgres/src/dynds/private.rs`) -- `data_sources$` DDL via `as_ddl()` method; note that `parent` and `id` columns are in the DDL but not in the struct's typed fields +- `VidBatcher` (`store/postgres/src/vid_batcher.rs`) -- adaptive batch iteration using PG histogram statistics +- `copy.rs` pattern -- progress reporting, batch operation lifecycle; already handles both entity tables and `data_sources$` copying +- `InputSchema::parse(spec_version, raw, id)` (`graph/src/schema/input/mod.rs:965`) -- schema reconstruction from text +- `DeploymentSearch` (`node/src/manager/deployment.rs`) -- CLI deployment resolution (supports name, Qm hash, sgdN namespace) +- `create_deployment` (`store/postgres/src/deployment.rs:1224`) -- populates `head`, `deployment`, and `subgraph_manifest` tables +- `DeploymentCreate` + `SubgraphManifestEntity` (`graph/src/data/subgraph/schema.rs:103`) -- structs needed by `create_deployment` +- `deployment_entity()` (`store/postgres/src/detail.rs`) -- reads deployment metadata into `SubgraphDeploymentEntity`; note that `start_block_*` is in the DB table (`StoredSubgraphManifest`) but only partially exposed through `SubgraphDeploymentEntity` +- `IndexList::load()` + `CreateIndex::to_sql()` (`store/postgres/src/relational/index.rs`) -- loads and serializes indexes +- Existing `dump.rs` (`store/postgres/src/relational/dump.rs`) -- metadata serialization types: `Manifest`, `BlockPtr`, `Health`, `Error`, `Control` (to be renamed `Metadata`) + +## Implementation Order + +1. Schema mapping + metadata types (foundation, unit-testable) +2. Parquet writer (dump from PG) + graphman `dump` command +3. Incremental append support +4. Parquet reader (restore to PG) + graphman `restore` command +5. Ongoing dump integration (run as part of graph-node, not just CLI) diff --git a/docs/specs/gnd-cli-expansion.md b/docs/specs/gnd-cli-expansion.md new file mode 100644 index 00000000000..7d0cf0d37fe --- /dev/null +++ b/docs/specs/gnd-cli-expansion.md @@ -0,0 +1,891 @@ +# Spec: gnd CLI Expansion + +This spec describes the expansion of `gnd` (Graph Node Dev) to include functionality currently provided by the TypeScript-based `graph-cli`. The goal is a drop-in replacement for subgraph development workflows, implemented in Rust and integrated with graph-node's existing infrastructure. + +## Goals + +- **Drop-in replacement**: Same commands, flags, output format, and exit codes as `graph-cli` +- **Identical code generation**: AssemblyScript output must be byte-for-byte identical after formatting +- **Ethereum only**: Initial scope limited to Ethereum protocol +- **Reuse graph-node internals**: Leverage existing manifest parsing, validation, IPFS client, etc. + +## Non-Goals + +- Multi-protocol support (NEAR, Cosmos, Arweave, Substreams) - future work +- Rollout/migration plan from TS CLI +- Performance optimization beyond reasonable behavior + +## Commands + +### Command Matrix + +| Command | TS CLI | gnd | Notes | +| --------- | ------ | --- | ------------------------------------------ | +| `codegen` | Yes | Yes | Generate AssemblyScript types | +| `build` | Yes | Yes | Compile to WASM | +| `deploy` | Yes | Yes | Deploy to Graph Node | +| `init` | Yes | Yes | Scaffold new subgraph | +| `add` | Yes | Yes | Add datasource to existing subgraph | +| `remove` | Yes | Yes | Unregister subgraph name | +| `create` | Yes | Yes | Register subgraph name | +| `auth` | Yes | Yes | Set deploy key | +| `publish` | Yes | Yes | Publish to decentralized network | +| `test` | Yes | Yes | Run Matchstick tests | +| `clean` | Yes | Yes | Remove build artifacts | +| `dev` | No | Yes | Run graph-node in dev mode (existing gnd) | +| `local` | Yes | No | Skipped - use existing test infrastructure | +| `node` | Yes | No | Skipped - use graphman for node management | + +### Known Differences from TS CLI + +**Commands:** + +1. **`local` command**: Not implemented. Users should use existing integration test infrastructure. +2. **`node` subcommand**: Not implemented. Use `graphman` for node management operations. +3. **`--uncrashable` flag on codegen**: Not implemented. Float Capital's uncrashable helper generation is a niche third-party feature. +4. **Debug output**: Uses `RUST_LOG` environment variable instead of `DEBUG=graph-cli:*`. + +**Code generation** (intentional, documented in verification tests): 5. **Int8 import**: gnd always imports `Int8` for simplicity, even when not used. 6. **Trailing commas**: gnd uses trailing commas in multi-line constructs. 7. **2D array accessors**: gnd uses correct `toStringMatrix()` while graph-cli has a bug using `toStringArray()` for 2D GraphQL array types. 8. **Tuple/struct component names**: gnd correctly extracts component names from raw ABI JSON, while graph-cli's ethabi parser loses nested component names for some tuple types. 9. **Subgraph data source file naming**: gnd uses the data source name (`subgraph-SourceName.ts`) instead of IPFS hash (`subgraph-QmHash.ts`) for stable import paths that don't break when source subgraphs are redeployed. + +## CLI Interface + +### Binary and Invocation + +``` +gnd [options] [arguments] +``` + +The binary name is `gnd`. All graph-cli commands become gnd subcommands. + +### Version Output + +``` +$ gnd --version +gnd 0.1.0 (graph-cli compatible: 0.98.1) +``` + +Shows both gnd version and the graph-cli version it emulates. + +### Flag Compatibility + +All flags must match the TS CLI exactly: + +- Same long names (`--output-dir`) +- Same short names (`-o`) +- Same defaults +- Same validation behavior + +Reference: Each command section below lists flags with references to TS CLI source. + +### Output Format + +Output must match TS CLI format exactly, including: + +- Spinner/progress indicators +- Success checkmarks (`✔`) +- Step descriptions +- File paths displayed +- Error formatting (information must match; exact wording may differ) + +Reference: `/packages/cli/src/command-helpers/spinner.ts` + +### Exit Codes + +Exit codes must match TS CLI behavior: + +- `0`: Success +- `1`: Error (validation, compilation, deployment failure, etc.) + +### Configuration Files + +Use same paths and formats as TS CLI: + +| File | Path | Purpose | +| -------------- | ------------------------------ | ----------------------------- | +| Auth tokens | `~/.graphprotocol/` | Deploy keys and access tokens | +| Network config | `networks.json` (project root) | Network-specific addresses | + +Reference: `/packages/cli/src/command-helpers/auth.ts` + +## Command Specifications + +### `gnd codegen` + +Generates AssemblyScript types from subgraph manifest. + +**Usage:** + +``` +gnd codegen [subgraph-manifest] +``` + +**Arguments:** + +- `subgraph-manifest`: Path to manifest file (default: `subgraph.yaml`) + +**Flags:** +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--output-dir` | `-o` | `generated/` | Output directory for generated types | +| `--skip-migrations` | | `false` | Skip subgraph migrations | +| `--watch` | `-w` | `false` | Regenerate on file changes | +| `--ipfs` | `-i` | `https://api.thegraph.com/ipfs/` | IPFS node URL | +| `--help` | `-h` | | Show help | + +**Behavior:** + +1. Load and validate manifest +2. Apply migrations (unless `--skip-migrations`) +3. Assert minimum API version (0.0.5) and graph-ts version (0.25.0) +4. Generate entity classes from GraphQL schema +5. Generate ABI bindings for each contract +6. Generate template datasource bindings +7. Format output with prettier +8. Write to output directory + +**Output Structure:** + +``` +generated/ +├── schema.ts # Entity classes +├── / +│ └── .ts # ABI bindings +├── templates/ +│ └── / +│ └── .ts # Template ABI bindings +└── subgraph-.ts # Entity types for subgraph data sources +``` + +**TS CLI References:** + +- Command: `/packages/cli/src/commands/codegen.ts` +- Type generator: `/packages/cli/src/type-generator.ts` +- Schema codegen: `/packages/cli/src/codegen/schema.ts` +- ABI codegen: `/packages/cli/src/protocols/ethereum/codegen/abi.ts` + +### `gnd build` + +Compiles subgraph to WASM. + +**Usage:** + +``` +gnd build [subgraph-manifest] +``` + +**Arguments:** + +- `subgraph-manifest`: Path to manifest file (default: `subgraph.yaml`) + +**Flags:** +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--output-dir` | `-o` | `build/` | Output directory | +| `--output-format` | `-t` | `wasm` | Output format: `wasm` or `wast` | +| `--skip-migrations` | | `false` | Skip subgraph migrations | +| `--watch` | `-w` | `false` | Rebuild on file changes | +| `--ipfs` | `-i` | | IPFS node URL (uploads if provided) | +| `--network` | | | Network to use from networks.json | +| `--network-file` | | `networks.json` | Path to networks config | +| `--help` | `-h` | | Show help | + +**Behavior:** + +1. Run codegen (unless types already exist) +2. Apply migrations (unless `--skip-migrations`) +3. Resolve network-specific values from networks.json +4. Shell out to `asc` (AssemblyScript compiler) for each mapping +5. Copy ABIs and schema to build directory +6. Copy template ABIs to build/templates// directories +7. Generate build manifest +8. Optionally upload to IPFS + +**Build Output Structure:** + +``` +build/ +├── schema.graphql +├── subgraph.yaml +├── / +│ ├── .wasm +│ └── .json # ABI +└── templates/ + └── / + ├── .wasm + └── .json # ABI +``` + +**Note:** When multiple data sources or templates share the same mapping.ts file, gnd compiles it once and copies the resulting WASM to all required output locations. + +**TS CLI References:** + +- Command: `/packages/cli/src/commands/build.ts` +- Compiler: `/packages/cli/src/compiler/index.ts` + +### `gnd deploy` + +Deploys subgraph to a Graph Node. + +**Usage:** + +``` +gnd deploy [subgraph-name] [subgraph-manifest] +``` + +**Arguments:** + +- `subgraph-name`: Name to deploy as (e.g., `user/subgraph`) +- `subgraph-manifest`: Path to manifest file (default: `subgraph.yaml`) + +**Flags:** +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--product` | | | Product: `subgraph-studio` or `hosted-service` | +| `--studio` | | `false` | Shorthand for `--product subgraph-studio` | +| `--node` | `-g` | | Graph Node URL | +| `--ipfs` | `-i` | | IPFS node URL | +| `--access-token` | | | Access token for authentication | +| `--deploy-key` | | | Deploy key (alias for access-token) | +| `--version-label` | `-l` | | Version label | +| `--headers` | | | Additional HTTP headers (JSON) | +| `--debug-fork` | | | Fork subgraph for debugging | +| `--skip-migrations` | | `false` | Skip subgraph migrations | +| `--network` | | | Network from networks.json | +| `--network-file` | | `networks.json` | Path to networks config | +| `--output-dir` | `-o` | `build/` | Build output directory | +| `--help` | `-h` | | Show help | + +**Deploy Targets:** + +- Local Graph Node (via `--node` and `--ipfs`) +- Subgraph Studio (`--product subgraph-studio` or `--studio`) +- Hosted Service (`--product hosted-service`) +- Decentralized network (via `publish` command) + +**Behavior:** + +1. Build subgraph (runs build command) +2. Upload build artifacts to IPFS +3. Send deployment request to Graph Node via JSON-RPC + +**TS CLI References:** + +- Command: `/packages/cli/src/commands/deploy.ts` + +### `gnd init` + +Scaffolds a new subgraph project. + +**Usage:** + +``` +gnd init [directory] +``` + +**Arguments:** + +- `directory`: Directory to create subgraph in + +**Flags:** +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--protocol` | | | Protocol: `ethereum` | +| `--product` | | | Product for deployment | +| `--studio` | | `false` | Initialize for Subgraph Studio | +| `--from-contract` | | | Contract address to generate from | +| `--from-example` | | | Example subgraph to clone | +| `--contract-name` | | | Name for the contract | +| `--index-events` | | `false` | Index all contract events | +| `--start-block` | | | Start block for indexing | +| `--network` | | `mainnet` | Network name | +| `--abi` | | | Path to ABI file | +| `--spkg` | | | Path to Substreams package | +| `--allow-simple-name` | | `false` | Allow simple subgraph names | +| `--help` | `-h` | | Show help | + +**Example Subgraphs:** + +The `--from-example ` flag clones from `https://github.com/graphprotocol/graph-tooling/tree/main/examples`. +The example name is **required** - no default is provided. Available examples include: +`ethereum-gravatar`, `aggregations`, `ethereum-basic-event-handlers`, `substreams-powered-subgraph`, etc. +Legacy name `ethereum/gravatar` is automatically converted to `ethereum-gravatar`. + +**Behavior:** + +1. Prompt for missing information (protocol, network, contract, etc.) +2. Fetch ABI from Etherscan/Sourcify if `--from-contract` and no `--abi` +3. Generate scaffold: + - `subgraph.yaml` manifest + - `schema.graphql` with entities for events + - `src/mapping.ts` with event handlers + - `package.json` with dependencies + - `tsconfig.json` + - ABIs directory +4. Optionally initialize git repository +5. Install dependencies + +**External APIs:** + +- Etherscan API: Fetch verified contract ABIs +- Sourcify API: Fetch verified contract ABIs (fallback) +- Network registry: `@pinax/graph-networks-registry` for chain configuration + +**TS CLI References:** + +- Command: `/packages/cli/src/commands/init.ts` +- Scaffold: `/packages/cli/src/scaffold/index.ts` +- Schema generation: `/packages/cli/src/scaffold/schema.ts` +- Mapping generation: `/packages/cli/src/scaffold/mapping.ts` +- Etherscan client: `/packages/cli/src/command-helpers/contracts.ts` + +### `gnd add` + +Adds a new datasource to an existing subgraph. + +**Usage:** + +``` +gnd add
[subgraph-manifest] +``` + +**Arguments:** + +- `address`: Contract address +- `subgraph-manifest`: Path to manifest file (default: `subgraph.yaml`) + +**Flags:** +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--abi` | | | Path to ABI file | +| `--contract-name` | | | Name for the contract | +| `--merge-entities` | | `false` | Merge with existing entities | +| `--network-file` | | `networks.json` | Path to networks config | +| `--start-block` | | | Start block | +| `--help` | `-h` | | Show help | + +**TS CLI References:** + +- Command: `/packages/cli/src/commands/add.ts` + +### `gnd create` + +Registers a subgraph name with a Graph Node. + +**Usage:** + +``` +gnd create +``` + +**Arguments:** + +- `subgraph-name`: Name to register + +**Flags:** +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--node` | `-g` | | Graph Node URL | +| `--access-token` | | | Access token | +| `--help` | `-h` | | Show help | + +**TS CLI References:** + +- Command: `/packages/cli/src/commands/create.ts` + +### `gnd remove` + +Unregisters a subgraph name from a Graph Node. + +**Usage:** + +``` +gnd remove +``` + +**Arguments:** + +- `subgraph-name`: Name to unregister + +**Flags:** +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--node` | `-g` | | Graph Node URL | +| `--access-token` | | | Access token | +| `--help` | `-h` | | Show help | + +**TS CLI References:** + +- Command: `/packages/cli/src/commands/remove.ts` + +### `gnd auth` + +Sets the deploy key for a Graph Node. + +**Usage:** + +``` +gnd auth +``` + +**Arguments:** + +- `deploy-key`: Deploy key to store + +**Flags:** +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--product` | | | Product: `subgraph-studio` or `hosted-service` | +| `--studio` | | `false` | Shorthand for subgraph-studio | +| `--help` | `-h` | | Show help | + +**Behavior:** +Stores deploy key in `~/.graphprotocol/` for later use by deploy/publish commands. + +**TS CLI References:** + +- Command: `/packages/cli/src/commands/auth.ts` +- Auth helpers: `/packages/cli/src/command-helpers/auth.ts` + +### `gnd publish` + +Publishes a subgraph to The Graph's decentralized network. + +**Usage:** + +``` +gnd publish [subgraph-manifest] +``` + +**Arguments:** + +- `subgraph-manifest`: Path to manifest file (default: `subgraph.yaml`) + +**Flags:** +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--subgraph-id` | | | Subgraph ID to publish to | +| `--ipfs` | `-i` | | IPFS node URL | +| `--protocol-network` | | | Protocol network (e.g., `arbitrum-one`) | +| `--help` | `-h` | | Show help | + +**TS CLI References:** + +- Command: `/packages/cli/src/commands/publish.ts` + +### `gnd test` + +Runs Matchstick tests for the subgraph. + +**Usage:** + +``` +gnd test [datasource] +``` + +**Arguments:** + +- `datasource`: Specific datasource to test (optional) + +**Flags:** +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--coverage` | `-c` | `false` | Run with coverage | +| `--docker` | `-d` | `false` | Run in Docker container | +| `--force` | `-f` | `false` | Force recompilation | +| `--logs` | `-l` | `false` | Show logs | +| `--recompile` | `-r` | `false` | Recompile before testing | +| `--version` | `-v` | | Matchstick version | +| `--help` | `-h` | | Show help | + +**Behavior:** + +1. Download Matchstick binary (if not present) +2. Shell out to Matchstick with appropriate flags +3. Report test results + +**TS CLI References:** + +- Command: `/packages/cli/src/commands/test.ts` + +### `gnd clean` + +Removes build artifacts and generated files. + +**Usage:** + +``` +gnd clean +``` + +**Flags:** +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--codegen-dir` | | `generated/` | Codegen output directory | +| `--build-dir` | | `build/` | Build output directory | +| `--help` | `-h` | | Show help | + +**Behavior:** +Removes `generated/` and `build/` directories (or custom paths if specified). + +**TS CLI References:** + +- Command: `/packages/cli/src/commands/clean.ts` + +### `gnd dev` + +Runs graph-node in development mode with file watching. + +This is the existing `gnd` functionality, preserved as a subcommand. The implementation can be adjusted to fit the new subcommand structure. + +**Usage:** + +``` +gnd dev [options] +``` + +**Flags:** +Preserve existing gnd flags, adjusted as needed for subcommand structure. + +## Code Generation + +Code generation is the most complex component and must produce byte-for-byte identical output to the TS CLI (after prettier formatting). + +### Generated File Types + +#### 1. Entity Classes (`schema.ts`) + +Generated from GraphQL schema. Each entity type becomes an AssemblyScript class with: + +- Constructor +- Static `load(id)` method +- Static `loadInBlock(id)` method +- `save()` method +- Getters/setters for each field +- Proper type mappings (GraphQL → AssemblyScript) + +**TS CLI Reference:** `/packages/cli/src/codegen/schema.ts` + +#### 2. ABI Bindings (`.ts`) + +Generated from contract ABI. Includes: + +- Event classes with typed parameters +- Function call result classes +- Contract class with typed call methods +- Proper Ethereum type mappings + +**Tuple/Struct Handling:** +When an event or function has tuple parameters with named components (e.g., a struct), gnd generates proper struct classes with named getters: + +```typescript +class AssetTransfer__ParamsAssetStruct extends ethereum.Tuple { + get addr(): Address { + return this[0].toAddress(); + } + get amount(): BigInt { + return this[1].toBigInt(); + } + get active(): boolean { + return this[2].toBoolean(); + } +} +``` + +gnd parses the raw ABI JSON to extract component names, which ethabi loses during parsing. + +**TS CLI Reference:** `/packages/cli/src/protocols/ethereum/codegen/abi.ts` + +#### 3. Template Bindings + +Generated for template datasources with the same structure as ABI bindings. + +**TS CLI Reference:** `/packages/cli/src/codegen/template.ts` + +#### 4. Subgraph Data Source Bindings (`subgraph-.ts`) + +Generated for `kind: subgraph` data sources. When a manifest contains a subgraph data source: + +```yaml +dataSources: + - kind: subgraph + name: SourceSubgraph + source: + address: "QmRWTEejPDDwALaquFGm6X2GBbbh5osYDXwCRRkoZ6KQhb" +``` + +gnd will: + +1. Validate that all subgraph data source names are unique (error if duplicates found) +2. Fetch the referenced subgraph's manifest from IPFS +3. Extract and fetch the schema from the manifest +4. Generate entity types (without store methods) to `generated/subgraph-{DataSourceName}.ts` + +**Note:** gnd uses the data source name (e.g., `SourceSubgraph`) for the generated filename rather than the IPFS hash. This produces stable import paths like `import { Entity } from '../generated/subgraph-SourceSubgraph'` that don't break when the source subgraph is redeployed with a new hash. This differs from graph-cli which uses the IPFS hash. + +The generated types include entity classes with getters for all fields, but no `save()`, `load()`, or `loadInBlock()` methods since these are read-only types from the source subgraph. + +**TS CLI Reference:** `/packages/cli/src/type-generator.ts` lines 72-136 + +### Type Mappings + +#### GraphQL → AssemblyScript + +| GraphQL | AssemblyScript | +| ---------------- | -------------- | +| `ID` | `string` | +| `String` | `string` | +| `Int` | `i32` | +| `BigInt` | `BigInt` | +| `BigDecimal` | `BigDecimal` | +| `Bytes` | `Bytes` | +| `Boolean` | `boolean` | +| `[T]` | `Array` | +| Entity reference | `string` (ID) | + +**TS CLI Reference:** `/packages/cli/src/codegen/schema.ts` (look for type mapping functions) + +#### Ethereum ABI → AssemblyScript + +| Solidity | AssemblyScript | +| --------- | --------------- | +| `address` | `Address` | +| `bool` | `boolean` | +| `bytes` | `Bytes` | +| `bytesN` | `Bytes` | +| `intN` | `BigInt` | +| `uintN` | `BigInt` | +| `string` | `string` | +| `T[]` | `Array` | +| tuple | Generated class | + +**TS CLI Reference:** `/packages/cli/src/protocols/ethereum/codegen/abi.ts` + +### API Version Handling + +Different `apiVersion` values in the manifest affect code generation. gnd must support all versions that the TS CLI supports. + +**TS CLI Reference:** `/packages/cli/src/codegen/` (version-specific logic throughout) + +### Formatting + +All generated code must be formatted with prettier before writing: + +- Shell out to `prettier` with same configuration as TS CLI +- Parser: `typescript` + +## Migrations + +Migrations update older manifest formats to newer versions. gnd must implement all migrations that TS CLI supports. + +**Migration Chain:** + +``` +0.0.1 → 0.0.2 → 0.0.3 → 0.0.4 → 0.0.5 → ... → current +``` + +**TS CLI Reference:** `/packages/cli/src/migrations/` + +Each migration is a transformation function that: + +1. Checks manifest version +2. Applies necessary changes +3. Updates version number + +## External Dependencies + +### Runtime Dependencies (shell out) + +| Tool | Purpose | Required | +| ------------ | ----------------------- | ------------- | +| `asc` | AssemblyScript compiler | For `build` | +| `prettier` | Code formatting | For `codegen` | +| `matchstick` | Test runner | For `test` | + +### Network APIs + +| API | Purpose | +| -------------------------------- | --------------------------------------- | +| Etherscan | Fetch verified contract ABIs | +| Sourcify | Fetch verified contract ABIs (fallback) | +| `@pinax/graph-networks-registry` | Network configuration (chain IDs, etc.) | + +The network registry should be fetched at runtime to get current network configurations. + +### graph-node Reuse + +| Component | graph-node Location | Purpose | +| ------------------- | ------------------------------------- | --------------------------- | +| Manifest parsing | `graph/src/data/subgraph/` | Load subgraph.yaml | +| Manifest validation | `graph/src/data/subgraph/` | Validate manifest structure | +| GraphQL schema | `graph/src/schema/input/` | Parse schema.graphql | +| IPFS client | `graph/src/ipfs/` | Upload to IPFS | +| Link resolver | `graph/src/components/link_resolver/` | Resolve file references | +| File watcher | `gnd/src/watcher.rs` | Watch mode | + +Refactor graph-node components as needed to make them reusable. + +## Module Structure + +``` +gnd/src/ +├── main.rs # Entry point, clap setup +├── lib.rs +├── formatter.rs # Prettier integration for code formatting +├── prompt.rs # Interactive prompts (network selection, etc.) +├── watcher.rs # File watcher for --watch modes +├── commands/ +│ ├── mod.rs +│ ├── add.rs +│ ├── auth.rs +│ ├── build.rs +│ ├── clean.rs +│ ├── codegen.rs +│ ├── create.rs +│ ├── deploy.rs +│ ├── dev.rs # Existing gnd functionality +│ ├── init.rs +│ ├── publish.rs +│ ├── remove.rs +│ └── test.rs +├── codegen/ +│ ├── mod.rs +│ ├── abi.rs # ABI binding generation +│ ├── schema.rs # Entity class generation +│ ├── template.rs # Template binding generation +│ ├── types.rs # Type conversion utilities +│ └── typescript.rs # AST builders for TypeScript/AssemblyScript +├── scaffold/ +│ ├── mod.rs +│ ├── manifest.rs # Generate subgraph.yaml +│ ├── mapping.rs # Generate mapping.ts +│ └── schema.rs # Generate schema.graphql +├── migrations/ +│ ├── mod.rs +│ ├── api_version.rs # API version migrations +│ ├── spec_version.rs # Spec version migrations +│ └── versions.rs # Version definitions +├── compiler/ +│ ├── mod.rs +│ └── asc.rs # Shell out to asc +├── services/ +│ ├── mod.rs +│ ├── contract.rs # ABI fetching (Etherscan, Blockscout, Sourcify, registry) +│ ├── graph_node.rs # Graph Node JSON-RPC client +│ └── ipfs.rs # IPFS client for uploads +├── config/ +│ ├── mod.rs +│ └── networks.rs # networks.json handling +└── output/ + ├── mod.rs + └── spinner.rs # Progress/spinner output +``` + +## Testing + +### Current Status + +- **166 unit tests** passing (`cargo test -p gnd --lib`) +- **12 codegen verification tests** passing (`cargo test -p gnd --test codegen_verification`) +- **7 CLI command tests** passing (`cargo test -p gnd --test cli_commands`) +- Test fixtures from graph-cli validation tests in `gnd/tests/fixtures/codegen_verification/` + +### Test Strategy + +1. **Unit tests**: Test individual functions (type mappings, migrations, etc.) +2. **Snapshot tests**: Compare generated output against TS CLI output +3. **Integration tests**: End-to-end command execution + +### Test Corpus + +Use the same test corpus as graph-cli. Tests must cover at least everything the TS CLI tests cover. + +**TS CLI Test Location:** `/packages/cli/tests/` + +### Codegen Verification Tests + +Located in `gnd/tests/codegen_verification.rs`, these tests verify compatibility with graph-cli: + +1. Copy fixture to temp directory (excluding `generated/`) +2. Run `gnd codegen` on the fixture +3. Compare output against expected `generated/` directory +4. Allow known differences (Int8 import, trailing commas, 2D array fix) + +Fixtures can be regenerated with `./tests/fixtures/regenerate.sh`. + +### Edge Cases + +When edge case bugs are discovered in the TS CLI, gnd should fix them rather than replicate them. Document any behavioral differences that result from bug fixes. + +### CLI Integration Tests + +Located in `tests/tests/gnd_cli_tests.rs`, these tests verify gnd works as a drop-in replacement for graph-cli by running the integration test suite with `GRAPH_CLI` environment variable pointing to the gnd binary. + +**How it works:** + +- The integration test infrastructure uses `CONFIG.graph_cli` for deployment commands +- Setting `GRAPH_CLI=../target/debug/gnd` makes tests use gnd instead of graph-cli +- `Subgraph::deploy()` calls `gnd codegen`, `gnd create`, `gnd deploy` + +**Commands tested:** + +- `gnd codegen` - Generate AssemblyScript types +- `gnd create` - Register subgraph name with Graph Node +- `gnd deploy` - Deploy subgraph to Graph Node + +**Running:** + +```bash +just test-gnd-cli +``` + +### Standalone Command Tests + +Located in `gnd/tests/cli_commands.rs`, these tests verify commands that don't require a running Graph Node: + +**Commands tested:** + +- `gnd init` - Scaffold generation (--from-example, --from-contract, --from-subgraph) +- `gnd add` - Add datasource to existing subgraph +- `gnd build` - WASM compilation + +**Running:** + +```bash +just test-gnd-commands +``` + +## Dependencies to Add + +### Cargo.toml additions + +```toml +[dependencies] +# CLI framework (already present) +clap = { version = "...", features = ["derive"] } + +# Interactive prompts (for init) +inquire = "..." + +# HTTP client (for Etherscan, Sourcify, registry) +reqwest = { version = "...", features = ["json"] } + +# Progress/spinner output +indicatif = "..." + +# Template rendering (for scaffold) +minijinja = "..." + +# JSON handling +serde_json = "..." +``` + +## Open Questions + +None at this time. All major decisions have been made. + +## References + +- TS CLI repository: https://github.com/graphprotocol/graph-tooling +- TS CLI source: `/packages/cli/src/` +- Local checkout: `/home/lutter/code/subgraphs/graph-cli` +- Original gnd expansion plan: `/LOCAL/plans/gnd-cli-expansion.md` diff --git a/docs/specs/runner-refactor.md b/docs/specs/runner-refactor.md new file mode 100644 index 00000000000..ee36ebe3342 --- /dev/null +++ b/docs/specs/runner-refactor.md @@ -0,0 +1,380 @@ +# Subgraph Runner Simplification Spec + +## Problem Statement + +`core/src/subgraph/runner.rs` is complex and hard to modify. Key issues: + +1. **Duplicated trigger processing** (lines 616-656 vs 754-790): Nearly identical loops +2. **Control flow confusion**: Nested loops in `run_inner` with 6 exit paths +3. **State management**: Mixed patterns (mutable fields, `std::mem::take`, drains) +4. **`process_block` monolith**: ~260 lines handling triggers, dynamic DS, offchain, persistence + +## Design Decisions + +| Aspect | Decision | +|--------|----------| +| Control flow | Enum-based FSM for full runner lifecycle | +| Trigger processing | New `TriggerRunner` component | +| Block processing | Pipeline with explicitly defined stages | +| State management | Mutable accumulator with checkpoints | +| Breaking changes | Moderate (internal APIs can change) | + +## Target Architecture + +### 1. Runner State Machine + +Replace nested loops in `run_inner` with an explicit enum FSM covering the full lifecycle: + +```rust +enum RunnerState { + /// Initial state, ready to start block stream + Initializing, + + /// Block stream active, waiting for next event + AwaitingBlock { + block_stream: Cancelable>>, + }, + + /// Processing a block through the pipeline + ProcessingBlock { + block: BlockWithTriggers, + cursor: FirehoseCursor, + }, + + /// Handling a revert event + Reverting { + to_ptr: BlockPtr, + cursor: FirehoseCursor, + }, + + /// Restarting block stream (new filters, store restart, etc.) + Restarting { + reason: RestartReason, + }, + + /// Terminal state + Stopped { + reason: StopReason, + }, +} + +enum RestartReason { + DynamicDataSourceCreated, + DataSourceExpired, + StoreError, + PossibleReorg, +} + +enum StopReason { + MaxEndBlockReached, + Canceled, + Unassigned, +} +``` + +The main loop becomes: + +```rust +async fn run(mut self) -> Result<(), SubgraphRunnerError> { + loop { + self.state = match self.state { + RunnerState::Initializing => self.initialize().await?, + RunnerState::AwaitingBlock { stream } => self.await_block(stream).await?, + RunnerState::ProcessingBlock { block, cursor } => { + self.process_block(block, cursor).await? + } + RunnerState::Reverting { to_ptr, cursor } => { + self.handle_revert(to_ptr, cursor).await? + } + RunnerState::Restarting { reason } => self.restart(reason).await?, + RunnerState::Stopped { reason } => return self.finalize(reason).await, + }; + } +} +``` + +### 2. Block Processing Pipeline + +Replace the `process_block` monolith with explicit stages: + +```rust +/// Pipeline stages for block processing +mod pipeline { + pub struct TriggerMatchStage; + pub struct TriggerExecuteStage; + pub struct DynamicDataSourceStage; + pub struct OffchainTriggerStage; + pub struct PersistStage; +} + +/// Result of block processing pipeline +pub struct BlockProcessingResult { + pub action: Action, + pub block_state: BlockState, +} + +impl SubgraphRunner { + async fn process_block( + &mut self, + block: BlockWithTriggers, + cursor: FirehoseCursor, + ) -> Result { + let block = Arc::new(block.block); + let triggers = block.trigger_data; + + // Stage 1: Match triggers to hosts and decode + let runnables = self.match_triggers(&block, triggers).await?; + + // Stage 2: Execute triggers (unified for initial + dynamic DS) + let mut block_state = self.execute_triggers(&block, runnables).await?; + + // Checkpoint before dynamic DS processing + let checkpoint = block_state.checkpoint(); + + // Stage 3: Process dynamic data sources (loop until none created) + block_state = self.process_dynamic_data_sources(&block, &cursor, block_state).await?; + + // Stage 4: Handle offchain triggers + let offchain_result = self.process_offchain_triggers(&block, &mut block_state).await?; + + // Stage 5: Persist to store + self.persist_block_state(block_state, offchain_result).await?; + + // Determine next state + Ok(self.determine_next_state()) + } +} +``` + +### 3. Error Handling Strategy + +Consolidate scattered error handling into explicit classification: + +```rust +/// Unified error classification for trigger processing +pub enum ProcessingErrorKind { + /// Stop processing, persist PoI only + Deterministic(anyhow::Error), + /// Retry with backoff, attempt to unfail + NonDeterministic(anyhow::Error), + /// Restart block stream cleanly (don't persist) + PossibleReorg(anyhow::Error), +} + +impl ProcessingError { + /// Classify error once, use classification throughout + pub fn kind(&self) -> ProcessingErrorKind { ... } + + /// Whether this error should stop processing the current block + pub fn should_stop_processing(&self) -> bool { + matches!(self.kind(), ProcessingErrorKind::Deterministic(_)) + } + + /// Whether this error requires a clean restart + pub fn should_restart(&self) -> bool { + matches!(self.kind(), ProcessingErrorKind::PossibleReorg(_)) + } + + /// Whether this error is retryable with backoff + pub fn is_retryable(&self) -> bool { + matches!(self.kind(), ProcessingErrorKind::NonDeterministic(_)) + } +} +``` + +**Key Invariant (must be preserved):** +``` +Deterministic → Stop processing block, persist PoI only +NonDeterministic → Retry with backoff +PossibleReorg → Restart cleanly (don't persist) +``` + +Currently this logic is scattered across: +- `process_block` early return for PossibleReorg (line 664-677) +- Dynamic data sources error mapping (line 792-802) +- `transact_block_state` (line 405-430) +- `handle_offchain_triggers` (line 1180-1190) + +Consolidating into helper methods eliminates these scattered special cases. + +### 4. TriggerRunner Component + +Extract trigger execution into a dedicated component: + +```rust +/// Handles matching, decoding, and executing triggers +pub struct TriggerRunner<'a, C: Blockchain, T: RuntimeHostBuilder> { + decoder: &'a Decoder, + processor: &'a dyn TriggerProcessor, + logger: &'a Logger, + metrics: &'a SubgraphMetrics, + debug_fork: &'a Option>, + instrument: bool, +} + +impl<'a, C, T> TriggerRunner<'a, C, T> +where + C: Blockchain, + T: RuntimeHostBuilder, +{ + /// Execute triggers against hosts, accumulating state + pub async fn execute( + &self, + block: &Arc, + runnables: Vec>, + mut block_state: BlockState, + proof_of_indexing: &SharedProofOfIndexing, + causality_region: &PoICausalityRegion, + ) -> Result { + for runnable in runnables { + block_state = self.processor + .process_trigger( + self.logger, + runnable.hosted_triggers, + block, + block_state, + proof_of_indexing, + causality_region, + self.debug_fork, + self.metrics, + self.instrument, + ) + .await + .map_err(|e| e.add_trigger_context(&runnable.trigger))?; + } + Ok(block_state) + } +} +``` + +This eliminates the duplicated loops (lines 616-656 and 754-790). + +### 5. State Management with Checkpoints + +**Explicit Input/Output Types for Pipeline Stages:** + +```rust +/// Input to trigger processing - makes dependencies explicit +struct TriggerProcessingContext<'a> { + block: &'a Arc, + proof_of_indexing: &'a SharedProofOfIndexing, + causality_region: &'a PoICausalityRegion, +} + +/// Output from trigger processing - makes results explicit +struct TriggerProcessingResult { + block_state: BlockState, + restart_needed: bool, +} +``` + +**Add checkpoint capability to `BlockState` for rollback scenarios:** + +```rust +impl BlockState { + /// Create a lightweight checkpoint for rollback + pub fn checkpoint(&self) -> BlockStateCheckpoint { + BlockStateCheckpoint { + created_data_sources_count: self.created_data_sources.len(), + persisted_data_sources_count: self.persisted_data_sources.len(), + // Note: entity_cache changes cannot be easily checkpointed + // Rollback clears the cache (acceptable per current behavior) + } + } + + /// Restore state to checkpoint (partial rollback) + pub fn restore(&mut self, checkpoint: BlockStateCheckpoint) { + self.created_data_sources.truncate(checkpoint.created_data_sources_count); + self.persisted_data_sources.truncate(checkpoint.persisted_data_sources_count); + // Entity cache is cleared on rollback (matches current behavior) + } +} +``` + +### 6. File Structure + +``` +core/src/subgraph/ +├── runner.rs # Main SubgraphRunner with FSM +├── runner/ +│ ├── mod.rs +│ ├── state.rs # RunnerState enum and transitions +│ ├── pipeline.rs # Pipeline stage definitions +│ └── trigger_runner.rs # TriggerRunner component +├── context.rs # IndexingContext (unchanged) +├── inputs.rs # IndexingInputs (unchanged) +└── state.rs # IndexingState (unchanged) +``` + +## Deferred Concerns + +### Fishy Block Refetch (Preserve Behavior) + +The TODO at lines 721-729 notes unclear behavior around block refetching in the dynamic DS loop. The restructure preserves this behavior without attempting to fix it. Investigate separately. + +## Key Interfaces + +### RunnerState Transitions + +``` +Initializing ──────────────────────────────────┐ + │ │ + v │ +AwaitingBlock ◄─────────────────────────────────┤ + │ │ + ├── ProcessBlock event ──► ProcessingBlock │ + │ │ │ + │ ├── success ┼──► AwaitingBlock + │ │ │ + │ └── restart ┼──► Restarting + │ │ + ├── Revert event ──────────► Reverting ────┤ + │ │ + ├── Error ─────────────────► Restarting ───┤ + │ │ + └── Cancel/MaxBlock ───────► Stopped │ + │ +Restarting ─────────────────────────────────────┘ +``` + +### Pipeline Data Flow + +``` +BlockWithTriggers + │ + v +┌──────────────────┐ +│ TriggerMatchStage│ ─► Vec +└──────────────────┘ + │ + v +┌────────────────────┐ +│ TriggerExecuteStage│ ─► BlockState (mutated) +└────────────────────┘ + │ + v (loop while has_created_data_sources) +┌─────────────────────────┐ +│ DynamicDataSourceStage │ ─► BlockState (mutated), new hosts added +└─────────────────────────┘ + │ + v +┌─────────────────────┐ +│ OffchainTriggerStage│ ─► offchain_mods, processed_ds +└─────────────────────┘ + │ + v +┌─────────────┐ +│ PersistStage│ ─► Store transaction +└─────────────┘ +``` + +## Verification + +After implementation, verify: + +1. **Unit tests pass**: `just test-unit` +2. **Runner tests pass**: `just test-runner` +3. **Lint clean**: `just lint` (zero warnings) +4. **Build succeeds**: `just check --release` + +For behavioral verification, the existing runner tests should catch regressions. No new integration tests required for a refactor that preserves behavior. diff --git a/flake.lock b/flake.lock index d8c4d140a34..7d69cf5ac8c 100644 --- a/flake.lock +++ b/flake.lock @@ -60,16 +60,17 @@ "nixpkgs": "nixpkgs" }, "locked": { - "lastModified": 1756199436, - "narHash": "sha256-tkLoAk2BkFIwxp9YrtcUeWugGQjiubbiZx/YGGnVrz4=", + "lastModified": 1760001025, + "narHash": "sha256-j/jPNlW2W4zwV2rSqgw5ts1VoTXVe5n3CNXFvtQhK3Q=", "owner": "shazow", "repo": "foundry.nix", - "rev": "2d28ea426c27166c8169e114eff4a5adcc00548d", + "rev": "b7adb89167832516589c899addcd25ca2a78dcfe", "type": "github" }, "original": { "owner": "shazow", "repo": "foundry.nix", + "rev": "b7adb89167832516589c899addcd25ca2a78dcfe", "type": "github" } }, diff --git a/flake.nix b/flake.nix index e0e7e6aeef9..933a35ef04c 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,7 @@ { inputs = { nixpkgs.url = "github:nixos/nixpkgs/nixpkgs-unstable"; - foundry.url = "github:shazow/foundry.nix"; + foundry.url = "github:shazow/foundry.nix/b7adb89167832516589c899addcd25ca2a78dcfe"; fenix = { url = "github:nix-community/fenix"; inputs.nixpkgs.follows = "nixpkgs"; @@ -90,6 +90,7 @@ inherit port dataDir; initialScript = { before = '' + CREATE ROLE postgres WITH LOGIN; CREATE USER \"${user}\" WITH PASSWORD '${password}' SUPERUSER; ''; }; @@ -140,6 +141,11 @@ database = "graph-test"; }; + # Set PGUSER so psql connects as the OS user (matching initdb's superuser) + settings.processes."postgres-unit-init".environment = { + PGUSER = builtins.getEnv "USER"; + }; + services.ipfs."ipfs-unit" = { enable = true; dataDir = "./.data/unit/ipfs"; @@ -172,6 +178,11 @@ database = "graph-node"; }; + # Set PGUSER so psql connects as the OS user (matching initdb's superuser) + settings.processes."postgres-integration-init".environment = { + PGUSER = builtins.getEnv "USER"; + }; + services.ipfs."ipfs-integration" = { enable = true; dataDir = "./.data/integration/ipfs"; @@ -186,7 +197,10 @@ timestamp = 1743944919; gasLimit = 100000000000; baseFee = 1; - blockTime = 2; + blockTime = null; + state = "./.data/integration/anvil/state.json"; + stateInterval = 30; + preserveHistoricalStates = true; }; }; }; diff --git a/gnd/Cargo.toml b/gnd/Cargo.toml index f8748246d64..754949a4722 100644 --- a/gnd/Cargo.toml +++ b/gnd/Cargo.toml @@ -7,25 +7,86 @@ edition.workspace = true name = "gnd" path = "src/main.rs" +[[test]] +name = "cli_commands" +path = "tests/cli_commands.rs" + +[[test]] +name = "codegen_verification" +path = "tests/codegen_verification.rs" + +[[test]] +name = "gnd_test" +path = "tests/gnd_test.rs" + [dependencies] # Core graph dependencies graph = { path = "../graph" } +graph-chain-ethereum = { path = "../chain/ethereum" } graph-core = { path = "../core" } graph-node = { path = "../node" } +graph-graphql = { path = "../graphql" } +graph-store-postgres = { path = "../store/postgres" } + +# Test command dependencies +hex = "0.4" +async-trait = { workspace = true } +tower = { workspace = true } # Direct dependencies from current dev.rs anyhow = { workspace = true } clap = { workspace = true } -env_logger = "0.11.8" -git-testament = "0.2" -lazy_static = "1.5.0" +clap_complete = { workspace = true } +env_logger = { workspace = true } +git-testament = { workspace = true } +lazy_static = { workspace = true } +serde = { workspace = true } tokio = { workspace = true } -tokio-util.workspace = true +tokio-util = { workspace = true } # File watching notify = "8.2.0" globset = "0.4.18" + +# Config and auth +url = { workspace = true } +serde_json = { workspace = true } pq-sys = { version = "0.7.5", features = ["bundled"] } +# HTTP client for Graph Node API +reqwest = { workspace = true } +thiserror = { workspace = true } + +# Console output +similar = "3" +indicatif = { workspace = true } +console = { workspace = true } + +# Code generation +graphql-tools = { workspace = true } +regex = { workspace = true } +serde_yaml = { workspace = true } +Inflector = { workspace = true } + +# Migrations +semver = { workspace = true } + +# Build command +sha1 = "0.11" +wasmparser = { workspace = true } + +# Interactive prompts +inquire = "0.9" + +# Temp directories for init +tempfile = "3" + +# Browser opening for publish command +open = "5" + [target.'cfg(unix)'.dependencies] pgtemp = { git = "https://github.com/graphprotocol/pgtemp", branch = "initdb-args" } + +[dev-dependencies] +tempfile = "3" +walkdir = "2" diff --git a/gnd/README.md b/gnd/README.md new file mode 100644 index 00000000000..375e1ef0e01 --- /dev/null +++ b/gnd/README.md @@ -0,0 +1,524 @@ +# gnd - Graph Node Dev CLI + +A drop-in replacement for `graph-cli` written in Rust. `gnd` provides all the subgraph development commands with identical output, flags, and behavior. + +## Installation + +Build from source: + +```bash +cargo build -p gnd --release +``` + +The binary will be at `target/release/gnd`. + +## Quick Start + +```bash +# Create a new subgraph from a contract +gnd init --from-contract 0x1234... --network mainnet my-subgraph + +# Generate AssemblyScript types +gnd codegen + +# Build the subgraph +gnd build + +# Deploy to a local Graph Node +gnd create --node http://localhost:8020 my-name/my-subgraph +gnd deploy --node http://localhost:8020 --ipfs http://localhost:5001 -l v0.0.1 my-name/my-subgraph + +# Or deploy to Subgraph Studio +gnd auth YOUR_DEPLOY_KEY +gnd deploy -l v0.0.1 my-name/my-subgraph +``` + +## Commands + +### `gnd init` + +Create a new subgraph with basic scaffolding. + +```bash +gnd init [SUBGRAPH_NAME] [DIRECTORY] +``` + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--protocol` | | Protocol: `ethereum`, `near`, `cosmos`, `arweave`, `substreams` | +| `--from-contract` | | Create from an existing contract address | +| `--from-example` | | Create from an example subgraph template | +| `--from-subgraph` | | Create from an existing deployed subgraph | +| `--contract-name` | | Name for the contract (with `--from-contract`) | +| `--index-events` | | Index all contract events as entities | +| `--network` | | Network the contract is deployed to | +| `--start-block` | | Block number to start indexing from | +| `--abi` | | Path to the contract ABI file | +| `--node` | `-g` | Graph Node URL | +| `--ipfs` | `-i` | IPFS node URL | +| `--skip-install` | | Skip installing npm dependencies | +| `--skip-git` | | Skip initializing a Git repository | + +**Examples:** + +```bash +# Interactive mode (prompts for all options) +gnd init + +# From contract with ABI fetched from Etherscan +gnd init --from-contract 0x1234... --network mainnet my-subgraph + +# From contract with local ABI +gnd init --from-contract 0x1234... --abi ./MyContract.json my-subgraph + +# From an existing deployed subgraph +gnd init --from-subgraph QmHash... my-subgraph +``` + +### `gnd codegen` + +Generate AssemblyScript types from the subgraph manifest. + +```bash +gnd codegen [MANIFEST] +``` + +**Arguments:** +- `MANIFEST`: Path to subgraph manifest (default: `subgraph.yaml`) + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--output-dir` | `-o` | Output directory (default: `generated/`) | +| `--skip-migrations` | | Skip manifest migrations | +| `--watch` | `-w` | Regenerate on file changes | +| `--ipfs` | `-i` | IPFS node URL | + +**Examples:** + +```bash +# Generate types +gnd codegen + +# Generate to custom directory +gnd codegen -o src/generated/ + +# Watch mode +gnd codegen --watch +``` + +### `gnd build` + +Compile the subgraph to WASM. + +```bash +gnd build [MANIFEST] +``` + +**Arguments:** +- `MANIFEST`: Path to subgraph manifest (default: `subgraph.yaml`) + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--output-dir` | `-o` | Output directory (default: `build/`) | +| `--output-format` | `-t` | Output format: `wasm` or `wast` (default: `wasm`) | +| `--skip-migrations` | | Skip manifest migrations | +| `--watch` | `-w` | Rebuild on file changes | +| `--ipfs` | `-i` | IPFS node URL (uploads if provided) | +| `--network` | | Network from networks.json | +| `--network-file` | | Path to networks config (default: `networks.json`) | + +**Examples:** + +```bash +# Build subgraph +gnd build + +# Build and upload to IPFS +gnd build --ipfs http://localhost:5001 + +# Build for specific network +gnd build --network mainnet +``` + +### `gnd deploy` + +Deploy a subgraph to a Graph Node. + +```bash +gnd deploy [MANIFEST] +``` + +**Arguments:** +- `SUBGRAPH_NAME`: Name to deploy as (e.g., `user/subgraph`) +- `MANIFEST`: Path to subgraph manifest (default: `subgraph.yaml`) + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--node` | `-g` | Graph Node URL (defaults to Subgraph Studio) | +| `--ipfs` | `-i` | IPFS node URL | +| `--deploy-key` | | Deploy key for authentication | +| `--version-label` | `-l` | Version label for the deployment (required in non-interactive mode) | +| `--ipfs-hash` | | IPFS hash of already-uploaded manifest | +| `--output-dir` | `-o` | Build output directory (default: `build/`) | +| `--skip-migrations` | | Skip manifest migrations | +| `--network` | | Network from networks.json | +| `--network-file` | | Path to networks config | +| `--debug-fork` | | Fork subgraph ID for debugging | + +If `--version-label` is omitted in an interactive terminal, `gnd deploy` prompts for it. +In non-interactive environments (CI/scripts), you must pass `--version-label`. + +**Examples:** + +```bash +# Deploy to Subgraph Studio (uses saved auth key) +gnd deploy -l v1.0.0 my-name/my-subgraph + +# Deploy to local Graph Node +gnd deploy --node http://localhost:8020 --ipfs http://localhost:5001 -l v1.0.0 my-name/my-subgraph + +# Deploy without --version-label in an interactive terminal (prompts) +gnd deploy my-name/my-subgraph +``` + +### `gnd publish` + +Publish a subgraph to The Graph's decentralized network. + +```bash +gnd publish [MANIFEST] +``` + +**Arguments:** +- `MANIFEST`: Path to subgraph manifest (default: `subgraph.yaml`) + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--ipfs` | `-i` | IPFS node URL | +| `--ipfs-hash` | | Skip build, use existing IPFS hash | +| `--subgraph-id` | | Subgraph ID for updating existing subgraphs | +| `--protocol-network` | | Network: `arbitrum-one` or `arbitrum-sepolia` | +| `--api-key` | | API key (required when updating existing) | +| `--output-dir` | `-o` | Build output directory | +| `--skip-migrations` | | Skip manifest migrations | +| `--network` | | Network from networks.json | +| `--network-file` | | Path to networks config | + +**Examples:** + +```bash +# Publish new subgraph +gnd publish + +# Update existing subgraph +gnd publish --subgraph-id Qm... --api-key YOUR_KEY +``` + +### `gnd add` + +Add a new data source to an existing subgraph. + +```bash +gnd add
[MANIFEST] +``` + +**Arguments:** +- `ADDRESS`: Contract address to add +- `MANIFEST`: Path to subgraph manifest (default: `subgraph.yaml`) + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--abi` | | Path to the contract ABI | +| `--contract-name` | | Name for the new data source | +| `--merge-entities` | | Merge with existing entities of same name | +| `--network` | | Network the contract is deployed to | +| `--start-block` | | Block number to start indexing from | + +**Examples:** + +```bash +# Add contract with ABI from Etherscan +gnd add 0x1234... --network mainnet + +# Add contract with local ABI +gnd add 0x1234... --abi ./NewContract.json --contract-name MyContract +``` + +### `gnd create` + +Register a subgraph name with a Graph Node. + +```bash +gnd create --node +``` + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--node` | `-g` | Graph Node URL (required) | +| `--access-token` | | Access token for authentication | + +### `gnd remove` + +Unregister a subgraph name from a Graph Node. + +```bash +gnd remove --node +``` + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--node` | `-g` | Graph Node URL (required) | +| `--access-token` | | Access token for authentication | + +### `gnd auth` + +Store a deploy key for authentication. + +```bash +gnd auth +``` + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--node` | `-g` | Graph Node URL (default: Subgraph Studio) | + +Keys are stored in `~/.graph-cli.json`. + +### `gnd test` + +Run subgraph tests. + +```bash +gnd test [TEST_FILES...] +``` + +**Arguments:** +- `PATHS`: Test JSON files or directories to scan. Defaults to `tests/` when nothing is specified. + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--manifest` | `-m` | Path to subgraph manifest (default: `subgraph.yaml`) | +| `--skip-build` | `-s` | Skip building the subgraph before testing | +| `--postgres-url` | | PostgreSQL connection URL (env: `POSTGRES_URL`) | +| `--matchstick` | | Use legacy Matchstick runner (**deprecated** — migrate to JSON-based tests) | +| `--docker` | `-d` | Run Matchstick in Docker (requires `--matchstick`) | +| `--coverage` | `-c` | Run with coverage reporting (requires `--matchstick`) | +| `--recompile` | `-r` | Force recompilation (requires `--matchstick`) | +| `--force` | `-f` | Force redownload of Matchstick binary (requires `--matchstick`) | + +**Examples:** + +```bash +# Run all tests in tests/ directory (default) +gnd test + +# Run specific test files +gnd test transfer.json approval.json +gnd test tests/transfer.json + +# Scan a custom directory +gnd test my-tests/ + +# Use a different manifest +gnd test -m subgraph.staging.yaml tests/transfer.json + +# Skip automatic build +gnd test -s +``` + +### `gnd clean` + +Remove build artifacts and generated files. + +```bash +gnd clean +``` + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--codegen-dir` | | Codegen directory (default: `generated/`) | +| `--build-dir` | | Build directory (default: `build/`) | + +### `gnd dev` + +Run graph-node in development mode with file watching. + +```bash +gnd dev [OPTIONS] +``` + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--watch` | | Watch build directory for changes | +| `--manifests` | | Subgraph manifest locations | +| `--sources` | | Source manifest locations for aliases | +| `--database-dir` | | Database directory (default: `./build`) | +| `--postgres-url` | | PostgreSQL connection URL | +| `--ethereum-rpc` | | Ethereum RPC URL | +| `--ipfs` | | IPFS node URL | + +### `gnd indexer` + +Manage indexer operations via [`indexer-cli`](https://github.com/graphprotocol/indexer/tree/main/packages/indexer-cli). + +Requires `graph-indexer` to be installed and on `$PATH`: + +```bash +npm install -g @graphprotocol/indexer-cli +``` + +```bash +gnd indexer [args...] +``` + +**Help:** + +There are two ways to get help: + +| Command | What it shows | +|---------|---------------| +| `gnd indexer --help` | gnd's own help for the indexer subcommand (works without `graph-indexer` installed) | +| `gnd indexer help` | Full `graph-indexer` help with all available commands (requires `graph-indexer`) | + +**Examples:** + +```bash +# Check indexer status +gnd indexer status --network arbitrum-one + +# Manage indexing rules +gnd indexer rules get all --network mainnet +gnd indexer rules set decisionBasis always --network mainnet + +# Manage allocations +gnd indexer allocations get --network arbitrum-one + +# Manage cost models +gnd indexer cost get + +# View available indexer commands +gnd indexer help + +# Check graph-indexer version +gnd indexer version +``` + +### `gnd completions` + +Generate shell completions. + +```bash +gnd completions +``` + +**Arguments:** +- `SHELL`: One of `bash`, `elvish`, `fish`, `powershell`, `zsh` + +**Examples:** + +```bash +# Bash +gnd completions bash > ~/.bash_completion.d/gnd + +# Zsh +gnd completions zsh > ~/.zfunc/_gnd + +# Fish +gnd completions fish > ~/.config/fish/completions/gnd.fish +``` + +## Configuration Files + +### `~/.graph-cli.json` + +Stores deploy keys for different Graph Node URLs. Created by `gnd auth`. + +### `networks.json` + +Network-specific configuration for contract addresses and start blocks: + +```json +{ + "mainnet": { + "MyContract": { + "address": "0x1234...", + "startBlock": 12345678 + } + }, + "sepolia": { + "MyContract": { + "address": "0x5678...", + "startBlock": 1000000 + } + } +} +``` + +Use with `--network mainnet` on build/deploy commands. + +## Differences from graph-cli + +`gnd` is designed as a drop-in replacement for `graph-cli`. Some intentional differences: + +### Commands Not Implemented + +- **`local`**: Use graph-node's integration test infrastructure instead +- **`node`**: Use `graphman` for node management operations + +### Code Generation Differences + +These are documented and tested: + +1. **Int8 import**: Always imported for simplicity, even when not used +2. **Trailing commas**: Used in multi-line constructs +3. **2D array accessors**: Uses correct `toStringMatrix()` (fixes a bug in graph-cli) + +### Other Differences + +- **Debug logging**: Uses `RUST_LOG` environment variable instead of `DEBUG=graph-cli:*` +- **`--uncrashable` flag**: Not implemented (Float Capital third-party feature) + +## Environment Variables + +| Variable | Description | +|----------|-------------| +| `RUST_LOG` | Debug logging level (e.g., `RUST_LOG=gnd=debug`) | +| `POSTGRES_URL` | PostgreSQL URL for `gnd dev` | +| `ETHEREUM_RPC` | Ethereum RPC URL for `gnd dev` | + +## Version + +```bash +gnd --version +# gnd 0.1.0 (graph-cli compatible: 0.98.1) +``` + +Shows both the gnd version and the graph-cli version it emulates. + +## License + +Apache-2.0 OR MIT diff --git a/gnd/docs/gnd-test.md b/gnd/docs/gnd-test.md new file mode 100644 index 00000000000..6fcc118fc29 --- /dev/null +++ b/gnd/docs/gnd-test.md @@ -0,0 +1,939 @@ +# gnd test + +Mock-based subgraph test runner that feeds JSON-defined blocks through real graph-node infrastructure (store, WASM runtime, trigger processing) with only the blockchain layer mocked. + +## Quick Start + +```bash +# Run all tests in tests/ directory +gnd test + +# Run a specific test file +gnd test tests/transfer.json + +# Skip automatic build (if subgraph already built) +gnd test --skip-build + +# Use legacy Matchstick runner +gnd test --matchstick +``` + +## Test File Format + +Tests are JSON files that define: +- Mock blockchain blocks with events +- Mock Ethereum RPC responses (`eth_call`, `eth_getBalance`, `eth_getCode`) +- GraphQL assertions to validate entity state + +Place test files in a `tests/` directory with `.json` or `.test.json` extension. + +### Basic Example + +```json +{ + "name": "Transfer creates entity", + "blocks": [ + { + "number": 1, + "timestamp": 1672531200, + "events": [ + { + "address": "0x1234...", + "event": "Transfer(address indexed from, address indexed to, uint256 value)", + "params": { + "from": "0xaaaa...", + "to": "0xbbbb...", + "value": "1000" + } + } + ], + "ethCalls": [ + { + "address": "0x1234...", + "function": "balanceOf(address)(uint256)", + "params": ["0xaaaa..."], + "returns": ["1000000000000000000"] + } + ] + } + ], + "assertions": [ + { + "query": "{ transfer(id: \"1\") { from to value } }", + "expected": { + "transfer": { + "from": "0xaaaa...", + "to": "0xbbbb...", + "value": "1000" + } + } + } + ] +} +``` + +## Block Fields + +| Field | Required | Default | Description | +|-------|----------|---------|-------------| +| `number` | No | Auto-increments from lowest defined `startBlock` in the manifest file, or from `0` if no `startBlock` are defined | Block number | +| `hash` | No | `keccak256(block_number)` | Block hash | +| `timestamp` | No | `block_number` | Unix timestamp | +| `baseFeePerGas` | No | None (pre-EIP-1559) | Base fee in wei | +| `events` | No | Empty array | Log events in this block | +| `ethCalls` | No | Empty array | Mock `eth_call` responses | +| `getBalanceCalls` | No | Empty array | Mock `eth_getBalance` responses for `ethereum.getBalance()` | +| `hasCodeCalls` | No | Empty array | Mock `eth_getCode` responses for `ethereum.hasCode()` | + +### Empty Blocks + +Empty blocks (no events) still trigger block handlers: + +```json +{ + "name": "Test block handlers", + "blocks": [ + { + "number": 1, + "events": [...] + }, + {} // Block 2 with no events - block handlers still fire + ] +} +``` + +## Event Fields + +| Field | Required | Default | Description | +|-------|----------|---------|-------------| +| `address` | Yes | — | Contract address (lowercase hex with 0x prefix) | +| `event` | Yes | — | Full event signature with `indexed` keywords | +| `params` | No | Empty object | Event parameter values | +| `txHash` | No | `keccak256(block_number \|\| log_index)` | Transaction hash | + +### Event Signature Format + +**Important:** Include `indexed` keywords in the signature: + +```json +{ + "event": "Transfer(address indexed from, address indexed to, uint256 value)" +} +``` + +Not: +```json +{ + "event": "Transfer(address,address,uint256)" // ❌ Missing indexed keywords +} +``` + +### Parameter Types + +Event parameters are automatically ABI-encoded based on the signature. Supported formats: + +```json +{ + "params": { + "from": "0xaaaa...", // address + "to": "0xbbbb...", // address + "value": "1000", // uint256 (string or number) + "amount": 1000, // uint256 (number) + "enabled": true, // bool + "data": "0x1234...", // bytes + "name": "Token" // string + } +} +``` + +## Transaction Receipts + +Mock receipts are constructed for every log trigger and attached only to handlers that declare `receipt: true` in the manifest, mirroring production behaviour. Handlers without `receipt: true` receive a null receipt — the same as on a real node. + +**Limitation:** Only `receipt.logs` reflects your test data. All other receipt fields (`from`, `to`, `gas_used`, `status`, etc.) are hardcoded stubs and do not correspond to real transaction data. If your handler reads those fields, the values will be fixed defaults regardless of what you put in the test JSON. + +### How receipts are built + +Every event gets a mock receipt attached automatically. The key rule is **`txHash` grouping**: + +- Events sharing the same `txHash` share **one receipt** — `event.receipt!.logs` contains all of their logs in declaration order. +- Events without an explicit `txHash` each get a unique auto-generated hash (`keccak256(block_number || log_index)`), so each gets its own single-log receipt. + +### Example: Two events sharing a receipt + +```json +{ + "events": [ + { + "address": "0x1234...", + "event": "Transfer(address indexed from, address indexed to, uint256 value)", + "params": { "from": "0xaaaa...", "to": "0xbbbb...", "value": "100" }, + "txHash": "0xdeadbeef0000000000000000000000000000000000000000000000000000000" + }, + { + "address": "0x1234...", + "event": "Transfer(address indexed from, address indexed to, uint256 value)", + "params": { "from": "0xbbbb...", "to": "0xcccc...", "value": "50" }, + "txHash": "0xdeadbeef0000000000000000000000000000000000000000000000000000000" + } + ] +} +``` + +Both handlers receive a receipt where `receipt.logs` has two entries, in declaration order. + +### Mock receipt defaults + +| Field | Value | +|-------|-------| +| `status` | success | +| `cumulative_gas_used` | `21000` | +| `gas_used` | `21000` | +| transaction type | `2` (EIP-1559) | +| `from` | `0x000...000` | +| `to` | `null` | +| `effective_gas_price` | `0` | + +Handlers without `receipt: true` in the manifest are unaffected — they never access `event.receipt`. + +## Block Handlers + +Block handlers are **automatically triggered** for every block. You don't need to specify block triggers in the JSON. + +### How Block Handlers Work + +The test runner auto-injects both `Start` and `End` block triggers for each block, ensuring all block handler filters work correctly: + +- **`once` filter** → Fires once at `startBlock` (via `Start` trigger) +- **No filter** → Fires on every block (via `End` trigger) +- **`polling` filter** → Fires every N blocks based on formula: `(block_number - startBlock) % every == 0` + +### Example: Basic Block Handlers + +```json +{ + "name": "Block handlers test", + "blocks": [ + {}, // Block 0 - both 'once' and regular block handlers fire + {} // Block 1 - only regular block handlers fire + ], + "assertions": [ + { + "query": "{ blocks { number } }", + "expected": { + "blocks": [ + {"number": "0"}, + {"number": "1"} + ] + } + }, + { + "query": "{ blockOnces { msg } }", + "expected": { + "blockOnces": [ + {"msg": "This fires only once at block 0"} + ] + } + } + ] +} +``` + +### Polling Block Handlers + +Polling handlers fire at regular intervals specified by the `every` parameter. The handler fires when: + +``` +(block_number - startBlock) % every == 0 +``` + +**Manifest example:** +```yaml +blockHandlers: + - handler: handleEveryThreeBlocks + filter: + kind: polling + every: 3 +``` + +**Test example (startBlock: 0):** +```json +{ + "name": "Polling handler test", + "blocks": [ + {}, // Block 0 - handler fires (0 % 3 == 0) + {}, // Block 1 - handler doesn't fire + {}, // Block 2 - handler doesn't fire + {}, // Block 3 - handler fires (3 % 3 == 0) + {}, // Block 4 - handler doesn't fire + {}, // Block 5 - handler doesn't fire + {} // Block 6 - handler fires (6 % 3 == 0) + ], + "assertions": [ + { + "query": "{ pollingBlocks(orderBy: number) { number } }", + "expected": { + "pollingBlocks": [ + {"number": "0"}, + {"number": "3"}, + {"number": "6"} + ] + } + } + ] +} +``` + +**With non-zero startBlock:** + +When your data source has `startBlock > 0`, the polling interval is calculated from that starting point. + +**Manifest:** +```yaml +dataSources: + - name: Token + source: + startBlock: 100 + mapping: + blockHandlers: + - handler: handlePolling + filter: + kind: polling + every: 5 +``` + +**Test:** +```json +{ + "name": "Polling from block 100", + "blocks": [ + {"number": 100}, // Fires: (100-100) % 5 == 0 + {"number": 101}, // Doesn't fire + {"number": 102}, // Doesn't fire + {"number": 103}, // Doesn't fire + {"number": 104}, // Doesn't fire + {"number": 105}, // Fires: (105-100) % 5 == 0 + {"number": 106}, // Doesn't fire + {"number": 107}, // Doesn't fire + {"number": 108}, // Doesn't fire + {"number": 109}, // Doesn't fire + {"number": 110} // Fires: (110-100) % 5 == 0 + ], + "assertions": [ + { + "query": "{ pollingBlocks(orderBy: number) { number } }", + "expected": { + "pollingBlocks": [ + {"number": "100"}, + {"number": "105"}, + {"number": "110"} + ] + } + } + ] +} +``` + +**Note:** The test runner automatically handles `startBlock > 0`, so blocks default to numbering from the manifest's `startBlock`. + +## eth_call Mocking + +Mock contract calls made from mapping handlers using `contract.call()`: + +```json +{ + "ethCalls": [ + { + "address": "0x1234...", + "function": "balanceOf(address)(uint256)", + "params": ["0xaaaa..."], + "returns": ["1000000000000000000"] + } + ] +} +``` + +### ethCall Fields + +| Field | Required | Description | +|-------|----------|-------------| +| `address` | Yes | Contract address | +| `function` | Yes | Full signature: `"functionName(inputTypes)(returnTypes)"` | +| `params` | Yes | Array of input parameters (as strings) | +| `returns` | Yes | Array of return values (as strings, ignored if `reverts: true`) | +| `reverts` | No | Default `false`. If `true`, the mock transport returns an RPC error | + +### Function Signature Format + +Use full signatures with input and return types: + +```json +{ + "function": "symbol()(string)", // No inputs, returns string + "function": "balanceOf(address)(uint256)", // One input, returns uint256 + "function": "decimals()(uint8)" // No inputs, returns uint8 +} +``` + +### Mocking Reverts + +```json +{ + "address": "0x1234...", + "function": "transfer(address,uint256)(bool)", + "params": ["0xaaaa...", "1000"], + "returns": [], + "reverts": true +} +``` + +### Real-World Example + +From the ERC20 test: + +```json +{ + "ethCalls": [ + { + "address": "0x731a10897d267e19b34503ad902d0a29173ba4b1", + "function": "symbol()(string)", + "params": [], + "returns": ["GRT"] + }, + { + "address": "0x731a10897d267e19b34503ad902d0a29173ba4b1", + "function": "name()(string)", + "params": [], + "returns": ["TheGraph"] + }, + { + "address": "0x731a10897d267e19b34503ad902d0a29173ba4b1", + "function": "balanceOf(address)(uint256)", + "params": ["0xaaaa000000000000000000000000000000000000"], + "returns": ["3000000000000000000"] + } + ] +} +``` + +## ethereum.getBalance() Mocking + +Mock balance lookups made from mapping handlers using `ethereum.getBalance()`: + +```json +{ + "getBalanceCalls": [ + { + "address": "0xaaaa000000000000000000000000000000000000", + "value": "1000000000000000000" + } + ] +} +``` + +### getBalanceCalls Fields + +| Field | Required | Description | +|-------|----------|-------------| +| `address` | Yes | Account address (checksummed or lowercase hex) | +| `value` | Yes | Balance in Wei as a decimal string | + +## ethereum.hasCode() Mocking + +Mock code existence checks made from mapping handlers using `ethereum.hasCode()`: + +```json +{ + "hasCodeCalls": [ + { + "address": "0x1234000000000000000000000000000000000000", + "hasCode": true + } + ] +} +``` + +### hasCodeCalls Fields + +| Field | Required | Description | +|-------|----------|-------------| +| `address` | Yes | Contract address (checksummed or lowercase hex) | +| `hasCode` | Yes | Whether the address has deployed bytecode | + +## File Data Sources + +Mock IPFS and Arweave file contents for file data source handlers. Files are defined at the top level of the test JSON (not inside blocks). + +### IPFS Files + +```json +{ + "name": "File data source test", + "files": [ + { + "cid": "QmExample...", + "content": "{\"name\": \"Token\", \"description\": \"A token\"}" + }, + { + "cid": "QmAnother...", + "file": "fixtures/metadata.json" + } + ], + "blocks": [...], + "assertions": [...] +} +``` + +#### files Fields + +| Field | Required | Description | +|-------|----------|-------------| +| `cid` | Yes | IPFS CID (`Qm...` or `bafy...`). The mock ignores hash/content relationship | +| `content` | One of `content`/`file` | Inline UTF-8 content | +| `file` | One of `content`/`file` | File path, resolved relative to the test JSON | + +### Arweave Files + +```json +{ + "name": "Arweave data source test", + "arweaveFiles": [ + { + "txId": "abc123", + "content": "{\"name\": \"Token\"}" + }, + { + "txId": "def456/metadata.json", + "file": "fixtures/arweave-data.json" + } + ], + "blocks": [...], + "assertions": [...] +} +``` + +#### arweaveFiles Fields + +| Field | Required | Description | +|-------|----------|-------------| +| `txId` | Yes | Arweave transaction ID or bundle path (e.g. `"txid/filename.json"`) | +| `content` | One of `content`/`file` | Inline UTF-8 content | +| `file` | One of `content`/`file` | File path, resolved relative to the test JSON | + +## Assertions + +GraphQL queries to validate the indexed entity state after processing all blocks. + +### Assertion Fields + +| Field | Required | Description | +|-------|----------|-------------| +| `query` | Yes | GraphQL query string | +| `expected` | Yes | Expected JSON response | + +### Comparison Behavior + +| Aspect | Behavior | +|--------|----------| +| Objects | Key-compared, order-insensitive | +| Arrays | **Order-insensitive** (set comparison) | +| String vs Number | Coerced — `"123"` matches `123` | +| Nulls/Booleans | Strict equality | +| Hex strings | **Must be lowercase** — graph-node returns all hex values in lowercase | + +**Important:** Arrays are compared as sets (order doesn't matter). + +**Important:** graph-node returns all hex-encoded values (addresses, transaction hashes, byte arrays) in **lowercase**. Expected values in assertions must match exactly — mixed-case hex will not match: + +```json +{ + "expected": { + "transfer": { + "from": "0xaaaa000000000000000000000000000000000000", // ✅ lowercase + "to": "0xBBBB000000000000000000000000000000000000" // ❌ will not match + } + } +} +``` + +Note: hex values in event inputs (`params`, `address`) are normalized automatically and can be mixed case. If you need ordered results, use `orderBy` in your GraphQL query: + +```json +{ + "query": "{ transfers(orderBy: timestamp, orderDirection: asc) { id from to value } }", + "expected": { ... } +} +``` + +### Multiple Assertions + +You can have multiple assertions per test. They run sequentially after all blocks are processed: + +```json +{ + "assertions": [ + { + "query": "{ tokens { id name symbol } }", + "expected": { ... } + }, + { + "query": "{ accounts { id balance } }", + "expected": { ... } + } + ] +} +``` + +### Nested Entity Queries + +Test relationships and nested entities: + +```json +{ + "query": "{ accounts { id balances { token { symbol } amount } } }", + "expected": { + "accounts": [ + { + "id": "0xbbbb...", + "balances": [ + { + "token": { "symbol": "GRT" }, + "amount": "5000000000000000000" + } + ] + } + ] + } +} +``` + +## startBlock Handling + +The test runner automatically reads `startBlock` from your subgraph manifest and handles it correctly — **no real blockchain connection needed**. + +### How It Works + +1. Extracts the **minimum `startBlock`** across all data sources in your manifest +2. If min > 0, creates a `start_block_override` to bypass graph-node's on-chain block validation +3. Test blocks without explicit `"number"` auto-increment starting from that minimum `startBlock` + +### Default Block Numbering + +The starting block number depends on your manifest: + +| Manifest Configuration | Test Block Numbers | +|----------------------|-------------------| +| `startBlock: 0` (or unset) | 0, 1, 2, ... | +| `startBlock: 100` | 100, 101, 102, ... | +| Multiple data sources: `startBlock: 50` and `startBlock: 200` | 50, 51, 52, ... (uses minimum) | + +### Example: Single Data Source + +**Manifest:** +```yaml +dataSources: + - name: Token + source: + startBlock: 1000 +``` + +**Test:** +```json +{ + "blocks": [ + {}, // Block 1000 (auto-numbered) + {} // Block 1001 (auto-numbered) + ] +} +``` + +### Example: Explicit Block Numbers + +Override auto-numbering by specifying `"number"`: + +```json +{ + "blocks": [ + { + "number": 5000, + "events": [...] + }, + { + "number": 5001, + "events": [...] + } + ] +} +``` + +### Multi-Data Source Testing + +When your subgraph has multiple data sources with different `startBlock` values, you may need to use explicit block numbers. + +**Scenario:** DataSource A at `startBlock: 50` (Transfer events), DataSource B at `startBlock: 200` (Approval events). You want to test only DataSource B. + +**Manifest:** +```yaml +dataSources: + - name: TokenTransfers + source: + startBlock: 50 + mapping: + eventHandlers: + - event: Transfer(...) + handler: handleTransfer + - name: TokenApprovals + source: + startBlock: 200 + mapping: + eventHandlers: + - event: Approval(...) + handler: handleApproval +``` + +**Test:** +```json +{ + "name": "Test Approval handler", + "blocks": [ + { + "number": 200, // Explicit number >= DataSource B's startBlock + "events": [ + { + "address": "0x5678...", + "event": "Approval(address indexed owner, address indexed spender, uint256 value)", + "params": { + "owner": "0xaaaa...", + "spender": "0xbbbb...", + "value": "500" + } + } + ] + }, + { + "number": 201, + "events": [...] + } + ] +} +``` + +**Why explicit numbers are needed:** +- Default numbering starts at the **minimum** `startBlock` across all data sources (50 in this case) +- Blocks 50-199 are below DataSource B's `startBlock: 200`, so its handlers won't fire +- Use explicit `"number": 200` to ensure the block is in DataSource B's active range + +**Note:** DataSource A is still "active" from block 50 onward, but it simply sees no matching Transfer events in blocks 200-201, so no handlers fire for it. This is normal behavior — graph-node doesn't error on inactive handlers. + +## Test Organization + +### Directory Structure + +``` +my-subgraph/ +├── subgraph.yaml +├── schema.graphql +├── src/ +│ └── mapping.ts +└── tests/ + ├── transfer.json + ├── approval.json + └── edge-cases.test.json +``` + +### Naming Conventions + +- Use `.json` or `.test.json` extension +- Descriptive names: `transfer.json`, `mint-burn.json`, `edge-cases.json` +- The test runner discovers all `*.json` and `*.test.json` files in the test directory + +## Known Limitations + +| Feature | Status | +|---------|--------| +| Log events | ✅ Supported | +| Block handlers (all filters) | ✅ Supported | +| `eth_call` mocking | ✅ Supported | +| `ethereum.getBalance()` mocking | ✅ Supported | +| `ethereum.hasCode()` mocking | ✅ Supported | +| Dynamic/template data sources | ✅ Supported | +| Transaction receipts (`receipt: true`) | ⚠️ Partial — `receipt.logs` is populated and grouped by `txHash`; other fields (gas, from, to, etc.) are hardcoded stubs (see [Transaction Receipts](#transaction-receipts)) | +| File data sources (IPFS + Arweave) | ✅ Supported | +| Call triggers (traces) | ❌ Not implemented | +| `--json` CI output | ❌ Not implemented | +| Parallel test execution | ❌ Not implemented | +| Test name filtering (`--filter`) | ❌ Not implemented | + +## Tips & Best Practices + +### Use Lowercase Hex in Assertions + +graph-node returns **all** hex-encoded values in lowercase — addresses, transaction hashes, and any `Bytes`/`ID` fields. Expected values in assertions must use lowercase hex: + +```json +{ + "expected": { + "transfer": { + "from": "0xaaaa000000000000000000000000000000000000" // ✅ lowercase + } + } +} +``` + +Not: +```json +{ + "expected": { + "transfer": { + "from": "0xAAAA000000000000000000000000000000000000" // ❌ will not match + } + } +} +``` + +Event inputs (`address`, `params`) are normalized automatically and can be mixed case. + +### Test One Thing at a Time + +Write focused tests that validate a single behavior: + +```json +// ✅ Good - tests one scenario +{ + "name": "Transfer event creates TransferEvent entity", + "blocks": [...], + "assertions": [...] +} +``` + +```json +// ❌ Avoid - tests too many things +{ + "name": "Test everything", + "blocks": [/* 50 blocks */], + "assertions": [/* 20 assertions */] +} +``` + +### Order GraphQL Results + +If your assertion needs specific ordering, use `orderBy`: + +```json +{ + "query": "{ transfers(first: 10, orderBy: timestamp, orderDirection: asc) { id } }", + "expected": { ... } +} +``` + +### Test Block Handlers with Empty Blocks + +Use empty blocks to test that block handlers fire even without events: + +```json +{ + "blocks": [ + {}, // Empty block - block handlers still fire + {} + ] +} +``` + +### Split Complex Tests + +Instead of one large test with many blocks, split into multiple focused test files: + +``` +tests/ +├── transfer-basic.json # Basic transfer functionality +├── transfer-zero-value.json # Edge case: zero value +└── transfer-same-account.json # Edge case: self-transfer +``` + +## Architecture + +The test runner reuses real graph-node infrastructure: + +``` +test.json + ↓ +Parse & ABI encode events + ↓ +Mock block stream (StaticStreamBuilder) + ↓ +Real graph-node indexer + ├── WASM runtime + ├── Trigger processing + └── Entity storage (pgtemp database) + ↓ +GraphQL queries → Assertions +``` + +**Key design principles:** + +- **Isolated database per test:** Each test gets a pgtemp database dropped on completion (default), or a shared persistent database with post-test cleanup (`--postgres-url`) +- **Mock transport layer:** A mock Alloy transport serves `eth_call`, `eth_getBalance`, and `eth_getCode` from test JSON data. All three flow through the real production code path — only the transport returns mock responses. Unmocked RPC calls fail immediately with a descriptive error. +- **No IPFS for manifest:** Uses `FileLinkResolver` to load manifest/WASM from build directory + +## Troubleshooting + +### Test Fails: "Entity not found" + +**Cause:** Handler didn't create the expected entity. + +**Fix:** +1. Check event signature matches ABI (include `indexed` keywords) +2. Verify contract address matches manifest +3. Check block number is >= data source's `startBlock` +4. Add debug logging to your mapping handler + +### Test Timeout + +**Cause:** Indexer took longer than 60 seconds (default timeout). + +**Fix:** +1. Reduce number of blocks in test +2. Simplify mapping logic +3. Check for infinite loops in handler code + +### Unmocked RPC Call + +**Cause:** A mapping handler calls `ethereum.call`, `ethereum.getBalance`, or `ethereum.hasCode` for a call that has no matching mock entry. + +**Symptom:** Test fails immediately with a descriptive error like: +``` +gnd test: unmocked eth_call to 0x1234... at block hash 0xabcd... +Add a matching 'ethCalls' entry to this block in your test JSON. +``` + +**Fix:** +1. Add the missing mock to the appropriate field in your test block (`ethCalls`, `getBalanceCalls`, or `hasCodeCalls`) +2. If the call is not supposed to happen, check the mapping logic — a code path may be executing unexpectedly + +### Block Handler Not Firing + +**Cause:** Block handlers auto-fire, but might be outside data source's active range. + +**Fix:** +1. Check data source's `startBlock` in manifest +2. Use explicit `"number"` in test blocks to ensure they're >= `startBlock` +3. Verify handler is defined in manifest's `blockHandlers` section + +## Legacy Matchstick Mode + +Fall back to the external Matchstick test runner for backward compatibility: + +```bash +gnd test --matchstick +``` + +This is useful if: +- You have existing Matchstick tests +- You need features not yet supported by the mock-based runner +- You're migrating gradually from Matchstick to the new test format + +## See Also + +- [Subgraph Manifest Documentation](https://thegraph.com/docs/en/developing/creating-a-subgraph/) +- [AssemblyScript Mapping API](https://thegraph.com/docs/en/developing/assemblyscript-api/) +- [GraphQL Schema](https://thegraph.com/docs/en/developing/creating-a-subgraph/#the-graph-ql-schema) diff --git a/gnd/docs/migrating-from-graph-cli.md b/gnd/docs/migrating-from-graph-cli.md new file mode 100644 index 00000000000..6203a6f2f21 --- /dev/null +++ b/gnd/docs/migrating-from-graph-cli.md @@ -0,0 +1,303 @@ +# Migrating from graph-cli to gnd + +`gnd` is designed as a drop-in replacement for `graph-cli`. This guide covers the differences and how to migrate your workflow. + +## TL;DR + +For most users, migration is straightforward: + +```bash +# Instead of: +graph codegen +graph build +graph deploy --studio my-subgraph + +# Use: +gnd codegen +gnd build +gnd deploy my-subgraph # defaults to Studio; prompts for version label in interactive terminals +``` + +## Command Mapping + +| graph-cli | gnd | Notes | +|-----------|-----|-------| +| `graph init` | `gnd init` | Identical flags | +| `graph codegen` | `gnd codegen` | Identical flags | +| `graph build` | `gnd build` | Identical flags | +| `graph deploy` | `gnd deploy` | Defaults to Studio if `--node` not provided; pass `--version-label` in non-interactive mode | +| `graph create` | `gnd create` | Identical flags | +| `graph remove` | `gnd remove` | Identical flags | +| `graph auth` | `gnd auth` | Identical flags | +| `graph add` | `gnd add` | Identical flags | +| `graph test` | `gnd test` | Identical flags | +| `graph clean` | `gnd clean` | Identical flags (new in graph-cli 0.80+) | +| `graph publish` | `gnd publish` | Identical flags | +| `graph indexer` | `gnd indexer` | Delegates to `graph-indexer` — requires `indexer-cli` installed | +| `graph local` | N/A | Not implemented - use graph-node's test infrastructure | +| `graph node` | N/A | Not implemented - use `graphman` | + +## What's the Same + +### Flag Compatibility + +All flags use the same names, short forms, and defaults: + +```bash +# Both work identically +graph codegen -o generated/ --skip-migrations +gnd codegen -o generated/ --skip-migrations + +graph build -t wasm --network mainnet +gnd build -t wasm --network mainnet + +graph deploy -l v1.0.0 --ipfs http://localhost:5001 my-subgraph +gnd deploy -l v1.0.0 --ipfs http://localhost:5001 my-subgraph +``` + +### Output Format + +Same success checkmarks, step descriptions, and progress indicators: + +``` +✔ Generate types for data source: Gravity (...) +✔ Write types to generated/Gravity/Gravity.ts (...) +``` + +### Exit Codes + +- `0` for success +- `1` for any error +- `gnd indexer` passes through the exit code from `graph-indexer` + +### Configuration Files + +`gnd` uses the same configuration files as `graph-cli`: + +- `~/.graph-cli.json` - Deploy keys (from `gnd auth`) +- `networks.json` - Network-specific addresses + +Your existing auth keys work with both tools. + +### Generated Code + +Code generation produces identical AssemblyScript output (after formatting). Your existing subgraph code will work without changes. + +## What's Different + +### Indexer Commands + +`gnd indexer` provides access to indexer management commands via [`indexer-cli`](https://github.com/graphprotocol/indexer/tree/main/packages/indexer-cli). Requires `graph-indexer` on `$PATH`: + +```bash +npm install -g @graphprotocol/indexer-cli + +gnd indexer status --network arbitrum-one +gnd indexer rules get all --network mainnet +gnd indexer allocations get --network arbitrum-one +``` + +Note: `gnd indexer --help` shows gnd's own help and works without `graph-indexer` installed. Use `gnd indexer help` to see the full list of `graph-indexer` commands. + +### Commands Not Available + +#### `graph local` + +Not implemented in `gnd`. Use graph-node's built-in integration test infrastructure or Docker compose setups instead. + +#### `graph node` + +Not implemented. Use `graphman` for node management operations: + +```bash +# Instead of graph node commands, use graphman: +graphman info subgraph-name +graphman reassign subgraph-name shard +``` + +### Debug Logging + +```bash +# graph-cli uses: +DEBUG=graph-cli:* graph codegen + +# gnd uses: +RUST_LOG=gnd=debug gnd codegen +``` + +### The `--uncrashable` Flag + +The `--uncrashable` flag on `codegen` (Float Capital's uncrashable helper generation) is not implemented. This is a third-party feature that most subgraphs don't use. + +### Minor Code Generation Differences + +These differences are intentional and documented: + +1. **Int8 import**: `gnd` always imports `Int8` in generated code for simplicity, even when not used. This doesn't affect functionality. + +2. **Trailing commas**: `gnd` uses trailing commas in multi-line constructs: + ```typescript + // gnd output + new Foo( + param1, + param2, // trailing comma + ) + + // graph-cli output + new Foo( + param1, + param2 + ) + ``` + +3. **2D array accessors**: `gnd` uses the correct `toStringMatrix()` method for 2D GraphQL arrays, fixing a bug in graph-cli that used `toStringArray()`. + +## Migration Steps + +### Step 1: Install gnd + +Build from source in the graph-node repository: + +```bash +cargo build -p gnd --release +# Binary at target/release/gnd +``` + +### Step 2: Verify Your Auth + +Your existing `~/.graph-cli.json` works with both tools. Verify with: + +```bash +# Check stored keys +cat ~/.graph-cli.json + +# Or just run a deploy dry-run to see if auth is working +gnd deploy --help +``` + +### Step 3: Test Locally + +Run your normal workflow with gnd: + +```bash +gnd codegen +gnd build +``` + +Compare the output to graph-cli if you want to verify: + +```bash +# graph-cli +graph codegen -o generated-graph/ +graph build -o build-graph/ + +# gnd +gnd codegen -o generated-gnd/ +gnd build -o build-gnd/ + +# Compare (should be identical after formatting) +diff -r generated-graph generated-gnd +diff -r build-graph build-gnd +``` + +### Step 4: Update CI/CD + +Replace `graph` with `gnd` in your CI configuration: + +```yaml +# Before +- run: graph codegen +- run: graph build +- run: graph deploy --studio -l ${{ github.sha }} ${{ secrets.SUBGRAPH_NAME }} + +# After +- run: gnd codegen +- run: gnd build +- run: gnd deploy -l ${{ github.sha }} ${{ secrets.SUBGRAPH_NAME }} +``` + +### Step 5: Update package.json Scripts (Optional) + +If you have npm scripts calling graph-cli: + +```json +{ + "scripts": { + "codegen": "gnd codegen", + "build": "gnd build", + "deploy": "gnd deploy -l $VERSION_LABEL" + } +} +``` + +Set `VERSION_LABEL` in your environment (for example, from a git tag or commit SHA). + +## Troubleshooting + +### "Command not found: gnd" + +Make sure the gnd binary is in your PATH: + +```bash +export PATH="$PATH:/path/to/graph-node/target/release" +``` + +### "prettier: command not found" + +`gnd codegen` shells out to `prettier` for formatting. Install it: + +```bash +npm install -g prettier +# or +pnpm add -g prettier +``` + +### "asc: command not found" + +`gnd build` shells out to the AssemblyScript compiler. Install it: + +```bash +npm install -g assemblyscript +# or have it in your subgraph's node_modules +npm install --save-dev assemblyscript +``` + +### Different Output After codegen + +If you notice differences in generated code: + +1. Check if it's one of the documented differences (Int8 import, trailing commas) +2. Run prettier on both outputs to normalize formatting +3. Report any unexpected differences as a bug + +### Authentication Issues + +Keys stored by graph-cli work with gnd and vice versa. If you have issues: + +```bash +# Re-authenticate +gnd auth YOUR_DEPLOY_KEY + +# Verify the key was saved +cat ~/.graph-cli.json +``` + +## Why Migrate? + +### Benefits of gnd + +- **Native performance**: Rust implementation is faster for large subgraphs +- **Integrated with graph-node**: Same codebase, consistent behavior +- **Better error messages**: Leverages graph-node's validation +- **`gnd dev` command**: Run graph-node in development mode + +### When to Keep Using graph-cli + +- If you need multi-protocol support (NEAR, Cosmos, Arweave, Substreams) - gnd currently focuses on Ethereum +- If you use the `--uncrashable` flag +- If you need the `graph local` or `graph node` commands + +## Getting Help + +- File issues at: https://github.com/graphprotocol/graph-node/issues +- Check gnd version: `gnd --version` diff --git a/gnd/npm/README.md b/gnd/npm/README.md new file mode 100644 index 00000000000..acca621b4dc --- /dev/null +++ b/gnd/npm/README.md @@ -0,0 +1,35 @@ +# The Graph CLI + +This package installs the CLI for developing subgraphs for [The Graph +Network](https://thegraph.com/subgraphs/) It supports all aspects of +[developing, testing, and deploying subgraphs](https://thegraph.com/docs/en/subgraphs/quick-start/) locally and +on the network. + +This package is a complete replacement for the older +[graph-cli](https://www.npmjs.com/package/@graphprotocol/graph-cli) which +will over time be migrated to be a simple wrapper for `gnd`. Older +documentation will reference `graph-cli` in its instructions; running +`alias graph=gnd` in the shell make it possible to follow these +instructions verbatim with `gnd`. + +Besides the tools to develop subgraphs, `gnd` also contains a version of +[graph-node](https://github.com/graphprotocol/graph-node) tailored to +running subgraphs locally via `gnd dev`. + + +## Getting started + +Run `npm install -g @graphprotocol/gnd` to install `gnd`. + +After installation, you can create and run an example subgraph locally with +```bash +gnd init --from-example ethereum-gravatar 'My new subgraph' new-subgraph +cd new-subgraph +gnd codegen +gnd build +gnd dev --ethereum-rpc 'mainnet:' +``` + +If you have an existing contract for which you want to write a subgraph, +have a look at `gnd help init` and the `--from-contract` option. You can +also simply run `gnd init` and follow the prompts to set up your subgraph. diff --git a/gnd/npm/bin/gnd.js b/gnd/npm/bin/gnd.js new file mode 100644 index 00000000000..82a7ae5bf09 --- /dev/null +++ b/gnd/npm/bin/gnd.js @@ -0,0 +1,33 @@ +#!/usr/bin/env node + +const { execFileSync } = require("child_process"); +const path = require("path"); + +const PLATFORMS = { + "darwin-arm64": "@graphprotocol/gnd-darwin-arm64", + "darwin-x64": "@graphprotocol/gnd-darwin-x64", + "linux-arm64": "@graphprotocol/gnd-linux-arm64", + "linux-x64": "@graphprotocol/gnd-linux-x64", + "win32-x64": "@graphprotocol/gnd-win32-x64", +}; + +const key = `${process.platform}-${process.arch}`; +const pkg = PLATFORMS[key]; +if (!pkg) { + console.error(`Unsupported platform: ${key}`); + process.exit(1); +} + +const bin = process.platform === "win32" ? "gnd.exe" : "gnd"; +const binPath = path.join( + require.resolve(`${pkg}/package.json`), + "..", + "bin", + bin +); + +try { + execFileSync(binPath, process.argv.slice(2), { stdio: "inherit" }); +} catch (e) { + process.exit(e.status ?? 1); +} diff --git a/gnd/src/abi.rs b/gnd/src/abi.rs new file mode 100644 index 00000000000..0a5fa695cf4 --- /dev/null +++ b/gnd/src/abi.rs @@ -0,0 +1,101 @@ +//! ABI normalization utilities. +//! +//! Handles extraction of bare ABI arrays from various artifact formats +//! (raw arrays, Hardhat/Foundry, Truffle). + +use anyhow::{Context, Result, anyhow}; + +/// Normalize ABI JSON to extract the actual ABI array from various artifact formats. +/// +/// Supports: +/// - Raw ABI array: `[{...}]` +/// - Foundry/Hardhat format: `{"abi": [...], ...}` +/// - Truffle format: `{"compilerOutput": {"abi": [...], ...}, ...}` +pub fn normalize_abi_json(abi_str: &str) -> Result { + let value: serde_json::Value = + serde_json::from_str(abi_str).context("Failed to parse ABI JSON")?; + + // Case 1: Already an array - return as-is + if value.is_array() { + return Ok(value); + } + + // Case 2: Object with "abi" field (Foundry/Hardhat format) + if let Some(abi) = value.get("abi") + && abi.is_array() + { + return Ok(abi.clone()); + } + + // Case 3: Object with "compilerOutput.abi" field (Truffle format) + if let Some(compiler_output) = value.get("compilerOutput") + && let Some(abi) = compiler_output.get("abi") + && abi.is_array() + { + return Ok(abi.clone()); + } + + Err(anyhow!( + "Invalid ABI format: expected an array or an object with 'abi' field" + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_normalize_abi_json_raw_array() { + let raw_abi = r#"[{"type": "event", "name": "Transfer"}]"#; + let result = normalize_abi_json(raw_abi).unwrap(); + assert!(result.is_array()); + assert_eq!(result.as_array().unwrap().len(), 1); + } + + #[test] + fn test_normalize_abi_json_hardhat_format() { + let hardhat_abi = r#"{ + "_format": "hh-sol-artifact-1", + "contractName": "MyContract", + "abi": [{"type": "event", "name": "Transfer"}], + "bytecode": "0x..." + }"#; + let result = normalize_abi_json(hardhat_abi).unwrap(); + assert!(result.is_array()); + assert_eq!(result.as_array().unwrap().len(), 1); + assert_eq!( + result.as_array().unwrap()[0].get("name").unwrap(), + "Transfer" + ); + } + + #[test] + fn test_normalize_abi_json_truffle_format() { + let truffle_abi = r#"{ + "contractName": "MyContract", + "compilerOutput": { + "abi": [{"type": "event", "name": "Transfer"}] + } + }"#; + let result = normalize_abi_json(truffle_abi).unwrap(); + assert!(result.is_array()); + assert_eq!(result.as_array().unwrap().len(), 1); + assert_eq!( + result.as_array().unwrap()[0].get("name").unwrap(), + "Transfer" + ); + } + + #[test] + fn test_normalize_abi_json_invalid_format() { + let invalid_abi = r#"{"contractName": "MyContract"}"#; + let result = normalize_abi_json(invalid_abi); + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("Invalid ABI format") + ); + } +} diff --git a/gnd/src/codegen/abi.rs b/gnd/src/codegen/abi.rs new file mode 100644 index 00000000000..9b881fdcbe9 --- /dev/null +++ b/gnd/src/codegen/abi.rs @@ -0,0 +1,1955 @@ +//! ABI code generation for Ethereum contracts. +//! +//! Generates AssemblyScript bindings from contract ABIs: +//! - Event classes with typed parameters +//! - Call classes for function calls with inputs/outputs +//! - Contract class with typed call methods + +use std::collections::HashMap; + +use graph::abi::{ + DynSolType, Event, EventParam, Function, FunctionExt, JsonAbi, Param, StateMutability, +}; +use regex::Regex; + +use super::typescript::{self as ts, Class, ClassMember, Method, ModuleImports, Param as TsParam}; +use crate::shared::{capitalize, handle_reserved_word}; + +/// Resolve a `Param`'s type to `DynSolType`. +fn resolve_param_type(param: &Param) -> DynSolType { + param + .selector_type() + .parse::() + .expect("valid ABI type") +} + +/// Resolve an `EventParam`'s type to `DynSolType`. +fn resolve_event_param_type(param: &EventParam) -> DynSolType { + param + .selector_type() + .parse::() + .expect("valid ABI type") +} + +const GRAPH_TS_MODULE: &str = "@graphprotocol/graph-ts"; + +/// ABI code generator. +pub struct AbiCodeGenerator { + contract: JsonAbi, + name: String, +} + +impl AbiCodeGenerator { + /// Create a new ABI code generator. + pub fn new(contract: JsonAbi, name: impl Into) -> Self { + let mut name = name.into(); + // Sanitize name to be a valid class name + let re = Regex::new(r#"[!@#$%^&*()+\-=\[\]{};':\"|,.<>/?]+"#).unwrap(); + name = re.replace_all(&name, "_").to_string(); + Self { contract, name } + } + + /// Generate module imports for the ABI file. + pub fn generate_module_imports(&self) -> Vec { + vec![ModuleImports::new( + vec![ + "ethereum".to_string(), + "JSONValue".to_string(), + "TypedMap".to_string(), + "Entity".to_string(), + "Bytes".to_string(), + "Address".to_string(), + "BigInt".to_string(), + ], + GRAPH_TS_MODULE, + )] + } + + /// Generate all types from the ABI. + pub fn generate_types(&self) -> Vec { + let mut classes = Vec::new(); + classes.extend(self.generate_event_types()); + classes.extend(self.generate_smart_contract_class()); + classes.extend(self.generate_call_types()); + classes + } + + /// Generate event type classes. + fn generate_event_types(&self) -> Vec { + let mut classes = Vec::new(); + let events = self.disambiguate_events(); + + for (event, alias) in events { + let event_class_name = alias.clone(); + let mut tuple_classes = Vec::new(); + + // Generate params class + let params_class_name = [&event_class_name, "__Params"].concat(); + let mut params_class = ts::klass(¶ms_class_name).exported(); + params_class.add_member(ClassMember::new("_event", &event_class_name)); + params_class.add_method(Method::new( + "constructor", + vec![TsParam::new("event", ts::NamedType::new(&event_class_name))], + None, + "this._event = event", + )); + + // Generate getters for event params + let inputs = self.disambiguate_event_params(&event.inputs, "param"); + for (index, (param, param_name)) in inputs.iter().enumerate() { + let param_object = self.generate_event_param( + param, + param_name, + index, + &event_class_name, + &mut tuple_classes, + ); + params_class.add_method(param_object); + } + + // Generate event class + let mut event_class = ts::klass(&event_class_name) + .exported() + .extends("ethereum.Event"); + event_class.add_method(Method::new( + "get params", + vec![], + Some(ts::NamedType::new(¶ms_class_name).into()), + format!("return new {}(this)", params_class_name), + )); + + classes.push(event_class); + classes.push(params_class); + classes.extend(tuple_classes); + } + + classes + } + + /// Generate the smart contract class with call methods. + fn generate_smart_contract_class(&self) -> Vec { + let mut classes = Vec::new(); + + let mut contract_class = ts::klass(&self.name) + .exported() + .extends("ethereum.SmartContract"); + + // Add static bind method + let contract_name = &self.name; + contract_class.add_static_method(ts::StaticMethod::new( + "bind", + vec![TsParam::new("address", ts::NamedType::new("Address"))], + ts::NamedType::new(&self.name), + format!("return new {}('{}', address)", contract_name, contract_name), + )); + + // Get callable functions and sort alphabetically for deterministic output + let mut functions = self.get_callable_functions(); + functions.sort_by(|a, b| a.name.cmp(&b.name)); + let disambiguated = self.disambiguate_functions(&functions); + + for (func, alias) in disambiguated { + let (method, try_method, result_classes) = self.generate_function_methods(func, &alias); + contract_class.add_method(method); + contract_class.add_method(try_method); + classes.extend(result_classes); + } + + classes.push(contract_class); + classes + } + + /// Generate call type classes. + fn generate_call_types(&self) -> Vec { + let mut classes = Vec::new(); + let mut functions = self.get_call_functions(); + functions.sort_by(|a, b| a.name.cmp(&b.name)); + let disambiguated = self.disambiguate_call_functions(&functions); + + for (func, alias) in disambiguated { + let cap_alias = capitalize(&alias); + let call_class_name = format!("{}Call", cap_alias); + let mut tuple_classes = Vec::new(); + + // Generate inputs class + let inputs_class_name = [&call_class_name, "__Inputs"].concat(); + let mut inputs_class = ts::klass(&inputs_class_name).exported(); + inputs_class.add_member(ClassMember::new("_call", &call_class_name)); + inputs_class.add_method(Method::new( + "constructor", + vec![TsParam::new("call", ts::NamedType::new(&call_class_name))], + None, + "this._call = call", + )); + + let inputs = self.disambiguate_params(&func.inputs, "value"); + for (index, (param, param_name)) in inputs.iter().enumerate() { + let getter = self.generate_input_output_getter( + param, + param_name, + index, + &call_class_name, + "call", + "inputValues", + &mut tuple_classes, + ); + inputs_class.add_method(getter); + } + + // Generate outputs class + let outputs_class_name = [&call_class_name, "__Outputs"].concat(); + let mut outputs_class = ts::klass(&outputs_class_name).exported(); + outputs_class.add_member(ClassMember::new("_call", &call_class_name)); + outputs_class.add_method(Method::new( + "constructor", + vec![TsParam::new("call", ts::NamedType::new(&call_class_name))], + None, + "this._call = call", + )); + + let outputs = self.disambiguate_params(&func.outputs, "value"); + for (index, (param, param_name)) in outputs.iter().enumerate() { + let getter = self.generate_input_output_getter( + param, + param_name, + index, + &call_class_name, + "call", + "outputValues", + &mut tuple_classes, + ); + outputs_class.add_method(getter); + } + + // Generate call class + let mut call_class = ts::klass(&call_class_name) + .exported() + .extends("ethereum.Call"); + call_class.add_method(Method::new( + "get inputs", + vec![], + Some(ts::NamedType::new(&inputs_class_name).into()), + format!("return new {}(this)", inputs_class_name), + )); + call_class.add_method(Method::new( + "get outputs", + vec![], + Some(ts::NamedType::new(&outputs_class_name).into()), + format!("return new {}(this)", outputs_class_name), + )); + + classes.push(call_class); + classes.push(inputs_class); + classes.push(outputs_class); + classes.extend(tuple_classes); + } + + classes + } + + /// Generate a getter method for an event parameter. + fn generate_event_param( + &self, + param: &EventParam, + name: &str, + index: usize, + event_class_name: &str, + tuple_classes: &mut Vec, + ) -> Method { + let param_type = resolve_event_param_type(param); + + // Handle indexed params - strings, bytes and arrays are hashed to bytes32 + let value_type = if param.indexed { + indexed_input_type(¶m_type) + } else { + param_type.clone() + }; + + if contains_tuple_type(&value_type) { + self.generate_tuple_getter_for_event_param( + param, + ¶m_type, + name, + index, + event_class_name, + "event", + "parameters", + tuple_classes, + ) + } else { + let asc_type = asc_type_for_ethereum(&value_type); + let access = format!("this._event.parameters[{}].value", index); + let conversion = ethereum_to_asc(&access, &value_type, None); + Method::new( + format!("get {}", name), + vec![], + Some(ts::TypeExpr::Raw(asc_type)), + format!("return {}", conversion), + ) + } + } + + /// Generate a getter for call inputs/outputs. + #[allow(clippy::too_many_arguments)] + fn generate_input_output_getter( + &self, + param: &Param, + name: &str, + index: usize, + parent_class: &str, + parent_type: &str, + parent_field: &str, + tuple_classes: &mut Vec, + ) -> Method { + let param_type = resolve_param_type(param); + if contains_tuple_type(¶m_type) { + self.generate_tuple_getter( + param, + ¶m_type, + name, + index, + parent_class, + parent_type, + parent_field, + tuple_classes, + ) + } else { + let asc_type = asc_type_for_ethereum(¶m_type); + let access = format!("this._{}.{}[{}].value", parent_type, parent_field, index); + let conversion = ethereum_to_asc(&access, ¶m_type, None); + Method::new( + format!("get {}", name), + vec![], + Some(ts::TypeExpr::Raw(asc_type)), + format!("return {}", conversion), + ) + } + } + + /// Generate a tuple getter and its associated classes (for `Param`). + #[allow(clippy::too_many_arguments)] + fn generate_tuple_getter( + &self, + param: &Param, + param_type: &DynSolType, + name: &str, + index: usize, + parent_class: &str, + parent_type: &str, + parent_field: &str, + tuple_classes: &mut Vec, + ) -> Method { + let cap_name = capitalize(name); + let tuple_identifier = format!("{}{}", parent_class, cap_name); + let tuple_class_name = if parent_field == "outputValues" { + format!("{}OutputStruct", tuple_identifier) + } else { + format!("{}Struct", tuple_identifier) + }; + + let is_tuple = matches!(param_type, DynSolType::Tuple(_)); + let access_code = if parent_type == "tuple" { + format!("this[{}]", index) + } else { + format!("this._{}.{}[{}].value", parent_type, parent_field, index) + }; + + let return_value = ethereum_to_asc(&access_code, param_type, Some(&tuple_class_name)); + + let return_type = if is_tuple_matrix_type(param_type) { + format!("Array>", tuple_class_name) + } else if is_tuple_array_type(param_type) { + format!("Array<{}>", tuple_class_name) + } else { + tuple_class_name.clone() + }; + + let body = if is_tuple { + format!("return changetype<{}>({})", tuple_class_name, return_value) + } else { + format!("return {}", return_value) + }; + + // Generate tuple class from Param's components + let components = get_tuple_param_components(param); + if !components.is_empty() { + let mut tuple_class = ts::klass(&tuple_class_name) + .exported() + .extends("ethereum.Tuple"); + + let component_params = disambiguate_tuple_components(components); + for (idx, (component, component_name)) in component_params.iter().enumerate() { + let component_getter = self.generate_tuple_component_getter( + component, + component_name, + idx, + &tuple_identifier, + tuple_classes, + ); + tuple_class.add_method(component_getter); + } + + tuple_classes.push(tuple_class); + } + + Method::new( + format!("get {}", name), + vec![], + Some(ts::TypeExpr::Raw(return_type)), + body, + ) + } + + /// Generate a tuple getter and its associated classes (for `EventParam`). + #[allow(clippy::too_many_arguments)] + fn generate_tuple_getter_for_event_param( + &self, + param: &EventParam, + param_type: &DynSolType, + name: &str, + index: usize, + parent_class: &str, + parent_type: &str, + parent_field: &str, + tuple_classes: &mut Vec, + ) -> Method { + let cap_name = capitalize(name); + let tuple_identifier = format!("{}{}", parent_class, cap_name); + let tuple_class_name = if parent_field == "outputValues" { + format!("{}OutputStruct", tuple_identifier) + } else { + format!("{}Struct", tuple_identifier) + }; + + let is_tuple = matches!(param_type, DynSolType::Tuple(_)); + let access_code = if parent_type == "tuple" { + format!("this[{}]", index) + } else { + format!("this._{}.{}[{}].value", parent_type, parent_field, index) + }; + + let return_value = ethereum_to_asc(&access_code, param_type, Some(&tuple_class_name)); + + let return_type = if is_tuple_matrix_type(param_type) { + format!("Array>", tuple_class_name) + } else if is_tuple_array_type(param_type) { + format!("Array<{}>", tuple_class_name) + } else { + tuple_class_name.clone() + }; + + let body = if is_tuple { + format!("return changetype<{}>({})", tuple_class_name, return_value) + } else { + format!("return {}", return_value) + }; + + // Generate tuple class from EventParam's components + // EventParam.components is Vec, same as Param.components + let components = ¶m.components; + if !components.is_empty() { + let mut tuple_class = ts::klass(&tuple_class_name) + .exported() + .extends("ethereum.Tuple"); + + let component_params = disambiguate_tuple_components(components); + for (idx, (component, component_name)) in component_params.iter().enumerate() { + let component_getter = self.generate_tuple_component_getter( + component, + component_name, + idx, + &tuple_identifier, + tuple_classes, + ); + tuple_class.add_method(component_getter); + } + + tuple_classes.push(tuple_class); + } + + Method::new( + format!("get {}", name), + vec![], + Some(ts::TypeExpr::Raw(return_type)), + body, + ) + } + + /// Generate a getter for a tuple component. + fn generate_tuple_component_getter( + &self, + param: &Param, + name: &str, + index: usize, + parent_class: &str, + tuple_classes: &mut Vec, + ) -> Method { + let param_type = resolve_param_type(param); + if contains_tuple_type(¶m_type) { + self.generate_tuple_getter( + param, + ¶m_type, + name, + index, + parent_class, + "tuple", + "", + tuple_classes, + ) + } else { + let asc_type = asc_type_for_ethereum(¶m_type); + let access = format!("this[{}]", index); + let conversion = ethereum_to_asc(&access, ¶m_type, None); + Method::new( + format!("get {}", name), + vec![], + Some(ts::TypeExpr::Raw(asc_type)), + format!("return {}", conversion), + ) + } + } + + /// Generate methods for a callable function. + fn generate_function_methods( + &self, + func: &Function, + alias: &str, + ) -> (Method, Method, Vec) { + let mut result_classes = Vec::new(); + let fn_signature = func.signature_compat(); + let contract_name = &self.name; + let tuple_result_parent_type = [contract_name, "__", alias, "Result"].concat(); + let tuple_input_parent_type = [contract_name, "__", alias, "Input"].concat(); + + // Disambiguate outputs + let outputs = self.disambiguate_params(&func.outputs, "value"); + + // Determine return type + let (return_type, simple_return_type) = if outputs.len() > 1 { + // Multiple outputs - create a result struct + let result_class = self.generate_result_class( + &outputs, + &tuple_result_parent_type, + &mut result_classes, + ); + result_classes.push(result_class.clone()); + (result_class.name.clone(), false) + } else if !outputs.is_empty() { + let (param, _) = &outputs[0]; + let param_type = resolve_param_type(param); + if contains_tuple_type(¶m_type) { + let tuple_name = self.generate_tuple_return_type( + param, + ¶m_type, + 0, + &tuple_result_parent_type, + &mut result_classes, + ); + (tuple_name, true) + } else { + (asc_type_for_ethereum(¶m_type), true) + } + } else { + ("void".to_string(), true) + }; + + // Disambiguate inputs + let inputs = self.disambiguate_params(&func.inputs, "param"); + + // Generate tuple types for inputs + for (index, (param, _)) in inputs.iter().enumerate() { + let param_type = resolve_param_type(param); + if contains_tuple_type(¶m_type) { + self.generate_tuple_class_for_input( + param, + index, + &tuple_input_parent_type, + &mut result_classes, + ); + } + } + + // Build params + let params: Vec = inputs + .iter() + .enumerate() + .map(|(index, (param, name))| { + let p_type = resolve_param_type(param); + let param_type_str = + get_param_type_for_input(&p_type, index, &tuple_input_parent_type); + TsParam::new(name.clone(), ts::TypeExpr::Raw(param_type_str)) + }) + .collect(); + + // Build call arguments + let call_args: Vec = inputs + .iter() + .map(|(param, name)| { + let p_type = resolve_param_type(param); + ethereum_from_asc(name, &p_type) + }) + .collect(); + + let func_name = &func.name; + let call_args_str = call_args.join(", "); + let super_inputs = format!("'{}', '{}', [{}]", func_name, fn_signature, call_args_str); + + // Generate method body + let method_body = self.generate_call_body( + &outputs, + &return_type, + simple_return_type, + &super_inputs, + &tuple_result_parent_type, + false, + ); + + let try_method_body = self.generate_call_body( + &outputs, + &return_type, + simple_return_type, + &super_inputs, + &tuple_result_parent_type, + true, + ); + + let method = Method::new( + alias.to_string(), + params.clone(), + Some(ts::TypeExpr::Raw(return_type.clone())), + method_body, + ); + + let try_method = Method::new( + format!("try_{}", alias), + params, + Some(ts::TypeExpr::Raw(format!( + "ethereum.CallResult<{}>", + return_type + ))), + try_method_body, + ); + + (method, try_method, result_classes) + } + + /// Generate call method body. + fn generate_call_body( + &self, + outputs: &[(&Param, String)], + return_type: &str, + simple_return_type: bool, + super_inputs: &str, + tuple_result_parent_type: &str, + is_try: bool, + ) -> String { + let nl = "\n"; + let (call_stmt, result_var) = if is_try { + let mut lines = Vec::new(); + lines.push(format!("let result = super.tryCall({})", super_inputs)); + lines.push(" if (result.reverted) {".to_string()); + lines.push(" return new ethereum.CallResult()".to_string()); + lines.push(" }".to_string()); + lines.push(" let value = result.value".to_string()); + (lines.join(nl), "value") + } else { + ( + format!("let result = super.call({})", super_inputs), + "result", + ) + }; + + let return_val = if simple_return_type { + if outputs.is_empty() { + String::new() + } else { + let (param, _) = &outputs[0]; + let p_type = resolve_param_type(param); + let tuple_name = if is_tuple_array_type(&p_type) { + Some(tuple_type_name(0, tuple_result_parent_type)) + } else { + None + }; + let val = ethereum_to_asc( + &format!("{}[0]", result_var), + &p_type, + tuple_name.as_deref(), + ); + if matches!(p_type, DynSolType::Tuple(_)) { + format!("changetype<{}>({})", return_type, val) + } else { + val + } + } + } else { + let conversions: Vec = outputs + .iter() + .enumerate() + .map(|(index, (param, _))| { + let p_type = resolve_param_type(param); + let tuple_name = if is_tuple_array_type(&p_type) { + Some(tuple_type_name(index, tuple_result_parent_type)) + } else { + None + }; + let val = ethereum_to_asc( + &format!("{}[{}]", result_var, index), + &p_type, + tuple_name.as_deref(), + ); + if matches!(p_type, DynSolType::Tuple(_)) { + let tn = tuple_type_name(index, tuple_result_parent_type); + format!("changetype<{}>({})", tn, val) + } else { + val + } + }) + .collect(); + let conv_str = conversions.join(", "); + format!("new {}({})", return_type, conv_str) + }; + + if is_try { + [ + &call_stmt, + nl, + " return ethereum.CallResult.fromValue(", + &return_val, + ")", + ] + .concat() + } else if outputs.is_empty() { + call_stmt + } else { + [&call_stmt, nl, nl, " return (", &return_val, ")"].concat() + } + } + + /// Generate a result class for multiple outputs. + fn generate_result_class( + &self, + outputs: &[(&Param, String)], + tuple_result_parent_type: &str, + result_classes: &mut Vec, + ) -> Class { + let class_name = tuple_result_parent_type.to_string(); + let mut klass = ts::klass(&class_name).exported(); + + // Add constructor + let constructor_params: Vec = outputs + .iter() + .enumerate() + .map(|(index, (param, _))| { + let p_type = resolve_param_type(param); + let param_type_str = + get_param_type_for_input(&p_type, index, tuple_result_parent_type); + TsParam::new(format!("value{}", index), ts::TypeExpr::Raw(param_type_str)) + }) + .collect(); + + let nl = "\n"; + let constructor_body: Vec = outputs + .iter() + .enumerate() + .map(|(index, _)| format!("this.value{} = value{}", index, index)) + .collect(); + + klass.add_method(Method::new( + "constructor", + constructor_params, + None, + constructor_body.join(&format!("{} ", nl)), + )); + + // Add toMap method + let map_entries: Vec = outputs + .iter() + .enumerate() + .map(|(index, (param, _))| { + let p_type = resolve_param_type(param); + let this_val = format!("this.value{}", index); + let from_asc = ethereum_from_asc(&this_val, &p_type); + format!("map.set('value{}', {})", index, from_asc) + }) + .collect(); + + let map_body = [ + "let map = new TypedMap()", + nl, + " ", + &map_entries.join(&format!("{} ", nl)), + nl, + " return map", + ] + .concat(); + + klass.add_method(Method::new( + "toMap", + vec![], + Some(ts::TypeExpr::Raw( + "TypedMap".to_string(), + )), + map_body, + )); + + // Add members + for (index, (param, _)) in outputs.iter().enumerate() { + let p_type = resolve_param_type(param); + let param_type_str = get_param_type_for_input(&p_type, index, tuple_result_parent_type); + klass.add_member(ClassMember::new(format!("value{}", index), param_type_str)); + } + + // Add getters for outputs + for (index, (param, _)) in outputs.iter().enumerate() { + let getter_name = if param.name.trim().is_empty() { + format!("getValue{}", index) + } else { + let cap = capitalize(¶m.name); + format!("get{}", cap) + }; + let p_type = resolve_param_type(param); + let param_type_str = get_param_type_for_input(&p_type, index, tuple_result_parent_type); + klass.add_method(Method::new( + getter_name, + vec![], + Some(ts::TypeExpr::Raw(param_type_str)), + format!("return this.value{}", index), + )); + } + + // Generate tuple classes for outputs + for (index, (param, _)) in outputs.iter().enumerate() { + let p_type = resolve_param_type(param); + if contains_tuple_type(&p_type) { + self.generate_tuple_class_for_input( + param, + index, + tuple_result_parent_type, + result_classes, + ); + } + } + + klass + } + + /// Generate tuple return type name and classes. + fn generate_tuple_return_type( + &self, + param: &Param, + param_type: &DynSolType, + index: usize, + parent_type: &str, + result_classes: &mut Vec, + ) -> String { + self.generate_tuple_class_for_input(param, index, parent_type, result_classes); + let tn = tuple_type_name(index, parent_type); + if is_tuple_array_type(param_type) { + format!("Array<{}>", tn) + } else if is_tuple_matrix_type(param_type) { + format!("Array>", tn) + } else { + tn + } + } + + /// Generate tuple class for an input/output. + fn generate_tuple_class_for_input( + &self, + param: &Param, + index: usize, + parent_type: &str, + result_classes: &mut Vec, + ) { + let tuple_class_name = tuple_type_name(index, parent_type); + let mut tuple_class = ts::klass(&tuple_class_name) + .exported() + .extends("ethereum.Tuple"); + + let components = get_tuple_param_components(param); + if !components.is_empty() { + let component_params = disambiguate_tuple_components(components); + for (idx, (component, component_name)) in component_params.iter().enumerate() { + let component_type = resolve_param_type(component); + let getter = if contains_tuple_type(&component_type) { + // Recursively generate tuple classes + let cap = capitalize(&format!("{}", index)); + let nested_parent = format!("{}Value{}", parent_type, cap); + self.generate_tuple_class_for_input( + component, + idx, + &nested_parent, + result_classes, + ); + let nested_tuple_name = tuple_type_name(idx, &nested_parent); + let access = format!("this[{}]", idx); + let conversion = + ethereum_to_asc(&access, &component_type, Some(&nested_tuple_name)); + let return_type = if is_tuple_array_type(&component_type) { + format!("Array<{}>", nested_tuple_name) + } else { + nested_tuple_name.clone() + }; + let body = if matches!(component_type, DynSolType::Tuple(_)) { + format!("return changetype<{}>({})", nested_tuple_name, conversion) + } else { + format!("return {}", conversion) + }; + Method::new( + format!("get {}", component_name), + vec![], + Some(ts::TypeExpr::Raw(return_type)), + body, + ) + } else { + let asc_type = asc_type_for_ethereum(&component_type); + let access = format!("this[{}]", idx); + let conversion = ethereum_to_asc(&access, &component_type, None); + Method::new( + format!("get {}", component_name), + vec![], + Some(ts::TypeExpr::Raw(asc_type)), + format!("return {}", conversion), + ) + }; + tuple_class.add_method(getter); + } + } + + result_classes.push(tuple_class); + } + + /// Get callable functions (view, pure, nonpayable, constant with outputs). + fn get_callable_functions(&self) -> Vec<&Function> { + self.contract + .functions() + .filter(|f| { + !f.outputs.is_empty() + && matches!( + f.state_mutability, + StateMutability::View | StateMutability::Pure | StateMutability::NonPayable + ) + }) + .collect() + } + + /// Get functions that can be used as calls (non-view, non-pure functions). + fn get_call_functions(&self) -> Vec<&Function> { + self.contract + .functions() + .filter(|f| { + matches!( + f.state_mutability, + StateMutability::NonPayable | StateMutability::Payable + ) + }) + .collect() + } + + /// Disambiguate events with duplicate names. + fn disambiguate_events(&self) -> Vec<(&Event, String)> { + let mut result = Vec::new(); + let mut collision_counter: HashMap = HashMap::new(); + + for event in self.contract.events() { + let name = handle_reserved_word(&event.name); + let counter = collision_counter.entry(name.clone()).or_insert(0); + let alias = if *counter == 0 { + name.clone() + } else { + format!("{}{}", name, counter) + }; + *counter += 1; + result.push((event, alias)); + } + + result + } + + /// Disambiguate functions. + fn disambiguate_functions<'a>( + &self, + functions: &[&'a Function], + ) -> Vec<(&'a Function, String)> { + let mut result = Vec::new(); + let mut collision_counter: HashMap = HashMap::new(); + + for func in functions { + let name = handle_reserved_word(&func.name); + let counter = collision_counter.entry(name.clone()).or_insert(0); + let alias = if *counter == 0 { + name.clone() + } else { + format!("{}{}", name, counter) + }; + *counter += 1; + result.push((*func, alias)); + } + + result + } + + /// Disambiguate call functions. + fn disambiguate_call_functions<'a>( + &self, + functions: &[&'a Function], + ) -> Vec<(&'a Function, String)> { + let mut result = Vec::new(); + let mut collision_counter: HashMap = HashMap::new(); + + for func in functions { + let name = if func.name.is_empty() { + "default".to_string() + } else { + handle_reserved_word(&func.name) + }; + let counter = collision_counter.entry(name.clone()).or_insert(0); + let alias = if *counter == 0 { + name.clone() + } else { + format!("{}{}", name, counter) + }; + *counter += 1; + result.push((*func, alias)); + } + + result + } + + /// Disambiguate event params. + fn disambiguate_event_params<'a>( + &self, + params: &'a [EventParam], + default_prefix: &str, + ) -> Vec<(&'a EventParam, String)> { + let mut result = Vec::new(); + let mut collision_counter: HashMap = HashMap::new(); + + for (index, param) in params.iter().enumerate() { + let name = if param.name.is_empty() { + format!("{}{}", default_prefix, index) + } else { + handle_reserved_word(¶m.name) + }; + let counter = collision_counter.entry(name.clone()).or_insert(0); + let disambiguated = if *counter == 0 { + name.clone() + } else { + format!("{}{}", name, counter) + }; + *counter += 1; + result.push((param, disambiguated)); + } + + result + } + + /// Disambiguate function params. + fn disambiguate_params<'a>( + &self, + params: &'a [Param], + default_prefix: &str, + ) -> Vec<(&'a Param, String)> { + let mut result = Vec::new(); + let mut collision_counter: HashMap = HashMap::new(); + + for (index, param) in params.iter().enumerate() { + let name = if param.name.is_empty() { + format!("{}{}", default_prefix, index) + } else { + handle_reserved_word(¶m.name) + }; + let counter = collision_counter.entry(name.clone()).or_insert(0); + let disambiguated = if *counter == 0 { + name.clone() + } else { + format!("{}{}", name, counter) + }; + *counter += 1; + result.push((param, disambiguated)); + } + + result + } +} + +/// Get tuple type name for a param. +fn tuple_type_name(index: usize, parent_type: &str) -> String { + format!("{}Value{}Struct", parent_type, index) +} + +/// Get the param type string for an input, handling tuples. +fn get_param_type_for_input(param_type: &DynSolType, index: usize, parent_type: &str) -> String { + if matches!(param_type, DynSolType::Tuple(_)) { + tuple_type_name(index, parent_type) + } else if is_tuple_matrix_type(param_type) { + let tn = tuple_type_name(index, parent_type); + format!("Array>", tn) + } else if is_tuple_array_type(param_type) { + let tn = tuple_type_name(index, parent_type); + format!("Array<{}>", tn) + } else { + asc_type_for_ethereum(param_type) + } +} + +/// Get the tuple components from a `Param`, following through arrays. +/// Returns the `components` from the innermost tuple `Param`. +fn get_tuple_param_components(param: &Param) -> &[Param] { + // If this param has components directly (tuple type), return them + if !param.components.is_empty() { + return ¶m.components; + } + // For array-of-tuple types, the components are on the param itself + // (alloy stores them on the outer param for tuple[] types) + &[] +} + +/// Disambiguate tuple components, reading names directly from `Param.name`. +fn disambiguate_tuple_components(components: &[Param]) -> Vec<(&Param, String)> { + components + .iter() + .enumerate() + .map(|(index, component)| { + let name = if component.name.is_empty() { + format!("value{}", index) + } else { + component.name.clone() + }; + (component, name) + }) + .collect() +} + +/// Get AssemblyScript type for an Ethereum type. +fn asc_type_for_ethereum(param_type: &DynSolType) -> String { + match param_type { + DynSolType::Address => "Address".to_string(), + DynSolType::Bool => "boolean".to_string(), + DynSolType::Bytes => "Bytes".to_string(), + DynSolType::FixedBytes(_) => "Bytes".to_string(), + DynSolType::Int(bits) => { + if *bits <= 32 { + "i32".to_string() + } else { + "BigInt".to_string() + } + } + DynSolType::Uint(bits) => { + if *bits <= 24 { + "i32".to_string() + } else { + "BigInt".to_string() + } + } + DynSolType::String => "string".to_string(), + DynSolType::Array(inner) => { + let inner_type = asc_type_for_ethereum(inner); + format!("Array<{}>", inner_type) + } + DynSolType::FixedArray(inner, _) => { + let inner_type = asc_type_for_ethereum(inner); + format!("Array<{}>", inner_type) + } + DynSolType::Tuple(_) => "ethereum.Tuple".to_string(), + _ => "ethereum.Tuple".to_string(), // Function and other future variants + } +} + +/// Convert ethereum value to AssemblyScript. +fn ethereum_to_asc(code: &str, param_type: &DynSolType, tuple_type: Option<&str>) -> String { + match param_type { + DynSolType::Address => format!("{}.toAddress()", code), + DynSolType::Bool => format!("{}.toBoolean()", code), + DynSolType::Bytes | DynSolType::FixedBytes(_) => format!("{}.toBytes()", code), + DynSolType::Int(bits) => { + if *bits <= 32 { + format!("{}.toI32()", code) + } else { + format!("{}.toBigInt()", code) + } + } + DynSolType::Uint(bits) => { + if *bits <= 24 { + format!("{}.toI32()", code) + } else { + format!("{}.toBigInt()", code) + } + } + DynSolType::String => format!("{}.toString()", code), + DynSolType::Array(inner) | DynSolType::FixedArray(inner, _) => match inner.as_ref() { + DynSolType::Address => format!("{}.toAddressArray()", code), + DynSolType::Bool => format!("{}.toBooleanArray()", code), + DynSolType::Bytes | DynSolType::FixedBytes(_) => { + format!("{}.toBytesArray()", code) + } + DynSolType::Int(bits) => { + if *bits <= 32 { + format!("{}.toI32Array()", code) + } else { + format!("{}.toBigIntArray()", code) + } + } + DynSolType::Uint(bits) => { + if *bits <= 24 { + format!("{}.toI32Array()", code) + } else { + format!("{}.toBigIntArray()", code) + } + } + DynSolType::String => format!("{}.toStringArray()", code), + DynSolType::Tuple(_) => { + if let Some(tuple_name) = tuple_type { + format!("{}.toTupleArray<{}>()", code, tuple_name) + } else { + format!("{}.toTupleArray()", code) + } + } + DynSolType::Array(inner2) | DynSolType::FixedArray(inner2, _) => { + ethereum_to_asc_matrix(code, inner2.as_ref(), tuple_type) + } + _ => format!("{}.toString()", code), // fallback for Function etc. + }, + DynSolType::Tuple(_) => format!("{}.toTuple()", code), + _ => format!("{}.toTuple()", code), // fallback for Function etc. + } +} + +/// Convert matrix type to AssemblyScript. +fn ethereum_to_asc_matrix(code: &str, inner_type: &DynSolType, tuple_type: Option<&str>) -> String { + match inner_type { + DynSolType::Address => format!("{}.toAddressMatrix()", code), + DynSolType::Bool => format!("{}.toBooleanMatrix()", code), + DynSolType::Bytes | DynSolType::FixedBytes(_) => format!("{}.toBytesMatrix()", code), + DynSolType::Int(bits) => { + if *bits <= 32 { + format!("{}.toI32Matrix()", code) + } else { + format!("{}.toBigIntMatrix()", code) + } + } + DynSolType::Uint(bits) => { + if *bits <= 24 { + format!("{}.toI32Matrix()", code) + } else { + format!("{}.toBigIntMatrix()", code) + } + } + DynSolType::String => format!("{}.toStringMatrix()", code), + DynSolType::Tuple(_) => { + if let Some(tuple_name) = tuple_type { + format!("{}.toTupleMatrix<{}>()", code, tuple_name) + } else { + format!("{}.toTupleMatrix()", code) + } + } + _ => format!("{}.toStringMatrix()", code), // fallback + } +} + +/// Convert AssemblyScript value to ethereum value. +fn ethereum_from_asc(code: &str, param_type: &DynSolType) -> String { + match param_type { + DynSolType::Address => format!("ethereum.Value.fromAddress({})", code), + DynSolType::Bool => format!("ethereum.Value.fromBoolean({})", code), + DynSolType::Bytes => format!("ethereum.Value.fromBytes({})", code), + DynSolType::FixedBytes(_) => format!("ethereum.Value.fromFixedBytes({})", code), + DynSolType::Int(bits) => { + if *bits <= 32 { + format!("ethereum.Value.fromI32({})", code) + } else { + format!("ethereum.Value.fromSignedBigInt({})", code) + } + } + DynSolType::Uint(bits) => { + if *bits <= 24 { + format!( + "ethereum.Value.fromUnsignedBigInt(BigInt.fromI32({}))", + code + ) + } else { + format!("ethereum.Value.fromUnsignedBigInt({})", code) + } + } + DynSolType::String => format!("ethereum.Value.fromString({})", code), + DynSolType::Array(inner) | DynSolType::FixedArray(inner, _) => { + ethereum_from_asc_array(code, inner.as_ref()) + } + DynSolType::Tuple(_) => format!("ethereum.Value.fromTuple({})", code), + _ => format!("ethereum.Value.fromTuple({})", code), // fallback + } +} + +/// Convert array to ethereum value. +fn ethereum_from_asc_array(code: &str, inner_type: &DynSolType) -> String { + match inner_type { + DynSolType::Address => format!("ethereum.Value.fromAddressArray({})", code), + DynSolType::Bool => format!("ethereum.Value.fromBooleanArray({})", code), + DynSolType::Bytes => format!("ethereum.Value.fromBytesArray({})", code), + DynSolType::FixedBytes(_) => format!("ethereum.Value.fromFixedBytesArray({})", code), + DynSolType::Int(bits) => { + if *bits <= 32 { + format!("ethereum.Value.fromI32Array({})", code) + } else { + format!("ethereum.Value.fromSignedBigIntArray({})", code) + } + } + DynSolType::Uint(bits) => { + if *bits <= 24 { + format!("ethereum.Value.fromI32Array({})", code) + } else { + format!("ethereum.Value.fromUnsignedBigIntArray({})", code) + } + } + DynSolType::String => format!("ethereum.Value.fromStringArray({})", code), + DynSolType::Tuple(_) => format!("ethereum.Value.fromTupleArray({})", code), + DynSolType::Array(inner2) | DynSolType::FixedArray(inner2, _) => { + ethereum_from_asc_matrix(code, inner2.as_ref()) + } + _ => format!("ethereum.Value.fromStringArray({})", code), // fallback + } +} + +/// Convert matrix to ethereum value. +fn ethereum_from_asc_matrix(code: &str, inner_type: &DynSolType) -> String { + match inner_type { + DynSolType::Address => format!("ethereum.Value.fromAddressMatrix({})", code), + DynSolType::Bool => format!("ethereum.Value.fromBooleanMatrix({})", code), + DynSolType::Bytes => format!("ethereum.Value.fromBytesMatrix({})", code), + DynSolType::FixedBytes(_) => format!("ethereum.Value.fromFixedBytesMatrix({})", code), + DynSolType::Int(bits) => { + if *bits <= 32 { + format!("ethereum.Value.fromI32Matrix({})", code) + } else { + format!("ethereum.Value.fromSignedBigIntMatrix({})", code) + } + } + DynSolType::Uint(bits) => { + if *bits <= 24 { + format!("ethereum.Value.fromI32Matrix({})", code) + } else { + format!("ethereum.Value.fromUnsignedBigIntMatrix({})", code) + } + } + DynSolType::String => format!("ethereum.Value.fromStringMatrix({})", code), + DynSolType::Tuple(_) => format!("ethereum.Value.fromTupleMatrix({})", code), + _ => format!("ethereum.Value.fromStringMatrix({})", code), // fallback + } +} + +/// Check if param type contains a tuple. +fn contains_tuple_type(param_type: &DynSolType) -> bool { + match param_type { + DynSolType::Tuple(_) => true, + DynSolType::Array(inner) | DynSolType::FixedArray(inner, _) => contains_tuple_type(inner), + _ => false, + } +} + +/// Check if param type is a tuple array. +fn is_tuple_array_type(param_type: &DynSolType) -> bool { + matches!( + param_type, + DynSolType::Array(inner) | DynSolType::FixedArray(inner, _) + if matches!(inner.as_ref(), DynSolType::Tuple(_)) + ) +} + +/// Check if param type is a tuple matrix (2D array). +fn is_tuple_matrix_type(param_type: &DynSolType) -> bool { + match param_type { + DynSolType::Array(inner) | DynSolType::FixedArray(inner, _) => is_tuple_array_type(inner), + _ => false, + } +} + +/// Handle indexed input type conversion. +fn indexed_input_type(param_type: &DynSolType) -> DynSolType { + // Strings, bytes, and arrays are encoded and hashed to bytes32 + match param_type { + DynSolType::String | DynSolType::Bytes | DynSolType::Tuple(_) => DynSolType::FixedBytes(32), + DynSolType::Array(_) | DynSolType::FixedArray(_, _) => DynSolType::FixedBytes(32), + _ => param_type.clone(), + } +} + +#[cfg(test)] +mod tests { + + use super::*; + + fn parse_abi(json: &str) -> JsonAbi { + serde_json::from_str(json).unwrap() + } + + #[test] + fn test_simple_event() { + let abi_json = r#"[ + { + "type": "event", + "name": "Transfer", + "inputs": [ + {"name": "from", "type": "address", "indexed": true}, + {"name": "to", "type": "address", "indexed": true}, + {"name": "value", "type": "uint256", "indexed": false} + ], + "anonymous": false + } + ]"#; + + let contract = parse_abi(abi_json); + let generator = AbiCodeGenerator::new(contract, "Token"); + let types = generator.generate_types(); + + assert!(types.iter().any(|c| c.name == "Transfer")); + assert!(types.iter().any(|c| c.name == "Transfer__Params")); + } + + #[test] + fn test_function_with_outputs() { + let abi_json = r#"[ + { + "type": "function", + "name": "balanceOf", + "inputs": [{"name": "owner", "type": "address"}], + "outputs": [{"name": "", "type": "uint256"}], + "stateMutability": "view" + } + ]"#; + + let contract = parse_abi(abi_json); + let generator = AbiCodeGenerator::new(contract, "Token"); + let types = generator.generate_types(); + + assert!(types.iter().any(|c| c.name == "Token")); + let token_class = types.iter().find(|c| c.name == "Token").unwrap(); + + assert!(token_class.methods.iter().any(|m| m.name == "balanceOf")); + assert!( + token_class + .methods + .iter() + .any(|m| m.name == "try_balanceOf") + ); + } + + #[test] + fn test_asc_type_for_ethereum() { + assert_eq!(asc_type_for_ethereum(&DynSolType::Address), "Address"); + assert_eq!(asc_type_for_ethereum(&DynSolType::Bool), "boolean"); + assert_eq!(asc_type_for_ethereum(&DynSolType::Uint(256)), "BigInt"); + assert_eq!(asc_type_for_ethereum(&DynSolType::Uint(8)), "i32"); + assert_eq!(asc_type_for_ethereum(&DynSolType::Int(32)), "i32"); + assert_eq!(asc_type_for_ethereum(&DynSolType::String), "string"); + assert_eq!(asc_type_for_ethereum(&DynSolType::Bytes), "Bytes"); + } + + #[test] + fn test_name_sanitization() { + let generator = AbiCodeGenerator::new(JsonAbi::default(), "Test!Contract@Name"); + assert_eq!(generator.name, "Test_Contract_Name"); + } + + #[test] + fn test_indexed_input_type() { + assert_eq!( + indexed_input_type(&DynSolType::String), + DynSolType::FixedBytes(32) + ); + assert_eq!( + indexed_input_type(&DynSolType::Bytes), + DynSolType::FixedBytes(32) + ); + assert_eq!( + indexed_input_type(&DynSolType::Array(Box::new(DynSolType::Uint(256)))), + DynSolType::FixedBytes(32) + ); + assert_eq!( + indexed_input_type(&DynSolType::Address), + DynSolType::Address + ); + assert_eq!( + indexed_input_type(&DynSolType::Uint(256)), + DynSolType::Uint(256) + ); + } + + /// Test that overloaded events (same name, different inputs) are disambiguated. + /// The TS CLI generates unique names like Transfer, Transfer1, Transfer2. + #[test] + fn test_overloaded_events() { + let abi_json = r#"[ + { + "type": "event", + "name": "Transfer", + "inputs": [], + "anonymous": false + }, + { + "type": "event", + "name": "Transfer", + "inputs": [{"name": "to", "type": "address", "indexed": false}], + "anonymous": false + }, + { + "type": "event", + "name": "Transfer", + "inputs": [ + {"name": "from", "type": "address", "indexed": false}, + {"name": "to", "type": "address", "indexed": false} + ], + "anonymous": false + } + ]"#; + + let contract = parse_abi(abi_json); + let generator = AbiCodeGenerator::new(contract, "Token"); + let types = generator.generate_types(); + + // Get all event class names + let event_names: Vec<&str> = types + .iter() + .filter(|c| c.extends == Some("ethereum.Event".to_string())) + .map(|c| c.name.as_str()) + .collect(); + + // Verify we have 3 distinct Transfer events with disambiguation + assert_eq!( + event_names.len(), + 3, + "Should have 3 Transfer event variants" + ); + + // Check that Transfer (with no suffix) exists + assert!( + event_names.contains(&"Transfer"), + "Should have base Transfer event" + ); + + // Check that numbered variants exist (Transfer1, Transfer2) + assert!( + event_names.contains(&"Transfer1"), + "Should have Transfer1 event" + ); + assert!( + event_names.contains(&"Transfer2"), + "Should have Transfer2 event" + ); + } + + /// Test that overloaded functions are disambiguated. + #[test] + fn test_overloaded_functions() { + let abi_json = r#"[ + { + "type": "function", + "name": "getSomething", + "inputs": [], + "outputs": [{"name": "result", "type": "bytes32"}], + "stateMutability": "view" + }, + { + "type": "function", + "name": "getSomething", + "inputs": [{"name": "owner", "type": "address"}], + "outputs": [{"name": "result", "type": "bytes32"}], + "stateMutability": "view" + } + ]"#; + + let contract = parse_abi(abi_json); + let generator = AbiCodeGenerator::new(contract, "Token"); + let types = generator.generate_types(); + + // Find the Token contract class + let token_class = types.iter().find(|c| c.name == "Token").unwrap(); + + // Get all method names (excluding try_ variants and bind) + let method_names: Vec<&str> = token_class + .methods + .iter() + .map(|m| m.name.as_str()) + .filter(|n| !n.starts_with("try_") && *n != "bind") + .collect(); + + // Verify we have disambiguated getSomething variants + assert!( + method_names.contains(&"getSomething"), + "Should have base getSomething method" + ); + assert!( + method_names.contains(&"getSomething1"), + "Should have getSomething1 method" + ); + } + + /// Test that tuple/struct types in function inputs and outputs are handled correctly. + #[test] + fn test_tuple_types_in_functions() { + let abi_json = r#"[ + { + "type": "function", + "name": "doSomething", + "inputs": [ + { + "name": "data", + "type": "tuple", + "components": [ + {"name": "owner", "type": "address"}, + {"name": "value", "type": "uint256"} + ] + } + ], + "outputs": [ + { + "name": "result", + "type": "tuple", + "components": [ + {"name": "success", "type": "bool"}, + {"name": "newValue", "type": "uint256"} + ] + } + ], + "stateMutability": "nonpayable" + } + ]"#; + + let contract = parse_abi(abi_json); + let generator = AbiCodeGenerator::new(contract, "TestContract"); + let types = generator.generate_types(); + + // Get class names + let class_names: Vec<&str> = types.iter().map(|c| c.name.as_str()).collect(); + + // Should have main contract class + assert!( + class_names.contains(&"TestContract"), + "Should have TestContract class" + ); + + // Should have struct classes for input and output tuples + // Input struct: TestContract__doSomethingInputValue0Struct + // Output struct: TestContract__doSomethingResultValue0Struct + let has_input_struct = class_names + .iter() + .any(|n| n.contains("Input") && n.contains("Struct")); + let has_output_struct = class_names + .iter() + .any(|n| n.contains("Result") && n.contains("Struct")); + + assert!( + has_input_struct, + "Should have input struct class, found: {:?}", + class_names + ); + assert!( + has_output_struct, + "Should have output/result struct class, found: {:?}", + class_names + ); + + // Verify the output struct extends ethereum.Tuple + let output_struct = types + .iter() + .find(|c| c.name.contains("Result") && c.name.contains("Struct")) + .expect("Should find output struct"); + assert_eq!( + output_struct.extends, + Some("ethereum.Tuple".to_string()), + "Output struct should extend ethereum.Tuple" + ); + + // Verify the output struct has getters for its components + // With alloy, component names are preserved from the ABI + let method_names: Vec<&str> = output_struct + .methods + .iter() + .map(|m| m.name.as_str()) + .collect(); + assert!( + method_names.iter().any(|n| n.contains("success")), + "Should have success getter for first component, found methods: {:?}", + method_names + ); + assert!( + method_names.iter().any(|n| n.contains("newValue")), + "Should have newValue getter for second component, found methods: {:?}", + method_names + ); + } + + /// Test that tuple types in events are handled correctly. + #[test] + fn test_tuple_types_in_events() { + let abi_json = r#"[ + { + "type": "event", + "name": "DataUpdated", + "inputs": [ + {"name": "id", "type": "uint256", "indexed": true}, + { + "name": "data", + "type": "tuple", + "indexed": false, + "components": [ + {"name": "timestamp", "type": "uint256"}, + {"name": "value", "type": "bytes32"} + ] + } + ], + "anonymous": false + } + ]"#; + + let contract = parse_abi(abi_json); + let generator = AbiCodeGenerator::new(contract, "TestContract"); + let types = generator.generate_types(); + + // Get class names + let class_names: Vec<&str> = types.iter().map(|c| c.name.as_str()).collect(); + + // Should have event class + assert!( + class_names.contains(&"DataUpdated"), + "Should have DataUpdated event class" + ); + + // Should have params class + assert!( + class_names.contains(&"DataUpdated__Params"), + "Should have DataUpdated__Params class" + ); + + // Should have struct class for the tuple parameter + let has_struct = class_names + .iter() + .any(|n| n.contains("Struct") && n.contains("DataUpdated")); + assert!( + has_struct, + "Should have struct class for tuple parameter, found: {:?}", + class_names + ); + } + + /// Test that nested tuple types (struct with struct field) are handled. + #[test] + fn test_nested_tuple_types() { + let abi_json = r#"[ + { + "type": "function", + "name": "getNestedData", + "inputs": [], + "outputs": [ + { + "name": "result", + "type": "tuple", + "components": [ + {"name": "id", "type": "uint256"}, + { + "name": "inner", + "type": "tuple", + "components": [ + {"name": "x", "type": "uint256"}, + {"name": "y", "type": "uint256"} + ] + } + ] + } + ], + "stateMutability": "view" + } + ]"#; + + let contract = parse_abi(abi_json); + let generator = AbiCodeGenerator::new(contract, "TestContract"); + let types = generator.generate_types(); + + // Get class names + let class_names: Vec<&str> = types.iter().map(|c| c.name.as_str()).collect(); + + // Should have main contract class + assert!( + class_names.contains(&"TestContract"), + "Should have TestContract class" + ); + + // Should have at least two struct classes (outer and inner) + let struct_count = class_names.iter().filter(|n| n.contains("Struct")).count(); + assert!( + struct_count >= 2, + "Should have at least 2 struct classes for nested tuple, found {} in {:?}", + struct_count, + class_names + ); + } + + /// Test that tuple arrays are handled correctly. + #[test] + fn test_tuple_array_types() { + let abi_json = r#"[ + { + "type": "function", + "name": "getAllItems", + "inputs": [], + "outputs": [ + { + "name": "items", + "type": "tuple[]", + "components": [ + {"name": "id", "type": "uint256"}, + {"name": "name", "type": "string"} + ] + } + ], + "stateMutability": "view" + } + ]"#; + + let contract = parse_abi(abi_json); + let generator = AbiCodeGenerator::new(contract, "TestContract"); + let types = generator.generate_types(); + + // Find the contract class + let contract_class = types.iter().find(|c| c.name == "TestContract").unwrap(); + + // Find the getAllItems method + let method = contract_class + .methods + .iter() + .find(|m| m.name == "getAllItems") + .expect("Should have getAllItems method"); + + // The return type should be an Array of the struct type + let return_type = method + .return_type + .as_ref() + .expect("Should have return type"); + let return_type_str = return_type.to_string(); + assert!( + return_type_str.starts_with("Array<"), + "Return type should be Array<...>, got: {}", + return_type_str + ); + } + + /// Test that array types in events are handled correctly. + #[test] + fn test_array_types_in_events() { + let abi_json = r#"[ + { + "type": "event", + "name": "Airdropped", + "inputs": [ + {"name": "sender", "type": "address", "indexed": true}, + {"name": "recipients", "type": "address[]", "indexed": false}, + {"name": "amounts", "type": "uint256[]", "indexed": false} + ], + "anonymous": false + } + ]"#; + + let contract = parse_abi(abi_json); + let generator = AbiCodeGenerator::new(contract, "Token"); + let types = generator.generate_types(); + + // Find the params class + let params_class = types + .iter() + .find(|c| c.name == "Airdropped__Params") + .expect("Should have Airdropped__Params class"); + + // Check that the array getters exist and have correct return types + let method_names: Vec<&str> = params_class + .methods + .iter() + .map(|m| m.name.as_str()) + .collect(); + + assert!( + method_names.iter().any(|n| n.contains("recipients")), + "Should have recipients getter, found: {:?}", + method_names + ); + assert!( + method_names.iter().any(|n| n.contains("amounts")), + "Should have amounts getter, found: {:?}", + method_names + ); + + // Verify the recipients getter returns Array
+ let recipients_getter = params_class + .methods + .iter() + .find(|m| m.name.contains("recipients")) + .expect("Should have recipients getter"); + let return_type_str = recipients_getter + .return_type + .as_ref() + .expect("Should have return type") + .to_string(); + assert!( + return_type_str.contains("Array
"), + "recipients should return Array
, got: {}", + return_type_str + ); + + // Verify the amounts getter returns Array + let amounts_getter = params_class + .methods + .iter() + .find(|m| m.name.contains("amounts")) + .expect("Should have amounts getter"); + let amounts_type_str = amounts_getter + .return_type + .as_ref() + .expect("Should have return type") + .to_string(); + assert!( + amounts_type_str.contains("Array"), + "amounts should return Array, got: {}", + amounts_type_str + ); + } + + /// Test that 2D array types (matrices) are handled correctly in functions. + #[test] + fn test_matrix_types_in_functions() { + let abi_json = r#"[ + { + "type": "function", + "name": "getMatrix", + "inputs": [], + "outputs": [ + {"name": "data", "type": "uint256[][]"} + ], + "stateMutability": "view" + } + ]"#; + + let contract = parse_abi(abi_json); + let generator = AbiCodeGenerator::new(contract, "TestContract"); + let types = generator.generate_types(); + + // Find the contract class + let contract_class = types.iter().find(|c| c.name == "TestContract").unwrap(); + + // Find the getMatrix method + let method = contract_class + .methods + .iter() + .find(|m| m.name == "getMatrix") + .expect("Should have getMatrix method"); + + // The return type should be Array> + let return_type = method + .return_type + .as_ref() + .expect("Should have return type"); + let return_type_str = return_type.to_string(); + assert!( + return_type_str.contains("Array>"), + "Return type should be Array>, got: {}", + return_type_str + ); + } +} diff --git a/gnd/src/codegen/mod.rs b/gnd/src/codegen/mod.rs new file mode 100644 index 00000000000..4c8f5fdbd98 --- /dev/null +++ b/gnd/src/codegen/mod.rs @@ -0,0 +1,20 @@ +//! Code generation for subgraph AssemblyScript types. +//! +//! This module generates AssemblyScript types from: +//! - GraphQL schema (entity classes) +//! - Contract ABIs (event and call bindings) +//! - Data source templates + +mod abi; +mod schema; +mod template; +mod types; +mod typescript; + +pub use abi::AbiCodeGenerator; +pub use schema::SchemaCodeGenerator; +pub use template::{Template, TemplateCodeGenerator, TemplateKind}; +pub use typescript::{ + ArrayType, Class, ClassMember, GENERATED_FILE_NOTE, Method, ModuleImports, NamedType, + NullableType, Param, StaticMethod, +}; diff --git a/gnd/src/codegen/schema.rs b/gnd/src/codegen/schema.rs new file mode 100644 index 00000000000..a8a276f1ca0 --- /dev/null +++ b/gnd/src/codegen/schema.rs @@ -0,0 +1,1289 @@ +//! Schema code generation. +//! +//! Generates AssemblyScript entity classes from GraphQL schemas. + +use anyhow::{Result, anyhow}; +use graphql_tools::parser::schema::{ + Definition, Document, Field, ObjectType, Type, TypeDefinition, +}; + +use super::types::{asc_type_for_value, value_from_asc, value_to_asc}; +use super::typescript::{ + self as ts, ArrayType, Class, Method, ModuleImports, NamedType, NullableType, Param, + StaticMethod, TypeExpr, +}; +use crate::shared::handle_reserved_word; + +/// Type of the ID field. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum IdFieldKind { + String, + Bytes, + Int8, +} + +impl IdFieldKind { + /// Get the AssemblyScript type name for this ID type. + pub fn type_name(&self) -> &'static str { + match self { + IdFieldKind::String => "string", + IdFieldKind::Bytes => "Bytes", + IdFieldKind::Int8 => "i64", + } + } + + /// Get the GraphQL type name for this ID type. + pub fn gql_type_name(&self) -> &'static str { + match self { + IdFieldKind::String => "String", + IdFieldKind::Bytes => "Bytes", + IdFieldKind::Int8 => "Int8", + } + } + + /// Get the code to create a Value from the ID. + pub fn value_from(&self) -> &'static str { + match self { + IdFieldKind::String => "Value.fromString(id)", + IdFieldKind::Bytes => "Value.fromBytes(id)", + IdFieldKind::Int8 => "Value.fromI64(id)", + } + } + + /// Get the ValueKind for this ID type. + pub fn value_kind(&self) -> &'static str { + match self { + IdFieldKind::String => "ValueKind.STRING", + IdFieldKind::Bytes => "ValueKind.BYTES", + IdFieldKind::Int8 => "ValueKind.INT8", + } + } + + /// Get the code to convert a Value to a string representation. + pub fn value_to_string(&self) -> &'static str { + match self { + IdFieldKind::String => "id.toString()", + IdFieldKind::Bytes => "id.toBytes().toHexString()", + IdFieldKind::Int8 => "id.toI64().toString()", + } + } + + /// Get the code to convert the ID to a string. + pub fn id_to_string_code(self) -> &'static str { + match self { + IdFieldKind::String => "id", + IdFieldKind::Bytes => "id.toHexString()", + IdFieldKind::Int8 => "id.toString()", + } + } + + /// Determine the ID field kind from a type name. + pub fn from_type_name(type_name: &str) -> Self { + match type_name { + "Bytes" => IdFieldKind::Bytes, + "Int8" => IdFieldKind::Int8, + _ => IdFieldKind::String, + } + } + + /// Returns true if this ID type supports auto-increment (optional constructor arg). + pub fn supports_auto(&self) -> bool { + matches!(self, IdFieldKind::Int8 | IdFieldKind::Bytes) + } + + /// Get the constructor parameter type (with nullability for auto-increment types). + pub fn constructor_param_type(&self) -> &'static str { + match self { + IdFieldKind::String => "string", + IdFieldKind::Bytes => "Bytes | null", + IdFieldKind::Int8 => "i64", // primitive, can't be nullable + } + } + + /// Get the default value for constructor parameter (for auto-increment types). + pub fn constructor_default(&self) -> Option<&'static str> { + match self { + IdFieldKind::String => None, + IdFieldKind::Bytes => Some("null"), + IdFieldKind::Int8 => Some("i64.MIN_VALUE"), + } + } + + /// Get the condition to check if id was provided (not auto-increment). + pub fn auto_check_condition(&self) -> &'static str { + match self { + IdFieldKind::String => "", // N/A + // For Bytes we end up generating 'if (id) { .. }' Using 'id != + // null' crashes the AssemblyScript compiler 0.19.23 because id + // has type 'Bytes | null' and the compiler doesn't strip the + // null from the type in the body of the if + IdFieldKind::Bytes => "id", + IdFieldKind::Int8 => "id != i64.MIN_VALUE", + } + } +} + +/// Get the base type name from a GraphQL type (stripping NonNull and List wrappers). +fn get_base_type_name(ty: &Type<'_, String>) -> String { + match ty { + Type::NamedType(name) => name.clone(), + Type::NonNullType(inner) => get_base_type_name(inner), + Type::ListType(inner) => get_base_type_name(inner), + } +} + +/// Check if a type is nullable (not wrapped in NonNull). +fn is_nullable(ty: &Type<'_, String>) -> bool { + !matches!(ty, Type::NonNullType(_)) +} + +/// Count the list nesting depth of a type. +/// `String` -> 0, `[String]` -> 1, `[[String]]` -> 2, etc. +fn list_depth(ty: &Type<'_, String>) -> u8 { + match ty { + Type::ListType(inner) => 1 + list_depth(inner), + Type::NonNullType(inner) => list_depth(inner), + Type::NamedType(_) => 0, + } +} + +/// Check if the innermost list members are nullable. +/// For `[String]` returns true, for `[String!]` returns false. +/// For non-list types, returns false. +fn is_list_member_nullable(ty: &Type<'_, String>) -> bool { + match ty { + Type::ListType(inner) => { + // Check the immediate inner type + is_nullable(inner) + } + Type::NonNullType(inner) => is_list_member_nullable(inner), + Type::NamedType(_) => false, + } +} + +/// Check if a field has the @derivedFrom directive. +fn is_derived_field(field: &Field<'_, String>) -> bool { + field.directives.iter().any(|d| d.name == "derivedFrom") +} + +/// Check if an object type has the @entity directive. +fn is_entity_type(obj: &ObjectType<'_, String>) -> bool { + obj.directives.iter().any(|d| d.name == "entity") +} + +/// Collected entity info for code generation. +struct EntityInfo { + name: String, + id_kind: IdFieldKind, + fields: Vec, +} + +/// Collected field info. +struct FieldInfo { + name: String, + is_derived: bool, + base_type: String, + is_nullable: bool, + /// The nesting depth of list wrappers. 0 = scalar, 1 = [T], 2 = [[T]], etc. + list_depth: u8, + /// Whether list members are nullable. Only meaningful when list_depth > 0. + member_nullable: bool, +} + +/// Schema code generator. +pub struct SchemaCodeGenerator { + entities: Vec, + entity_names: std::collections::HashSet, + /// Maps entity name to its ID field kind, for resolving entity reference types. + entity_id_kinds: std::collections::HashMap, +} + +impl SchemaCodeGenerator { + /// Create a new schema code generator from a parsed GraphQL document. + /// + /// Returns an error if the schema contains invalid patterns like non-nullable + /// lists with nullable members (e.g., `[Something]!`). + pub fn new(document: &Document<'_, String>) -> Result { + let mut entities = Vec::new(); + let mut entity_names = std::collections::HashSet::new(); + let mut entity_id_kinds = std::collections::HashMap::new(); + + // First pass: collect entity names and their ID types + for def in &document.definitions { + if let Definition::TypeDefinition(TypeDefinition::Object(obj)) = def + && is_entity_type(obj) + { + entity_names.insert(obj.name.clone()); + let id_field = obj.fields.iter().find(|f| f.name == "id"); + let id_kind = id_field + .map(|f| IdFieldKind::from_type_name(&get_base_type_name(&f.field_type))) + .unwrap_or(IdFieldKind::String); + entity_id_kinds.insert(obj.name.clone(), id_kind); + } + } + + // Second pass: collect entity info + for def in &document.definitions { + if let Definition::TypeDefinition(TypeDefinition::Object(obj)) = def + && is_entity_type(obj) + { + let name = obj.name.clone(); + + // Find ID field + let id_field = obj.fields.iter().find(|f| f.name == "id"); + let id_kind = id_field + .map(|f| IdFieldKind::from_type_name(&get_base_type_name(&f.field_type))) + .unwrap_or(IdFieldKind::String); + + // Collect field info + let fields: Vec<_> = obj + .fields + .iter() + .map(|f| FieldInfo { + name: f.name.clone(), + is_derived: is_derived_field(f), + base_type: get_base_type_name(&f.field_type), + is_nullable: is_nullable(&f.field_type), + list_depth: list_depth(&f.field_type), + member_nullable: is_list_member_nullable(&f.field_type), + }) + .collect(); + + entities.push(EntityInfo { + name, + id_kind, + fields, + }); + } + } + + // Validate: non-nullable lists must have non-nullable members + for entity in &entities { + for field in &entity.fields { + if field.list_depth > 0 && !field.is_nullable && field.member_nullable { + return Err(anyhow!( + "Codegen can't generate code for GraphQL field '{}' of type '[{}]!' since the inner type is nullable.\n\ + Suggestion: add an '!' to the inner type, e.g., '[{}!]!'", + field.name, + field.base_type, + field.base_type + )); + } + } + } + + Ok(Self { + entities, + entity_names, + entity_id_kinds, + }) + } + + /// Generate module imports for the schema file. + pub fn generate_module_imports(&self) -> Vec { + vec![ModuleImports::new( + vec![ + "TypedMap".to_string(), + "Entity".to_string(), + "Value".to_string(), + "ValueKind".to_string(), + "store".to_string(), + "Bytes".to_string(), + "BigInt".to_string(), + "BigDecimal".to_string(), + "Int8".to_string(), + ], + "@graphprotocol/graph-ts", + )] + } + + /// Generate entity classes from the schema. + pub fn generate_types(&self, generate_store_methods: bool) -> Vec { + self.entities + .iter() + .map(|entity| self.generate_entity_type(entity, generate_store_methods)) + .collect() + } + + /// Generate derived loaders for fields with @derivedFrom. + pub fn generate_derived_loaders(&self) -> Vec { + let mut loaders = Vec::new(); + let mut seen_types = std::collections::HashSet::new(); + + for entity in &self.entities { + for field in &entity.fields { + if field.is_derived && !seen_types.contains(&field.base_type) { + // Only generate loaders for entity types, not interfaces + if self.entity_names.contains(&field.base_type) { + seen_types.insert(field.base_type.clone()); + loaders.push(self.generate_derived_loader(&field.base_type)); + } + } + } + } + + loaders + } + + fn generate_entity_type(&self, entity: &EntityInfo, generate_store_methods: bool) -> Class { + let mut klass = ts::klass(&entity.name).exported().extends("Entity"); + + // Generate constructor + klass.add_method(self.generate_constructor(&entity.id_kind)); + + // Generate store methods + if generate_store_methods { + for method in self.generate_store_methods(&entity.name, &entity.id_kind) { + match method { + StoreMethod::Regular(m) => klass.add_method(m), + StoreMethod::Static(m) => klass.add_static_method(m), + } + } + } + + // Generate field getters and setters + for field in &entity.fields { + if let Some(getter) = self.generate_field_getter(&entity.name, field, &entity.id_kind) { + klass.add_method(getter); + } + if let Some(setter) = self.generate_field_setter(field) { + klass.add_method(setter); + } + } + + klass + } + + fn generate_constructor(&self, id_kind: &IdFieldKind) -> Method { + if id_kind.supports_auto() { + // For Int8 and Bytes, make id optional to support auto-increment + let param_type = id_kind.constructor_param_type(); + let default_value = id_kind.constructor_default().unwrap(); + let check_condition = id_kind.auto_check_condition(); + + Method::new( + "constructor", + vec![Param::with_default( + "id", + TypeExpr::Raw(param_type.to_string()), + default_value, + )], + None, + format!( + r#" + super() + if ({}) {{ + this.set('id', {}) + }}"#, + check_condition, + id_kind.value_from() + ), + ) + .with_doc("Leaving out the id argument uses an autoincrementing id.") + } else { + // For String IDs, keep the existing behavior (required parameter) + Method::new( + "constructor", + vec![Param::new("id", NamedType::new(id_kind.type_name()))], + None, + format!( + r#" + super() + this.set('id', {})"#, + id_kind.value_from() + ), + ) + } + } + + fn generate_store_methods(&self, entity_name: &str, id_kind: &IdFieldKind) -> Vec { + // Generate save() method - different for auto-increment vs string IDs + let save_method = if id_kind.supports_auto() { + // For Int8 and Bytes, check if id is null/unset and use "auto" as the key + Method::new( + "save", + vec![], + Some(NamedType::new("void").into()), + format!( + r#" + let id = this.get('id') + if (id == null || id.kind == ValueKind.NULL) {{ + store.set('{}', 'auto', this) + }} else {{ + assert(id.kind == {}, + `Entities of type {} must have an ID of type {} but the id '${{id.displayData()}}' is of type ${{id.displayKind()}}`) + store.set('{}', {}, this) + }}"#, + entity_name, + id_kind.value_kind(), + entity_name, + id_kind.gql_type_name(), + entity_name, + id_kind.value_to_string() + ), + ) + } else { + // For String IDs, keep the existing behavior (require ID) + Method::new( + "save", + vec![], + Some(NamedType::new("void").into()), + format!( + r#" + let id = this.get('id') + assert(id != null, + 'Cannot save {} entity without an ID') + if (id) {{ + assert(id.kind == {}, + `Entities of type {} must have an ID of type {} but the id '${{id.displayData()}}' is of type ${{id.displayKind()}}`) + store.set('{}', {}, this) + }}"#, + entity_name, + id_kind.value_kind(), + entity_name, + id_kind.gql_type_name(), + entity_name, + id_kind.value_to_string() + ), + ) + }; + + vec![ + StoreMethod::Regular(save_method), + // loadInBlock() static method + StoreMethod::Static(StaticMethod::new( + "loadInBlock", + vec![Param::new("id", NamedType::new(id_kind.type_name()))], + NullableType::new(NamedType::new(entity_name)), + format!( + r#" + return changetype<{} | null>(store.get_in_block('{}', {}))"#, + entity_name, + entity_name, + id_kind.id_to_string_code() + ), + )), + // load() static method + StoreMethod::Static(StaticMethod::new( + "load", + vec![Param::new("id", NamedType::new(id_kind.type_name()))], + NullableType::new(NamedType::new(entity_name)), + format!( + r#" + return changetype<{} | null>(store.get('{}', {}))"#, + entity_name, + entity_name, + id_kind.id_to_string_code() + ), + )), + ] + } + + fn generate_field_getter( + &self, + entity_name: &str, + field: &FieldInfo, + id_kind: &IdFieldKind, + ) -> Option { + let safe_name = handle_reserved_word(&field.name); + + // Handle derived fields + if field.is_derived { + return self.generate_derived_field_getter(entity_name, field, &safe_name, id_kind); + } + + let value_type = self.value_type_from_field(field); + let return_type = self.type_from_field(field); + let nullable = field.is_nullable; + + let primitive_default = match &return_type { + TypeExpr::Named(t) => t.get_primitive_default(), + _ => None, + }; + + let get_code = if nullable { + format!( + r#" + let value = this.get('{}') + if (!value || value.kind == ValueKind.NULL) {{ + return null + }} else {{ + return {} + }}"#, + field.name, + value_to_asc("value", &value_type) + ) + } else { + let null_handling = match primitive_default { + Some(default) => format!("return {}", default), + None => "throw new Error('Cannot return null for a required field.')".to_string(), + }; + format!( + r#" + let value = this.get('{}') + if (!value || value.kind == ValueKind.NULL) {{ + {} + }} else {{ + return {} + }}"#, + field.name, + null_handling, + value_to_asc("value", &value_type) + ) + }; + + Some(Method::new( + format!("get {}", safe_name), + vec![], + Some(return_type), + get_code, + )) + } + + fn generate_derived_field_getter( + &self, + entity_name: &str, + field: &FieldInfo, + safe_name: &str, + id_kind: &IdFieldKind, + ) -> Option { + let loader_name = format!("{}Loader", field.base_type); + + let id_conversion = match id_kind { + IdFieldKind::Bytes => "this.get('id')!.toBytes().toHexString()", + _ => "this.get('id')!.toString()", + }; + + Some(Method::new( + format!("get {}", safe_name), + vec![], + Some(NamedType::new(&loader_name).into()), + format!( + r#" + return new {}('{}', {}, '{}')"#, + loader_name, entity_name, id_conversion, field.name + ), + )) + } + + fn generate_field_setter(&self, field: &FieldInfo) -> Option { + // No setters for derived fields + if field.is_derived { + return None; + } + + let safe_name = handle_reserved_word(&field.name); + let value_type = self.value_type_from_field(field); + let param_type = self.type_from_field(field); + let nullable = field.is_nullable; + + let set_code = if nullable { + let inner_type = match ¶m_type { + TypeExpr::Nullable(n) => n.inner.to_string(), + other => other.to_string(), + }; + format!( + r#" + if (!value) {{ + this.unset('{}') + }} else {{ + this.set('{}', {}) + }}"#, + field.name, + field.name, + value_from_asc(&format!("<{}>value", inner_type), &value_type) + ) + } else { + format!( + r#" + this.set('{}', {})"#, + field.name, + value_from_asc("value", &value_type) + ) + }; + + Some(Method::new( + format!("set {}", safe_name), + vec![Param::new("value", param_type)], + None, + set_code, + )) + } + + fn generate_derived_loader(&self, type_name: &str) -> Class { + let loader_name = format!("{}Loader", type_name); + let mut klass = ts::klass(&loader_name).exported().extends("Entity"); + + // Add members + klass.add_member(ts::klass_member("_entity", "string")); + klass.add_member(ts::klass_member("_field", "string")); + klass.add_member(ts::klass_member("_id", "string")); + + // Add constructor + klass.add_method(Method::new( + "constructor", + vec![ + Param::new("entity", NamedType::new("string")), + Param::new("id", NamedType::new("string")), + Param::new("field", NamedType::new("string")), + ], + None, + r#" + super(); + this._entity = entity; + this._id = id; + this._field = field;"# + .to_string(), + )); + + // Add load() method + klass.add_method(Method::new( + "load", + vec![], + Some(TypeExpr::Raw(format!("{}[]", type_name))), + format!( + r#" + let value = store.loadRelated(this._entity, this._id, this._field); + return changetype<{}[]>(value);"#, + type_name + ), + )); + + klass + } + + /// Get the value type string for a field. + /// + /// Returns the GraphQL-style value type string: + /// - Scalars: `String`, `Int`, `BigInt`, etc. + /// - Arrays: `[String]`, `[Int]`, etc. + /// - Nested arrays: `[[String]]`, `[[Int]]`, etc. + /// - Entity references are converted to the referenced entity's ID type + fn value_type_from_field(&self, field: &FieldInfo) -> String { + let base = if let Some(id_kind) = self.entity_id_kinds.get(&field.base_type) { + id_kind.gql_type_name().to_string() + } else { + field.base_type.clone() + }; + + // Wrap with brackets for each level of list nesting + let mut result = base; + for _ in 0..field.list_depth { + result = format!("[{}]", result); + } + result + } + + /// Convert field info to an AssemblyScript TypeExpr. + /// + /// Creates the correct type expression including nested arrays: + /// - Scalars: `string`, `i32`, `BigInt`, etc. + /// - Arrays: `Array`, `Array`, etc. + /// - Nested arrays: `Array>`, etc. + fn type_from_field(&self, field: &FieldInfo) -> TypeExpr { + let type_name = if let Some(id_kind) = self.entity_id_kinds.get(&field.base_type) { + id_kind.type_name() + } else { + asc_type_for_value(&field.base_type) + }; + + let named = NamedType::new(type_name); + + if field.list_depth > 0 { + // Use ArrayType::with_depth to create nested array types + let array_type = ArrayType::with_depth(named, field.list_depth); + if field.is_nullable { + NullableType::new(array_type).into() + } else { + array_type + } + } else if field.is_nullable && !named.is_primitive() { + NullableType::new(named).into() + } else { + named.into() + } + } +} + +enum StoreMethod { + Regular(Method), + Static(StaticMethod), +} + +#[cfg(test)] +mod tests { + use super::*; + + use graphql_tools::parser::parse_schema; + + #[test] + fn test_simple_entity() { + let schema = r#" + type Transfer @entity { + id: ID! + from: Bytes! + to: Bytes! + value: BigInt! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + let classes = generator.generate_types(true); + assert_eq!(classes.len(), 1); + + let transfer = &classes[0]; + assert_eq!(transfer.name, "Transfer"); + assert_eq!(transfer.extends, Some("Entity".to_string())); + assert!(transfer.export); + } + + #[test] + fn test_nullable_field() { + let schema = r#" + type Token @entity { + id: ID! + name: String + symbol: String! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + let classes = generator.generate_types(true); + assert_eq!(classes.len(), 1); + + // Check that we have methods for nullable and non-nullable fields + let token = &classes[0]; + let method_names: Vec<_> = token.methods.iter().map(|m| m.name.as_str()).collect(); + assert!(method_names.contains(&"get name")); + assert!(method_names.contains(&"set name")); + assert!(method_names.contains(&"get symbol")); + assert!(method_names.contains(&"set symbol")); + } + + #[test] + fn test_id_field_types() { + assert_eq!(IdFieldKind::String.type_name(), "string"); + assert_eq!(IdFieldKind::Bytes.type_name(), "Bytes"); + assert_eq!(IdFieldKind::Int8.type_name(), "i64"); + } + + #[test] + fn test_auto_increment_support() { + // String IDs don't support auto-increment + assert!(!IdFieldKind::String.supports_auto()); + assert!(IdFieldKind::String.constructor_default().is_none()); + + // Int8 IDs support auto-increment + assert!(IdFieldKind::Int8.supports_auto()); + assert_eq!(IdFieldKind::Int8.constructor_param_type(), "i64"); + assert_eq!( + IdFieldKind::Int8.constructor_default(), + Some("i64.MIN_VALUE") + ); + assert_eq!( + IdFieldKind::Int8.auto_check_condition(), + "id != i64.MIN_VALUE" + ); + + // Bytes IDs support auto-increment + assert!(IdFieldKind::Bytes.supports_auto()); + assert_eq!(IdFieldKind::Bytes.constructor_param_type(), "Bytes | null"); + assert_eq!(IdFieldKind::Bytes.constructor_default(), Some("null")); + assert_eq!(IdFieldKind::Bytes.auto_check_condition(), "id"); + } + + #[test] + fn test_int8_id_auto_increment_codegen() { + let schema = r#" + type Counter @entity { + id: Int8! + value: BigInt! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + let classes = generator.generate_types(true); + assert_eq!(classes.len(), 1); + + let counter = &classes[0]; + let output = counter.to_string(); + + // Constructor should have optional id with sentinel default + assert!( + output.contains("constructor(id: i64 = i64.MIN_VALUE)"), + "Int8 ID constructor should have default sentinel value, got: {}", + output + ); + + // Constructor should have doc comment + assert!( + output.contains("Leaving out the id argument uses an autoincrementing id"), + "Constructor should have auto-increment doc comment" + ); + + // Constructor body should conditionally set id + assert!( + output.contains("if (id != i64.MIN_VALUE)"), + "Constructor should check for sentinel value" + ); + + // save() method should use "auto" when id is null + assert!( + output.contains("store.set('Counter', 'auto', this)"), + "save() should use 'auto' key for auto-increment, got: {}", + output + ); + } + + #[test] + fn test_bytes_id_auto_increment_codegen() { + let schema = r#" + type Event @entity { + id: Bytes! + data: String! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + let classes = generator.generate_types(true); + assert_eq!(classes.len(), 1); + + let event = &classes[0]; + let output = event.to_string(); + + // Constructor should have nullable id with null default + assert!( + output.contains("constructor(id: Bytes | null = null)"), + "Bytes ID constructor should have null default, got: {}", + output + ); + + // Constructor should have doc comment + assert!( + output.contains("Leaving out the id argument uses an autoincrementing id"), + "Constructor should have auto-increment doc comment" + ); + + // Constructor body should conditionally set id + assert!( + output.contains("if (id)"), + "Constructor should check for null" + ); + + // save() method should use "auto" when id is null + assert!( + output.contains("store.set('Event', 'auto', this)"), + "save() should use 'auto' key for auto-increment, got: {}", + output + ); + } + + #[test] + fn test_string_id_no_auto_increment() { + let schema = r#" + type User @entity { + id: ID! + name: String! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + let classes = generator.generate_types(true); + assert_eq!(classes.len(), 1); + + let user = &classes[0]; + let output = user.to_string(); + + // Constructor should have required id (no default) + assert!( + output.contains("constructor(id: string)"), + "String ID constructor should have required parameter, got: {}", + output + ); + + // Constructor should NOT have auto-increment doc comment + assert!( + !output.contains("autoincrementing"), + "String ID should not mention auto-increment" + ); + + // save() method should NOT use "auto" + assert!( + !output.contains("'auto'"), + "String ID save() should not use 'auto' key" + ); + + // save() should require ID + assert!( + output.contains("Cannot save User entity without an ID"), + "String ID save() should require ID" + ); + } + + #[test] + fn test_entity_reference() { + let schema = r#" + type User @entity { + id: ID! + name: String! + } + type Post @entity { + id: ID! + author: User! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + // The Post.author field should be treated as a string (entity ID reference) + assert!(generator.entity_names.contains("User")); + assert!(generator.entity_names.contains("Post")); + } + + #[test] + fn test_simple_array_field() { + let schema = r#" + type Token @entity { + id: ID! + holders: [String!]! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + let classes = generator.generate_types(true); + assert_eq!(classes.len(), 1); + + let token = &classes[0]; + let output = token.to_string(); + + // Verify array field getter/setter are generated + assert!( + output.contains("get holders()"), + "Should have holders getter" + ); + assert!( + output.contains("set holders("), + "Should have holders setter" + ); + + // Check the type is Array + assert!( + output.contains("Array"), + "Array field should use Array type" + ); + } + + #[test] + fn test_nested_array_field() { + let schema = r#" + type Matrix @entity { + id: ID! + stringMatrix: [[String!]!]! + intMatrix: [[Int!]!] + bigIntMatrix: [[BigInt!]!]! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + let classes = generator.generate_types(true); + assert_eq!(classes.len(), 1); + + let matrix = &classes[0]; + let output = matrix.to_string(); + + // Verify nested array field getter/setter are generated + assert!( + output.contains("get stringMatrix()"), + "Should have stringMatrix getter" + ); + assert!( + output.contains("set stringMatrix("), + "Should have stringMatrix setter" + ); + + // Check the type is Array> + assert!( + output.contains("Array>"), + "Nested array field should use Array> type, got: {}", + output + ); + + // Check that toStringMatrix() and fromStringMatrix() are used + assert!( + output.contains("toStringMatrix()"), + "Should use toStringMatrix() for nested string arrays" + ); + assert!( + output.contains("fromStringMatrix("), + "Should use Value.fromStringMatrix() for nested string arrays" + ); + + // Check BigInt matrix uses correct methods + assert!( + output.contains("Array>"), + "BigInt matrix should use Array> type" + ); + assert!( + output.contains("toBigIntMatrix()"), + "Should use toBigIntMatrix() for nested BigInt arrays" + ); + assert!( + output.contains("fromBigIntMatrix("), + "Should use Value.fromBigIntMatrix() for nested BigInt arrays" + ); + + // Check nullable nested array has correct type + assert!( + output.contains("Array> | null"), + "Nullable nested array should be Array> | null" + ); + } + + #[test] + fn test_list_depth() { + use graphql_tools::parser::parse_schema; + + // Helper to get list depth from schema field type + fn get_field_list_depth(schema_str: &str) -> u8 { + let doc = parse_schema::(schema_str).unwrap(); + for def in &doc.definitions { + if let Definition::TypeDefinition(TypeDefinition::Object(obj)) = def { + for field in &obj.fields { + if field.name == "field" { + return list_depth(&field.field_type); + } + } + } + } + panic!("Field not found"); + } + + // Scalar + assert_eq!( + get_field_list_depth("type T @entity { id: ID!, field: String! }"), + 0 + ); + + // Simple array + assert_eq!( + get_field_list_depth("type T @entity { id: ID!, field: [String!]! }"), + 1 + ); + + // Nested array (matrix) + assert_eq!( + get_field_list_depth("type T @entity { id: ID!, field: [[String!]!]! }"), + 2 + ); + + // Triple nested array + assert_eq!( + get_field_list_depth("type T @entity { id: ID!, field: [[[String!]!]!]! }"), + 3 + ); + } + + #[test] + fn test_value_type_from_field_nested() { + let schema = r#" + type Test @entity { + id: ID! + scalar: String! + array: [String!]! + matrix: [[String!]!]! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + // Find the entity + let entity = &generator.entities[0]; + + // Find each field and check its value type + let scalar_field = entity.fields.iter().find(|f| f.name == "scalar").unwrap(); + let array_field = entity.fields.iter().find(|f| f.name == "array").unwrap(); + let matrix_field = entity.fields.iter().find(|f| f.name == "matrix").unwrap(); + + assert_eq!(generator.value_type_from_field(scalar_field), "String"); + assert_eq!(generator.value_type_from_field(array_field), "[String]"); + assert_eq!(generator.value_type_from_field(matrix_field), "[[String]]"); + } + + #[test] + fn test_bytes_id_entity_reference() { + let schema = r#" + type Token @entity { + id: Bytes! + name: String! + } + type Balance @entity { + id: ID! + token: Token! + amount: BigInt! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + let classes = generator.generate_types(true); + let balance = classes.iter().find(|c| c.name == "Balance").unwrap(); + let output = balance.to_string(); + + // Getter should return Bytes and use toBytes() + assert!( + output.contains("value.toBytes()"), + "Bytes-ID entity reference getter should use toBytes(), got: {}", + output + ); + + // Setter should use Value.fromBytes() + assert!( + output.contains("Value.fromBytes("), + "Bytes-ID entity reference setter should use Value.fromBytes(), got: {}", + output + ); + + // Return type should be Bytes, not string + assert!( + output.contains("get token(): Bytes"), + "Bytes-ID entity reference getter should return Bytes, got: {}", + output + ); + } + + #[test] + fn test_int8_id_entity_reference() { + let schema = r#" + type Counter @entity { + id: Int8! + value: BigInt! + } + type Snapshot @entity { + id: ID! + counter: Counter! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + let classes = generator.generate_types(true); + let snapshot = classes.iter().find(|c| c.name == "Snapshot").unwrap(); + let output = snapshot.to_string(); + + // Getter should return i64 and use toI64() + assert!( + output.contains("value.toI64()"), + "Int8-ID entity reference getter should use toI64(), got: {}", + output + ); + + // Setter should use Value.fromI64() + assert!( + output.contains("Value.fromI64("), + "Int8-ID entity reference setter should use Value.fromI64(), got: {}", + output + ); + + // Return type should be i64 + assert!( + output.contains("get counter(): i64"), + "Int8-ID entity reference getter should return i64, got: {}", + output + ); + } + + #[test] + fn test_mixed_id_entity_references() { + let schema = r#" + type User @entity { + id: ID! + name: String! + } + type Token @entity { + id: Bytes! + owner: User! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + let classes = generator.generate_types(true); + let token = classes.iter().find(|c| c.name == "Token").unwrap(); + let output = token.to_string(); + + // Token.owner references User which has String ID + assert!( + output.contains("get owner(): string"), + "Reference to String-ID entity should use string type, got: {}", + output + ); + assert!( + output.contains("value.toString()"), + "Reference to String-ID entity should use toString(), got: {}", + output + ); + } + + #[test] + fn test_nullable_bytes_id_entity_reference() { + let schema = r#" + type Token @entity { + id: Bytes! + name: String! + } + type Balance @entity { + id: ID! + token: Token + amount: BigInt! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + let classes = generator.generate_types(true); + let balance = classes.iter().find(|c| c.name == "Balance").unwrap(); + let output = balance.to_string(); + + // Nullable Bytes reference should be `Bytes | null` + assert!( + output.contains("get token(): Bytes | null"), + "Nullable Bytes-ID reference should return Bytes | null, got: {}", + output + ); + } + + #[test] + fn test_derived_field_with_bytes_id_parent() { + let schema = r#" + type Token @entity { + id: Bytes! + balances: [Balance!]! @derivedFrom(field: "token") + } + type Balance @entity { + id: ID! + token: Token! + amount: BigInt! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + let classes = generator.generate_types(true); + let token = classes.iter().find(|c| c.name == "Token").unwrap(); + let output = token.to_string(); + + // Derived field getter on Bytes-ID entity should use toBytes().toHexString() + assert!( + output.contains("this.get('id')!.toBytes().toHexString()"), + "Derived field on Bytes-ID entity should use toBytes().toHexString(), got: {}", + output + ); + } +} diff --git a/gnd/src/codegen/template.rs b/gnd/src/codegen/template.rs new file mode 100644 index 00000000000..72a27f9e733 --- /dev/null +++ b/gnd/src/codegen/template.rs @@ -0,0 +1,286 @@ +//! Data source template code generation. +//! +//! Generates AssemblyScript classes for subgraph templates that allow +//! dynamic data source creation at runtime. + +use super::typescript::{self as ts, Class, ModuleImports, Param, StaticMethod}; + +/// The kind of a data source template. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TemplateKind { + /// Ethereum contract template + Ethereum, + /// IPFS file template + FileIpfs, + /// Arweave file template + FileArweave, +} + +impl TemplateKind { + /// Parse a template kind from a string (e.g., "ethereum/contract", "file/ipfs"). + pub fn from_str_kind(kind: &str) -> Option { + match kind { + "ethereum/contract" | "ethereum" => Some(TemplateKind::Ethereum), + "file/ipfs" => Some(TemplateKind::FileIpfs), + "file/arweave" => Some(TemplateKind::FileArweave), + _ => None, + } + } +} + +/// A data source template from the subgraph manifest. +pub struct Template { + /// The name of the template. + pub name: String, + /// The kind of template. + pub kind: TemplateKind, +} + +impl Template { + /// Create a new template. + pub fn new(name: impl Into, kind: TemplateKind) -> Self { + Self { + name: name.into(), + kind, + } + } +} + +const GRAPH_TS_MODULE: &str = "@graphprotocol/graph-ts"; + +/// Template code generator. +pub struct TemplateCodeGenerator { + templates: Vec