diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index e437415..98e5171 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -1,8 +1,13 @@ -# This file is autogenerated by maturin v1.8.6 +# This file is autogenerated by maturin>=v1.9.4,<2.0 # To update, run # # maturin generate-ci github # +# After all Python versions are installed, the --find-interpreter parameter of maturin build still fails to +# find all Python versions, resulting in some Python versions not being built. Therefore, the -i parameter +# is needed to manually specify Python versions. +# + name: CI on: @@ -19,144 +24,239 @@ permissions: contents: read jobs: - linux: + # Setup Python Environments Jobs + # These jobs will set up the Python environments for the other jobs to use. + setup-python-linux: + name: Setup Python Environments (Linux) runs-on: ${{ matrix.platform.runner }} strategy: matrix: platform: - - runner: ubuntu-22.04 + - runner: ubuntu-latest target: x86_64 - - runner: ubuntu-22.04 + - runner: ubuntu-latest target: x86 - - runner: ubuntu-22.04 + - runner: ubuntu-latest target: aarch64 - - runner: ubuntu-22.04 + - runner: ubuntu-latest target: armv7 - - runner: ubuntu-22.04 + - runner: ubuntu-latest target: s390x - - runner: ubuntu-22.04 + - runner: ubuntu-latest target: ppc64le + python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14", "pypy3.9", "pypy3.10", "pypy3.11" ] steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v5 + - name: Set up Python + uses: actions/setup-python@v6 with: - python-version: 3.x - - name: Build wheels + python-version: ${{ matrix.python-version }} + allow-prereleases: true + + setup-python-windows: + name: Setup Python Environments (Windows) + runs-on: ${{ matrix.platform.runner }} + strategy: + matrix: + platform: + - runner: windows-latest + target: x64 # 64-bit Python interpreter, note that 32-bit Python interpreter is not supported on Windows x86 + # Other Python versions need to be installed separately to build correctly + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + steps: + - uses: actions/checkout@v5 + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + architecture: ${{ matrix.platform.target }} + allow-prereleases: true + + setup-python-macos: + name: Setup Python Environments (macOS) + runs-on: ${{ matrix.platform.runner }} + strategy: + matrix: + platform: + - runner: macos-latest + target: x86_64 + - runner: macos-latest + target: aarch64 + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14", "pypy3.9", "pypy3.10", "pypy3.11" ] + steps: + - uses: actions/checkout@v5 + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + + # Build Jobs + # These jobs will build the wheels for all Python versions and upload them as artifacts. + linux: + runs-on: ubuntu-latest + needs: setup-python-linux + strategy: + matrix: + target: [x86_64, x86, aarch64, armv7, s390x, ppc64le] + steps: + - uses: actions/checkout@v5 + + - name: Build wheels for all Python versions uses: PyO3/maturin-action@v1 with: - target: ${{ matrix.platform.target }} - args: --release --out dist --find-interpreter + target: ${{ matrix.target }} + args: | + --release + --out target/wheels + -i python3.8 python3.9 python3.10 python3.11 python3.12 python3.13 python3.14 pypy3.9 pypy3.10 pypy3.11 sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} manylinux: auto + - name: Upload wheels uses: actions/upload-artifact@v4 with: - name: wheels-linux-${{ matrix.platform.target }} - path: dist + name: wheels-linux-${{ matrix.target }} + path: target/wheels musllinux: - runs-on: ${{ matrix.platform.runner }} + runs-on: ubuntu-latest + needs: setup-python-linux strategy: matrix: - platform: - - runner: ubuntu-22.04 - target: x86_64 - - runner: ubuntu-22.04 - target: x86 - - runner: ubuntu-22.04 - target: aarch64 - - runner: ubuntu-22.04 - target: armv7 + target: [x86_64, x86, aarch64, armv7] steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: 3.x - - name: Build wheels + - uses: actions/checkout@v5 + + - name: Build wheels for all Python versions uses: PyO3/maturin-action@v1 with: - target: ${{ matrix.platform.target }} - args: --release --out dist --find-interpreter + target: ${{ matrix.target }} + args: | + --release + --out target/wheels + -i python3.8 python3.9 python3.10 python3.11 python3.12 python3.13 python3.14 pypy3.9 pypy3.10 pypy3.11 sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} manylinux: musllinux_1_2 + - name: Upload wheels uses: actions/upload-artifact@v4 with: - name: wheels-musllinux-${{ matrix.platform.target }} - path: dist + name: wheels-musllinux-${{ matrix.target }} + path: target/wheels windows: runs-on: ${{ matrix.platform.runner }} + needs: setup-python-windows strategy: matrix: platform: - runner: windows-latest target: x64 - - runner: windows-latest - target: x86 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v5 + + - name: Install Python3.8、3.14 and multiple PyPy version + uses: actions/setup-python@v6 with: - python-version: 3.x + python-version: | + 3.8 + 3.14 + pypy3.9 + pypy3.10 + pypy3.11 architecture: ${{ matrix.platform.target }} - - name: Build wheels + allow-prereleases: true + + - name: Set PyPy paths + shell: pwsh + run: | + $PYPY_PATHS = @() + $versions = @("3.9", "3.10", "3.11") + + foreach ($version in $versions) { + $command = "pypy$version" + $pypyCommand = Get-Command $command -ErrorAction SilentlyContinue + if ($pypyCommand) { + $path = "$($pypyCommand.Source)" + $PYPY_PATHS += $path + Write-Host "Found PyPy$version : $path" + } else { + Write-Host "PyPy$version not found" + } + } + + $PYPY_PATHS_STRING = $PYPY_PATHS -join " " + Write-Host "All PyPy paths: $PYPY_PATHS_STRING" + echo "PYPY_PATHS=$PYPY_PATHS_STRING" >> $env:GITHUB_ENV + + - name: Build wheels for all Python versions uses: PyO3/maturin-action@v1 with: target: ${{ matrix.platform.target }} - args: --release --out dist --find-interpreter + args: | + --release + --out target/wheels + -i python3.8 python3.9 python3.10 python3.11 python3.12 python3.13 python3.14 + -i ${{ env.PYPY_PATHS }} + sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} + - name: Upload wheels uses: actions/upload-artifact@v4 with: name: wheels-windows-${{ matrix.platform.target }} - path: dist + path: target/wheels + macos: - runs-on: ${{ matrix.platform.runner }} + runs-on: ${{ matrix.runner }} + needs: setup-python-macos strategy: matrix: - platform: - - runner: macos-13 + include: + - runner: macos-latest target: x86_64 - - runner: macos-14 + - runner: macos-latest target: aarch64 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: 3.x - - name: Build wheels + - uses: actions/checkout@v5 + + - name: Build wheels for all Python versions uses: PyO3/maturin-action@v1 with: - target: ${{ matrix.platform.target }} - args: --release --out dist --find-interpreter + target: ${{ matrix.target }} + args: | + --release + --out target/wheels + -i python3.8 python3.9 python3.10 python3.11 python3.12 python3.13 python3.14 pypy3.9 pypy3.10 pypy3.11 sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} + - name: Upload wheels uses: actions/upload-artifact@v4 with: - name: wheels-macos-${{ matrix.platform.target }} - path: dist + name: wheels-macos-${{ matrix.target }} + path: target/wheels sdist: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Build sdist uses: PyO3/maturin-action@v1 with: command: sdist - args: --out dist + args: --out target/wheels - name: Upload sdist uses: actions/upload-artifact@v4 with: name: wheels-sdist - path: dist + path: target/wheels release: name: Release runs-on: ubuntu-latest - if: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }} + if: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch'}} needs: [linux, musllinux, windows, macos, sdist] permissions: # Use to sign the release artifacts diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0d69b74..ce8c402 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,21 +7,50 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ ubuntu-latest, macos-latest ] - python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "pypy3.9", "pypy3.10" ] + os: [ ubuntu-latest, macos-latest, windows-latest ] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14", "pypy3.9", "pypy3.10", "pypy3.11" ] + include: + - os: ubuntu-latest + python-version: "3.14" + allow-failure: true + - os: macos-latest + python-version: "3.14" + allow-failure: true + - os: windows-latest + python-version: "3.14" + allow-failure: true + continue-on-error: ${{ matrix.allow-failure || false }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} + allow-prereleases: true + cache: pip + cache-dependency-path: pyproject.toml - name: Create virtualenv and install package + shell: bash run: | python -m venv env - source env/bin/activate - pip install --upgrade pip setuptools wheel maturin pytest virtualenv + if [[ "${{ matrix.python-version }}" == "pypy3.11" ]]; then + echo "Skipping pip upgrade for PyPy3.11" + else + python -m pip install --upgrade pip + fi + if [ "$RUNNER_OS" == "Windows" ]; then + source env/Scripts/activate + else + source env/bin/activate + fi + pip install -U setuptools wheel maturin pytest virtualenv maturin develop --release - name: Test + shell: bash run: | - source env/bin/activate + if [ "$RUNNER_OS" == "Windows" ]; then + source env/Scripts/activate + else + source env/bin/activate + fi python -m pytest diff --git a/Cargo.lock b/Cargo.lock index 1cd9389..40c439c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,16 @@ dependencies = [ "generic-array", ] +[[package]] +name = "cc" +version = "1.2.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1354349954c6fc9cb0deab020f27f783cf0b604e8bb754dc4658ecf0d29c35f" +dependencies = [ + "find-msvc-tools", + "shlex", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -61,6 +71,12 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "find-msvc-tools" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ced73b1dacfc750a6db6c0a0c3a3853c8b41997e2e2c563dc90804ae6867959" + [[package]] name = "generic-array" version = "0.14.7" @@ -91,9 +107,9 @@ checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "jsonpath-rust" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b37465feaf9d41f74df7da98c6c1c31ca8ea06d11b5bf7869c8f1ccc51a793f" +checksum = "633a7320c4bb672863a3782e89b9094ad70285e097ff6832cddd0ec615beadfa" dependencies = [ "pest", "pest_derive", @@ -104,9 +120,10 @@ dependencies = [ [[package]] name = "jsonpath_rust_bindings" -version = "1.0.2" +version = "1.1.0" dependencies = [ "jsonpath-rust", + "mimalloc", "pyo3", "pythonize", "serde_json", @@ -118,6 +135,16 @@ version = "0.2.149" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" +[[package]] +name = "libmimalloc-sys" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "667f4fec20f29dfc6bc7357c582d91796c169ad7e2fce709468aefeb2c099870" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "memchr" version = "2.6.4" @@ -133,11 +160,20 @@ dependencies = [ "autocfg", ] +[[package]] +name = "mimalloc" +version = "0.1.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1ee66a4b64c74f4ef288bcbb9192ad9c3feaad75193129ac8509af543894fd8" +dependencies = [ + "libmimalloc-sys", +] + [[package]] name = "once_cell" -version = "1.18.0" +version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] name = "pest" @@ -201,9 +237,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.25.0" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f239d656363bcee73afef85277f1b281e8ac6212a1d42aa90e55b90ed43c47a4" +checksum = "7ba0117f4212101ee6544044dae45abe1083d30ce7b29c4b5cbdfa2354e07383" dependencies = [ "indoc", "libc", @@ -218,19 +254,18 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.25.0" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "755ea671a1c34044fa165247aaf6f419ca39caa6003aee791a0df2713d8f1b6d" +checksum = "4fc6ddaf24947d12a9aa31ac65431fb1b851b8f4365426e182901eabfb87df5f" dependencies = [ - "once_cell", "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.25.0" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc95a2e67091e44791d4ea300ff744be5293f394f1bafd9f78c080814d35956e" +checksum = "025474d3928738efb38ac36d4744a74a400c901c7596199e20e45d98eb194105" dependencies = [ "libc", "pyo3-build-config", @@ -238,9 +273,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.25.0" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a179641d1b93920829a62f15e87c0ed791b6c8db2271ba0fd7c2686090510214" +checksum = "2e64eb489f22fe1c95911b77c44cc41e7c19f3082fc81cce90f657cdc42ffded" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -250,9 +285,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.25.0" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dff85ebcaab8c441b0e3f7ae40a6963ecea8a9f5e74f647e33fcf5ec9a1e89e" +checksum = "100246c0ecf400b475341b8455a9213344569af29a3c841d29270e53102e0fcf" dependencies = [ "heck", "proc-macro2", @@ -263,9 +298,9 @@ dependencies = [ [[package]] name = "pythonize" -version = "0.25.0" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597907139a488b22573158793aa7539df36ae863eba300c75f3a0d65fc475e27" +checksum = "11e06e4cff9be2bbf2bddf28a486ae619172ea57e79787f856572878c62dcfe2" dependencies = [ "pyo3", "serde", @@ -357,6 +392,12 @@ dependencies = [ "digest", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "syn" version = "2.0.101" diff --git a/Cargo.toml b/Cargo.toml index dee4970..2173df1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,14 +1,29 @@ [package] name = "jsonpath_rust_bindings" -version = "1.0.2" +version = "1.1.0" edition = "2021" [lib] name = "jsonpath_rust_bindings" crate-type = ["cdylib"] +[profile.release] +opt-level = 3 +lto = "fat" +panic = "abort" +strip = true +codegen-units = 1 +debug = false + +[profile.dev] +opt-level = 1 +debug = true + [dependencies] -pyo3 = "0.25.0" -jsonpath-rust = "1.0.2" +pyo3 = { version = "0.26.0", features = ["extension-module"] } +jsonpath-rust = "1.0.4" serde_json = "1.0" -pythonize = "0.25.0" +pythonize = "0.26.0" + +[target.'cfg(any(not(target_os = "linux"),all(target_os = "linux", target_env = "musl")))'.dependencies] +mimalloc = "0.1.48" diff --git a/README.md b/README.md index 87f9db2..45addd2 100644 --- a/README.md +++ b/README.md @@ -1,134 +1,141 @@ -# jsonpath-rust-bindings - -![PyPI - Downloads](https://img.shields.io/pypi/dm/jsonpath-rust-bindings) -![GitHub Workflow Status (with event)](https://img.shields.io/github/actions/workflow/status/night-crawler/jsonpath-rust-bindings/CI.yml) -![GitHub Workflow Status (with event)](https://img.shields.io/github/actions/workflow/status/night-crawler/jsonpath-rust-bindings/test.yml?label=tests) -![piwheels (including prereleases)](https://img.shields.io/piwheels/v/jsonpath-rust-bindings) - -This package contains Python bindings for [jsonpath-rust](https://github.com/besok/jsonpath-rust) library by [besok](https://github.com/besok). - -The details regarding the JsonPath itself can be found [here](https://goessner.net/articles/JsonPath/). - -## Installation - -```bash -pip install jsonpath-rust-bindings -``` - -## Usage - -```python -from jsonpath_rust_bindings import Finder - -sample = { - "store": { - "book": [ - { - "category": "reference", - "author": "Nigel Rees", - "title": "Sayings of the Century", - "price": 8.95, - }, - { - "category": "fiction", - "author": "Evelyn Waugh", - "title": "Sword of Honour", - "price": 12.99, - }, - { - "category": "fiction", - "author": "Herman Melville", - "title": "Moby Dick", - "isbn": "0-553-21311-3", - "price": 8.99, - }, - { - "category": "fiction", - "author": "J. R. R. Tolkien", - "title": "The Lord of the Rings", - "isbn": "0-395-19395-8", - "price": 22.99, - }, - ], - "bicycle": {"color": "red", "price": 19.95}, - }, - "expensive": 10, -} - -queries = [ - '$.store.book[*].author', - '$..book[?(@.isbn)]', - '$.store.*', - '$..author', - '$.store..price', - '$..book[2]', - # '$..book[-2]', - '$..book[0,1]', - '$..book[:2]', - '$..book[1:2]', - '$..book[-2:]', - '$..book[2:]', - '$.store.book[?(@.price<10)]', - '$..book[?(@.price<=$.expensive)]', - "$..book[?(@.author ~= '.*Rees')].price", - '$..*', -] - -f = Finder(sample) - -for query in queries: - print(query, f.find(query), '\n') - -# You will see a bunch of found items like -# $..book[?(@.author ~= '.*Rees')].price [JsonPathResult(data=8.95, path=Some("$.['store'].['book'][0].['price']"), is_new_value=False)] - -``` - -`JsonPathResult` has the following attributes: - -- data: the found value -- path: the path to the found value - -`JsonPathResult` can't be constructed from Python; it is only returned by `Finder.find()`. - -## Caveats - -The current implementation is cloning the original `PyObject` data when converting it to the serde `Value`. -It happens each time you're creating a new `Finder` instance. Try to reuse the same `Finder` instance for querying if it's possible. - -Also, It has yet another consequence demonstrated in the following example: - -```python ->>> original_object_i_want_to_mutate = {'a': {'b': 'sample b'}} ->>> from jsonpath_rust_bindings import Finder ->>> f = Finder(original_object_i_want_to_mutate) ->>> b_dict = f.find('$.a')[0].data ->>> b_dict -{'b': 'sample b'} ->>> b_dict['new'] = 42 ->>> original_object_i_want_to_mutate -{'a': {'b': 'sample b'}} -``` - -## Development - -Clone the repository. - -Update maturin if needed: - -```bash -pip install maturin -U -``` - -Add your changes, then run: - -```bash -maturin develop -``` - - -Update CI: - -```bash -maturin generate-ci github > ./.github/workflows/CI.yml -``` +# jsonpath-rust-bindings + +![PyPI - Downloads](https://img.shields.io/pypi/dm/jsonpath-rust-bindings) +![GitHub Workflow Status (with event)](https://img.shields.io/github/actions/workflow/status/night-crawler/jsonpath-rust-bindings/CI.yml) +![GitHub Workflow Status (with event)](https://img.shields.io/github/actions/workflow/status/night-crawler/jsonpath-rust-bindings/test.yml?label=tests) +![piwheels (including prereleases)](https://img.shields.io/piwheels/v/jsonpath-rust-bindings) + +This package contains Python bindings for [jsonpath-rust](https://github.com/besok/jsonpath-rust) library by [besok](https://github.com/besok). + +The details regarding the JsonPath itself can be found [here](https://goessner.net/articles/JsonPath/). + +## Installation + +```bash +pip install jsonpath-rust-bindings +``` + +## Usage + +```python +from jsonpath_rust_bindings import Finder + +sample = { + "store": { + "book": [ + { + "category": "reference", + "author": "Nigel Rees", + "title": "Sayings of the Century", + "price": 8.95, + }, + { + "category": "fiction", + "author": "Evelyn Waugh", + "title": "Sword of Honour", + "price": 12.99, + }, + { + "category": "fiction", + "author": "Herman Melville", + "title": "Moby Dick", + "isbn": "0-553-21311-3", + "price": 8.99, + }, + { + "category": "fiction", + "author": "J. R. R. Tolkien", + "title": "The Lord of the Rings", + "isbn": "0-395-19395-8", + "price": 22.99, + }, + ], + "bicycle": {"color": "red", "price": 19.95}, + }, + "expensive": 10, +} + +queries = [ + '$.store.book[*].author', + '$..book[?(@.isbn)]', + '$.store.*', + '$..author', + '$.store..price', + '$..book[2]', + # '$..book[-2]', + '$..book[0,1]', + '$..book[:2]', + '$..book[1:2]', + '$..book[-2:]', + '$..book[2:]', + '$.store.book[?(@.price<10)]', + '$..book[?(@.price<=$.expensive)]', + "$..book[?(@.author ~= '.*Rees')].price", + '$..*', +] + +f = Finder(sample) + +for query in queries: + # print(query, f.find(query), '\n') + print(query) + print(f.find(query)) + print(f.find_data(query)) + print(f.find_absolute_path(query)) + +# You will see a bunch of found items like +# $.store.book[*].author +# [JsonPathResult(data='Nigel Rees', path="$['store']['book'][0]['author']"), JsonPathResult(data='Evelyn Waugh', path="$['store']['book'][1]['author']"), JsonPathResult(data='Herman Melville', path="$['store']['book'][2]['author']"), JsonPathResult(data='J. R. R. Tolkien', path="$['store']['book'][3]['author']")] +# ['Nigel Rees', 'Evelyn Waugh', 'Herman Melville', 'J. R. R. Tolkien'] +# ["$['store']['book'][0]['author']", "$['store']['book'][1]['author']", "$['store']['book'][2]['author']", "$['store']['book'][3]['author']"] + +``` + +`JsonPathResult` has the following attributes: + +- data: the found value +- path: the path to the found value + +`JsonPathResult` can't be constructed from Python; it is only returned by `Finder.find()`. + +## Caveats + +The current implementation is cloning the original `PyObject` data when converting it to the serde `Value`. +It happens each time you're creating a new `Finder` instance. Try to reuse the same `Finder` instance for querying if it's possible. + +Also, It has yet another consequence demonstrated in the following example: + +```python +>>> original_object_i_want_to_mutate = {'a': {'b': 'sample b'}} +>>> from jsonpath_rust_bindings import Finder +>>> f = Finder(original_object_i_want_to_mutate) +>>> b_dict = f.find('$.a')[0].data +>>> b_dict +{'b': 'sample b'} +>>> b_dict['new'] = 42 +>>> original_object_i_want_to_mutate +{'a': {'b': 'sample b'}} +``` + +## Development + +Clone the repository. + +Update maturin if needed: + +```bash +pip install maturin -U +``` + +Add your changes, then run: + +```bash +maturin develop +``` + + +Update CI: + +```bash +maturin generate-ci github > ./.github/workflows/CI.yml +``` diff --git a/jsonpath_rust_bindings.pyi b/jsonpath_rust_bindings.pyi index 1283aa7..e9febc2 100644 --- a/jsonpath_rust_bindings.pyi +++ b/jsonpath_rust_bindings.pyi @@ -16,3 +16,7 @@ class Finder: ) -> None: ... def find(self, query: str) -> List[JsonPathResult]: ... + + def find_data(self, query: str) -> List: ... + + def find_absolute_path(self, query: str) -> List[str]: ... diff --git a/pyproject.toml b/pyproject.toml index 8edc998..d49d926 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["maturin>=1.3,<2.0"] +requires = ["maturin>=1.9.4,<2.0"] build-backend = "maturin" [project] @@ -17,8 +17,9 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Topic :: Software Development :: Libraries", - "Topic :: Software Development :: Libraries :: Python Modules" + "Topic :: Software Development :: Libraries :: Python Modules", ] dynamic = ["version"] maintainer = "Igor Kalishevsky" @@ -27,4 +28,3 @@ project-url = { homepage = "https://github.com/night-crawler/jsonpath-rust-bindi [tool.maturin] features = ["pyo3/extension-module"] -maturin = "==1.3.1" diff --git a/src/lib.rs b/src/lib.rs index 690821b..6c63660 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,22 +6,45 @@ use pyo3::prelude::*; use pythonize::{depythonize, pythonize}; use serde_json::Value; +#[cfg(any( + not(target_os = "linux"), + all(target_os = "linux", target_env = "musl") +))] +use mimalloc::MiMalloc; +#[cfg(any( + not(target_os = "linux"), + all(target_os = "linux", target_env = "musl") +))] +#[global_allocator] +static GLOBAL: MiMalloc = MiMalloc; + +#[cfg(all(target_os = "linux", not(target_env = "musl")))] +use std::alloc::System; +#[cfg(all(target_os = "linux", not(target_env = "musl")))] +#[global_allocator] +static GLOBAL: System = System; + +// JSONPath query result containing found data and path #[pyclass(frozen)] struct JsonPathResult { + // Found data value #[pyo3(get)] - data: Option, + data: Option>, + // Path of the data in JSON #[pyo3(get)] path: Option, } #[pymethods] impl JsonPathResult { + // Returns string representation of JsonPathResult fn __repr__(slf: PyRef<'_, Self>) -> PyResult { repr_json_path_result(slf) } } +// JSONPath finder for executing queries on JSON data #[pyclass(frozen)] struct Finder { value: Value, @@ -30,32 +53,50 @@ struct Finder { #[pymethods] impl Finder { #[new] - fn py_new(obj: PyObject) -> PyResult { + fn py_new(obj: Py) -> PyResult { Ok(Self { value: parse_py_object(obj)?, }) } + // Execute JSONPath query, return list of results containing data and paths fn find(self_: PyRef<'_, Self>, query: String) -> PyResult> { - find_internal(&self_.value, &query, |_| true) + find_internal_path_value(&self_.value, &query, |_| true) } + + // Execute JSONPath query, return only found data values + fn find_data(self_: PyRef<'_, Self>, query: String) -> PyResult>> { + find_internal_data(&self_.value, &query, |_| true) + } + + // Execute JSONPath query, return only found absolute paths + fn find_absolute_path(self_: PyRef<'_, Self>, query: String) -> PyResult> { + find_internal_path(&self_.value, &query, |_| true) + } +} + +// Execute JSONPath query and return processed results +fn execute_query<'a>( + value: &'a Value, + query: &str, + predicate: impl Fn(&QueryRef) -> bool, +) -> PyResult>> { + let parsed_query = parse_query(query)?; + let processed = js_path_process(&parsed_query, value) + .map_err(|err| PyValueError::new_err(err.to_string()))?; + + Ok(processed.into_iter().filter(predicate).collect()) } -fn find_internal( +// Execute query and return JsonPathResult list +fn find_internal_path_value( value: &Value, query: &str, predicate: impl Fn(&QueryRef) -> bool, ) -> PyResult> { - let query = parse_query(query)?; - let processed = match js_path_process(&query, value) { - Ok(p) => p, - Err(err) => { - return Err(PyValueError::new_err(err.to_string())); - } - }; - let filtered = processed.into_iter().filter(predicate); - - Python::with_gil(|py| { + let filtered = execute_query(value, query, predicate)?; + + Python::attach(|py| { filtered .into_iter() .map(|v| map_json_path_value(py, v)) @@ -63,44 +104,76 @@ fn find_internal( }) } +// Execute query and return data value list +fn find_internal_data( + value: &Value, + query: &str, + predicate: impl Fn(&QueryRef) -> bool, +) -> PyResult>> { + let filtered = execute_query(value, query, predicate)?; + + Python::attach(|py| { + filtered + .into_iter() + .map(|v| map_json_value(py, v)) + .collect() + }) +} + +// Execute query and return path string list +fn find_internal_path( + value: &Value, + query: &str, + predicate: impl Fn(&QueryRef) -> bool, +) -> PyResult> { + let filtered = execute_query(value, query, predicate)?; + filtered.into_iter().map(|v| map_json_path(v)).collect() +} + +// Map QueryRef to JsonPathResult fn map_json_path_value(py: Python, jpv: QueryRef) -> PyResult { let path = jpv.clone().path(); let val = jpv.val(); - let res = JsonPathResult { + Ok(JsonPathResult { data: Some(pythonize(py, val)?.into_pyobject(py)?.unbind()), path: Some(path), - }; + }) +} - Ok(res) +// Extract path string from QueryRef +fn map_json_path(jpv: QueryRef) -> PyResult { + Ok(jpv.path()) } +// Convert value in QueryRef to Python object +fn map_json_value(py: Python, jpv: QueryRef) -> PyResult> { + let val = jpv.val(); + Ok(pythonize(py, val)?.into_pyobject(py)?.unbind()) +} + +// Parse JSONPath query string directly without caching fn parse_query(query: &str) -> PyResult { - match parse_json_path(query) { - Ok(inst) => Ok(inst), - Err(err) => Err(PyValueError::new_err(format!("{err:?}"))), - } + parse_json_path(query).map_err(|err| PyValueError::new_err(format!("{err:?}"))) } -fn parse_py_object(obj: PyObject) -> PyResult { - Python::with_gil(|py| { - let any = obj.downcast_bound::(py)?.clone().into_any(); - let value = depythonize(&any)?; - Ok(value) +// Convert Python object to serde_json::Value +fn parse_py_object(obj: Py) -> PyResult { + Python::attach(|py| { + let any = obj.bind(py); + depythonize(any).map_err(|e| PyValueError::new_err(e.to_string())) }) } +// Generate string representation for JsonPathResult fn repr_json_path_result(slf: PyRef<'_, JsonPathResult>) -> PyResult { let data_repr = slf .data .as_ref() - .map(|data| Python::with_gil(|py| format!("{:?}", data.bind(py)))) + .map(|data| Python::attach(|py| format!("{:?}", data.bind(py)))) .unwrap_or_default(); - let path_repr = match &slf.path { - Some(path) => path, - None => "None", - }; + let path_repr = slf.path.as_deref().unwrap_or("None"); Ok(format!( "JsonPathResult(data={data_repr}, path={path_repr:?})", )) diff --git a/tests/test_bindings.py b/tests/test_bindings.py index 30a2d3b..a91ac0b 100644 --- a/tests/test_bindings.py +++ b/tests/test_bindings.py @@ -1,99 +1,200 @@ -import pytest -from jsonpath_rust_bindings import Finder -from jsonpath_rust_bindings import JsonPathResult - - -@pytest.fixture -def sample_data() -> dict: - return { - 'store': { - 'book': [ - { - 'category': 'reference', - 'author': 'Nigel Rees', - 'title': 'Sayings of the Century', - 'price': 8.95 - }, - { - 'category': 'fiction', - 'author': 'Evelyn Waugh', - 'title': 'Sword of Honour', - 'price': 12.99 - }, - { - 'category': 'fiction', - 'author': 'Herman Melville', - 'title': 'Moby Dick', - 'isbn': '0-553-21311-3', - 'price': 8.99 - }, - { - 'category': 'fiction', - 'author': 'J. R. R. Tolkien', - 'title': 'The Lord of the Rings', - 'isbn': '0-395-19395-8', - 'price': 22.99 - } - ], - 'bicycle': { - 'color': 'red', - 'price': 19.95 - } - }, - 'expensive': 10 - } - - -def test_sanity(sample_data): - finder = Finder(sample_data) - res = finder.find('$..book[?(@.price<=$.expensive)]') - assert len(res) == 2 - assert isinstance(res[0], JsonPathResult) - - -def test_exceptions(sample_data): - finder = Finder(sample_data) - - with pytest.raises(ValueError): - finder.find('fail') - - -def test_repr(sample_data): - finder = Finder(sample_data) - result = str(finder.find('$.store.bicycle.color')[0]) - assert result == """JsonPathResult(data='red', path="$['store']['bicycle']['color']")""" - - -def test_smoke_queries(sample_data): - queries = [ - '$.store.book[*].author', - '$..book[?(@.isbn)]', - '$.store.*', - '$..author', - '$.store..price', - '$..book[2]', - '$..book[-2]', - '$..book[0,1]', - '$..book[:2]', - '$..book[1:2]', - '$..book[-2:]', - '$..book[2:]', - '$.store.book[?(@.price<10)]', - '$..book[?(@.price<=$.expensive)]', - # "$..book[?@.author ~= '(?i)REES']", - '$..*', - ] - - f = Finder(sample_data) - - for query in queries: - print(query) - print(f.find(query), '\n') - print('----------') - - -def test_overflow(): - big_number = 18446744005107584948 - f = Finder({"test": big_number}) - res = f.find('$.test')[0].data - assert res == big_number +import pytest +from jsonpath_rust_bindings import Finder +from jsonpath_rust_bindings import JsonPathResult + + +@pytest.fixture +def sample_data() -> dict: + return { + "store": { + "book": [ + { + "category": "reference", + "author": "Nigel Rees", + "title": "Sayings of the Century", + "price": 8.95, + }, + { + "category": "fiction", + "author": "Evelyn Waugh", + "title": "Sword of Honour", + "price": 12.99, + }, + { + "category": "fiction", + "author": "Herman Melville", + "title": "Moby Dick", + "isbn": "0-553-21311-3", + "price": 8.99, + }, + { + "category": "fiction", + "author": "J. R. R. Tolkien", + "title": "The Lord of the Rings", + "isbn": "0-395-19395-8", + "price": 22.99, + }, + ], + "bicycle": {"color": "red", "price": 19.95}, + }, + "expensive": 10, + } + + +def test_sanity(sample_data): + finder = Finder(sample_data) + res = finder.find("$..book[?(@.price<=$.expensive)]") + assert len(res) == 2 + assert isinstance(res[0], JsonPathResult) + + +def test_exceptions(sample_data): + finder = Finder(sample_data) + + with pytest.raises(ValueError): + finder.find("fail") + + +def test_repr(sample_data): + finder = Finder(sample_data) + result = str(finder.find("$.store.bicycle.color")[0]) + assert ( + result + == """JsonPathResult(data='red', path="$['store']['bicycle']['color']")""" + ) + + +def test_all_queries(sample_data): + queries = [ + "$.store.book[*].author", + "$..book[?(@.isbn)]", + "$.store.*", + "$..author", + "$.store..price", + "$..book[2]", + "$..book[-2]", + "$..book[0,1]", + "$..book[:2]", + "$..book[1:2]", + "$..book[-2:]", + "$..book[2:]", + "$.store.book[?(@.price<10)]", + "$..book[?(@.price<=$.expensive)]", + # "$..book[?@.author ~= '(?i)REES']", + "$..*", + ] + queries_results = [ + """[JsonPathResult(data='Nigel Rees', path="$['store']['book'][0]['author']"), +JsonPathResult(data='Evelyn Waugh', path="$['store']['book'][1]['author']"), +JsonPathResult(data='Herman Melville', path="$['store']['book'][2]['author']"), +JsonPathResult(data='J. R. R. Tolkien', path="$['store']['book'][3]['author']")]""", + """[JsonPathResult(data={'author': 'Herman Melville', 'category': 'fiction', 'isbn': '0-553-21311-3', 'price': 8.99, 'title': 'Moby Dick'}, +path="$['store']['book'][2]"), +JsonPathResult(data={'author': 'J. R. R. Tolkien', 'category': 'fiction', 'isbn': '0-395-19395-8', 'price': 22.99, 'title': 'The Lord of the Rings'}, +path="$['store']['book'][3]")]""", + """[JsonPathResult(data={'color': 'red', 'price': 19.95}, path="$['store']['bicycle']"), +JsonPathResult(data=[{'author': 'Nigel Rees', 'category': 'reference', 'price': 8.95, 'title': 'Sayings of the Century'}, +{'author': 'Evelyn Waugh', 'category': 'fiction', 'price': 12.99, 'title': 'Sword of Honour'}, +{'author': 'Herman Melville', 'category': 'fiction', 'isbn': '0-553-21311-3', 'price': 8.99, 'title': 'Moby Dick'}, +{'author': 'J. R. R. Tolkien', 'category': 'fiction', 'isbn': '0-395-19395-8', 'price': 22.99, 'title': 'The Lord of the Rings'}], +path="$['store']['book']")]""", + """[JsonPathResult(data='Nigel Rees', path="$['store']['book'][0]['author']"), +JsonPathResult(data='Evelyn Waugh', path="$['store']['book'][1]['author']"), +JsonPathResult(data='Herman Melville', path="$['store']['book'][2]['author']"), +JsonPathResult(data='J. R. R. Tolkien', path="$['store']['book'][3]['author']")]""", + """[JsonPathResult(data=19.95, path="$['store']['bicycle']['price']"), +JsonPathResult(data=8.95, path="$['store']['book'][0]['price']"), +JsonPathResult(data=12.99, path="$['store']['book'][1]['price']"), +JsonPathResult(data=8.99, path="$['store']['book'][2]['price']"), +JsonPathResult(data=22.99, path="$['store']['book'][3]['price']")]""", + """[JsonPathResult(data={'author': 'Herman Melville', 'category': 'fiction', 'isbn': '0-553-21311-3', 'price': 8.99, 'title': 'Moby Dick'}, +path="$['store']['book'][2]")]""", + """[JsonPathResult(data={'author': 'Herman Melville', 'category': 'fiction', 'isbn': '0-553-21311-3', 'price': 8.99, 'title': 'Moby Dick'}, +path="$['store']['book'][2]")]""", + """[JsonPathResult(data={'author': 'Nigel Rees', 'category': 'reference', 'price': 8.95, 'title': 'Sayings of the Century'}, +path="$['store']['book'][0]"), +JsonPathResult(data={'author': 'Evelyn Waugh', 'category': 'fiction', 'price': 12.99, 'title': 'Sword of Honour'}, path="$['store']['book'][1]")]""", + """[JsonPathResult(data={'author': 'Nigel Rees', 'category': 'reference', 'price': 8.95, 'title': 'Sayings of the Century'}, +path="$['store']['book'][0]"), +JsonPathResult(data={'author': 'Evelyn Waugh', 'category': 'fiction', 'price': 12.99, 'title': 'Sword of Honour'}, path="$['store']['book'][1]")]""", + """[JsonPathResult(data={'author': 'Evelyn Waugh', 'category': 'fiction', 'price': 12.99, 'title': 'Sword of Honour'}, +path="$['store']['book'][1]")]""", + """[JsonPathResult(data={'author': 'Herman Melville', 'category': 'fiction', 'isbn': '0-553-21311-3', 'price': 8.99, 'title': 'Moby Dick'}, +path="$['store']['book'][2]"), +JsonPathResult(data={'author': 'J. R. R. Tolkien', 'category': 'fiction', 'isbn': '0-395-19395-8', 'price': 22.99, 'title': 'The Lord of the Rings'}, +path="$['store']['book'][3]")]""", + """[JsonPathResult(data={'author': 'Herman Melville', 'category': 'fiction', 'isbn': '0-553-21311-3', 'price': 8.99, 'title': 'Moby Dick'}, +path="$['store']['book'][2]"), +JsonPathResult(data={'author': 'J. R. R. Tolkien', 'category': 'fiction', 'isbn': '0-395-19395-8', 'price': 22.99, 'title': 'The Lord of the Rings'}, +path="$['store']['book'][3]")]""", + """[JsonPathResult(data={'author': 'Nigel Rees', 'category': 'reference', 'price': 8.95, 'title': 'Sayings of the Century'}, +path="$['store']['book'][0]"), +JsonPathResult(data={'author': 'Herman Melville', 'category': 'fiction', 'isbn': '0-553-21311-3', 'price': 8.99, 'title': 'Moby Dick'}, +path="$['store']['book'][2]")]""", + """[JsonPathResult(data={'author': 'Nigel Rees', 'category': 'reference', 'price': 8.95, 'title': 'Sayings of the Century'}, +path="$['store']['book'][0]"), +JsonPathResult(data={'author': 'Herman Melville', 'category': 'fiction', 'isbn': '0-553-21311-3', 'price': 8.99, 'title': 'Moby Dick'}, +path="$['store']['book'][2]")]""", + """[JsonPathResult(data=10, path="$['expensive']"), +JsonPathResult(data={'bicycle': {'color': 'red', 'price': 19.95}, +'book': [{'author': 'Nigel Rees', 'category': 'reference', 'price': 8.95, 'title': 'Sayings of the Century'}, +{'author': 'Evelyn Waugh', 'category': 'fiction', 'price': 12.99, 'title': 'Sword of Honour'}, +{'author': 'Herman Melville', 'category': 'fiction', 'isbn': '0-553-21311-3', 'price': 8.99, 'title': 'Moby Dick'}, +{'author': 'J. R. R. Tolkien', 'category': 'fiction', 'isbn': '0-395-19395-8', 'price': 22.99, 'title': 'The Lord of the Rings'}]}, +path="$['store']"), +JsonPathResult(data={'color': 'red', 'price': 19.95}, path="$['store']['bicycle']"), +JsonPathResult(data=[{'author': 'Nigel Rees', 'category': 'reference', 'price': 8.95, 'title': 'Sayings of the Century'}, +{'author': 'Evelyn Waugh', 'category': 'fiction', 'price': 12.99, 'title': 'Sword of Honour'}, +{'author': 'Herman Melville', 'category': 'fiction', 'isbn': '0-553-21311-3', 'price': 8.99, 'title': 'Moby Dick'}, +{'author': 'J. R. R. Tolkien', 'category': 'fiction', 'isbn': '0-395-19395-8', 'price': 22.99, 'title': 'The Lord of the Rings'}], +path="$['store']['book']"), +JsonPathResult(data='red', path="$['store']['bicycle']['color']"), +JsonPathResult(data=19.95, path="$['store']['bicycle']['price']"), +JsonPathResult(data={'author': 'Nigel Rees', 'category': 'reference', 'price': 8.95, 'title': 'Sayings of the Century'}, +path="$['store']['book'][0]"), +JsonPathResult(data={'author': 'Evelyn Waugh', 'category': 'fiction', 'price': 12.99, 'title': 'Sword of Honour'}, path="$['store']['book'][1]"), +JsonPathResult(data={'author': 'Herman Melville', 'category': 'fiction', 'isbn': '0-553-21311-3', 'price': 8.99, 'title': 'Moby Dick'}, +path="$['store']['book'][2]"), +JsonPathResult(data={'author': 'J. R. R. Tolkien', 'category': 'fiction', 'isbn': '0-395-19395-8', 'price': 22.99, 'title': 'The Lord of the Rings'}, +path="$['store']['book'][3]"), +JsonPathResult(data='Nigel Rees', path="$['store']['book'][0]['author']"), +JsonPathResult(data='reference', path="$['store']['book'][0]['category']"), +JsonPathResult(data=8.95, path="$['store']['book'][0]['price']"), +JsonPathResult(data='Sayings of the Century', path="$['store']['book'][0]['title']"), +JsonPathResult(data='Evelyn Waugh', path="$['store']['book'][1]['author']"), +JsonPathResult(data='fiction', path="$['store']['book'][1]['category']"), +JsonPathResult(data=12.99, path="$['store']['book'][1]['price']"), +JsonPathResult(data='Sword of Honour', path="$['store']['book'][1]['title']"), +JsonPathResult(data='Herman Melville', path="$['store']['book'][2]['author']"), +JsonPathResult(data='fiction', path="$['store']['book'][2]['category']"), +JsonPathResult(data='0-553-21311-3', path="$['store']['book'][2]['isbn']"), +JsonPathResult(data=8.99, path="$['store']['book'][2]['price']"), +JsonPathResult(data='Moby Dick', path="$['store']['book'][2]['title']"), +JsonPathResult(data='J. R. R. Tolkien', path="$['store']['book'][3]['author']"), +JsonPathResult(data='fiction', path="$['store']['book'][3]['category']"), +JsonPathResult(data='0-395-19395-8', path="$['store']['book'][3]['isbn']"), +JsonPathResult(data=22.99, path="$['store']['book'][3]['price']"), +JsonPathResult(data='The Lord of the Rings', path="$['store']['book'][3]['title']")]""", + ] + f = Finder(sample_data) + res = [] + res_data = [] + res_absolute_path = [] + for query, queries_result in zip(queries, queries_results): + temp_res = f.find(query) + res.extend(temp_res) + res_data.extend(f.find_data(query)) + res_absolute_path.extend(f.find_absolute_path(query)) + assert str(temp_res) == queries_result.replace("\n", "") + assert [r.data for r in res] == res_data + assert [r.path for r in res] == res_absolute_path + # print(query) + # print(f.find(query), '\n') + # print('----------') + + +def test_overflow(): + big_number = 18446744005107584948 + f = Finder({"test": big_number}) + res = f.find('$.test')[0].data + assert res == big_number