diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 77fe875e..432ead70 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,22 +13,14 @@ env: CARGO_TERM_COLOR: always jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - - uses: pre-commit/action@v3.0.0 - build-linux-armv7: runs-on: [self-hosted, linux, arm] - needs: [lint] steps: - name: Setup python run: | pyenv global system python --version - - uses: actions/checkout@v3 + - uses: actions/checkout@v2 - name: Build run: cargo build --verbose - name: Run tests @@ -36,13 +28,12 @@ jobs: build: runs-on: ${{ matrix.os }} - needs: [lint] strategy: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v2 - name: Install Dependencies run: sudo apt install libunwind-dev if: runner.os == 'Linux' @@ -60,14 +51,13 @@ jobs: - name: Test (retry#2) run: cargo test --release if: steps.test1.outcome=='failure' - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v2 with: python-version: 3.9 - name: Build Wheel run: | pip install --upgrade maturin maturin build --release -o dist - if: runner.os != 'Linux' - name: Build Wheel - universal2 env: DEVELOPER_DIR: /Applications/Xcode.app/Contents/Developer @@ -76,14 +66,13 @@ jobs: run: | rustup target add aarch64-apple-darwin pip install --upgrade maturin - maturin build --release -o dist --target universal2-apple-darwin + maturin build --release -o dist --universal2 if: matrix.os == 'macos-latest' - name: Rename Wheels run: | python3 -c "import shutil; import glob; wheels = glob.glob('dist/*.whl'); [shutil.move(wheel, wheel.replace('py3', 'py2.py3')) for wheel in wheels if 'py2' not in wheel]" - if: runner.os != 'Linux' - name: Upload wheels - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v2 with: name: wheels path: dist @@ -91,7 +80,6 @@ jobs: build-linux-cross: runs-on: ubuntu-latest - needs: [lint] strategy: fail-fast: false matrix: @@ -102,64 +90,60 @@ jobs: RUSTUP_HOME: /root/.rustup CARGO_HOME: /root/.cargo steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v2 - name: Build run: | python3 -m pip install --upgrade maturin maturin build --release -o dist --target $RUST_MUSL_CROSS_TARGET - maturin sdist -o dist - name: Rename Wheels run: | python3 -c "import shutil; import glob; wheels = glob.glob('dist/*.whl'); [shutil.move(wheel, wheel.replace('py3', 'py2.py3')) for wheel in wheels if 'py2' not in wheel]" - name: Upload wheels - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v2 with: name: wheels path: dist build-freebsd: - runs-on: ubuntu-22.04 - needs: [lint] + runs-on: macos-10.15 timeout-minutes: 30 strategy: matrix: - box: - - freebsd-14 + include: + - box: fbsd_13_1 + release: FreeBSD-13.1-STABLE + url: https://github.com/rbspy/freebsd-vagrant-box/releases/download/20220703/fbsd_13_1.box steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v2 - name: Cache Vagrant box - uses: actions/cache@v3.0.4 + uses: actions/cache@v2 with: - path: .vagrant.d - key: ${{ matrix.box }}-vagrant-boxes-20231115-${{ hashFiles('ci/Vagrantfile') }} + path: ~/.vagrant.d + key: ${{ matrix.box }}-vagrant-boxes-${{ hashFiles('ci/Vagrantfile') }} restore-keys: | - ${{ matrix.box }}-vagrant-boxes-20231115- + ${{ matrix.box }}-vagrant- - name: Cache Cargo and build artifacts - uses: actions/cache@v3.0.4 + uses: actions/cache@v2.1.4 with: path: build-artifacts.tar - key: ${{ matrix.box }}-cargo-20231115-${{ hashFiles('**/Cargo.lock') }} + key: ${{ matrix.box }}-cargo-${{ hashFiles('**/Cargo.lock') }} restore-keys: | - ${{ matrix.box }}-cargo-20231115- - - name: Display CPU info - run: lscpu - - name: Install VM tools - run: | - sudo apt-get install -qq -o=Dpkg::Use-Pty=0 moreutils - sudo chronic apt-get install -qq -o=Dpkg::Use-Pty=0 vagrant virtualbox qemu libvirt-daemon-system + ${{ matrix.box }}-cargo- - name: Set up VM - shell: sudo bash {0} run: | - vagrant plugin install vagrant-libvirt + brew install vagrant + vagrant plugin install vagrant-vbguest vagrant plugin install vagrant-scp + ln -sf ci/Vagrantfile Vagrantfile - vagrant status - vagrant up --no-tty --provider libvirt ${{ matrix.box }} + + if [ ! -d ~/.vagrant.d/boxes/rbspy-VAGRANTSLASH-${{ matrix.release }} ]; then + vagrant box add --no-tty rbspy/${{ matrix.release }} ${{ matrix.url }} + fi + vagrant up ${{ matrix.box }} - name: Build and test - shell: sudo bash {0} run: vagrant ssh ${{ matrix.box }} -- bash /vagrant/ci/test_freebsd.sh - name: Retrieve build artifacts for caching purposes - shell: sudo bash {0} run: | vagrant scp ${{ matrix.box }}:/vagrant/build-artifacts.tar build-artifacts.tar ls -ahl build-artifacts.tar @@ -168,7 +152,7 @@ jobs: tar xf build-artifacts.tar target/release/py-spy mv target/release/py-spy py-spy-x86_64-unknown-freebsd - name: Upload Binaries - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v2 with: name: py-spy-x86_64-unknown-freebsd path: py-spy-x86_64-unknown-freebsd @@ -180,30 +164,15 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.5.4, 3.5.9, 3.5.10, 3.6.7, 3.6.8, 3.6.9, 3.6.10, 3.6.11, 3.6.12, 3.6.13, 3.6.14, 3.6.15, 3.7.1, 3.7.2, 3.7.3, 3.7.4, 3.7.5, 3.7.6, 3.7.7, 3.7.8, 3.7.9, 3.7.10, 3.7.11, 3.7.12, 3.7.13, 3.7.14, 3.7.15, 3.7.16, 3.7.17, 3.8.0, 3.8.1, 3.8.2, 3.8.3, 3.8.4, 3.8.5, 3.8.6, 3.8.7, 3.8.8, 3.8.9, 3.8.10, 3.8.11, 3.8.12, 3.8.13, 3.8.14, 3.8.15, 3.8.16, 3.8.17, 3.8.18, 3.9.0, 3.9.1, 3.9.2, 3.9.3, 3.9.4, 3.9.5, 3.9.6, 3.9.7, 3.9.8, 3.9.9, 3.9.10, 3.9.11, 3.9.12, 3.9.13, 3.9.14, 3.9.15, 3.9.16, 3.9.17, 3.9.18, 3.10.0, 3.10.1, 3.10.2, 3.10.3, 3.10.4, 3.10.5, 3.10.6, 3.10.7, 3.10.8, 3.10.9, 3.10.10, 3.10.11, 3.10.12, 3.10.13, 3.11.0, 3.11.1, 3.11.2, 3.11.3, 3.11.4, 3.11.5] + python-version: [2.7.17, 2.7.18, 3.5.4, 3.5.9, 3.5.10, 3.6.7, 3.6.8, 3.6.9, 3.6.10, 3.6.11, 3.6.12, 3.6.13, 3.6.14, 3.6.15, 3.7.1, 3.7.5, 3.7.6, 3.7.7, 3.7.8, 3.7.9, 3.7.10, 3.7.11, 3.7.12, 3.7.13, 3.8.0, 3.8.1, 3.8.2, 3.8.3, 3.8.4, 3.8.5, 3.8.6, 3.8.7, 3.8.8, 3.8.9, 3.8.10, 3.8.11, 3.8.12, 3.8.13, 3.9.0, 3.9.1, 3.9.2, 3.9.3, 3.9.4, 3.9.5, 3.9.6, 3.9.7, 3.9.8, 3.9.9, 3.9.10, 3.9.11, 3.9.12, 3.9.13, 3.10.0, 3.10.1, 3.10.2, 3.10.3, 3.10.4, 3.10.5, 3.10.6, 3.10.7, 3.11.0-beta.5] # TODO: also test windows - os: [ubuntu-20.04, macos-latest] - # some versions of python can't be tested on GHA with osx because of SIP: - exclude: - - os: macos-latest - python-version: 3.11.0 - - os: macos-latest - python-version: 3.11.1 - - os: macos-latest - python-version: 3.11.2 - - os: macos-latest - python-version: 3.11.3 - - os: macos-latest - python-version: 3.11.4 - - os: macos-latest - python-version: 3.11.5 - + os: [ubuntu-latest, macos-latest] steps: - uses: actions/checkout@v2 - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v2 with: name: wheels - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Install wheel @@ -248,8 +217,8 @@ jobs: # only test out relatively recent versions of python pyenv-python-version: [3.7.10, 3.8.9, 3.9.4] steps: - - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 + - uses: actions/checkout@v2 + - uses: actions/download-artifact@v2 with: name: wheels - name: Setup pyenv @@ -271,7 +240,7 @@ jobs: if: "startsWith(github.ref, 'refs/tags/')" needs: [test-wheels, test-wheel-linux-armv7] steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v2 with: name: wheels - name: Create GitHub Release @@ -282,7 +251,7 @@ jobs: - name: Install Dependencies run: sudo apt install libunwind-dev if: runner.os == 'Linux' - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v2 with: python-version: 3.9 - name: Push to PyPi @@ -293,7 +262,7 @@ jobs: pip install --upgrade wheel pip setuptools twine twine upload * rm * - - uses: actions/checkout@v3 + - uses: actions/checkout@v2 - name: Push to crates.io env: CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} diff --git a/.github/workflows/update_python_test.yml b/.github/workflows/update_python_test.yml index 7f12fc9e..bff0ce59 100644 --- a/.github/workflows/update_python_test.yml +++ b/.github/workflows/update_python_test.yml @@ -7,10 +7,10 @@ jobs: update-dep: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v2 with: ssh-key: ${{ secrets.SSH_PRIVATE_KEY }} - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v2 with: python-version: 3.9 - name: Install @@ -18,11 +18,11 @@ jobs: - name: Scan for new python versions run: python ci/update_python_test_versions.py - name: Create Pull Request - uses: peter-evans/create-pull-request@v4 + uses: peter-evans/create-pull-request@v3 with: commit-message: Update tested python versions title: Update tested python versions - branch: update-python-versions + branch: update-python-versons labels: | skip-changelog dependencies diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 100644 index e7e6d301..00000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1,21 +0,0 @@ -repos: - - repo: https://github.com/codespell-project/codespell - rev: v2.2.4 - hooks: - - id: codespell - additional_dependencies: [tomli] - args: ["--toml", "pyproject.toml"] - exclude: (?x)^(ci/testdata.*|images.*)$ - - repo: https://github.com/doublify/pre-commit-rust - rev: v1.0 - hooks: - - id: fmt - - id: cargo-check - - repo: local - hooks: - - id: cargo-clippy - name: cargo clippy - entry: cargo clippy -- -D warnings - language: system - files: \.rs$ - pass_filenames: false diff --git a/Cargo.lock b/Cargo.lock index 5a493fa7..4f9c02ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,14 +4,13 @@ version = 3 [[package]] name = "addr2line" -version = "0.20.0" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" +checksum = "6ca9b76e919fd83ccfb509f51b28c333c0e03f2221616e347a129215cec4e4a9" dependencies = [ "cpp_demangle", "fallible-iterator", "gimli", - "memmap2", "object", "rustc-demangle", "smallvec", @@ -25,11 +24,10 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "ahash" -version = "0.8.3" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" dependencies = [ - "cfg-if", "getrandom", "once_cell", "version_check", @@ -37,88 +35,36 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.0.2" +version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" dependencies = [ "memchr", ] [[package]] -name = "android-tzdata" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" - -[[package]] -name = "android_system_properties" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" -dependencies = [ - "libc", -] - -[[package]] -name = "anstream" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "is-terminal", - "utf8parse", -] - -[[package]] -name = "anstyle" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41ed9a86bf92ae6580e0a31281f65a1b1d867c0cc68d5346e2ae128dddfa6a7d" - -[[package]] -name = "anstyle-parse" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e765fd216e48e067936442276d1d57399e37bce53c264d6fefbe298080cb57ee" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-query" -version = "1.0.0" +name = "ansi_term" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" dependencies = [ - "windows-sys 0.48.0", -] - -[[package]] -name = "anstyle-wincon" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" -dependencies = [ - "anstyle", - "windows-sys 0.48.0", + "winapi", ] [[package]] name = "anyhow" -version = "1.0.71" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" +checksum = "bb07d2053ccdbe10e2af2995a2f116c1330396493dc1269f6a91d0ae82e19704" [[package]] name = "arrayvec" -version = "0.7.2" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" +checksum = "cd9fd44efafa8690358b7408d253adf110036b88f55672a933f01d616ad9b1b9" +dependencies = [ + "nodrop", +] [[package]] name = "atty" @@ -126,7 +72,7 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ - "hermit-abi 0.1.19", + "hermit-abi", "libc", "winapi", ] @@ -139,44 +85,24 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "bindgen" -version = "0.64.0" +version = "0.59.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4243e6031260db77ede97ad86c27e501d646a27ab57b59a574f725d98ab1fb4" +checksum = "2bd2a9a458e8f4304c52c43ebb0cfbd520289f8379a52e329a38afda99bf8eb8" dependencies = [ - "bitflags 1.3.2", - "cexpr", - "clang-sys", - "lazy_static", - "lazycell", - "peeking_take_while", - "proc-macro2", - "quote", - "regex", - "rustc-hash", - "shlex", - "syn 1.0.109", -] - -[[package]] -name = "bindgen" -version = "0.68.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "726e4313eb6ec35d2730258ad4e15b547ee75d6afaa1361a922e78e59b7d8078" -dependencies = [ - "bitflags 2.4.1", + "bitflags", "cexpr", "clang-sys", + "clap 2.34.0", + "env_logger", "lazy_static", "lazycell", "log", "peeking_take_while", - "prettyplease", "proc-macro2", "quote", "regex", "rustc-hash", "shlex", - "syn 2.0.38", "which", ] @@ -186,35 +112,17 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" -[[package]] -name = "bitflags" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" - -[[package]] -name = "bumpalo" -version = "3.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" - [[package]] name = "bytemuck" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17febce684fd15d89027105661fec94afb475cb995fbc59d2865198446ba2eea" - -[[package]] -name = "byteorder" -version = "1.4.3" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "a5377c8865e74a160d21f29c2d40669f53286db6eab59b88540cbb12ffc8b835" [[package]] name = "cc" -version = "1.0.79" +version = "1.0.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" [[package]] name = "cexpr" @@ -233,24 +141,22 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.26" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" dependencies = [ - "android-tzdata", - "iana-time-zone", - "js-sys", + "libc", + "num-integer", "num-traits", "time", - "wasm-bindgen", "winapi", ] [[package]] name = "clang-sys" -version = "1.6.1" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f" +checksum = "5a050e2153c5be08febd6734e29298e844fdb0fa21aeddd63b4eb7baa106c69b" dependencies = [ "glob", "libc", @@ -259,78 +165,57 @@ dependencies = [ [[package]] name = "clap" -version = "3.2.25" +version = "2.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" dependencies = [ + "ansi_term", "atty", - "bitflags 1.3.2", - "clap_derive 3.2.25", - "clap_lex 0.2.4", - "indexmap 1.9.3", - "once_cell", - "strsim", - "termcolor", - "terminal_size", - "textwrap", + "bitflags", + "strsim 0.8.0", + "textwrap 0.11.0", + "unicode-width", + "vec_map", ] [[package]] name = "clap" -version = "4.3.2" +version = "3.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "401a4694d2bf92537b6867d94de48c4842089645fdcdf6c71865b175d836e9c2" +checksum = "44bbe24bbd31a185bc2c4f7c2abe80bea13a20d57ee4e55be70ac512bdc76417" dependencies = [ - "clap_builder", - "clap_derive 4.3.2", + "atty", + "bitflags", + "clap_derive", + "clap_lex", + "indexmap", "once_cell", -] - -[[package]] -name = "clap_builder" -version = "4.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72394f3339a76daf211e57d4bcb374410f3965dcc606dd0e03738c7888766980" -dependencies = [ - "anstream", - "anstyle", - "bitflags 1.3.2", - "clap_lex 0.5.0", - "strsim", + "strsim 0.10.0", + "termcolor", + "terminal_size", + "textwrap 0.15.0", ] [[package]] name = "clap_complete" -version = "3.2.5" +version = "3.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f7a2e0a962c45ce25afce14220bc24f9dade0a1787f185cecf96bfba7847cd8" +checksum = "ead064480dfc4880a10764488415a97fdd36a4cf1bb022d372f02e8faf8386e1" dependencies = [ - "clap 3.2.25", + "clap 3.2.15", ] [[package]] name = "clap_derive" -version = "3.2.25" +version = "3.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae6371b8bdc8b7d3959e9cf7b22d4435ef3e79e138688421ec654acf8c81b008" +checksum = "9ba52acd3b0a5c33aeada5cdaa3267cdc7c594a98731d4268cdc1532f4264cb4" dependencies = [ "heck", "proc-macro-error", "proc-macro2", "quote", - "syn 1.0.109", -] - -[[package]] -name = "clap_derive" -version = "4.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8cd2b2a819ad6eec39e8f1d6b53001af1e5469f8c177579cdaeb313115b825f" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn 2.0.38", + "syn", ] [[package]] @@ -342,42 +227,25 @@ dependencies = [ "os_str_bytes", ] -[[package]] -name = "clap_lex" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" - -[[package]] -name = "colorchoice" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" - [[package]] name = "console" -version = "0.15.7" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" +checksum = "89eab4d20ce20cea182308bca13088fecea9c05f6776cf287205d41a0ed3c847" dependencies = [ "encode_unicode", - "lazy_static", "libc", + "once_cell", + "terminal_size", "unicode-width", - "windows-sys 0.45.0", + "winapi", ] -[[package]] -name = "core-foundation-sys" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" - [[package]] name = "cpp_demangle" -version = "0.4.1" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c76f98bdfc7f66172e6c7065f981ebb576ffc903fe4c0561d9f0c2509226dc6" +checksum = "eeaa953eaad386a53111e47172c2fedba671e5684c8dd601a5f474f4f118710f" dependencies = [ "cfg-if", ] @@ -393,9 +261,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.8" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" dependencies = [ "cfg-if", "crossbeam-utils", @@ -403,41 +271,41 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.15" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" +checksum = "51887d4adc7b564537b15adcfb307936f8075dfcd5f00dde9a9f1d29383682bc" dependencies = [ "cfg-if", + "once_cell", ] [[package]] name = "ctrlc" -version = "3.4.0" +version = "3.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a011bbe2c35ce9c1f143b7af6f94f29a167beb4cd1d29e6740ce836f723120e" +checksum = "b37feaa84e6861e00a1f5e5aa8da3ee56d605c9992d33e082786754828e20865" dependencies = [ "nix", - "windows-sys 0.48.0", + "winapi", ] [[package]] name = "dashmap" -version = "5.4.0" +version = "5.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc" +checksum = "3495912c9c1ccf2e18976439f4443f3fee0fd61f424ff99fde6a66b15ecb448f" dependencies = [ "cfg-if", - "hashbrown 0.12.3", + "hashbrown", "lock_api", - "once_cell", "parking_lot_core", ] [[package]] name = "either" -version = "1.8.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +checksum = "3f107b87b6afc2a64fd13cac55fe06d6c8859f12d4b14cbcdd2c67d0976781be" [[package]] name = "encode_unicode" @@ -447,23 +315,17 @@ checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" [[package]] name = "env_logger" -version = "0.10.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" +checksum = "0b2cf0344971ee6c64c31be0d530793fba457d322dfec2810c453d0ef228f9c3" dependencies = [ + "atty", "humantime", - "is-terminal", "log", "regex", "termcolor", ] -[[package]] -name = "equivalent" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" - [[package]] name = "errno" version = "0.2.8" @@ -475,17 +337,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "errno" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" -dependencies = [ - "errno-dragonfly", - "libc", - "windows-sys 0.48.0", -] - [[package]] name = "errno-dragonfly" version = "0.1.2" @@ -504,18 +355,18 @@ checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" [[package]] name = "fastrand" -version = "1.9.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499" dependencies = [ "instant", ] [[package]] name = "flate2" -version = "1.0.26" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" +checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6" dependencies = [ "crc32fast", "miniz_oxide", @@ -523,9 +374,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.10" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6" dependencies = [ "cfg-if", "libc", @@ -534,9 +385,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.27.2" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad0a93d233ebf96623465aad4046a8d3aa4da22d4f4beba5388838c8a434bbb4" +checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d" dependencies = [ "fallible-iterator", "stable_deref_trait", @@ -544,26 +395,15 @@ dependencies = [ [[package]] name = "glob" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" - -[[package]] -name = "goblin" -version = "0.6.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d6b4de4a8eb6c46a8c77e1d3be942cb9a8bf073c22374578e5ba4b08ed0ff68" -dependencies = [ - "log", - "plain", - "scroll", -] +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" [[package]] name = "goblin" -version = "0.7.1" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f27c1b4369c2cd341b5de549380158b105a04c331be5db9110eef7b6d2742134" +checksum = "91766b1121940d622933a13e20665857648681816089c9bc2075c4b75a6e4f6b" dependencies = [ "log", "plain", @@ -575,27 +415,15 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - -[[package]] -name = "hashbrown" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" dependencies = [ "ahash", ] -[[package]] -name = "hashbrown" -version = "0.14.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" - [[package]] name = "heck" -version = "0.4.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" [[package]] name = "hermit-abi" @@ -606,91 +434,52 @@ dependencies = [ "libc", ] -[[package]] -name = "hermit-abi" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" - [[package]] name = "humantime" version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" -[[package]] -name = "iana-time-zone" -version = "0.1.57" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "wasm-bindgen", - "windows", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" -dependencies = [ - "cc", -] - [[package]] name = "indexmap" -version = "1.9.3" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e" dependencies = [ "autocfg", - "hashbrown 0.12.3", -] - -[[package]] -name = "indexmap" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" -dependencies = [ - "equivalent", - "hashbrown 0.14.2", + "hashbrown", ] [[package]] name = "indicatif" -version = "0.17.7" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb28741c9db9a713d93deb3bb9515c20788cef5815265bee4980e87bde7e0f25" +checksum = "2d207dc617c7a380ab07ff572a6e52fa202a2a8f355860ac9c38e23f8196be1b" dependencies = [ "console", - "instant", + "lazy_static", "number_prefix", - "portable-atomic", - "unicode-width", + "regex", ] [[package]] name = "inferno" -version = "0.11.17" +version = "0.11.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c50453ec3a6555fad17b1cd1a80d16af5bc7cb35094f64e429fd46549018c6a3" +checksum = "9709543bd6c25fdc748da2bed0f6855b07b7e93a203ae31332ac2101ab2f4782" dependencies = [ "ahash", - "clap 4.3.2", + "atty", + "clap 3.2.15", "crossbeam-channel", "crossbeam-utils", "dashmap", "env_logger", - "indexmap 2.0.2", - "is-terminal", - "itoa", + "indexmap", + "itoa 1.0.2", "log", "num-format", + "num_cpus", "once_cell", "quick-xml", "rgb", @@ -706,43 +495,17 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "io-lifetimes" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" -dependencies = [ - "hermit-abi 0.3.1", - "libc", - "windows-sys 0.48.0", -] - -[[package]] -name = "is-terminal" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" -dependencies = [ - "hermit-abi 0.3.1", - "io-lifetimes", - "rustix", - "windows-sys 0.48.0", -] - [[package]] name = "itoa" -version = "1.0.6" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" +checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" [[package]] -name = "js-sys" -version = "0.3.63" +name = "itoa" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f37a4a5928311ac501dee68b3c7613a1037d0edb30c8e5427bd832d55d1b790" -dependencies = [ - "wasm-bindgen", -] +checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" [[package]] name = "lazy_static" @@ -758,15 +521,15 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.146" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f92be4933c13fd498862a9e02a3055f8a8d9c039ce33db97306fd5a6caa7f29b" +checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836" [[package]] name = "libloading" -version = "0.7.4" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +checksum = "efbc0f03f9a775e9f6aed295c6a1ba2253c5757a9e03d55c6caa46a681abcddd" dependencies = [ "cfg-if", "winapi", @@ -774,43 +537,36 @@ dependencies = [ [[package]] name = "libm" -version = "0.2.7" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" +checksum = "33a33a362ce288760ec6a508b94caaec573ae7d3bbbd91b87aa0bad4456839db" [[package]] name = "libproc" -version = "0.13.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b18cbf29f8ff3542ba22bdce9ac610fcb75d74bb4e2b306b2a2762242025b4f" +checksum = "6466fc1f834276563fbbd4be1c24236ef92bb9efdbd4691e07f1cf85a0b407f0" dependencies = [ - "bindgen 0.64.0", - "errno 0.2.8", + "errno", "libc", ] [[package]] name = "libproc" -version = "0.14.2" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "229004ebba9d1d5caf41623f1523b6d52abb47d9f6ab87f7e6fc992e3b854aef" +checksum = "0b799ad155d75ce914c467ee5627b62247c20d4aedbd446f821484cebf3cded7" dependencies = [ - "bindgen 0.68.1", - "errno 0.3.1", + "bindgen", + "errno", "libc", ] -[[package]] -name = "linux-raw-sys" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" - [[package]] name = "lock_api" -version = "0.4.10" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53" dependencies = [ "autocfg", "scopeguard", @@ -818,17 +574,20 @@ dependencies = [ [[package]] name = "log" -version = "0.4.18" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if", +] [[package]] name = "lru" -version = "0.10.0" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03f1160296536f10c833a82dca22267d5486734230d47bf00bf435885814ba1e" +checksum = "e999beba7b6e8345721bd280141ed958096a2e4abdf74f67ff4ce49b4b54e47a" dependencies = [ - "hashbrown 0.13.2", + "hashbrown", ] [[package]] @@ -871,15 +630,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "memmap2" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" -dependencies = [ - "libc", -] - [[package]] name = "minimal-lexical" version = "0.2.1" @@ -888,30 +638,35 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.1" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +checksum = "6f5c75688da582b8ffc1f1799e9db273f32133c49e048f614d22ec3256773ccc" dependencies = [ "adler", ] [[package]] name = "nix" -version = "0.26.2" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" +checksum = "195cdbc1741b8134346d515b3a56a1c94b0912758009cfd53f99ea0f57b065fc" dependencies = [ - "bitflags 1.3.2", + "bitflags", "cfg-if", "libc", - "static_assertions", ] +[[package]] +name = "nodrop" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" + [[package]] name = "nom" -version = "7.1.3" +version = "7.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" dependencies = [ "memchr", "minimal-lexical", @@ -919,12 +674,22 @@ dependencies = [ [[package]] name = "num-format" -version = "0.4.4" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" +checksum = "bafe4179722c2894288ee77a9f044f02811c86af699344c498b0840c698a2465" dependencies = [ "arrayvec", - "itoa", + "itoa 0.4.8", +] + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", ] [[package]] @@ -937,6 +702,16 @@ dependencies = [ "libm", ] +[[package]] +name = "num_cpus" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "number_prefix" version = "0.4.0" @@ -945,38 +720,37 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "object" -version = "0.31.1" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" +checksum = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53" dependencies = [ "flate2", "memchr", - "ruzstd", ] [[package]] name = "once_cell" -version = "1.18.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1" [[package]] name = "os_str_bytes" -version = "6.5.0" +version = "6.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ceedf44fb00f2d1984b0bc98102627ce622e083e49a5bacdb3e514fa4238e267" +checksum = "648001efe5d5c0102d8cea768e348da85d90af8ba91f0bea908f157951493cd4" [[package]] name = "parking_lot_core" -version = "0.9.8" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-targets 0.48.0", + "windows-sys", ] [[package]] @@ -991,27 +765,11 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" -[[package]] -name = "portable-atomic" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b559898e0b4931ed2d3b959ab0c2da4d99cc644c4b0b1a35b4d344027f474023" - [[package]] name = "ppv-lite86" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" - -[[package]] -name = "prettyplease" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" -dependencies = [ - "proc-macro2", - "syn 2.0.38", -] +checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" [[package]] name = "proc-macro-error" @@ -1022,7 +780,7 @@ dependencies = [ "proc-macro-error-attr", "proc-macro2", "quote", - "syn 1.0.109", + "syn", "version_check", ] @@ -1039,23 +797,23 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.69" +version = "1.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" +checksum = "c278e965f1d8cf32d6e0e96de3d3e79712178ae67986d9cf9151f51e95aac89b" dependencies = [ "unicode-ident", ] [[package]] name = "proc-maps" -version = "0.3.2" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ec8fdc22cb95c02f6a26a91fb1cd60a7a115916c2ed3b09d0a312e11785bd57" +checksum = "a2f62c16ccc63ce2f590b17b8b9b33616f59631b8982ad52ed21e7f6d936c409" dependencies = [ "anyhow", - "bindgen 0.68.1", + "bindgen", "libc", - "libproc 0.14.2", + "libproc 0.10.0", "mach2", "winapi", ] @@ -1066,13 +824,13 @@ version = "0.3.14" dependencies = [ "anyhow", "chrono", - "clap 3.2.25", + "clap 3.2.15", "clap_complete", "console", "cpp_demangle", "ctrlc", "env_logger", - "goblin 0.7.1", + "goblin", "indicatif", "inferno", "lazy_static", @@ -1081,7 +839,6 @@ dependencies = [ "lru", "memmap", "proc-maps", - "py-spy-testdata", "rand", "rand_distr", "regex", @@ -1094,26 +851,20 @@ dependencies = [ "winapi", ] -[[package]] -name = "py-spy-testdata" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "188970d7e0a2e1c4e17d48c821de4d3ad37df749306b249020adc92c1c40a8a1" - [[package]] name = "quick-xml" -version = "0.26.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd" +checksum = "9279fbdacaad3baf559d8cabe0acc3d06e30ea14931af31af79578ac0946decc" dependencies = [ "memchr", ] [[package]] name = "quote" -version = "1.0.28" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +checksum = "3bcdf212e9776fbcb2d23ab029360416bb1706b1aea2d1a5ba002727cbcab804" dependencies = [ "proc-macro2", ] @@ -1141,9 +892,9 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.6.4" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" dependencies = [ "getrandom", ] @@ -1160,9 +911,9 @@ dependencies = [ [[package]] name = "read-process-memory" -version = "0.1.6" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8497683b2f0b6887786f1928c118f26ecc6bb3d78bbb6ed23e8e7ba110af3bb0" +checksum = "a103589f0c267c68328595f1ff45647e9d4b3ee6b9c1c51dcdc9b4b55ca504c5" dependencies = [ "libc", "log", @@ -1172,18 +923,18 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.3.5" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ - "bitflags 1.3.2", + "bitflags", ] [[package]] name = "regex" -version = "1.8.4" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f" +checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b" dependencies = [ "aho-corasick", "memchr", @@ -1192,21 +943,21 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.7.2" +version = "0.6.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" +checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" [[package]] name = "remoteprocess" -version = "0.4.12" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91114d769bd6dffc9565c01bbba121ca223efba7fdbc4c57b63fd91c1ea8478e" +checksum = "ab91eb6b1c4c507a08d6fb6e2f38982a1e09923d19023506cba25aefcfc6e46f" dependencies = [ "addr2line", - "goblin 0.6.1", + "goblin", "lazy_static", "libc", - "libproc 0.13.0", + "libproc 0.12.0", "log", "mach", "mach_o_sys", @@ -1219,20 +970,29 @@ dependencies = [ "winapi", ] +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + [[package]] name = "rgb" -version = "0.8.36" +version = "0.8.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20ec2d3e3fc7a92ced357df9cebd5a10b6fb2aa1ee797bf7e9ce2f17dffc8f59" +checksum = "c3b221de559e4a29df3b957eec92bc0de6bc8eaf6ca9cfed43e5e1d67ff65a34" dependencies = [ "bytemuck", ] [[package]] name = "rustc-demangle" -version = "0.1.23" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" [[package]] name = "rustc-hash" @@ -1240,36 +1000,11 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" -[[package]] -name = "rustix" -version = "0.37.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4eb579851244c2c03e7c24f501c3432bed80b8f720af1d6e5b0e0f01555a035" -dependencies = [ - "bitflags 1.3.2", - "errno 0.3.1", - "io-lifetimes", - "libc", - "linux-raw-sys", - "windows-sys 0.48.0", -] - -[[package]] -name = "ruzstd" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a15e661f0f9dac21f3494fe5d23a6338c0ac116a2d22c2b63010acd89467ffe" -dependencies = [ - "byteorder", - "thiserror", - "twox-hash", -] - [[package]] name = "ryu" -version = "1.0.13" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695" [[package]] name = "scopeguard" @@ -1294,33 +1029,33 @@ checksum = "bdbda6ac5cd1321e724fa9cee216f3a61885889b896f073b8f82322789c5250e" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn", ] [[package]] name = "serde" -version = "1.0.163" +version = "1.0.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2" +checksum = "fc855a42c7967b7c369eb5860f7164ef1f6f81c20c7cc1141f2a604e18723b03" [[package]] name = "serde_derive" -version = "1.0.163" +version = "1.0.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" +checksum = "6f2122636b9fe3b81f1cb25099fcf2d3f542cdb1d45940d56c713158884a05da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn", ] [[package]] name = "serde_json" -version = "1.0.96" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" +checksum = "82c2c1fdcd807d1098552c5b9a36e425e42e9fbd7c6a37a8425f390f781f7fa7" dependencies = [ - "itoa", + "itoa 1.0.2", "ryu", "serde", ] @@ -1333,9 +1068,9 @@ checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" [[package]] name = "smallvec" -version = "1.10.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1" [[package]] name = "stable_deref_trait" @@ -1343,12 +1078,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - [[package]] name = "str_stack" version = "0.1.0" @@ -1357,26 +1086,21 @@ checksum = "9091b6114800a5f2141aee1d1b9d6ca3592ac062dc5decb3764ec5895a47b4eb" [[package]] name = "strsim" -version = "0.10.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" [[package]] -name = "syn" -version = "1.0.109" +name = "strsim" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "2.0.38" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" +checksum = "c50aef8a904de4c23c788f104b7dddc7d6f79c647c7c8ce4cc8f73eb0ca773dd" dependencies = [ "proc-macro2", "quote", @@ -1385,35 +1109,35 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.6.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" +checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" dependencies = [ - "autocfg", "cfg-if", "fastrand", + "libc", "redox_syscall", - "rustix", - "windows-sys 0.48.0", + "remove_dir_all", + "winapi", ] [[package]] name = "termcolor" -version = "1.2.0" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" dependencies = [ "winapi-util", ] [[package]] name = "terminal_size" -version = "0.2.6" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e6bf6f19e9f8ed8d4048dc22981458ebcf406d67e94cd422e5ecd73d63b3237" +checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df" dependencies = [ - "rustix", - "windows-sys 0.48.0", + "libc", + "winapi", ] [[package]] @@ -1427,71 +1151,50 @@ dependencies = [ [[package]] name = "textwrap" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" -dependencies = [ - "terminal_size", -] - -[[package]] -name = "thiserror" -version = "1.0.40" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" dependencies = [ - "thiserror-impl", + "unicode-width", ] [[package]] -name = "thiserror-impl" -version = "1.0.40" +name = "textwrap" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.38", + "terminal_size", ] [[package]] name = "time" -version = "0.1.45" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" dependencies = [ "libc", "wasi 0.10.0+wasi-snapshot-preview1", "winapi", ] -[[package]] -name = "twox-hash" -version = "1.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" -dependencies = [ - "cfg-if", - "static_assertions", -] - [[package]] name = "unicode-ident" -version = "1.0.9" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" +checksum = "15c61ba63f9235225a22310255a29b806b907c9b8c964bcbd0a2c70f3f2deea7" [[package]] name = "unicode-width" -version = "0.1.10" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" [[package]] -name = "utf8parse" -version = "0.2.1" +name = "vec_map" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" [[package]] name = "version_check" @@ -1511,69 +1214,15 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" -[[package]] -name = "wasm-bindgen" -version = "0.2.86" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bba0e8cb82ba49ff4e229459ff22a191bbe9a1cb3a341610c9c33efc27ddf73" -dependencies = [ - "cfg-if", - "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.86" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b04bc93f9d6bdee709f6bd2118f57dd6679cf1176a1af464fca3ab0d66d8fb" -dependencies = [ - "bumpalo", - "log", - "once_cell", - "proc-macro2", - "quote", - "syn 2.0.38", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.86" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14d6b024f1a526bb0234f52840389927257beb670610081360e5a03c5df9c258" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.86" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e128beba882dd1eb6200e1dc92ae6c5dbaa4311aa7bb211ca035779e5efc39f8" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.38", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.86" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed9d5b4305409d1fc9482fee2d7f9bcbf24b3972bf59817ef757e23982242a93" - [[package]] name = "which" -version = "4.4.0" +version = "4.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" +checksum = "5c4fb54e6113b6a8772ee41c3404fb0301ac79604489467e0a9ce1f3e97c24ae" dependencies = [ "either", + "lazy_static", "libc", - "once_cell", ] [[package]] @@ -1607,143 +1256,45 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" -dependencies = [ - "windows-targets 0.48.0", -] - -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", -] - [[package]] name = "windows-sys" -version = "0.48.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" dependencies = [ - "windows-targets 0.48.0", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_msvc", ] -[[package]] -name = "windows-targets" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - -[[package]] -name = "windows-targets" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" -dependencies = [ - "windows_aarch64_gnullvm 0.48.0", - "windows_aarch64_msvc 0.48.0", - "windows_i686_gnu 0.48.0", - "windows_i686_msvc 0.48.0", - "windows_x86_64_gnu 0.48.0", - "windows_x86_64_gnullvm 0.48.0", - "windows_x86_64_msvc 0.48.0", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" - [[package]] name = "windows_aarch64_msvc" -version = "0.42.2" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" - -[[package]] -name = "windows_i686_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" +checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" [[package]] name = "windows_i686_gnu" -version = "0.48.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" [[package]] name = "windows_i686_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" - -[[package]] -name = "windows_i686_msvc" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" +checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" [[package]] name = "windows_x86_64_gnu" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" [[package]] name = "windows_x86_64_msvc" -version = "0.48.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" diff --git a/Cargo.toml b/Cargo.toml index cdfbd569..c6a1aad7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,29 +17,26 @@ clap = {version="3.2", features=["wrap_help", "cargo", "derive"]} clap_complete="3.2" console = "0.15" ctrlc = "3" -indicatif = "0.17" -env_logger = "0.10" -goblin = "0.7.1" -inferno = "0.11.17" +indicatif = "0.16" +env_logger = "0.9" +goblin = "0.5.3" +inferno = "0.11.7" lazy_static = "1.4.0" libc = "0.2" log = "0.4" -lru = "0.10" +lru = "0.7" regex = ">=1.6.0" -tempfile = "3.6.0" -proc-maps = "0.3.2" +tempfile = "3.0.3" +proc-maps = "0.2.1" memmap = "0.7.0" -cpp_demangle = "0.4" +cpp_demangle = "0.3" serde = {version="1.0", features=["rc"]} serde_derive = "1.0" serde_json = "1.0" rand = "0.8" rand_distr = "0.4" -remoteprocess = {version="0.4.12", features=["unwind"]} -chrono = "0.4.26" - -[dev-dependencies] -py-spy-testdata = "0.1.0" +remoteprocess = {version="0.4.10", features=["unwind"]} +chrono = "0.4.19" [target.'cfg(unix)'.dependencies] termios = "0.3.3" diff --git a/README.md b/README.md index e8b80ca9..713eea98 100755 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ py-spy is extremely low overhead: it is written in Rust for speed and doesn't ru in the same process as the profiled Python program. This means py-spy is safe to use against production Python code. py-spy works on Linux, OSX, Windows and FreeBSD, and supports profiling all recent versions of the CPython -interpreter (versions 2.3-2.7 and 3.3-3.11). +interpreter (versions 2.3-2.7 and 3.3-3.10). ## Installation diff --git a/build.rs b/build.rs index cec961ca..3d8e7bdd 100644 --- a/build.rs +++ b/build.rs @@ -8,6 +8,6 @@ fn main() { match env::var("CARGO_CFG_TARGET_OS").unwrap().as_ref() { "windows" => println!("cargo:rustc-cfg=unwind"), "linux" => println!("cargo:rustc-cfg=unwind"), - _ => {} + _ => { } } } diff --git a/ci/Vagrantfile b/ci/Vagrantfile index c22b36ac..cdb30d3c 100644 --- a/ci/Vagrantfile +++ b/ci/Vagrantfile @@ -1,27 +1,20 @@ Vagrant.configure("2") do |config| - config.vm.define "freebsd-14" do |c| - c.vm.box = "roboxes/freebsd14" + config.vm.define "fbsd_12_2" do |fbsd_12_2| + fbsd_12_2.vm.box = "rbspy/FreeBSD-12.2-STABLE" end - - config.vm.boot_timeout = 600 - - config.vm.provider "libvirt" do |qe| - # https://vagrant-libvirt.github.io/vagrant-libvirt/configuration.html - qe.driver = "kvm" - qe.cpus = 3 - qe.memory = 8192 + config.vm.define "fbsd_13_1" do |c| + c.vm.box = "rbspy/FreeBSD-13.1-STABLE" end config.vm.synced_folder ".", "/vagrant", type: "rsync", rsync__exclude: [".git", ".vagrant.d"] config.vm.provision "shell", inline: <<~SHELL - set -e - pkg install -y curl bash python llvm - chsh -s /usr/local/bin/bash vagrant - pw groupmod wheel -m vagrant - su -l vagrant <<'EOF' - curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal --default-toolchain 1.73.0 - EOF + pkg install -y python devel/llvm90 SHELL + + config.vm.provider "virtualbox" do |v| + v.memory = 4096 + v.cpus = 2 + end end diff --git a/ci/test_freebsd.sh b/ci/test_freebsd.sh index 5eff807c..556cf5ea 100755 --- a/ci/test_freebsd.sh +++ b/ci/test_freebsd.sh @@ -1,16 +1,18 @@ #!/usr/bin/env bash -source "$HOME/.cargo/env" +source ~/.bash_profile set -e python --version cargo --version +export CARGO_HOME="/vagrant/.cargo" +mkdir -p $CARGO_HOME + cd /vagrant if [ -f build-artifacts.tar ]; then - echo "Unpacking cached build artifacts..." tar xf build-artifacts.tar rm -f build-artifacts.tar fi @@ -18,9 +20,4 @@ fi cargo build --release --workspace --all-targets cargo test --release -set +e tar cf build-artifacts.tar target -tar rf build-artifacts.tar "$HOME/.cargo/git" -tar rf build-artifacts.tar "$HOME/.cargo/registry" - -exit 0 diff --git a/ci/testdata/cython_test.pyx b/ci/testdata/cython_test.pyx index 8f93b45e..0d226182 100644 --- a/ci/testdata/cython_test.pyx +++ b/ci/testdata/cython_test.pyx @@ -5,7 +5,7 @@ from cython cimport floating cpdef sqrt(floating value): # solve for the square root of value by finding the zeros of - # 'x * x - value = 0' using newtons method + # 'x * x - value = 0' using newtons meethod cdef double x = value / 2 for _ in range(8): x -= (x * x - value) / (2 * x) diff --git a/ci/update_python_test_versions.py b/ci/update_python_test_versions.py index 958daa41..94d8dd51 100644 --- a/ci/update_python_test_versions.py +++ b/ci/update_python_test_versions.py @@ -15,7 +15,7 @@ def get_github_python_versions(): raw_versions = [v["version"] for v in versions_json] versions = [] for version_str in raw_versions: - if "-" in version_str: + if "-" in version_str and version_str != "3.11.0-beta.5": continue major, minor, patch = parse_version(version_str) @@ -38,7 +38,6 @@ def get_github_python_versions(): pathlib.Path(__file__).parent.parent / ".github" / "workflows" / "build.yml" ) - transformed = [] for line in open(build_yml): if line.startswith(" python-version: ["): @@ -50,17 +49,5 @@ def get_github_python_versions(): line = newversions transformed.append(line) - # also automatically exclude v3.11.* from running on OSX, - # since it currently fails in GHA on SIP errors - exclusions = [] - for v in versions: - if v.startswith("3.11"): - exclusions.append(" - os: macos-latest\n") - exclusions.append(f" python-version: {v}\n") - test_wheels = transformed.index(" test-wheels:\n") - first_line = transformed.index(" exclude:\n", test_wheels) - last_line = transformed.index("\n", first_line) - transformed = transformed[:first_line+1] + exclusions + transformed[last_line:] - with open(build_yml, "w") as o: o.write("".join(transformed)) diff --git a/generate_bindings.py b/generate_bindings.py index 6323a440..48617318 100644 --- a/generate_bindings.py +++ b/generate_bindings.py @@ -153,7 +153,6 @@ def extract_bindings(cpython_path, version, configure=False): o.write("#![allow(clippy::default_trait_access)]\n") o.write("#![allow(clippy::cast_lossless)]\n") o.write("#![allow(clippy::trivially_copy_pass_by_ref)]\n\n") - o.write("#![allow(clippy::upper_case_acronyms)]\n\n") o.write(open(os.path.join(cpython_path, "bindgen_output.rs")).read()) diff --git a/pyproject.toml b/pyproject.toml index 10aba3db..9bf6ba44 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["maturin>=1.0,<2.0"] +requires = ["maturin>=0.11,<0.12"] build-backend = "maturin" [project] @@ -16,7 +16,3 @@ classifiers = [ [tool.maturin] bindings = "bin" - -[tool.codespell] -ignore-words-list = "crate" -skip = "./.git,./.github,./target,./ci/testdata,./images/" diff --git a/src/binary_parser.rs b/src/binary_parser.rs index 057fed67..c3194e2c 100644 --- a/src/binary_parser.rs +++ b/src/binary_parser.rs @@ -1,8 +1,10 @@ + use std::collections::HashMap; use std::fs::File; use std::path::Path; use anyhow::Error; +use goblin; use goblin::Object; use memmap::Mmap; @@ -13,7 +15,7 @@ pub struct BinaryInfo { pub bss_size: u64, pub offset: u64, pub addr: u64, - pub size: u64, + pub size: u64 } impl BinaryInfo { @@ -24,7 +26,17 @@ impl BinaryInfo { } /// Uses goblin to parse a binary file, returns information on symbols/bss/adjusted offset etc -pub fn parse_binary(filename: &Path, addr: u64, size: u64) -> Result { +pub fn parse_binary(_pid: remoteprocess::Pid, filename: &Path, addr: u64, size: u64, _is_bin: bool) -> Result { + // on linux the process could be running in docker, access the filename through procfs + // if filename is the binary executable (not libpython) - take it from /proc/pid/exe, which works + // across namespaces just like /proc/pid/root, and also if the file was deleted. + #[cfg(target_os="linux")] + let filename = &std::path::PathBuf::from(&if _is_bin { + format!("/proc/{}/exe", _pid) + } else { + format!("/proc/{}/root{}", _pid, filename.display()) + }); + let offset = addr; let mut symbols = HashMap::new(); @@ -40,18 +52,12 @@ pub fn parse_binary(filename: &Path, addr: u64, size: u64) -> Result mach, goblin::mach::Mach::Fat(fat) => { - let arch = fat - .iter_arches() - .find(|arch| match arch { + let arch = fat.iter_arches().find(|arch| + match arch { Ok(arch) => arch.is_64(), - Err(_) => false, - }) - .ok_or_else(|| { - format_err!( - "Failed to find 64 bit arch in FAT archive in {}", - filename.display() - ) - })??; + Err(_) => false + } + ).ok_or_else(|| format_err!("Failed to find 64 bit arch in FAT archive in {}", filename.display()))??; let bytes = &buffer[arch.offset as usize..][..arch.size as usize]; goblin::mach::MachO::parse(bytes, 0)? } @@ -73,60 +79,31 @@ pub fn parse_binary(filename: &Path, addr: u64, size: u64) -> Result { - let strtab = elf.shdr_strtab; - let bss_header = elf - .section_headers + let bss_header = elf.section_headers .iter() - // filter down to things that are both NOBITS sections and are named .bss - .filter(|header| header.sh_type == goblin::elf::section_header::SHT_NOBITS) - .filter(|header| { - strtab - .get_at(header.sh_name) - .map_or(true, |name| name == ".bss") - }) - // if we have multiple sections here, take the largest - .max_by_key(|header| header.sh_size) - .ok_or_else(|| { - format_err!( - "Failed to find BSS section header in {}", - filename.display() - ) - })?; - - let program_header = elf - .program_headers + .find(|ref header| header.sh_type == goblin::elf::section_header::SHT_NOBITS) + .ok_or_else(|| format_err!("Failed to find BSS section header in {}", filename.display()))?; + + let program_header = elf.program_headers .iter() - .find(|header| { - header.p_type == goblin::elf::program_header::PT_LOAD - && header.p_flags & goblin::elf::program_header::PF_X != 0 - }) - .ok_or_else(|| { - format_err!( - "Failed to find executable PT_LOAD program header in {}", - filename.display() - ) - })?; + .find(|ref header| + header.p_type == goblin::elf::program_header::PT_LOAD && + header.p_flags & goblin::elf::program_header::PF_X != 0) + .ok_or_else(|| format_err!("Failed to find executable PT_LOAD program header in {}", filename.display()))?; // p_vaddr may be larger than the map address in case when the header has an offset and // the map address is relatively small. In this case we can default to 0. - let offset = offset.saturating_sub(program_header.p_vaddr); + let offset = offset.checked_sub(program_header.p_vaddr).unwrap_or(0); for sym in elf.syms.iter() { let name = elf.strtab[sym.st_name].to_string(); @@ -136,49 +113,36 @@ pub fn parse_binary(filename: &Path, addr: u64, size: u64) -> Result { for export in pe.exports { if let Some(name) = export.name { if let Some(export_offset) = export.offset { - symbols.insert(name.to_string(), export_offset as u64 + offset); + symbols.insert(name.to_string(), export_offset as u64 + offset as u64); } } } pe.sections .iter() - .find(|section| section.name.starts_with(b".data")) - .ok_or_else(|| { - format_err!( - "Failed to find .data section in PE binary of {}", - filename.display() - ) - }) + .find(|ref section| section.name.starts_with(b".data")) + .ok_or_else(|| format_err!("Failed to find .data section in PE binary of {}", filename.display())) .map(|data_section| { let bss_addr = u64::from(data_section.virtual_address) + offset; let bss_size = u64::from(data_section.virtual_size); - BinaryInfo { - filename: filename.to_owned(), - symbols, - bss_addr, - bss_size, - offset, - addr, - size, - } + BinaryInfo{filename: filename.to_owned(), symbols, bss_addr, bss_size, offset, addr, size} }) + }, + _ => { + Err(format_err!("Unhandled binary type")) } - _ => Err(format_err!("Unhandled binary type")), } } diff --git a/src/chrometrace.rs b/src/chrometrace.rs deleted file mode 100644 index 24463800..00000000 --- a/src/chrometrace.rs +++ /dev/null @@ -1,122 +0,0 @@ -use std::cmp::min; -use std::collections::HashMap; -use std::io::Write; -use std::time::Instant; - -use anyhow::Error; -use serde_derive::Serialize; - -use crate::stack_trace::Frame; -use crate::stack_trace::StackTrace; - -#[derive(Clone, Debug, Serialize)] -struct Args { - pub filename: String, - pub line: Option, -} - -#[derive(Clone, Debug, Serialize)] -struct Event { - pub args: Args, - pub cat: String, - pub name: String, - pub ph: String, - pub pid: u64, - pub tid: u64, - pub ts: u64, -} - -pub struct Chrometrace { - events: Vec, - start_ts: Instant, - prev_traces: HashMap, - show_linenumbers: bool, -} - -impl Chrometrace { - pub fn new(show_linenumbers: bool) -> Chrometrace { - Chrometrace { - events: Vec::new(), - start_ts: Instant::now(), - prev_traces: HashMap::new(), - show_linenumbers, - } - } - - // Return whether these frames are similar enough such that we should merge - // them, instead of creating separate events for them. - fn should_merge_frames(&self, a: &Frame, b: &Frame) -> bool { - a.name == b.name && a.filename == b.filename && (!self.show_linenumbers || a.line == b.line) - } - - fn event(&self, trace: &StackTrace, frame: &Frame, phase: &str, ts: u64) -> Event { - Event { - tid: trace.thread_id, - pid: trace.pid as u64, - name: frame.name.to_string(), - cat: "py-spy".to_owned(), - ph: phase.to_owned(), - ts, - args: Args { - filename: frame.filename.to_string(), - line: if self.show_linenumbers { - Some(frame.line as u32) - } else { - None - }, - }, - } - } - - pub fn increment(&mut self, trace: &StackTrace) -> std::io::Result<()> { - let now = self.start_ts.elapsed().as_micros() as u64; - - // Load the previous frames for this thread. - let prev_frames = self - .prev_traces - .remove(&trace.thread_id) - .map(|t| t.frames) - .unwrap_or_default(); - - // Find the index where we first see new frames. - let new_idx = prev_frames - .iter() - .rev() - .zip(trace.frames.iter().rev()) - .position(|(a, b)| !self.should_merge_frames(a, b)) - .unwrap_or(min(prev_frames.len(), trace.frames.len())); - - // Publish end events for the previous frames that got dropped in the - // most recent trace. - for frame in prev_frames.iter().rev().skip(new_idx).rev() { - self.events.push(self.event(trace, frame, "E", now)); - } - - // Publish start events for frames that got added in the most recent - // trace. - for frame in trace.frames.iter().rev().skip(new_idx) { - self.events.push(self.event(trace, frame, "B", now)); - } - - // Save this stack trace for the next iteration. - self.prev_traces.insert(trace.thread_id, trace.clone()); - - Ok(()) - } - - pub fn write(&self, w: &mut dyn Write) -> Result<(), Error> { - let mut events = Vec::new(); - events.extend(self.events.to_vec()); - - // Add end events for any unfinished slices. - let now = self.start_ts.elapsed().as_micros() as u64; - for trace in self.prev_traces.values() { - for frame in &trace.frames { - events.push(self.event(trace, frame, "E", now)); - } - } - - writeln!(w, "{}", serde_json::to_string(&events)?)?; - Ok(()) - } -} diff --git a/src/config.rs b/src/config.rs index d8c936f9..bffb871c 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,11 +1,8 @@ -use clap::{ - crate_description, crate_name, crate_version, value_parser, Arg, ArgEnum, Command, - PossibleValue, -}; +use clap::{ArgEnum, Arg, Command, crate_description, crate_name, crate_version, PossibleValue, value_parser}; use remoteprocess::Pid; /// Options on how to collect samples from a python process -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, Eq, PartialEq)] pub struct Config { /// Whether or not we should stop the python process when taking samples. /// Setting this to false will reduce the performance impact on the target @@ -42,6 +39,8 @@ pub struct Config { #[doc(hidden)] pub subprocesses: bool, #[doc(hidden)] + pub whitelist: Option>, + #[doc(hidden)] pub gil_only: bool, #[doc(hidden)] pub hide_progress: bool, @@ -55,10 +54,6 @@ pub struct Config { pub full_filenames: bool, #[doc(hidden)] pub lineno: LineNo, - #[doc(hidden)] - pub refresh_seconds: f64, - #[doc(hidden)] - pub core_filename: Option, } #[allow(non_camel_case_types)] @@ -66,8 +61,7 @@ pub struct Config { pub enum FileFormat { flamegraph, raw, - speedscope, - chrometrace, + speedscope } impl FileFormat { @@ -91,55 +85,40 @@ impl std::str::FromStr for FileFormat { } } + + #[derive(Debug, Clone, Eq, PartialEq)] pub enum LockingStrategy { NonBlocking, #[allow(dead_code)] AlreadyLocked, - Lock, + Lock } #[derive(Debug, Clone, Eq, PartialEq)] pub enum RecordDuration { Unlimited, - Seconds(u64), + Seconds(u64) } #[derive(Debug, Clone, Eq, PartialEq, Copy)] pub enum LineNo { NoLine, - First, - LastInstruction, + FirstLineNo, + LastInstruction } impl Default for Config { /// Initializes a new Config object with default parameters #[allow(dead_code)] fn default() -> Config { - Config { - pid: None, - python_program: None, - filename: None, - format: None, - command: String::from("top"), - blocking: LockingStrategy::Lock, - show_line_numbers: false, - sampling_rate: 100, - duration: RecordDuration::Unlimited, - native: false, - gil_only: false, - include_idle: false, - include_thread_ids: false, - hide_progress: false, - capture_output: true, - dump_json: false, - dump_locals: 0, - subprocesses: false, - full_filenames: false, - lineno: LineNo::LastInstruction, - refresh_seconds: 1.0, - core_filename: None, - } + Config{pid: None, python_program: None, filename: None, format: None, + command: String::from("top"), + blocking: LockingStrategy::Lock, show_line_numbers: false, sampling_rate: 100, + duration: RecordDuration::Unlimited, native: false, + gil_only: false, include_idle: false, include_thread_ids: false, + hide_progress: false, capture_output: true, dump_json: false, dump_locals: 0, subprocesses: false, + whitelist: None, full_filenames: false, lineno: LineNo::LastInstruction } } } @@ -147,24 +126,24 @@ impl Config { /// Uses clap to set config options from commandline arguments pub fn from_commandline() -> Config { let args: Vec = std::env::args().collect(); - Config::from_args(&args).unwrap_or_else(|e| e.exit()) + Config::from_args(&args).unwrap_or_else( |e| e.exit() ) } pub fn from_args(args: &[String]) -> clap::Result { // pid/native/nonblocking/rate/python_program/subprocesses/full_filenames arguments can be // used across various subcommand - define once here let pid = Arg::new("pid") - .short('p') - .long("pid") - .value_name("pid") - .help("PID of a running python program to spy on") - .takes_value(true); + .short('p') + .long("pid") + .value_name("pid") + .help("PID of a running python program to spy on") + .takes_value(true); #[cfg(unwind)] let native = Arg::new("native") - .short('n') - .long("native") - .help("Collect stack traces from native extensions written in Cython, C or C++"); + .short('n') + .long("native") + .help("Collect stack traces from native extensions written in Cython, C or C++"); #[cfg(not(target_os="freebsd"))] let nonblocking = Arg::new("nonblocking") @@ -173,107 +152,94 @@ impl Config { the performance impact of sampling, but may lead to inaccurate results"); let rate = Arg::new("rate") - .short('r') - .long("rate") - .value_name("rate") - .help("The number of samples to collect per second") - .default_value("100") - .takes_value(true); + .short('r') + .long("rate") + .value_name("rate") + .help("The number of samples to collect per second") + .default_value("100") + .takes_value(true); let subprocesses = Arg::new("subprocesses") - .short('s') - .long("subprocesses") - .help("Profile subprocesses of the original process"); - - let full_filenames = Arg::new("full_filenames").long("full-filenames").help( - "Show full Python filenames, instead of shortening to show only the package part", - ); + .short('s') + .long("subprocesses") + .help("Profile subprocesses of the original process"); + + let whitelist = Arg::new("whitelist") + .short('w') + .long("whitelist") + .help("A comma separated list of subprocess names to spy on") + .takes_value(true); + + + let full_filenames = Arg::new("full_filenames") + .long("full-filenames") + .help("Show full Python filenames, instead of shortening to show only the package part"); let program = Arg::new("python_program") - .help("commandline of a python program to run") - .multiple_values(true); + .help("commandline of a python program to run") + .multiple_values(true); let idle = Arg::new("idle") - .short('i') - .long("idle") - .help("Include stack traces for idle threads"); + .short('i') + .long("idle") + .help("Include stack traces for idle threads"); let gil = Arg::new("gil") - .short('g') - .long("gil") - .help("Only include traces that are holding on to the GIL"); - - let top_delay = Arg::new("delay") - .long("delay") - .value_name("seconds") - .help("Delay between 'top' refreshes.") - .default_value("1.0") - .value_parser(clap::value_parser!(f64)) - .takes_value(true); + .short('g') + .long("gil") + .help("Only include traces that are holding on to the GIL"); let record = Command::new("record") .about("Records stack trace information to a flamegraph, speedscope or raw file") .arg(program.clone()) .arg(pid.clone().required_unless_present("python_program")) .arg(full_filenames.clone()) - .arg( - Arg::new("output") - .short('o') - .long("output") - .value_name("filename") - .help("Output filename") - .takes_value(true) - .required(false), - ) - .arg( - Arg::new("format") - .short('f') - .long("format") - .value_name("format") - .help("Output file format") - .takes_value(true) - .possible_values(FileFormat::possible_values()) - .ignore_case(true) - .default_value("flamegraph"), - ) - .arg( - Arg::new("duration") - .short('d') - .long("duration") - .value_name("duration") - .help("The number of seconds to sample for") - .default_value("unlimited") - .takes_value(true), - ) + .arg(Arg::new("output") + .short('o') + .long("output") + .value_name("filename") + .help("Output filename") + .takes_value(true) + .required(false)) + .arg(Arg::new("format") + .short('f') + .long("format") + .value_name("format") + .help("Output file format") + .takes_value(true) + .possible_values(FileFormat::possible_values()) + .ignore_case(true) + .default_value("flamegraph")) + .arg(Arg::new("duration") + .short('d') + .long("duration") + .value_name("duration") + .help("The number of seconds to sample for") + .default_value("unlimited") + .takes_value(true)) .arg(rate.clone()) .arg(subprocesses.clone()) - .arg(Arg::new("function").short('F').long("function").help( - "Aggregate samples by function's first line number, instead of current line number", - )) - .arg( - Arg::new("nolineno") - .long("nolineno") - .help("Do not show line numbers"), - ) - .arg( - Arg::new("threads") - .short('t') - .long("threads") - .help("Show thread ids in the output"), - ) + .arg(whitelist.clone()) + .arg(Arg::new("function") + .short('F') + .long("function") + .help("Aggregate samples by function's first line number, instead of current line number")) + .arg(Arg::new("nolineno") + .long("nolineno") + .help("Do not show line numbers")) + .arg(Arg::new("threads") + .short('t') + .long("threads") + .help("Show thread ids in the output")) .arg(gil.clone()) .arg(idle.clone()) - .arg( - Arg::new("capture") - .long("capture") - .hide(true) - .help("Captures output from child process"), - ) - .arg( - Arg::new("hideprogress") - .long("hideprogress") - .hide(true) - .help("Hides progress bar (useful for showing error output on record)"), - ); + .arg(Arg::new("capture") + .long("capture") + .hide(true) + .help("Captures output from child process")) + .arg(Arg::new("hideprogress") + .long("hideprogress") + .hide(true) + .help("Hides progress bar (useful for showing error output on record)")); let top = Command::new("top") .about("Displays a top like view of functions consuming CPU") @@ -281,32 +247,15 @@ impl Config { .arg(pid.clone().required_unless_present("python_program")) .arg(rate.clone()) .arg(subprocesses.clone()) + .arg(whitelist.clone()) .arg(full_filenames.clone()) .arg(gil.clone()) - .arg(idle.clone()) - .arg(top_delay.clone()); - - #[cfg(target_os = "linux")] - let dump_pid = pid.clone().required_unless_present("core"); - - #[cfg(not(target_os = "linux"))] - let dump_pid = pid.clone().required(true); + .arg(idle.clone()); let dump = Command::new("dump") .about("Dumps stack traces for a target program to stdout") - .arg(dump_pid); - - #[cfg(target_os = "linux")] - let dump = dump.arg( - Arg::new("core") - .short('c') - .long("core") - .help("Filename of coredump to display python stack traces from") - .value_name("core") - .takes_value(true), - ); - - let dump = dump.arg(full_filenames.clone()) + .arg(pid.clone().required(true)) + .arg(full_filenames.clone()) .arg(Arg::new("locals") .short('l') .long("locals") @@ -316,16 +265,15 @@ impl Config { .short('j') .long("json") .help("Format output as JSON")) - .arg(subprocesses.clone()); + .arg(subprocesses.clone()) + .arg(whitelist.clone()); let completions = Command::new("completions") .about("Generate shell completions") .hide(true) - .arg( - Arg::new("shell") - .value_parser(value_parser!(clap_complete::Shell)) - .help("Shell type"), - ); + .arg(Arg::new("shell") + .value_parser(value_parser!(clap_complete::Shell)) + .help("Shell type")); // add native unwinding if appropriate #[cfg(unwind)] @@ -336,11 +284,11 @@ impl Config { let dump = dump.arg(native.clone()); // Nonblocking isn't an option for freebsd, remove - #[cfg(not(target_os = "freebsd"))] + #[cfg(not(target_os="freebsd"))] let record = record.arg(nonblocking.clone()); - #[cfg(not(target_os = "freebsd"))] + #[cfg(not(target_os="freebsd"))] let top = top.arg(nonblocking.clone()); - #[cfg(not(target_os = "freebsd"))] + #[cfg(not(target_os="freebsd"))] let dump = dump.arg(nonblocking.clone()); let mut app = Command::new(crate_name!()) @@ -366,41 +314,26 @@ impl Config { config.sampling_rate = matches.value_of_t("rate")?; config.duration = match matches.value_of("duration") { Some("unlimited") | None => RecordDuration::Unlimited, - Some(seconds) => { - RecordDuration::Seconds(seconds.parse().expect("invalid duration")) - } + Some(seconds) => RecordDuration::Seconds(seconds.parse().expect("invalid duration")) }; config.format = Some(matches.value_of_t("format")?); config.filename = matches.value_of("output").map(|f| f.to_owned()); config.show_line_numbers = matches.occurrences_of("nolineno") == 0; - config.lineno = if matches.occurrences_of("nolineno") > 0 { - LineNo::NoLine - } else if matches.occurrences_of("function") > 0 { - LineNo::First - } else { - LineNo::LastInstruction - }; + config.lineno = if matches.occurrences_of("nolineno") > 0 { LineNo::NoLine } else if matches.occurrences_of("function") > 0 { LineNo::FirstLineNo } else { LineNo::LastInstruction }; config.include_thread_ids = matches.occurrences_of("threads") > 0; - if matches.occurrences_of("nolineno") > 0 && matches.occurrences_of("function") > 0 - { + if matches.occurrences_of("nolineno") > 0 && matches.occurrences_of("function") > 0 { eprintln!("--function & --nolinenos can't be used together"); std::process::exit(1); } config.hide_progress = matches.occurrences_of("hideprogress") > 0; - } + }, "top" => { config.sampling_rate = matches.value_of_t("rate")?; - config.refresh_seconds = *matches.get_one::("delay").unwrap(); - } + }, "dump" => { config.dump_json = matches.occurrences_of("json") > 0; config.dump_locals = matches.occurrences_of("locals"); - - #[cfg(target_os = "linux")] - { - config.core_filename = matches.value_of("core").map(|f| f.to_owned()); - } - } + }, "completions" => { let shell = matches.get_one::("shell").unwrap(); let app_name = app.get_name().to_string(); @@ -412,22 +345,23 @@ impl Config { match subcommand { "record" | "top" => { - config.python_program = matches - .values_of("python_program") - .map(|vals| vals.map(|v| v.to_owned()).collect()); + config.python_program = matches.values_of("python_program").map(|vals| { + vals.map(|v| v.to_owned()).collect() + }); config.gil_only = matches.occurrences_of("gil") > 0; config.include_idle = matches.occurrences_of("idle") > 0; - } + }, _ => {} } config.subprocesses = matches.occurrences_of("subprocesses") > 0; config.command = subcommand.to_owned(); + config.whitelist = matches.value_of("whitelist") + .map(|p| p.split(',').map(|s| String::from(s.trim())).filter(|s| !s.is_empty()).collect()); + // options that can be shared between subcommands - config.pid = matches - .value_of("pid") - .map(|p| p.parse().expect("invalid pid")); + config.pid = matches.value_of("pid").map(|p| p.parse().expect("invalid pid")); config.full_filenames = matches.occurrences_of("full_filenames") > 0; if cfg!(unwind) { config.native = matches.occurrences_of("native") > 0; @@ -440,7 +374,7 @@ impl Config { if matches.occurrences_of("nonblocking") > 0 { // disable native profiling if invalidly asked for - if config.native { + if config.native { eprintln!("Can't get native stack traces with the --nonblocking option."); std::process::exit(1); } @@ -452,26 +386,22 @@ impl Config { if config.native && config.subprocesses { // the native extension profiling code relies on dbghelp library, which doesn't // seem to work when connecting to multiple processes. disallow - eprintln!( - "Can't get native stack traces with the ---subprocesses option on windows." - ); + eprintln!("Can't get native stack traces with the ---subprocesses option on windows."); std::process::exit(1); } } - #[cfg(target_os = "freebsd")] + #[cfg(target_os="freebsd")] { - if config.pid.is_some() { - if std::env::var("PYSPY_ALLOW_FREEBSD_ATTACH").is_err() { + if config.pid.is_some() { + if std::env::var("PYSPY_ALLOW_FREEBSD_ATTACH").is_err() { eprintln!("On FreeBSD, running py-spy can cause an exception in the profiled process if the process \ is calling 'socket.connect'."); eprintln!("While this is fixed in recent versions of python, you need to acknowledge the risk here by \ setting an environment variable PYSPY_ALLOW_FREEBSD_ATTACH to run this command."); - eprintln!( - "\nSee https://github.com/benfred/py-spy/issues/147 for more information" - ); + eprintln!("\nSee https://github.com/benfred/py-spy/issues/147 for more information"); std::process::exit(-1); - } + } } } Ok(config) @@ -482,7 +412,7 @@ impl Config { mod tests { use super::*; fn get_config(cmd: &str) -> clap::Result { - #[cfg(target_os = "freebsd")] + #[cfg(target_os="freebsd")] std::env::set_var("PYSPY_ALLOW_FREEBSD_ATTACH", "1"); let args: Vec = cmd.split_whitespace().map(|x| x.to_owned()).collect(); Config::from_args(&args) @@ -502,26 +432,17 @@ mod tests { assert_eq!(config, short_config); // missing the --pid argument should fail - assert_eq!( - get_config("py-spy record -o foo").unwrap_err().kind, - clap::ErrorKind::MissingRequiredArgument - ); + assert_eq!(get_config("py-spy record -o foo").unwrap_err().kind, + clap::ErrorKind::MissingRequiredArgument); // but should work when passed a python program let program_config = get_config("py-spy r -o foo -- python test.py").unwrap(); - assert_eq!( - program_config.python_program, - Some(vec![String::from("python"), String::from("test.py")]) - ); + assert_eq!(program_config.python_program, Some(vec![String::from("python"), String::from("test.py")])); assert_eq!(program_config.pid, None); // passing an invalid file format should fail - assert_eq!( - get_config("py-spy r -p 1234 -o foo -f unknown") - .unwrap_err() - .kind, - clap::ErrorKind::InvalidValue - ); + assert_eq!(get_config("py-spy r -p 1234 -o foo -f unknown").unwrap_err().kind, + clap::ErrorKind::InvalidValue); // test out overriding these params by setting flags assert_eq!(config.include_idle, false); @@ -546,10 +467,8 @@ mod tests { assert_eq!(config, short_config); // missing the --pid argument should fail - assert_eq!( - get_config("py-spy dump").unwrap_err().kind, - clap::ErrorKind::MissingRequiredArgument - ); + assert_eq!(get_config("py-spy dump").unwrap_err().kind, + clap::ErrorKind::MissingRequiredArgument); } #[test] @@ -566,9 +485,7 @@ mod tests { #[test] fn test_parse_args() { - assert_eq!( - get_config("py-spy dude").unwrap_err().kind, - clap::ErrorKind::UnrecognizedSubcommand - ); + assert_eq!(get_config("py-spy dude").unwrap_err().kind, + clap::ErrorKind::UnrecognizedSubcommand); } } diff --git a/src/console_viewer.rs b/src/console_viewer.rs index ed49c8b3..fcc706fc 100644 --- a/src/console_viewer.rs +++ b/src/console_viewer.rs @@ -1,15 +1,16 @@ +use std; use std::collections::HashMap; +use std::vec::Vec; use std::io; use std::io::{Read, Write}; -use std::sync::{atomic, Arc, Mutex}; +use std::sync::{Mutex, Arc, atomic}; use std::thread; -use std::vec::Vec; use anyhow::Error; -use console::{style, Term}; +use console::{Term, style}; use crate::config::Config; -use crate::stack_trace::{Frame, StackTrace}; +use crate::stack_trace::{StackTrace, Frame}; use crate::version::Version; pub struct ConsoleViewer { @@ -22,16 +23,14 @@ pub struct ConsoleViewer { options: Arc>, stats: Stats, subprocesses: bool, - config: Config, + config: Config } impl ConsoleViewer { - pub fn new( - show_linenumbers: bool, - python_command: &str, - version: &Option, - config: &Config, - ) -> io::Result { + pub fn new(show_linenumbers: bool, + python_command: &str, + version: &Option, + config: &Config) -> io::Result { let sampling_rate = 1.0 / (config.sampling_rate as f64); let running = Arc::new(atomic::AtomicBool::new(true)); let options = Arc::new(Mutex::new(Options::new(show_linenumbers))); @@ -56,7 +55,7 @@ impl ConsoleViewer { '2' => options.sort_column = 2, '3' => options.sort_column = 3, '4' => options.sort_column = 4, - _ => {} + _ => {}, } options.reset_style = previous_usage != options.usage; @@ -64,17 +63,13 @@ impl ConsoleViewer { } }); - Ok(ConsoleViewer { - console_config: os_impl::ConsoleConfig::new()?, - version: version.clone(), - command: python_command.to_owned(), - running, - options, - sampling_rate, - subprocesses: config.subprocesses, - stats: Stats::new(), - config: config.clone(), - }) + Ok(ConsoleViewer{console_config: os_impl::ConsoleConfig::new()?, + version: version.clone(), + command: python_command.to_owned(), + running, options, sampling_rate, + subprocesses: config.subprocesses, + stats: Stats::new(), + config: config.clone()}) } pub fn increment(&mut self, traces: &[StackTrace]) -> Result<(), Error> { @@ -106,10 +101,7 @@ impl ConsoleViewer { } update_function_statistics(&mut self.stats.line_counts, trace, |frame| { - let filename = match &frame.short_filename { - Some(f) => f, - None => &frame.filename, - }; + let filename = match &frame.short_filename { Some(f) => &f, None => &frame.filename }; if frame.line != 0 { format!("{} ({}:{})", frame.name, filename, frame.line) } else { @@ -118,10 +110,7 @@ impl ConsoleViewer { }); update_function_statistics(&mut self.stats.function_counts, trace, |frame| { - let filename = match &frame.short_filename { - Some(f) => f, - None => &frame.filename, - }; + let filename = match &frame.short_filename { Some(f) => &f, None => &frame.filename }; format!("{} ({})", frame.name, filename) }); } @@ -133,13 +122,8 @@ impl ConsoleViewer { // Get the top aggregate function calls (either by line or by function as ) let mut options = self.options.lock().unwrap(); options.dirty = false; - let counts = if options.show_linenumbers { - &self.stats.line_counts - } else { - &self.stats.function_counts - }; - let mut counts: Vec<(&FunctionStatistics, &str)> = - counts.iter().map(|(x, y)| (y, x.as_ref())).collect(); + let counts = if options.show_linenumbers { &self.stats.line_counts } else { &self.stats.function_counts }; + let mut counts:Vec<(&FunctionStatistics, &str)> = counts.iter().map(|(x,y)| (y, x.as_ref())).collect(); // TODO: subsort ? match options.sort_column { @@ -147,7 +131,7 @@ impl ConsoleViewer { 2 => counts.sort_unstable_by(|a, b| b.0.current_total.cmp(&a.0.current_total)), 3 => counts.sort_unstable_by(|a, b| b.0.overall_own.cmp(&a.0.overall_own)), 4 => counts.sort_unstable_by(|a, b| b.0.overall_total.cmp(&a.0.overall_total)), - _ => panic!("unknown sort column. this really shouldn't happen"), + _ => panic!("unknown sort column. this really shouldn't happen") } let term = Term::stdout(); let (height, width) = term.size(); @@ -180,33 +164,23 @@ impl ConsoleViewer { } if self.subprocesses { - out!( - "Collecting samples from '{}' and subprocesses", - style(&self.command).green() - ); + out!("Collecting samples from '{}' and subprocesses", style(&self.command).green()); } else { - out!( - "Collecting samples from '{}' (python v{})", - style(&self.command).green(), - self.version.as_ref().unwrap() - ); + out!("Collecting samples from '{}' (python v{})", style(&self.command).green(), self.version.as_ref().unwrap()); } let error_rate = self.stats.errors as f64 / self.stats.overall_samples as f64; if error_rate >= 0.01 && self.stats.overall_samples > 100 { let error_string = self.stats.last_error.as_ref().unwrap(); - out!( - "Total Samples {}, Error Rate {:.2}% ({})", - style(self.stats.overall_samples).bold(), - style(error_rate * 100.0).bold().red(), - style(error_string).bold() - ); + out!("Total Samples {}, Error Rate {:.2}% ({})", + style(self.stats.overall_samples).bold(), + style(error_rate * 100.0).bold().red(), + style(error_string).bold()); } else { - out!("Total Samples {}", style(self.stats.overall_samples).bold()); + out!("Total Samples {}", style(self.stats.overall_samples).bold()); } - out!( - "GIL: {:.2}%, Active: {:>.2}%, Threads: {}{}", + out!("GIL: {:.2}%, Active: {:>.2}%, Threads: {}{}", style(100.0 * self.stats.gil as f64 / self.stats.current_samples as f64).bold(), style(100.0 * self.stats.active as f64 / self.stats.current_samples as f64).bold(), style(self.stats.threads).bold(), @@ -214,8 +188,7 @@ impl ConsoleViewer { format!(", Processes {}", style(self.stats.processes).bold()) } else { "".to_owned() - } - ); + }); out!(); @@ -240,91 +213,51 @@ impl ConsoleViewer { // If we aren't at least 50 characters wide, lets use two lines per entry // Otherwise, truncate the filename so that it doesn't wrap around to the next line - let header_lines = if width > 50 { - header_lines - } else { - header_lines + height as usize / 2 - }; - let max_function_width = if width > 50 { width - 35 } else { width }; - - out!( - "{:>7}{:>8}{:>9}{:>11}{:width$}", - percent_own_header, - percent_total_header, - time_own_header, - time_total_header, - function_header, - width = max_function_width - ); + let header_lines = if width > 50 { header_lines } else { header_lines + height as usize / 2 }; + let max_function_width = if width > 50 { width as usize - 35 } else { width as usize }; + + out!("{:>7}{:>8}{:>9}{:>11}{:width$}", percent_own_header, percent_total_header, + time_own_header, time_total_header, function_header, width=max_function_width); let mut written = 0; for (samples, label) in counts.iter().take(height as usize - header_lines) { - out!( - "{:>6.2}% {:>6.2}% {:>7}s {:>8}s {:.width$}", + out!("{:>6.2}% {:>6.2}% {:>7}s {:>8}s {:.width$}", 100.0 * samples.current_own as f64 / (self.stats.current_samples as f64), 100.0 * samples.current_total as f64 / (self.stats.current_samples as f64), display_time(samples.overall_own as f64 * self.sampling_rate), display_time(samples.overall_total as f64 * self.sampling_rate), - label, - width = max_function_width - 2 - ); - written += 1; + label, width=max_function_width - 2); + written += 1; } - for _ in written..height as usize - header_lines { + for _ in written.. height as usize - header_lines { out!(); } out!(); if options.usage { - out!( - "{:width$}", - style(" Keyboard Shortcuts ").reverse(), - width = width - ); + out!("{:width$}", style(" Keyboard Shortcuts ").reverse(), width=width as usize); out!(); out!("{:^12}{:<}", style("key").green(), style("action").green()); - out!( - "{:^12}{:<}", - "1", - "Sort by %Own (% of time currently spent in the function)" - ); - out!( - "{:^12}{:<}", - "2", - "Sort by %Total (% of time currently in the function and its children)" - ); - out!( - "{:^12}{:<}", - "3", - "Sort by OwnTime (Overall time spent in the function)" - ); - out!( - "{:^12}{:<}", - "4", - "Sort by TotalTime (Overall time spent in the function and its children)" - ); - out!( - "{:^12}{:<}", - "L,l", - "Toggle between aggregating by line number or by function" - ); + out!("{:^12}{:<}", "1", "Sort by %Own (% of time currently spent in the function)"); + out!("{:^12}{:<}", "2", "Sort by %Total (% of time currently in the function and its children)"); + out!("{:^12}{:<}", "3", "Sort by OwnTime (Overall time spent in the function)"); + out!("{:^12}{:<}", "4", "Sort by TotalTime (Overall time spent in the function and its children)"); + out!("{:^12}{:<}", "L,l", "Toggle between aggregating by line number or by function"); out!("{:^12}{:<}", "R,r", "Reset statistics"); out!("{:^12}{:<}", "X,x", "Exit this help screen"); out!(); //println!("{:^12}{:<}", "Control-C", "Quit py-spy"); } else { - out!( - "Press {} to quit, or {} for help.", - style("Control-C").bold().reverse(), - style("?").bold().reverse() - ); + out!("Press {} to quit, or {} for help.", + style("Control-C").bold().reverse(), + style("?").bold().reverse()); } std::io::stdout().flush()?; Ok(()) } - pub fn increment_error(&mut self, err: &Error) -> Result<(), Error> { + pub fn increment_error(&mut self, err: &Error) -> Result<(), Error> { self.maybe_reset(); self.stats.errors += 1; self.stats.last_error = Some(format!("{}", err)); @@ -340,10 +273,8 @@ impl ConsoleViewer { // update faster if we only have a few samples, or if we changed options match self.stats.overall_samples { 10 | 100 | 500 => true, - _ => { - self.options.lock().unwrap().dirty - || self.stats.elapsed >= self.config.refresh_seconds - } + _ => self.options.lock().unwrap().dirty || + self.stats.elapsed >= 1.0 } } @@ -380,16 +311,11 @@ struct FunctionStatistics { current_own: u64, current_total: u64, overall_own: u64, - overall_total: u64, + overall_total: u64 } -fn update_function_statistics( - counts: &mut HashMap, - trace: &StackTrace, - key_func: K, -) where - K: Fn(&Frame) -> String, -{ +fn update_function_statistics(counts: &mut HashMap, trace: &StackTrace, key_func: K) + where K: Fn(&Frame) -> String { // we need to deduplicate (so we don't overcount cumulative stats with recursive function calls) let mut current = HashMap::new(); for (i, frame) in trace.frames.iter().enumerate() { @@ -398,12 +324,8 @@ fn update_function_statistics( } for (key, order) in current { - let entry = counts.entry(key).or_insert_with(|| FunctionStatistics { - current_own: 0, - current_total: 0, - overall_own: 0, - overall_total: 0, - }); + let entry = counts.entry(key).or_insert_with(|| FunctionStatistics{current_own: 0, current_total: 0, + overall_own: 0, overall_total: 0}); entry.current_total += 1; entry.overall_total += 1; @@ -441,34 +363,16 @@ struct Stats { impl Options { fn new(show_linenumbers: bool) -> Options { - Options { - dirty: false, - usage: false, - reset: false, - sort_column: 3, - show_linenumbers, - reset_style: false, - } + Options{dirty: false, usage: false, reset: false, sort_column: 3, show_linenumbers, reset_style: false} } } impl Stats { fn new() -> Stats { - Stats { - current_samples: 0, - overall_samples: 0, - elapsed: 0., - errors: 0, - late_samples: 0, - threads: 0, - processes: 0, - gil: 0, - active: 0, - line_counts: HashMap::new(), - function_counts: HashMap::new(), - last_error: None, - last_delay: None, - } + Stats{current_samples: 0, overall_samples: 0, elapsed: 0., + errors: 0, late_samples: 0, threads: 0, processes: 0, gil: 0, active: 0, + line_counts: HashMap::new(), function_counts: HashMap::new(), + last_error: None, last_delay: None} } pub fn reset_current(&mut self) { @@ -517,11 +421,11 @@ for doing this: #[cfg(unix)] mod os_impl { use super::*; - use termios::{tcsetattr, Termios, ECHO, ICANON, TCSANOW}; + use termios::{Termios, TCSANOW, ECHO, ICANON, tcsetattr}; pub struct ConsoleConfig { termios: Termios, - stdin: i32, + stdin: i32 } impl ConsoleConfig { @@ -541,7 +445,7 @@ mod os_impl { println!(); } - Ok(ConsoleConfig { termios, stdin }) + Ok(ConsoleConfig{termios, stdin}) } pub fn reset_cursor(&self) -> io::Result<()> { @@ -562,21 +466,19 @@ mod os_impl { #[cfg(windows)] mod os_impl { use super::*; - use winapi::shared::minwindef::DWORD; - use winapi::um::consoleapi::{GetConsoleMode, SetConsoleMode}; - use winapi::um::handleapi::INVALID_HANDLE_VALUE; - use winapi::um::processenv::GetStdHandle; + use winapi::shared::minwindef::{DWORD}; + use winapi::um::winnt::{HANDLE}; use winapi::um::winbase::{STD_INPUT_HANDLE, STD_OUTPUT_HANDLE}; - use winapi::um::wincon::{ - FillConsoleOutputAttribute, GetConsoleScreenBufferInfo, SetConsoleCursorPosition, - CONSOLE_SCREEN_BUFFER_INFO, COORD, ENABLE_ECHO_INPUT, ENABLE_LINE_INPUT, - }; - use winapi::um::winnt::HANDLE; + use winapi::um::processenv::GetStdHandle; + use winapi::um::handleapi::INVALID_HANDLE_VALUE; + use winapi::um::consoleapi::{GetConsoleMode, SetConsoleMode}; + use winapi::um::wincon::{ENABLE_LINE_INPUT, ENABLE_ECHO_INPUT, CONSOLE_SCREEN_BUFFER_INFO, SetConsoleCursorPosition, + GetConsoleScreenBufferInfo, COORD, FillConsoleOutputAttribute}; pub struct ConsoleConfig { stdin: HANDLE, mode: DWORD, - top_left: COORD, + top_left: COORD } impl ConsoleConfig { @@ -613,17 +515,9 @@ mod os_impl { // Figure out a consistent spot in the terminal buffer to write output to let mut top_left = csbi.dwCursorPosition; top_left.X = 0; - top_left.Y = if top_left.Y > height { - top_left.Y - height - } else { - 0 - }; - - Ok(ConsoleConfig { - stdin, - mode, - top_left, - }) + top_left.Y = if top_left.Y > height { top_left.Y - height } else { 0 }; + + Ok(ConsoleConfig{stdin, mode, top_left}) } } @@ -649,17 +543,8 @@ mod os_impl { } let mut written: DWORD = 0; - let console_size = ((1 + csbi.srWindow.Bottom - csbi.srWindow.Top) - * (csbi.srWindow.Right - csbi.srWindow.Left)) - as DWORD; - if FillConsoleOutputAttribute( - stdout, - csbi.wAttributes, - console_size, - self.top_left, - &mut written, - ) == 0 - { + let console_size = ((1 + csbi.srWindow.Bottom - csbi.srWindow.Top) * (csbi.srWindow.Right - csbi.srWindow.Left)) as DWORD; + if FillConsoleOutputAttribute(stdout, csbi.wAttributes, console_size, self.top_left, &mut written) == 0 { return Err(io::Error::last_os_error()); } Ok(()) @@ -669,9 +554,7 @@ mod os_impl { impl Drop for ConsoleConfig { fn drop(&mut self) { - unsafe { - SetConsoleMode(self.stdin, self.mode); - } + unsafe { SetConsoleMode(self.stdin, self.mode); } } } } diff --git a/src/coredump.rs b/src/coredump.rs deleted file mode 100644 index 53940bdb..00000000 --- a/src/coredump.rs +++ /dev/null @@ -1,470 +0,0 @@ -use std::collections::HashMap; -use std::ffi::OsStr; -use std::fs::File; -use std::io::Read; -use std::os::unix::ffi::OsStrExt; -use std::path::Path; -use std::path::PathBuf; - -use anyhow::{Context, Error, Result}; -use console::style; -use log::info; -use remoteprocess::ProcessMemory; - -use crate::binary_parser::{parse_binary, BinaryInfo}; -use crate::config::Config; -use crate::dump::print_trace; -use crate::python_bindings::{ - v2_7_15, v3_10_0, v3_11_0, v3_3_7, v3_5_5, v3_6_6, v3_7_0, v3_8_0, v3_9_5, -}; -use crate::python_data_access::format_variable; -use crate::python_interpreters::InterpreterState; -use crate::python_process_info::{ - get_interpreter_address, get_python_version, get_threadstate_address, is_python_lib, - ContainsAddr, PythonProcessInfo, -}; -use crate::python_threading::thread_names_from_interpreter; -use crate::stack_trace::{get_stack_traces, StackTrace}; -use crate::version::Version; - -#[derive(Debug, Clone)] -pub struct CoreMapRange { - pub pathname: Option, - pub segment: goblin::elf::ProgramHeader, -} - -// Defines accessors to match those in proc_maps. However, can't use the -// proc_maps trait since is private -impl CoreMapRange { - pub fn size(&self) -> usize { - self.segment.p_memsz as usize - } - pub fn start(&self) -> usize { - self.segment.p_vaddr as usize - } - pub fn filename(&self) -> Option<&Path> { - self.pathname.as_deref() - } - pub fn is_exec(&self) -> bool { - self.segment.is_executable() - } - pub fn is_write(&self) -> bool { - self.segment.is_write() - } - pub fn is_read(&self) -> bool { - self.segment.is_read() - } -} - -impl ContainsAddr for Vec { - fn contains_addr(&self, addr: usize) -> bool { - self.iter() - .any(|map| (addr >= map.start()) && (addr < (map.start() + map.size()))) - } -} - -pub struct CoreDump { - filename: PathBuf, - contents: Vec, - maps: Vec, - psinfo: Option, - status: Vec, -} - -impl CoreDump { - pub fn new>(filename: P) -> Result { - let filename = filename.as_ref(); - let mut file = File::open(filename)?; - let mut contents = Vec::new(); - file.read_to_end(&mut contents)?; - let elf = goblin::elf::Elf::parse(&contents)?; - - let notes = elf - .iter_note_headers(&contents) - .ok_or_else(|| format_err!("no note segment found"))?; - - let mut filenames = HashMap::new(); - let mut psinfo = None; - let mut status = Vec::new(); - for note in notes.flatten() { - if note.n_type == goblin::elf::note::NT_PRPSINFO { - psinfo = Some(unsafe { *(note.desc.as_ptr() as *const elfcore::elf_prpsinfo) }); - } else if note.n_type == goblin::elf::note::NT_PRSTATUS { - let thread_status = - unsafe { *(note.desc.as_ptr() as *const elfcore::elf_prstatus) }; - status.push(thread_status); - } else if note.n_type == goblin::elf::note::NT_FILE { - let data = note.desc; - let ptrs = data.as_ptr() as *const usize; - - let count = unsafe { *ptrs }; - let _page_size = unsafe { *ptrs.offset(1) }; - - let string_table = &data[(std::mem::size_of::() * (2 + count * 3))..]; - - for (i, filename) in string_table.split(|chr| *chr == 0).enumerate() { - if i < count { - let i = i as isize; - let start = unsafe { *ptrs.offset(i * 3 + 2) }; - let _end = unsafe { *ptrs.offset(i * 3 + 3) }; - let _page_offset = unsafe { *ptrs.offset(i * 3 + 4) }; - - let pathname = Path::new(&OsStr::from_bytes(filename)).to_path_buf(); - filenames.insert(start, pathname); - } - } - } - } - - let mut maps = Vec::new(); - for ph in elf.program_headers { - if ph.p_type == goblin::elf::program_header::PT_LOAD { - let pathname = filenames.get(&(ph.p_vaddr as _)); - let map = CoreMapRange { - pathname: pathname.cloned(), - segment: ph, - }; - info!( - "map: {:016x}-{:016x} {}{}{} {}", - map.start(), - map.start() + map.size(), - if map.is_read() { 'r' } else { '-' }, - if map.is_write() { 'w' } else { '-' }, - if map.is_exec() { 'x' } else { '-' }, - map.filename() - .unwrap_or(&std::path::PathBuf::from("")) - .display() - ); - - maps.push(map); - } - } - - Ok(CoreDump { - filename: filename.to_owned(), - contents, - maps, - psinfo, - status, - }) - } -} - -impl ProcessMemory for CoreDump { - fn read(&self, addr: usize, buf: &mut [u8]) -> Result<(), remoteprocess::Error> { - let start = addr as u64; - let _end = (addr + buf.len()) as u64; - - for map in &self.maps { - // TODO: one issue here is the bss addr spans multiple mmap segments - so checking the 'end' - // here means we skip it. Instead we're just checking if the start address exists in - // the segment - let ph = &map.segment; - if start >= ph.p_vaddr && start <= (ph.p_vaddr + ph.p_memsz) { - let offset = (start - ph.p_vaddr + ph.p_offset) as usize; - buf.copy_from_slice(&self.contents[offset..(offset + buf.len())]); - return Ok(()); - } - } - - let io_error = std::io::Error::from_raw_os_error(libc::EFAULT); - Err(remoteprocess::Error::IOError(io_error)) - } -} - -pub struct PythonCoreDump { - core: CoreDump, - version: Version, - interpreter_address: usize, - threadstate_address: usize, -} - -impl PythonCoreDump { - pub fn new>(filename: P) -> Result { - let core = CoreDump::new(filename)?; - let maps = &core.maps; - - // Get the python binary from the maps, and parse it - let (python_filename, python_binary) = { - let map = maps - .iter() - .find(|m| m.filename().is_some() & m.is_exec()) - .ok_or_else(|| format_err!("Failed to get binary from coredump"))?; - let python_filename = map.filename().unwrap(); - let python_binary = parse_binary(python_filename, map.start() as _, map.size() as _); - info!("Found python binary @ {}", python_filename.display()); - (python_filename.to_owned(), python_binary) - }; - - // get the libpython binary (if any) from maps - let libpython_binary = { - let libmap = maps.iter().find(|m| { - if let Some(pathname) = m.filename() { - if let Some(pathname) = pathname.to_str() { - return is_python_lib(pathname) && m.is_exec(); - } - } - false - }); - - let mut libpython_binary: Option = None; - if let Some(libpython) = libmap { - if let Some(filename) = &libpython.filename() { - info!("Found libpython binary @ {}", filename.display()); - let parsed = - parse_binary(filename, libpython.start() as u64, libpython.size() as u64)?; - libpython_binary = Some(parsed); - } - } - libpython_binary - }; - - // If we have a libpython binary - we can tolerate failures on parsing the main python binary. - let python_binary = match libpython_binary { - None => Some(python_binary.context("Failed to parse python binary")?), - _ => python_binary.ok(), - }; - - let python_info = PythonProcessInfo { - python_binary, - libpython_binary, - maps: Box::new(core.maps.clone()), - python_filename, - dockerized: false, - }; - - let version = - get_python_version(&python_info, &core).context("failed to get python version")?; - info!("Got python version {}", version); - - let interpreter_address = get_interpreter_address(&python_info, &core, &version)?; - info!("Found interpreter at 0x{:016x}", interpreter_address); - - // lets us figure out which thread has the GIL - let config = Config::default(); - let threadstate_address = get_threadstate_address(&python_info, &version, &config)?; - info!("found threadstate at 0x{:016x}", threadstate_address); - - Ok(PythonCoreDump { - core, - version, - interpreter_address, - threadstate_address, - }) - } - - pub fn get_stack(&self, config: &Config) -> Result, Error> { - if config.native { - return Err(format_err!( - "Native unwinding isn't yet supported with coredumps" - )); - } - - if config.subprocesses { - return Err(format_err!( - "Subprocesses can't be used for getting stacktraces from coredumps" - )); - } - - // different versions have different layouts, check as appropriate - match self.version { - Version { - major: 2, - minor: 3..=7, - .. - } => self._get_stack::(config), - Version { - major: 3, minor: 3, .. - } => self._get_stack::(config), - Version { - major: 3, - minor: 4..=5, - .. - } => self._get_stack::(config), - Version { - major: 3, minor: 6, .. - } => self._get_stack::(config), - Version { - major: 3, minor: 7, .. - } => self._get_stack::(config), - Version { - major: 3, minor: 8, .. - } => self._get_stack::(config), - Version { - major: 3, minor: 9, .. - } => self._get_stack::(config), - Version { - major: 3, - minor: 10, - .. - } => self._get_stack::(config), - Version { - major: 3, - minor: 11, - .. - } => self._get_stack::(config), - _ => Err(format_err!( - "Unsupported version of Python: {}", - self.version - )), - } - } - - fn _get_stack(&self, config: &Config) -> Result, Error> { - let interp: I = self.core.copy_struct(self.interpreter_address)?; - - let mut traces = - get_stack_traces(&interp, &self.core, self.threadstate_address, Some(config))?; - let thread_names = thread_names_from_interpreter(&interp, &self.core, &self.version).ok(); - - for trace in &mut traces { - if let Some(ref thread_names) = thread_names { - trace.thread_name = thread_names.get(&trace.thread_id).cloned(); - } - - for frame in &mut trace.frames { - if let Some(locals) = frame.locals.as_mut() { - let max_length = (128 * config.dump_locals) as isize; - for local in locals { - let repr = format_variable::( - &self.core, - &self.version, - local.addr, - max_length, - ); - local.repr = Some(repr.unwrap_or_else(|_| "?".to_owned())); - } - } - } - } - Ok(traces) - } - - pub fn print_traces(&self, traces: &Vec, config: &Config) -> Result<(), Error> { - if config.dump_json { - println!("{}", serde_json::to_string_pretty(&traces)?); - return Ok(()); - } - - if let Some(status) = self.core.status.first() { - println!( - "Signal {}: {}", - style(status.pr_cursig).bold().yellow(), - self.core.filename.display() - ); - } - - if let Some(psinfo) = self.core.psinfo { - println!( - "Process {}: {}", - style(psinfo.pr_pid).bold().yellow(), - OsStr::from_bytes(&psinfo.pr_psargs).to_string_lossy() - ); - } - println!("Python v{}", style(&self.version).bold()); - println!(); - for trace in traces.iter().rev() { - print_trace(trace, false); - } - Ok(()) - } -} - -mod elfcore { - #[repr(C)] - #[derive(Debug, Copy, Clone)] - pub struct elf_siginfo { - pub si_signo: ::std::os::raw::c_int, - pub si_code: ::std::os::raw::c_int, - pub si_errno: ::std::os::raw::c_int, - } - - #[repr(C)] - #[derive(Debug, Copy, Clone)] - pub struct timeval { - pub tv_sec: ::std::os::raw::c_long, - pub tv_usec: ::std::os::raw::c_long, - } - - #[repr(C)] - #[derive(Debug, Copy, Clone)] - pub struct elf_prstatus { - pub pr_info: elf_siginfo, - pub pr_cursig: ::std::os::raw::c_short, - pub pr_sigpend: ::std::os::raw::c_ulong, - pub pr_sighold: ::std::os::raw::c_ulong, - pub pr_pid: ::std::os::raw::c_int, - pub pr_ppid: ::std::os::raw::c_int, - pub pr_pgrp: ::std::os::raw::c_int, - pub pr_sid: ::std::os::raw::c_int, - pub pr_utime: timeval, - pub pr_stime: timeval, - pub pr_cutime: timeval, - pub pr_cstime: timeval, - // TODO: has registers next for thread next - don't need them right now, but if we want to do - // unwinding we will - } - - #[repr(C)] - #[derive(Debug, Copy, Clone)] - pub struct elf_prpsinfo { - pub pr_state: ::std::os::raw::c_char, - pub pr_sname: ::std::os::raw::c_char, - pub pr_zomb: ::std::os::raw::c_char, - pub pr_nice: ::std::os::raw::c_char, - pub pr_flag: ::std::os::raw::c_ulong, - pub pr_uid: ::std::os::raw::c_uint, - pub pr_gid: ::std::os::raw::c_uint, - pub pr_pid: ::std::os::raw::c_int, - pub pr_ppid: ::std::os::raw::c_int, - pub pr_pgrp: ::std::os::raw::c_int, - pub pr_sid: ::std::os::raw::c_int, - pub pr_fname: [::std::os::raw::c_uchar; 16usize], - pub pr_psargs: [::std::os::raw::c_uchar; 80usize], - } -} - -#[cfg(test)] -mod test { - use super::*; - use py_spy_testdata::get_coredump_path; - - #[cfg(target_pointer_width = "64")] - #[test] - fn test_coredump() { - // we won't have the python binary for the core dump here, - // so we can't (yet) figure out the interpreter address & version. - // Manually specify here to test out instead - let core = CoreDump::new(&get_coredump_path("python_3_9_threads")).unwrap(); - let version = Version { - major: 3, - minor: 9, - patch: 13, - release_flags: "".to_owned(), - build_metadata: None, - }; - let python_core = PythonCoreDump { - core, - version, - interpreter_address: 0x000055a8293dbe20, - threadstate_address: 0x000055a82745fe18, - }; - - let config = Config::default(); - let traces = python_core.get_stack(&config).unwrap(); - - // should have two threads - assert_eq!(traces.len(), 2); - - let main_thread = &traces[1]; - assert_eq!(main_thread.frames.len(), 1); - assert_eq!(main_thread.frames[0].name, ""); - assert_eq!(main_thread.thread_name, Some("MainThread".to_owned())); - - let child_thread = &traces[0]; - assert_eq!(child_thread.frames.len(), 5); - assert_eq!(child_thread.frames[0].name, "dump_sum"); - assert_eq!(child_thread.frames[0].line, 16); - assert_eq!(child_thread.thread_name, Some("child_thread".to_owned())); - } -} diff --git a/src/cython.rs b/src/cython.rs index 3497f614..1a0a9814 100644 --- a/src/cython.rs +++ b/src/cython.rs @@ -1,11 +1,13 @@ -use regex::Regex; + +use std; use std::collections::{BTreeMap, HashMap}; +use regex::Regex; use anyhow::Error; use lazy_static::lazy_static; -use crate::stack_trace::Frame; use crate::utils::resolve_filename; +use crate::stack_trace::Frame; pub struct SourceMaps { maps: HashMap>, @@ -14,7 +16,7 @@ pub struct SourceMaps { impl SourceMaps { pub fn new() -> SourceMaps { let maps = HashMap::new(); - SourceMaps { maps } + SourceMaps{maps} } pub fn translate(&mut self, frame: &mut Frame) { @@ -40,7 +42,8 @@ impl SourceMaps { } return false; } - true + + return true; } // loads the corresponding cython source map for the frame @@ -64,7 +67,7 @@ impl SourceMaps { } struct SourceMap { - lookup: BTreeMap, + lookup: BTreeMap } impl SourceMap { @@ -73,11 +76,7 @@ impl SourceMap { SourceMap::from_contents(&contents, filename, module) } - pub fn from_contents( - contents: &str, - cpp_filename: &str, - module: &Option, - ) -> Result { + pub fn from_contents(contents: &str, cpp_filename: &str, module: &Option) -> Result { lazy_static! { static ref RE: Regex = Regex::new(r#"^\s*/\* "(.+\..+)":([0-9]+)"#).unwrap(); } @@ -87,7 +86,7 @@ impl SourceMap { let mut line_count = 0; for (lineno, line) in contents.lines().enumerate() { - if let Some(captures) = RE.captures(line) { + if let Some(captures) = RE.captures(&line) { let cython_file = captures.get(1).map_or("", |m| m.as_str()); let cython_line = captures.get(2).map_or("", |m| m.as_str()); @@ -109,7 +108,7 @@ impl SourceMap { } lookup.insert(line_count + 1, ("".to_owned(), 0)); - Ok(SourceMap { lookup }) + Ok(SourceMap{lookup}) } pub fn lookup(&self, lineno: u32) -> Option<&(String, u32)> { @@ -117,38 +116,25 @@ impl SourceMap { // handle EOF Some((_, (_, 0))) => None, Some((_, val)) => Some(val), - None => None, + None => None } } } pub fn ignore_frame(name: &str) -> bool { - let ignorable = [ - "__Pyx_PyFunction_FastCallDict", - "__Pyx_PyObject_CallOneArg", - "__Pyx_PyObject_Call", - "__Pyx_PyObject_Call", - "__pyx_FusedFunction_call", - ]; + let ignorable = ["__Pyx_PyFunction_FastCallDict", "__Pyx_PyObject_CallOneArg", + "__Pyx_PyObject_Call", "__Pyx_PyObject_Call", "__pyx_FusedFunction_call"]; ignorable.iter().any(|&f| f == name) } pub fn demangle(name: &str) -> &str { // slice off any leading cython prefix. - let prefixes = [ - "__pyx_fuse_1_0__pyx_pw", - "__pyx_fuse_0__pyx_f", - "__pyx_fuse_1__pyx_f", - "__pyx_pf", - "__pyx_pw", - "__pyx_f", - "___pyx_f", - "___pyx_pw", - ]; + let prefixes = ["__pyx_fuse_1_0__pyx_pw", "__pyx_fuse_0__pyx_f", "__pyx_fuse_1__pyx_f", + "__pyx_pf", "__pyx_pw", "__pyx_f", "___pyx_f", "___pyx_pw"]; let mut current = match prefixes.iter().find(|&prefix| name.starts_with(prefix)) { Some(prefix) => &name[prefix.len()..], - None => return name, + None => return name }; let mut next = current; @@ -162,8 +148,8 @@ pub fn demangle(name: &str) -> &str { } let mut digit_index = 1; - for ch in chars { - if !ch.is_ascii_digit() { + while let Some(ch) = chars.next() { + if !ch.is_digit(10) { break; } digit_index += 1; @@ -180,8 +166,8 @@ pub fn demangle(name: &str) -> &str { break; } next = &next[digits + digit_index..]; - } - Err(_) => break, + }, + Err(_) => { break } }; } debug!("cython_demangle(\"{}\") -> \"{}\"", name, current); @@ -189,15 +175,11 @@ pub fn demangle(name: &str) -> &str { current } -fn resolve_cython_file( - cpp_filename: &str, - cython_filename: &str, - module: &Option, -) -> String { +fn resolve_cython_file(cpp_filename: &str, cython_filename: &str, module: &Option) -> String { let cython_path = std::path::PathBuf::from(cython_filename); if let Some(ext) = cython_path.extension() { let mut path_buf = std::path::PathBuf::from(cpp_filename); - path_buf.set_extension(ext); + path_buf.set_extension(&ext); if path_buf.ends_with(&cython_path) && path_buf.exists() { return path_buf.to_string_lossy().to_string(); } @@ -205,9 +187,10 @@ fn resolve_cython_file( match module { Some(module) => { - resolve_filename(cython_filename, module).unwrap_or_else(|| cython_filename.to_owned()) - } - None => cython_filename.to_owned(), + resolve_filename(cython_filename, module) + .unwrap_or_else(|| cython_filename.to_owned()) + }, + None => cython_filename.to_owned() } } @@ -217,58 +200,34 @@ mod tests { #[test] fn test_demangle() { // all of these were wrong at certain points when writing cython_demangle =( - assert_eq!( - demangle("__pyx_pf_8implicit_4_als_30_least_squares_cg"), - "_least_squares_cg" - ); - assert_eq!( - demangle("__pyx_pw_8implicit_4_als_5least_squares_cg"), - "least_squares_cg" - ); - assert_eq!( - demangle("__pyx_fuse_1_0__pyx_pw_8implicit_4_als_31_least_squares_cg"), - "_least_squares_cg" - ); - assert_eq!( - demangle("__pyx_f_6mtrand_cont0_array"), - "mtrand_cont0_array" - ); + assert_eq!(demangle("__pyx_pf_8implicit_4_als_30_least_squares_cg"), "_least_squares_cg"); + assert_eq!(demangle("__pyx_pw_8implicit_4_als_5least_squares_cg"), "least_squares_cg"); + assert_eq!(demangle("__pyx_fuse_1_0__pyx_pw_8implicit_4_als_31_least_squares_cg"), "_least_squares_cg"); + assert_eq!(demangle("__pyx_f_6mtrand_cont0_array"), "mtrand_cont0_array"); // in both of these cases we should ideally slice off the module (_als/bpr), but it gets tricky // implementation wise - assert_eq!( - demangle("__pyx_fuse_0__pyx_f_8implicit_4_als_axpy"), - "_als_axpy" - ); - assert_eq!( - demangle("__pyx_fuse_1__pyx_f_8implicit_3bpr_has_non_zero"), - "bpr_has_non_zero" - ); + assert_eq!(demangle("__pyx_fuse_0__pyx_f_8implicit_4_als_axpy"), "_als_axpy"); + assert_eq!(demangle("__pyx_fuse_1__pyx_f_8implicit_3bpr_has_non_zero"), "bpr_has_non_zero"); } #[test] fn test_source_map() { - let map = SourceMap::from_contents( - include_str!("../ci/testdata/cython_test.c"), - "cython_test.c", - &None, - ) - .unwrap(); + let map = SourceMap::from_contents(include_str!("../ci/testdata/cython_test.c"), "cython_test.c", &None).unwrap(); // we don't have info on cython line numbers until line 1261 assert_eq!(map.lookup(1000), None); // past the end of the file should also return none assert_eq!(map.lookup(10000), None); - let lookup = |lineno: u32, cython_file: &str, cython_line: u32| match map.lookup(lineno) { - Some((file, line)) => { - assert_eq!(file, cython_file); - assert_eq!(line, &cython_line); - } - None => { - panic!( - "Failed to lookup line {} (expected {}:{})", - lineno, cython_file, cython_line - ); + let lookup = |lineno: u32, cython_file: &str, cython_line: u32| { + match map.lookup(lineno) { + Some((file, line)) => { + assert_eq!(file, cython_file); + assert_eq!(line, &cython_line); + }, + None => { + panic!("Failed to lookup line {} (expected {}:{})", lineno, cython_file, cython_line); + } } }; lookup(1298, "cython_test.pyx", 6); diff --git a/src/dump.rs b/src/dump.rs index 39624e6d..b20a762b 100644 --- a/src/dump.rs +++ b/src/dump.rs @@ -1,9 +1,8 @@ use anyhow::Error; -use console::{style, Term}; +use console::{Term, style}; use crate::config::Config; use crate::python_spy::PythonSpy; -use crate::stack_trace::StackTrace; use remoteprocess::Pid; @@ -12,39 +11,66 @@ pub fn print_traces(pid: Pid, config: &Config, parent: Option) -> Result<() if config.dump_json { let traces = process.get_stack_traces()?; println!("{}", serde_json::to_string_pretty(&traces)?); - return Ok(()); + return Ok(()) } - println!( - "Process {}: {}", + println!("Process {}: {}", style(process.pid).bold().yellow(), - process.process.cmdline()?.join(" ") - ); + process.process.cmdline()?.join(" ")); - println!( - "Python v{} ({})", + println!("Python v{} ({})", style(&process.version).bold(), - style(process.process.exe()?).dim() - ); + style(process.process.exe()?).dim()); if let Some(parentpid) = parent { let parentprocess = remoteprocess::Process::new(parentpid)?; - println!( - "Parent Process {}: {}", + println!("Parent Process {}: {}", style(parentpid).bold().yellow(), - parentprocess.cmdline()?.join(" ") - ); + parentprocess.cmdline()?.join(" ")); } - println!(); + println!(""); + let traces = process.get_stack_traces()?; + for trace in traces.iter().rev() { - print_trace(trace, true); + let thread_id = trace.format_threadid(); + match trace.thread_name.as_ref() { + Some(name) => { + println!("Thread {} ({}): \"{}\"", style(thread_id).bold().yellow(), trace.status_str(), name); + } + None => { + println!("Thread {} ({})", style(thread_id).bold().yellow(), trace.status_str()); + } + }; + + for frame in &trace.frames { + let filename = match &frame.short_filename { Some(f) => &f, None => &frame.filename }; + if frame.line != 0 { + println!(" {} ({}:{})", style(&frame.name).green(), style(&filename).cyan(), style(frame.line).dim()); + } else { + println!(" {} ({})", style(&frame.name).green(), style(&filename).cyan()); + } + + if let Some(locals) = &frame.locals { + let mut shown_args = false; + let mut shown_locals = false; + for local in locals { + if local.arg && !shown_args { + println!(" {}", style("Arguments:").dim()); + shown_args = true; + } else if !local.arg && !shown_locals { + println!(" {}", style("Locals:").dim()); + shown_locals = true; + } + + let repr = local.repr.as_ref().map(String::as_str).unwrap_or("?"); + println!(" {}: {}", local.name, repr); + } + } + } + if config.subprocesses { - for (childpid, parentpid) in process - .process - .child_processes() - .expect("failed to get subprocesses") - { + for (childpid, parentpid) in process.process.child_processes().expect("failed to get subprocesses") { let term = Term::stdout(); let (_, width) = term.size(); @@ -53,74 +79,10 @@ pub fn print_traces(pid: Pid, config: &Config, parent: Option) -> Result<() // though we could end up printing grandchild processes multiple times. Limit down // to just once if parentpid == pid { - print_traces(childpid, config, Some(parentpid))?; + print_traces(childpid, &config, Some(parentpid))?; } } } } Ok(()) } - -pub fn print_trace(trace: &StackTrace, include_activity: bool) { - let thread_id = trace.format_threadid(); - - let status = if include_activity { - format!(" ({})", trace.status_str()) - } else if trace.owns_gil { - " (gil)".to_owned() - } else { - "".to_owned() - }; - - match trace.thread_name.as_ref() { - Some(name) => { - println!( - "Thread {}{}: \"{}\"", - style(thread_id).bold().yellow(), - status, - name - ); - } - None => { - println!("Thread {}{}", style(thread_id).bold().yellow(), status); - } - }; - - for frame in &trace.frames { - let filename = match &frame.short_filename { - Some(f) => f, - None => &frame.filename, - }; - if frame.line != 0 { - println!( - " {} ({}:{})", - style(&frame.name).green(), - style(&filename).cyan(), - style(frame.line).dim() - ); - } else { - println!( - " {} ({})", - style(&frame.name).green(), - style(&filename).cyan() - ); - } - - if let Some(locals) = &frame.locals { - let mut shown_args = false; - let mut shown_locals = false; - for local in locals { - if local.arg && !shown_args { - println!(" {}", style("Arguments:").dim()); - shown_args = true; - } else if !local.arg && !shown_locals { - println!(" {}", style("Locals:").dim()); - shown_locals = true; - } - - let repr = local.repr.as_deref().unwrap_or("?"); - println!(" {}: {}", local.name, repr); - } - } - } -} diff --git a/src/flamegraph.rs b/src/flamegraph.rs index 795f0503..ef33c379 100644 --- a/src/flamegraph.rs +++ b/src/flamegraph.rs @@ -26,8 +26,10 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -use std::collections::HashMap; use std::io::Write; +use std; +use std::collections::HashMap; + use anyhow::Error; use inferno::flamegraph::{Direction, Options}; @@ -41,47 +43,32 @@ pub struct Flamegraph { impl Flamegraph { pub fn new(show_linenumbers: bool) -> Flamegraph { - Flamegraph { - counts: HashMap::new(), - show_linenumbers, - } + Flamegraph { counts: HashMap::new(), show_linenumbers } } pub fn increment(&mut self, trace: &StackTrace) -> std::io::Result<()> { // convert the frame into a single ';' delimited String - let frame = trace - .frames - .iter() - .rev() - .map(|frame| { - let filename = match &frame.short_filename { - Some(f) => f, - None => &frame.filename, - }; - if self.show_linenumbers && frame.line != 0 { - format!("{} ({}:{})", frame.name, filename, frame.line) - } else if !filename.is_empty() { - format!("{} ({})", frame.name, filename) - } else { - frame.name.clone() - } - }) - .collect::>() - .join(";"); + let frame = trace.frames.iter().rev().map(|frame| { + let filename = match &frame.short_filename { Some(f) => &f, None => &frame.filename }; + if self.show_linenumbers && frame.line != 0 { + format!("{} ({}:{})", frame.name, filename, frame.line) + } else if filename.len() > 0 { + format!("{} ({})", frame.name, filename) + } else { + frame.name.clone() + } + }).collect::>().join(";"); // update counts for that frame *self.counts.entry(frame).or_insert(0) += 1; Ok(()) } fn get_lines(&self) -> Vec { - self.counts - .iter() - .map(|(k, v)| format!("{} {}", k, v)) - .collect() + self.counts.iter().map(|(k, v)| format!("{} {}", k, v)).collect() } pub fn write(&self, w: &mut dyn Write) -> Result<(), Error> { - let mut opts = Options::default(); + let mut opts = Options::default(); opts.direction = Direction::Inverted; opts.min_width = 0.1; opts.title = std::env::args().collect::>().join(" "); diff --git a/src/lib.rs b/src/lib.rs index 424e253e..8373c3d2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -29,29 +29,25 @@ extern crate anyhow; #[macro_use] extern crate log; -pub mod binary_parser; pub mod config; -#[cfg(target_os = "linux")] -pub mod coredump; +pub mod binary_parser; #[cfg(unwind)] mod cython; -pub mod dump; #[cfg(unwind)] mod native_stack_trace; mod python_bindings; -mod python_data_access; mod python_interpreters; -pub mod python_process_info; -pub mod python_spy; +mod python_spy; +mod python_data_access; mod python_threading; pub mod sampler; -pub mod stack_trace; +mod stack_trace; pub mod timer; mod utils; mod version; -pub use config::Config; pub use python_spy::PythonSpy; -pub use remoteprocess::Pid; -pub use stack_trace::Frame; +pub use config::Config; pub use stack_trace::StackTrace; +pub use stack_trace::Frame; +pub use remoteprocess::Pid; diff --git a/src/main.rs b/src/main.rs index f965cc71..ca3a10a0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,27 +3,23 @@ extern crate anyhow; #[macro_use] extern crate log; -mod binary_parser; -mod chrometrace; mod config; -mod console_viewer; -#[cfg(target_os = "linux")] -mod coredump; +mod dump; +mod binary_parser; #[cfg(unwind)] mod cython; -mod dump; -mod flamegraph; #[cfg(unwind)] mod native_stack_trace; mod python_bindings; -mod python_data_access; mod python_interpreters; -mod python_process_info; mod python_spy; +mod python_data_access; mod python_threading; -mod sampler; -mod speedscope; mod stack_trace; +mod console_viewer; +mod flamegraph; +mod speedscope; +mod sampler; mod timer; mod utils; mod version; @@ -33,40 +29,40 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::Duration; -use anyhow::Error; use console::style; +use anyhow::Error; -use config::{Config, FileFormat, RecordDuration}; +use stack_trace::{StackTrace, Frame}; use console_viewer::ConsoleViewer; -use stack_trace::{Frame, StackTrace}; +use config::{Config, FileFormat, RecordDuration}; -use chrono::{Local, SecondsFormat}; +use chrono::{SecondsFormat, Local}; #[cfg(unix)] fn permission_denied(err: &Error) -> bool { err.chain().any(|cause| { if let Some(ioerror) = cause.downcast_ref::() { ioerror.kind() == std::io::ErrorKind::PermissionDenied - } else if let Some(remoteprocess::Error::IOError(ioerror)) = - cause.downcast_ref::() - { + } else if let Some(remoteprocess::Error::IOError(ioerror)) = cause.downcast_ref::() { ioerror.kind() == std::io::ErrorKind::PermissionDenied - } else { + }else { false } }) } -fn sample_console(pid: remoteprocess::Pid, config: &Config) -> Result<(), Error> { +fn sample_console(pid: remoteprocess::Pid, + config: &Config) -> Result<(), Error> { let sampler = sampler::Sampler::new(pid, config)?; let display = match remoteprocess::Process::new(pid)?.cmdline() { Ok(cmdline) => cmdline.join(" "), - Err(_) => format!("Pid {}", pid), + Err(_) => format!("Pid {}", pid) }; - let mut console = - ConsoleViewer::new(config.show_line_numbers, &display, &sampler.version, config)?; + let mut console = ConsoleViewer::new(config.show_line_numbers, &display, + &sampler.version, + config)?; for sample in sampler { if let Some(elapsed) = sample.late { console.increment_late_sample(elapsed); @@ -109,15 +105,6 @@ impl Recorder for flamegraph::Flamegraph { } } -impl Recorder for chrometrace::Chrometrace { - fn increment(&mut self, trace: &StackTrace) -> Result<(), Error> { - Ok(self.increment(trace)?) - } - fn write(&self, w: &mut dyn Write) -> Result<(), Error> { - self.write(w) - } -} - pub struct RawFlamegraph(flamegraph::Flamegraph); impl Recorder for RawFlamegraph { @@ -132,17 +119,10 @@ impl Recorder for RawFlamegraph { fn record_samples(pid: remoteprocess::Pid, config: &Config) -> Result<(), Error> { let mut output: Box = match config.format { - Some(FileFormat::flamegraph) => { - Box::new(flamegraph::Flamegraph::new(config.show_line_numbers)) - } - Some(FileFormat::speedscope) => Box::new(speedscope::Stats::new(config)), - Some(FileFormat::raw) => Box::new(RawFlamegraph(flamegraph::Flamegraph::new( - config.show_line_numbers, - ))), - Some(FileFormat::chrometrace) => { - Box::new(chrometrace::Chrometrace::new(config.show_line_numbers)) - } - None => return Err(format_err!("A file format is required to record samples")), + Some(FileFormat::flamegraph) => Box::new(flamegraph::Flamegraph::new(config.show_line_numbers)), + Some(FileFormat::speedscope) => Box::new(speedscope::Stats::new(config)), + Some(FileFormat::raw) => Box::new(RawFlamegraph(flamegraph::Flamegraph::new(config.show_line_numbers))), + None => return Err(format_err!("A file format is required to record samples")) }; let filename = match config.filename.clone() { @@ -152,19 +132,18 @@ fn record_samples(pid: remoteprocess::Pid, config: &Config) -> Result<(), Error> Some(FileFormat::flamegraph) => "svg", Some(FileFormat::speedscope) => "json", Some(FileFormat::raw) => "txt", - Some(FileFormat::chrometrace) => "json", - None => return Err(format_err!("A file format is required to record samples")), + None => return Err(format_err!("A file format is required to record samples")) }; let local_time = Local::now().to_rfc3339_opts(SecondsFormat::Secs, true); let name = match config.python_program.as_ref() { Some(prog) => prog[0].to_string(), - None => match config.pid.as_ref() { + None => match config.pid.as_ref() { Some(pid) => pid.to_string(), - None => String::from("unknown"), - }, + None => String::from("unknown") + } }; format!("{}-{}.{}", name, local_time, ext) - } + } }; let sampler = sampler::Sampler::new(pid, config)?; @@ -180,17 +159,11 @@ fn record_samples(pid: remoteprocess::Pid, config: &Config) -> Result<(), Error> let max_intervals = match &config.duration { RecordDuration::Unlimited => { - println!( - "{}Sampling process {} times a second. Press Control-C to exit.", - lede, config.sampling_rate - ); + println!("{}Sampling process {} times a second. Press Control-C to exit.", lede, config.sampling_rate); None - } + }, RecordDuration::Seconds(sec) => { - println!( - "{}Sampling process {} times a second for {} seconds. Press Control-C to exit.", - lede, config.sampling_rate, sec - ); + println!("{}Sampling process {} times a second for {} seconds. Press Control-C to exit.", lede, config.sampling_rate, sec); Some(sec * config.sampling_rate) } }; @@ -200,17 +173,12 @@ fn record_samples(pid: remoteprocess::Pid, config: &Config) -> Result<(), Error> (true, _) => ProgressBar::hidden(), (false, RecordDuration::Seconds(samples)) => ProgressBar::new(*samples), (false, RecordDuration::Unlimited) => { - #[allow(clippy::let_and_return)] let progress = ProgressBar::new_spinner(); // The spinner on windows doesn't look great: was replaced by a [?] character at least on // my system. Replace unicode spinners with just how many seconds have elapsed #[cfg(windows)] - progress.set_style( - indicatif::ProgressStyle::default_spinner() - .template("[{elapsed}] {msg}") - .unwrap(), - ); + progress.set_style(indicatif::ProgressStyle::default_spinner().template("[{elapsed}] {msg}")); progress } }; @@ -272,23 +240,12 @@ fn record_samples(pid: remoteprocess::Pid, config: &Config) -> Result<(), Error> if config.include_thread_ids { let threadid = trace.format_threadid(); - let thread_fmt = if let Some(thread_name) = &trace.thread_name { - format!("thread ({}): {}", threadid, thread_name) - } else { - format!("thread ({})", threadid) - }; - trace.frames.push(Frame { - name: thread_fmt, + trace.frames.push(Frame{name: format!("thread ({})", threadid), filename: String::from(""), - module: None, - short_filename: None, - line: 0, - locals: None, - is_entry: true, - }); + module: None, short_filename: None, line: 0, locals: None}); } - if let Some(process_info) = trace.process_info.as_ref() { + if let Some(process_info) = trace.process_info.as_ref().map(|x| x) { trace.frames.push(process_info.to_frame()); let mut parent = process_info.parent.as_ref(); while parent.is_some() { @@ -300,7 +257,7 @@ fn record_samples(pid: remoteprocess::Pid, config: &Config) -> Result<(), Error> } samples += 1; - output.increment(trace)?; + output.increment(&trace)?; } if let Some(sampling_errors) = sample.sampling_errors { @@ -327,43 +284,27 @@ fn record_samples(pid: remoteprocess::Pid, config: &Config) -> Result<(), Error> } { - let mut out_file = std::fs::File::create(&filename)?; - output.write(&mut out_file)?; + let mut out_file = std::fs::File::create(&filename)?; + output.write(&mut out_file)?; } match config.format.as_ref().unwrap() { FileFormat::flamegraph => { - println!( - "{}Wrote flamegraph data to '{}'. Samples: {} Errors: {}", - lede, filename, samples, errors - ); + println!("{}Wrote flamegraph data to '{}'. Samples: {} Errors: {}", lede, filename, samples, errors); // open generated flame graph in the browser on OSX (theory being that on linux // you might be SSH'ed into a server somewhere and this isn't desired, but on // that is pretty unlikely for osx) (note to self: xdg-open will open on linux) #[cfg(target_os = "macos")] std::process::Command::new("open").arg(&filename).spawn()?; - } - FileFormat::speedscope => { - println!( - "{}Wrote speedscope file to '{}'. Samples: {} Errors: {}", - lede, filename, samples, errors - ); + }, + FileFormat::speedscope => { + println!("{}Wrote speedscope file to '{}'. Samples: {} Errors: {}", lede, filename, samples, errors); println!("{}Visit https://www.speedscope.app/ to view", lede); - } + }, FileFormat::raw => { - println!( - "{}Wrote raw flamegraph data to '{}'. Samples: {} Errors: {}", - lede, filename, samples, errors - ); + println!("{}Wrote raw flamegraph data to '{}'. Samples: {} Errors: {}", lede, filename, samples, errors); println!("{}You can use the flamegraph.pl script from https://github.com/brendangregg/flamegraph to generate a SVG", lede); } - FileFormat::chrometrace => { - println!( - "{}Wrote chrome trace to '{}'. Samples: {} Errors: {}", - lede, filename, samples, errors - ); - println!("{}Visit chrome://tracing to view", lede); - } }; Ok(()) @@ -371,12 +312,12 @@ fn record_samples(pid: remoteprocess::Pid, config: &Config) -> Result<(), Error> fn run_spy_command(pid: remoteprocess::Pid, config: &config::Config) -> Result<(), Error> { match config.command.as_ref() { - "dump" => { + "dump" => { dump::print_traces(pid, config, None)?; - } + }, "record" => { record_samples(pid, config)?; - } + }, "top" => { sample_console(pid, config)?; } @@ -391,7 +332,7 @@ fn run_spy_command(pid: remoteprocess::Pid, config: &config::Config) -> Result<( fn pyspy_main() -> Result<(), Error> { let config = config::Config::from_commandline(); - #[cfg(target_os = "macos")] + #[cfg(target_os="macos")] { if unsafe { libc::geteuid() } != 0 { eprintln!("This program requires root on OSX."); @@ -400,18 +341,11 @@ fn pyspy_main() -> Result<(), Error> { } } - #[cfg(target_os = "linux")] - { - if let Some(ref core_filename) = config.core_filename { - let core = coredump::PythonCoreDump::new(std::path::Path::new(&core_filename))?; - let traces = core.get_stack(&config)?; - return core.print_traces(&traces, &config); - } - } - if let Some(pid) = config.pid { run_spy_command(pid, &config)?; - } else if let Some(ref subprocess) = config.python_program { + } + + else if let Some(ref subprocess) = config.python_program { // Dump out stdout/stderr from the process to a temp file, so we can view it later if needed let mut process_output = tempfile::NamedTempFile::new()?; @@ -422,10 +356,7 @@ fn pyspy_main() -> Result<(), Error> { if unsafe { libc::geteuid() } == 0 { if let Ok(sudo_uid) = std::env::var("SUDO_UID") { use std::os::unix::process::CommandExt; - info!( - "Dropping root and running python command as {}", - std::env::var("SUDO_USER")? - ); + info!("Dropping root and running python command as {}", std::env::var("SUDO_USER")?); command.uid(sudo_uid.parse::()?); } } @@ -434,17 +365,15 @@ fn pyspy_main() -> Result<(), Error> { let mut command = command.args(&subprocess[1..]); if config.capture_output { - command = command - .stdin(std::process::Stdio::null()) + command = command.stdin(std::process::Stdio::null()) .stdout(process_output.reopen()?) .stderr(process_output.reopen()?) } - let mut command = command - .spawn() + let mut command = command.spawn() .map_err(|e| format_err!("Failed to create process '{}': {}", subprocess[0], e))?; - #[cfg(target_os = "macos")] + #[cfg(target_os="macos")] { // sleep just in case: https://jvns.ca/blog/2018/01/28/mac-freeze/ std::thread::sleep(Duration::from_millis(50)); @@ -453,10 +382,10 @@ fn pyspy_main() -> Result<(), Error> { // check exit code of subprocess std::thread::sleep(Duration::from_millis(1)); - let success = match command.try_wait()? { + let success = match command.try_wait()? { Some(exit) => exit.success(), // if process hasn't finished, assume success - None => true, + None => true }; // if we failed for any reason, dump out stderr from child process here @@ -480,37 +409,34 @@ fn pyspy_main() -> Result<(), Error> { } fn main() { - env_logger::builder() - .format_timestamp_nanos() - .try_init() - .unwrap(); + env_logger::builder().format_timestamp_nanos().try_init().unwrap(); if let Err(err) = pyspy_main() { #[cfg(unix)] { - if permission_denied(&err) { - // Got a permission denied error, if we're not running as root - ask to use sudo - if unsafe { libc::geteuid() } != 0 { - eprintln!("Permission Denied: Try running again with elevated permissions by going 'sudo env \"PATH=$PATH\" !!'"); - std::process::exit(1); - } + if permission_denied(&err) { + // Got a permission denied error, if we're not running as root - ask to use sudo + if unsafe { libc::geteuid() } != 0 { + eprintln!("Permission Denied: Try running again with elevated permissions by going 'sudo env \"PATH=$PATH\" !!'"); + std::process::exit(1); + } - // We got a permission denied error running as root, check to see if we're running - // as docker, and if so ask the user to check the SYS_PTRACE capability is added - // Otherwise, fall through to the generic error handling - #[cfg(target_os = "linux")] - if let Ok(cgroups) = std::fs::read_to_string("/proc/self/cgroup") { - if cgroups.contains("/docker/") { - eprintln!("Permission Denied"); - eprintln!("\nIt looks like you are running in a docker container. Please make sure \ + // We got a permission denied error running as root, check to see if we're running + // as docker, and if so ask the user to check the SYS_PTRACE capability is added + // Otherwise, fall through to the generic error handling + #[cfg(target_os="linux")] + if let Ok(cgroups) = std::fs::read_to_string("/proc/self/cgroup") { + if cgroups.contains("/docker/") { + eprintln!("Permission Denied"); + eprintln!("\nIt looks like you are running in a docker container. Please make sure \ you started your container with the SYS_PTRACE capability. See \ https://github.com/benfred/py-spy#how-do-i-run-py-spy-in-docker for \ more details"); - std::process::exit(1); - } + std::process::exit(1); } } } + } eprintln!("Error: {}", err); for (i, suberror) in err.chain().enumerate() { diff --git a/src/native_stack_trace.rs b/src/native_stack_trace.rs index ba7e6c64..d4dd1981 100644 --- a/src/native_stack_trace.rs +++ b/src/native_stack_trace.rs @@ -1,15 +1,14 @@ -use anyhow::Error; use std::collections::HashSet; -use std::num::NonZeroUsize; +use anyhow::Error; -use cpp_demangle::{BorrowedSymbol, DemangleOptions}; +use cpp_demangle::{DemangleOptions, BorrowedSymbol}; +use remoteprocess::{self, Pid}; use lazy_static::lazy_static; use lru::LruCache; -use remoteprocess::{self, Pid}; use crate::binary_parser::BinaryInfo; use crate::cython; -use crate::stack_trace::Frame; +use crate::stack_trace::{Frame}; use crate::utils::resolve_filename; pub struct NativeStack { @@ -26,34 +25,22 @@ pub struct NativeStack { } impl NativeStack { - pub fn new( - pid: Pid, - python: Option, - libpython: Option, - ) -> Result { + pub fn new(pid: Pid, python: Option, libpython: Option) -> Result { let cython_maps = cython::SourceMaps::new(); let process = remoteprocess::Process::new(pid)?; let unwinder = process.unwinder()?; let symbolicator = process.symbolicator()?; - Ok(NativeStack { - cython_maps, - unwinder, - symbolicator, - should_reload: false, - python, - libpython, - process, - symbol_cache: LruCache::new(NonZeroUsize::new(65536).unwrap()), - }) + return Ok(NativeStack{cython_maps, unwinder, symbolicator, should_reload: false, + python, + libpython, + process, + symbol_cache: LruCache::new(65536) + }); } - pub fn merge_native_thread( - &mut self, - frames: &Vec, - thread: &remoteprocess::Thread, - ) -> Result, Error> { + pub fn merge_native_thread(&mut self, frames: &Vec, thread: &remoteprocess::Thread) -> Result, Error> { if self.should_reload { self.symbolicator.reload()?; self.should_reload = false; @@ -63,46 +50,34 @@ impl NativeStack { let native_stack = self.get_thread(thread)?; // TODO: merging the two stack together could happen outside of thread lock - self.merge_native_stack(frames, native_stack) + return self.merge_native_stack(frames, native_stack); } - pub fn merge_native_stack( - &mut self, - frames: &Vec, - native_stack: Vec, - ) -> Result, Error> { + pub fn merge_native_stack(&mut self, frames: &Vec, native_stack: Vec) -> Result, Error> { let mut python_frame_index = 0; let mut merged = Vec::new(); // merge the native_stack and python stack together for addr in native_stack { // check in the symbol cache if we have looked up this symbol yet - let cached_symbol = self.symbol_cache.get(&addr).cloned(); + let cached_symbol = self.symbol_cache.get(&addr).map(|f| f.clone()); // merges a remoteprocess::StackFrame into the current merged vec - let is_python_addr = self.python.as_ref().map_or(false, |m| m.contains(addr)) - || self.libpython.as_ref().map_or(false, |m| m.contains(addr)); + let is_python_addr = self.python.as_ref().map_or(false, |m| m.contains(addr)) || + self.libpython.as_ref().map_or(false, |m| m.contains(addr)); let merge_frame = &mut |frame: &remoteprocess::StackFrame| { match self.get_merge_strategy(is_python_addr, frame) { - MergeType::Ignore => {} + MergeType::Ignore => {}, MergeType::MergeNativeFrame => { if let Some(python_frame) = self.translate_native_frame(frame) { merged.push(python_frame); } - } + }, MergeType::MergePythonFrame => { // if we have a corresponding python frame for the evalframe // merge it into the stack. (if we're out of bounds a later // check will pick up - and report overall totals mismatch) - - // Merge all python frames until we hit one with `is_entry`. - while python_frame_index < frames.len() { + if python_frame_index < frames.len() { merged.push(frames[python_frame_index].clone()); - - if frames[python_frame_index].is_entry { - break; - } - - python_frame_index += 1; } python_frame_index += 1; } @@ -121,37 +96,22 @@ impl NativeStack { let mut symbolicated_count = 0; let mut first_frame = None; - self.symbolicator - .symbolicate( - addr, - !is_python_addr, - &mut |frame: &remoteprocess::StackFrame| { - symbolicated_count += 1; - if symbolicated_count == 1 { - first_frame = Some(frame.clone()); - } - merge_frame(frame); - }, - ) - .unwrap_or_else(|e| { - if let remoteprocess::Error::NoBinaryForAddress(_) = e { - debug!( - "don't have a binary for symbols at 0x{:x} - reloading", - addr - ); - self.should_reload = true; - } - // if we can't symbolicate, just insert a stub here. - merged.push(Frame { - filename: "?".to_owned(), - name: format!("0x{:x}", addr), - line: 0, - short_filename: None, - module: None, - locals: None, - is_entry: true, - }); - }); + self.symbolicator.symbolicate(addr, !is_python_addr, &mut |frame: &remoteprocess::StackFrame| { + symbolicated_count += 1; + if symbolicated_count == 1 { + first_frame = Some(frame.clone()); + } + merge_frame(frame); + }).unwrap_or_else(|e| { + if let remoteprocess::Error::NoBinaryForAddress(_) = e { + debug!("don't have a binary for symbols at 0x{:x} - reloading", addr); + self.should_reload = true; + } + // if we can't symbolicate, just insert a stub here. + merged.push(Frame{filename: "?".to_owned(), + name: format!("0x{:x}", addr), + line: 0, short_filename: None, module: None, locals: None}); + }); if symbolicated_count == 1 { self.symbol_cache.put(addr, first_frame.unwrap()); @@ -174,17 +134,11 @@ impl NativeStack { // if we have seen exactly one more python frame in the native stack than the python stack - let it go. // (can happen when the python stack has been unwound, but haven't exited the PyEvalFrame function // yet) - info!( - "Have {} native and {} python threads in stack - allowing for now", - python_frame_index, - frames.len() - ); + info!("Have {} native and {} python threads in stack - allowing for now", + python_frame_index, frames.len()); } else { - return Err(format_err!( - "Failed to merge native and python frames (Have {} native and {} python)", - python_frame_index, - frames.len() - )); + return Err(format_err!("Failed to merge native and python frames (Have {} native and {} python)", + python_frame_index, frames.len())); } } @@ -196,11 +150,7 @@ impl NativeStack { Ok(merged) } - fn get_merge_strategy( - &self, - check_python: bool, - frame: &remoteprocess::StackFrame, - ) -> MergeType { + fn get_merge_strategy(&self, check_python: bool, frame: &remoteprocess::StackFrame) -> MergeType { if check_python { if let Some(ref function) = frame.function { // We want to include some internal python functions. For example, calls like time.sleep @@ -230,17 +180,17 @@ impl NativeStack { // which is replaced by the function from the python stack // note: we're splitting on both _ and . to handle symbols like // _PyEval_EvalFrameDefault.cold.2962 - let mut tokens = function.split(&['_', '.'][..]).filter(|&x| !x.is_empty()); + let mut tokens = function.split(&['_', '.'][..]).filter(|&x| x.len() > 0); match tokens.next() { - Some("PyEval") => match tokens.next() { - Some("EvalFrameDefault") => MergeType::MergePythonFrame, - Some("EvalFrameEx") => MergeType::MergePythonFrame, - _ => MergeType::Ignore, + Some("PyEval") => { + match tokens.next() { + Some("EvalFrameDefault") => MergeType::MergePythonFrame, + Some("EvalFrameEx") => MergeType::MergePythonFrame, + _ => MergeType::Ignore + } }, - Some(prefix) if WHITELISTED_PREFIXES.contains(prefix) => { - MergeType::MergeNativeFrame - } - _ => MergeType::Ignore, + Some(prefix) if WHITELISTED_PREFIXES.contains(prefix) => MergeType::MergeNativeFrame, + _ => MergeType::Ignore } } else { // is this correct? if we don't have a function name and in python binary should ignore? @@ -254,7 +204,7 @@ impl NativeStack { /// translates a native frame into a optional frame. none indicates we should ignore this frame fn translate_native_frame(&self, frame: &remoteprocess::StackFrame) -> Option { match &frame.function { - Some(func) => { + Some(func) => { if ignore_frame(func, &frame.module) { return None; } @@ -264,9 +214,11 @@ impl NativeStack { // try to resolve the filename relative to the module if given let filename = match frame.filename.as_ref() { - Some(filename) => resolve_filename(filename, &frame.module) - .unwrap_or_else(|| filename.clone()), - None => frame.module.clone(), + Some(filename) => { + resolve_filename(filename, &frame.module) + .unwrap_or_else(|| filename.clone()) + }, + None => frame.module.clone() }; let mut demangled = None; @@ -278,30 +230,18 @@ impl NativeStack { } } } - let name = demangled.as_ref().unwrap_or(func); + let name = demangled.as_ref().unwrap_or_else(|| &func); if cython::ignore_frame(name) { return None; } - let name = cython::demangle(name).to_owned(); - Some(Frame { - filename, - line, - name, - short_filename: None, - module: Some(frame.module.clone()), - locals: None, - is_entry: true, - }) + let name = cython::demangle(&name).to_owned(); + Some(Frame{filename, line, name, short_filename: None, module: Some(frame.module.clone()), locals: None}) + }, + None => { + Some(Frame{filename: frame.module.clone(), + name: format!("0x{:x}", frame.addr), locals: None, + line: 0, short_filename: None, module: Some(frame.module.clone())}) } - None => Some(Frame { - filename: frame.module.clone(), - name: format!("0x{:x}", frame.addr), - locals: None, - line: 0, - short_filename: None, - module: Some(frame.module.clone()), - is_entry: true, - }), } } @@ -318,12 +258,12 @@ impl NativeStack { enum MergeType { Ignore, MergePythonFrame, - MergeNativeFrame, + MergeNativeFrame } // the intent here is to remove top-level libc or pthreads calls // from the stack traces. This almost certainly can be done better -#[cfg(target_os = "linux")] +#[cfg(target_os="linux")] fn ignore_frame(function: &str, module: &str) -> bool { if function == "__libc_start_main" && module.contains("/libc") { return true; @@ -340,7 +280,7 @@ fn ignore_frame(function: &str, module: &str) -> bool { false } -#[cfg(target_os = "macos")] +#[cfg(target_os="macos")] fn ignore_frame(function: &str, module: &str) -> bool { if function == "_start" && module.contains("/libdyld.dylib") { return true; diff --git a/src/python_bindings/mod.rs b/src/python_bindings/mod.rs index 420e3c9a..e69156a0 100644 --- a/src/python_bindings/mod.rs +++ b/src/python_bindings/mod.rs @@ -1,12 +1,12 @@ pub mod v2_7_15; -pub mod v3_10_0; -pub mod v3_11_0; pub mod v3_3_7; pub mod v3_5_5; pub mod v3_6_6; pub mod v3_7_0; pub mod v3_8_0; pub mod v3_9_5; +pub mod v3_10_0; +pub mod v3_11_0; // currently the PyRuntime struct used from Python 3.7 on really can't be // exposed in a cross platform way using bindgen. PyRuntime has several mutex's @@ -23,231 +23,125 @@ pub mod pyruntime { #[cfg(target_arch = "x86")] pub fn get_interp_head_offset(version: &Version) -> usize { match version { - Version { - major: 3, - minor: 8, - patch: 0, - .. - } => match version.release_flags.as_ref() { - "a1" | "a2" => 16, - "a3" | "a4" => 20, - _ => 24, + Version{major: 3, minor: 8, patch: 0, ..} => { + match version.release_flags.as_ref() { + "a1" | "a2" => 16, + "a3" | "a4" => 20, + _ => 24 + } }, - Version { - major: 3, - minor: 8..=10, - .. - } => 24, - _ => 16, + Version{major: 3, minor: 8..=10, ..} => 24, + _ => 16 } } #[cfg(target_arch = "arm")] pub fn get_interp_head_offset(version: &Version) -> usize { match version { - Version { - major: 3, minor: 7, .. - } => 20, - _ => 28, + Version{major: 3, minor: 7, ..} => 20, + _ => 28 } } #[cfg(target_pointer_width = "64")] pub fn get_interp_head_offset(version: &Version) -> usize { match version { - Version { - major: 3, - minor: 8, - patch: 0, - .. - } => match version.release_flags.as_ref() { - "a1" | "a2" => 24, - _ => 32, + Version{major: 3, minor: 8, patch: 0, ..} => { + match version.release_flags.as_ref() { + "a1" | "a2" => 24, + _ => 32 + } }, - Version { - major: 3, - minor: 8..=10, - .. - } => 32, - Version { - major: 3, - minor: 11, - .. - } => 40, - _ => 24, + Version{major: 3, minor: 8..=10, ..} => 32, + Version{major: 3, minor: 11, ..} => 40, + _ => 24 } } // getting gilstate.tstate_current is different for all OS // and is also different for each python version, and even // between v3.8.0a1 and v3.8.0a2 =( - #[cfg(target_os = "macos")] + #[cfg(target_os="macos")] pub fn get_tstate_current_offset(version: &Version) -> Option { match version { - Version { - major: 3, - minor: 7, - patch: 0..=3, - .. - } => Some(1440), - Version { - major: 3, minor: 7, .. - } => Some(1528), - Version { - major: 3, - minor: 8, - patch: 0, - .. - } => match version.release_flags.as_ref() { - "a1" => Some(1432), - "a2" => Some(888), - "a3" | "a4" => Some(1448), - _ => Some(1416), + Version{major: 3, minor: 7, patch: 0..=3, ..} => Some(1440), + Version{major: 3, minor: 7, ..} => Some(1528), + Version{major: 3, minor: 8, patch: 0, ..} => { + match version.release_flags.as_ref() { + "a1" => Some(1432), + "a2" => Some(888), + "a3" | "a4" => Some(1448), + _ => Some(1416), + } }, - Version { - major: 3, minor: 8, .. - } => Some(1416), - Version { - major: 3, - minor: 9..=10, - .. - } => Some(616), - Version { - major: 3, - minor: 11, - .. - } => Some(624), - _ => None, + Version{major: 3, minor: 8, ..} => { Some(1416) }, + Version{major: 3, minor: 9..=10, ..} => { Some(616) }, + Version{major: 3, minor: 11, ..} => Some(624), + _ => None } } - #[cfg(all(target_os = "linux", target_arch = "x86"))] + #[cfg(all(target_os="linux", target_arch="x86"))] pub fn get_tstate_current_offset(version: &Version) -> Option { match version { - Version { - major: 3, minor: 7, .. - } => Some(796), - Version { - major: 3, - minor: 8, - patch: 0, - .. - } => match version.release_flags.as_ref() { - "a1" => Some(792), - "a2" => Some(512), - "a3" | "a4" => Some(800), - _ => Some(788), + Version{major: 3, minor: 7, ..} => Some(796), + Version{major: 3, minor: 8, patch: 0, ..} => { + match version.release_flags.as_ref() { + "a1" => Some(792), + "a2" => Some(512), + "a3" | "a4" => Some(800), + _ => Some(788) + } }, - Version { - major: 3, minor: 8, .. - } => Some(788), - Version { - major: 3, - minor: 9..=10, - .. - } => Some(352), - _ => None, + Version{major: 3, minor: 8, ..} => Some(788), + Version{major: 3, minor: 9..=10, ..} => Some(352), + _ => None } } - #[cfg(all(target_os = "linux", target_arch = "arm"))] + #[cfg(all(target_os="linux", target_arch="arm"))] pub fn get_tstate_current_offset(version: &Version) -> Option { match version { - Version { - major: 3, minor: 7, .. - } => Some(828), - Version { - major: 3, minor: 8, .. - } => Some(804), - Version { - major: 3, - minor: 9..=11, - .. - } => Some(364), - _ => None, + Version{major: 3, minor: 7, ..} => Some(828), + Version{major: 3, minor: 8, ..} => Some(804), + Version{major: 3, minor: 9..=11, ..} => Some(364), + _ => None } } - #[cfg(all(target_os = "linux", target_arch = "aarch64"))] + #[cfg(all(target_os="linux", target_arch="aarch64"))] pub fn get_tstate_current_offset(version: &Version) -> Option { match version { - Version { - major: 3, - minor: 7, - patch: 0..=3, - .. - } => Some(1408), - Version { - major: 3, minor: 7, .. - } => Some(1496), - Version { - major: 3, minor: 8, .. - } => Some(1384), - Version { - major: 3, - minor: 9..=10, - .. - } => Some(584), - Version { - major: 3, - minor: 11, - .. - } => Some(592), - _ => None, + Version{major: 3, minor: 7, patch: 0..=3, ..} => Some(1408), + Version{major: 3, minor: 7, ..} => Some(1496), + Version{major: 3, minor: 8, ..} => Some(1384), + Version{major: 3, minor: 9..=10, ..} => Some(584), + Version{major: 3, minor: 11, ..} => Some(592), + _ => None } } - #[cfg(all(target_os = "linux", target_arch = "x86_64"))] + #[cfg(all(target_os="linux", target_arch="x86_64"))] pub fn get_tstate_current_offset(version: &Version) -> Option { match version { - Version { - major: 3, - minor: 7, - patch: 0..=3, - .. - } => Some(1392), - Version { - major: 3, minor: 7, .. - } => Some(1480), - Version { - major: 3, - minor: 8, - patch: 0, - .. - } => match version.release_flags.as_ref() { - "a1" => Some(1384), - "a2" => Some(840), - "a3" | "a4" => Some(1400), - _ => Some(1368), - }, - Version { - major: 3, minor: 8, .. - } => match version.build_metadata.as_deref() { - Some("cinder") => Some(1384), - _ => Some(1368), - }, - Version { - major: 3, - minor: 9..=10, - .. - } => Some(568), - Version { - major: 3, - minor: 11, - .. - } => Some(576), - _ => None, + Version{major: 3, minor: 7, patch: 0..=3, ..} => Some(1392), + Version{major: 3, minor: 7, ..} => Some(1480), + Version{major: 3, minor: 8, patch: 0, ..} => { + match version.release_flags.as_ref() { + "a1" => Some(1384), + "a2" => Some(840), + "a3" | "a4" => Some(1400), + _ => Some(1368) + } + }, + Version{major: 3, minor: 8, ..} => Some(1368), + Version{major: 3, minor: 9..=10, ..} => Some(568), + Version{major: 3, minor: 11, ..} => Some(576), + _ => None } } - #[cfg(all( - target_os = "linux", - any( - target_arch = "powerpc64", - target_arch = "powerpc", - target_arch = "mips" - ) - ))] + #[cfg(all(target_os="linux", any(target_arch="powerpc64", target_arch="powerpc", target_arch="mips")))] pub fn get_tstate_current_offset(version: &Version) -> Option { None } @@ -255,80 +149,39 @@ pub mod pyruntime { #[cfg(windows)] pub fn get_tstate_current_offset(version: &Version) -> Option { match version { - Version { - major: 3, - minor: 7, - patch: 0..=3, - .. - } => Some(1320), - Version { - major: 3, - minor: 8, - patch: 0, - .. - } => match version.release_flags.as_ref() { - "a1" => Some(1312), - "a2" => Some(768), - "a3" | "a4" => Some(1328), - _ => Some(1296), + Version{major: 3, minor: 7, patch: 0..=3, ..} => Some(1320), + Version{major: 3, minor: 8, patch: 0, ..} => { + match version.release_flags.as_ref() { + "a1" => Some(1312), + "a2" => Some(768), + "a3" | "a4" => Some(1328), + _ => Some(1296) + } }, - Version { - major: 3, minor: 8, .. - } => Some(1296), - Version { - major: 3, - minor: 9..=10, - .. - } => Some(496), - Version { - major: 3, - minor: 11, - .. - } => Some(504), - _ => None, + Version{major: 3, minor: 8, ..} => Some(1296), + Version{major: 3, minor: 9..=10, ..} => Some(496), + Version{major: 3, minor: 11, ..} => Some(504), + _ => None } } - #[cfg(target_os = "freebsd")] + #[cfg(target_os="freebsd")] pub fn get_tstate_current_offset(version: &Version) -> Option { match version { - Version { - major: 3, - minor: 7, - patch: 0..=3, - .. - } => Some(1248), - Version { - major: 3, - minor: 7, - patch: 4..=7, - .. - } => Some(1336), - Version { - major: 3, - minor: 8, - patch: 0, - .. - } => match version.release_flags.as_ref() { - "a1" => Some(1240), - "a2" => Some(696), - "a3" | "a4" => Some(1256), - _ => Some(1224), + Version{major: 3, minor: 7, patch: 0..=3, ..} => Some(1248), + Version{major: 3, minor: 7, patch: 4..=7, ..} => Some(1336), + Version{major: 3, minor: 8, patch: 0, ..} => { + match version.release_flags.as_ref() { + "a1" => Some(1240), + "a2" => Some(696), + "a3" | "a4" => Some(1256), + _ => Some(1224) + } }, - Version { - major: 3, minor: 8, .. - } => Some(1224), - Version { - major: 3, - minor: 9..=10, - .. - } => Some(424), - Version { - major: 3, - minor: 11, - .. - } => Some(432), - _ => None, + Version{major: 3, minor: 8, ..} => Some(1224), + Version{major: 3, minor: 9..=10, ..} => Some(424), + Version{major: 3, minor: 11, ..} => Some(432), + _ => None } } } diff --git a/src/python_bindings/v2_7_15.rs b/src/python_bindings/v2_7_15.rs index 0b5e7afd..839e7ee9 100644 --- a/src/python_bindings/v2_7_15.rs +++ b/src/python_bindings/v2_7_15.rs @@ -7,7 +7,6 @@ #![allow(clippy::default_trait_access)] #![allow(clippy::cast_lossless)] #![allow(clippy::trivially_copy_pass_by_ref)] -#![allow(clippy::upper_case_acronyms)] /* automatically generated by rust-bindgen */ diff --git a/src/python_bindings/v3_10_0.rs b/src/python_bindings/v3_10_0.rs index 37611203..74da9649 100644 --- a/src/python_bindings/v3_10_0.rs +++ b/src/python_bindings/v3_10_0.rs @@ -7,7 +7,6 @@ #![allow(clippy::default_trait_access)] #![allow(clippy::cast_lossless)] #![allow(clippy::trivially_copy_pass_by_ref)] -#![allow(clippy::upper_case_acronyms)] /* automatically generated by rust-bindgen */ diff --git a/src/python_bindings/v3_11_0.rs b/src/python_bindings/v3_11_0.rs index ad57a72e..3bbd3570 100644 --- a/src/python_bindings/v3_11_0.rs +++ b/src/python_bindings/v3_11_0.rs @@ -7,7 +7,6 @@ #![allow(clippy::default_trait_access)] #![allow(clippy::cast_lossless)] #![allow(clippy::trivially_copy_pass_by_ref)] -#![allow(clippy::upper_case_acronyms)] /* automatically generated by rust-bindgen */ diff --git a/src/python_bindings/v3_3_7.rs b/src/python_bindings/v3_3_7.rs index 542d6e2c..99f5ddd2 100644 --- a/src/python_bindings/v3_3_7.rs +++ b/src/python_bindings/v3_3_7.rs @@ -7,7 +7,6 @@ #![allow(clippy::default_trait_access)] #![allow(clippy::cast_lossless)] #![allow(clippy::trivially_copy_pass_by_ref)] -#![allow(clippy::upper_case_acronyms)] /* automatically generated by rust-bindgen */ diff --git a/src/python_bindings/v3_4_8.rs b/src/python_bindings/v3_4_8.rs index 4f39c70d..8e1e6a0b 100644 --- a/src/python_bindings/v3_4_8.rs +++ b/src/python_bindings/v3_4_8.rs @@ -7,7 +7,6 @@ #![allow(clippy::default_trait_access)] #![allow(clippy::cast_lossless)] #![allow(clippy::trivially_copy_pass_by_ref)] -#![allow(clippy::upper_case_acronyms)] /* automatically generated by rust-bindgen */ diff --git a/src/python_bindings/v3_5_5.rs b/src/python_bindings/v3_5_5.rs index 02e47980..8ee1a774 100644 --- a/src/python_bindings/v3_5_5.rs +++ b/src/python_bindings/v3_5_5.rs @@ -7,7 +7,6 @@ #![allow(clippy::default_trait_access)] #![allow(clippy::cast_lossless)] #![allow(clippy::trivially_copy_pass_by_ref)] -#![allow(clippy::upper_case_acronyms)] /* automatically generated by rust-bindgen */ diff --git a/src/python_bindings/v3_6_6.rs b/src/python_bindings/v3_6_6.rs index 2fa2721b..22c28a3f 100644 --- a/src/python_bindings/v3_6_6.rs +++ b/src/python_bindings/v3_6_6.rs @@ -7,7 +7,6 @@ #![allow(clippy::default_trait_access)] #![allow(clippy::cast_lossless)] #![allow(clippy::trivially_copy_pass_by_ref)] -#![allow(clippy::upper_case_acronyms)] /* automatically generated by rust-bindgen */ diff --git a/src/python_bindings/v3_7_0.rs b/src/python_bindings/v3_7_0.rs index 7cd13800..6c9a24f8 100644 --- a/src/python_bindings/v3_7_0.rs +++ b/src/python_bindings/v3_7_0.rs @@ -7,7 +7,6 @@ #![allow(clippy::default_trait_access)] #![allow(clippy::cast_lossless)] #![allow(clippy::trivially_copy_pass_by_ref)] -#![allow(clippy::upper_case_acronyms)] /* automatically generated by rust-bindgen */ @@ -123,7 +122,7 @@ impl ::std::fmt::Debug for __IncompleteArrayField { impl ::std::clone::Clone for __IncompleteArrayField { #[inline] fn clone(&self) -> Self { - *self + Self::new() } } impl ::std::marker::Copy for __IncompleteArrayField {} diff --git a/src/python_bindings/v3_8_0.rs b/src/python_bindings/v3_8_0.rs index bced6874..433ed6c6 100644 --- a/src/python_bindings/v3_8_0.rs +++ b/src/python_bindings/v3_8_0.rs @@ -7,7 +7,6 @@ #![allow(clippy::default_trait_access)] #![allow(clippy::cast_lossless)] #![allow(clippy::trivially_copy_pass_by_ref)] -#![allow(clippy::upper_case_acronyms)] /* automatically generated by rust-bindgen */ @@ -123,7 +122,7 @@ impl ::std::fmt::Debug for __IncompleteArrayField { impl ::std::clone::Clone for __IncompleteArrayField { #[inline] fn clone(&self) -> Self { - *self + Self::new() } } impl ::std::marker::Copy for __IncompleteArrayField {} diff --git a/src/python_bindings/v3_9_5.rs b/src/python_bindings/v3_9_5.rs index 1f80b2d7..1930e146 100644 --- a/src/python_bindings/v3_9_5.rs +++ b/src/python_bindings/v3_9_5.rs @@ -7,7 +7,6 @@ #![allow(clippy::default_trait_access)] #![allow(clippy::cast_lossless)] #![allow(clippy::trivially_copy_pass_by_ref)] -#![allow(clippy::upper_case_acronyms)] /* automatically generated by rust-bindgen */ diff --git a/src/python_data_access.rs b/src/python_data_access.rs index 4249e40a..ac5e8afa 100644 --- a/src/python_data_access.rs +++ b/src/python_data_access.rs @@ -1,23 +1,16 @@ -#![allow(clippy::unnecessary_cast)] +use std; + use anyhow::Error; -use crate::python_interpreters::{ - BytesObject, InterpreterState, ListObject, Object, StringObject, TupleObject, TypeObject, -}; -use crate::version::Version; use remoteprocess::ProcessMemory; +use crate::python_interpreters::{StringObject, BytesObject, InterpreterState, Object, TypeObject, TupleObject, ListObject}; +use crate::version::Version; /// Copies a string from a target process. Attempts to handle unicode differences, which mostly seems to be working -pub fn copy_string( - ptr: *const T, - process: &P, -) -> Result { +pub fn copy_string(ptr: * const T, process: &P) -> Result { let obj = process.copy_pointer(ptr)?; if obj.size() >= 4096 { - return Err(format_err!( - "Refusing to copy {} chars of a string", - obj.size() - )); + return Err(format_err!("Refusing to copy {} chars of a string", obj.size())); } let kind = obj.kind(); @@ -27,28 +20,23 @@ pub fn copy_string( match (kind, obj.ascii()) { (4, _) => { #[allow(clippy::cast_ptr_alignment)] - let chars = unsafe { - std::slice::from_raw_parts(bytes.as_ptr() as *const char, bytes.len() / 4) - }; + let chars = unsafe { std::slice::from_raw_parts(bytes.as_ptr() as * const char, bytes.len() / 4) }; Ok(chars.iter().collect()) - } + }, (2, _) => { // UCS2 strings aren't used internally after v3.3: https://www.python.org/dev/peps/pep-0393/ // TODO: however with python 2.7 they could be added with --enable-unicode=ucs2 configure flag. // or with python 3.2 --with-wide-unicode=ucs2 Err(format_err!("ucs2 strings aren't supported yet!")) - } + }, (1, true) => Ok(String::from_utf8(bytes)?), - (1, false) => Ok(bytes.iter().map(|&b| b as char).collect()), - _ => Err(format_err!("Unknown string kind {}", kind)), + (1, false) => Ok(bytes.iter().map(|&b| { b as char }).collect()), + _ => Err(format_err!("Unknown string kind {}", kind)) } } /// Copies data from a PyBytesObject (currently only lnotab object) -pub fn copy_bytes( - ptr: *const T, - process: &P, -) -> Result, Error> { +pub fn copy_bytes(ptr: * const T, process: &P) -> Result, Error> { let obj = process.copy_pointer(ptr)?; let size = obj.size(); if size >= 65536 { @@ -58,11 +46,10 @@ pub fn copy_bytes( } /// Copies a i64 from a PyLongObject. Returns the value + if it overflowed -pub fn copy_long(process: &P, addr: usize) -> Result<(i64, bool), Error> { +pub fn copy_long(process: &remoteprocess::Process, addr: usize) -> Result<(i64, bool), Error> { // this is PyLongObject for a specific version of python, but this works since it's binary compatible // layout across versions we're targeting - let value = - process.copy_pointer(addr as *const crate::python_bindings::v3_7_0::PyLongObject)?; + let value = process.copy_pointer(addr as *const crate::python_bindings::v3_7_0::PyLongObject)?; let negative: i64 = if value.ob_base.ob_size < 0 { -1 } else { 1 }; let size = value.ob_base.ob_size * (negative as isize); match size { @@ -88,55 +75,38 @@ pub fn copy_long(process: &P, addr: usize) -> Result<(i64, boo Ok((negative * ret, false)) } // we don't support arbitrary sized integers yet, signal this by returning that we've overflowed - _ => Ok((value.ob_base.ob_size as i64, true)), + _ => Ok((value.ob_base.ob_size as i64, true)) } } /// Copies a i64 from a python 2.7 PyIntObject -pub fn copy_int(process: &P, addr: usize) -> Result { - let value = - process.copy_pointer(addr as *const crate::python_bindings::v2_7_15::PyIntObject)?; +pub fn copy_int(process: &remoteprocess::Process, addr: usize) -> Result { + let value = process.copy_pointer(addr as *const crate::python_bindings::v2_7_15::PyIntObject)?; Ok(value.ob_ival as i64) } /// Allows iteration of a python dictionary. Only supports python 3.6+ right now -pub struct DictIterator<'a, P: 'a> { - process: &'a P, +pub struct DictIterator<'a> { + process: &'a remoteprocess::Process, entries_addr: usize, kind: u8, index: usize, entries: usize, - values: usize, + values: usize } -impl<'a, P: ProcessMemory> DictIterator<'a, P> { - pub fn from_managed_dict( - process: &'a P, - version: &'a Version, - addr: usize, - tp_addr: usize, - ) -> Result, Error> { +impl<'a> DictIterator<'a> { + pub fn from_managed_dict(process: &'a remoteprocess::Process, version: &'a Version, addr: usize, tp_addr: usize) -> Result, Error> { // Handles logic of _PyObject_ManagedDictPointer in python 3.11 let values_addr: usize = process.copy_struct(addr - 4 * std::mem::size_of::())?; let dict_addr: usize = process.copy_struct(addr - 3 * std::mem::size_of::())?; if values_addr != 0 { - let ht: crate::python_bindings::v3_11_0::PyHeapTypeObject = - process.copy_struct(tp_addr)?; - let keys: crate::python_bindings::v3_11_0::PyDictKeysObject = - process.copy_struct(ht.ht_cached_keys as usize)?; - let entries_addr = ht.ht_cached_keys as usize - + (1 << keys.dk_log2_index_bytes) - + std::mem::size_of_val(&keys); - Ok(DictIterator { - process, - entries_addr, - index: 0, - kind: keys.dk_kind, - entries: keys.dk_nentries as usize, - values: values_addr, - }) + let ht: crate::python_bindings::v3_11_0::PyHeapTypeObject = process.copy_struct(tp_addr)?; + let keys: crate::python_bindings::v3_11_0::PyDictKeysObject = process.copy_struct(ht.ht_cached_keys as usize)?; + let entries_addr = ht.ht_cached_keys as usize + (1 << keys.dk_log2_index_bytes) + std::mem::size_of_val(&keys); + Ok(DictIterator{process, entries_addr, index: 0, kind: keys.dk_kind, entries: keys.dk_nentries as usize, values: values_addr}) } else if dict_addr != 0 { DictIterator::from(process, version, dict_addr) } else { @@ -144,35 +114,16 @@ impl<'a, P: ProcessMemory> DictIterator<'a, P> { } } - pub fn from( - process: &'a P, - version: &'a Version, - addr: usize, - ) -> Result, Error> { - match version { - Version { - major: 3, - minor: 11, - .. - } => { - let dict: crate::python_bindings::v3_11_0::PyDictObject = - process.copy_struct(addr)?; + pub fn from(process: &'a remoteprocess::Process, version: &'a Version, addr: usize) -> Result, Error> { + match version { + Version{major: 3, minor: 11, ..} => { + let dict: crate::python_bindings::v3_11_0::PyDictObject = process.copy_struct(addr)?; let keys = process.copy_pointer(dict.ma_keys)?; - let entries_addr = dict.ma_keys as usize - + (1 << keys.dk_log2_index_bytes) - + std::mem::size_of_val(&keys); - Ok(DictIterator { - process, - entries_addr, - index: 0, - kind: keys.dk_kind, - entries: keys.dk_nentries as usize, - values: dict.ma_values as usize, - }) - } - _ => { - let dict: crate::python_bindings::v3_7_0::PyDictObject = - process.copy_struct(addr)?; + let entries_addr = dict.ma_keys as usize + (1 << keys.dk_log2_index_bytes) + std::mem::size_of_val(&keys); + Ok(DictIterator{process, entries_addr, index: 0, kind: keys.dk_kind, entries: keys.dk_nentries as usize, values: dict.ma_values as usize}) + }, + _ => { + let dict: crate::python_bindings::v3_7_0::PyDictObject = process.copy_struct(addr)?; // Getting this going generically is tricky: there is a lot of variation on how dictionaries are handled // instead this just focuses on a single version, which works for python // 3.6/3.7/3.8/3.9/3.10 @@ -185,25 +136,17 @@ impl<'a, P: ProcessMemory> DictIterator<'a, P> { #[cfg(target_pointer_width = "64")] _ => 8, #[cfg(not(target_pointer_width = "64"))] - _ => 4, + _ => 4 }; let byteoffset = (keys.dk_size * index_size) as usize; - let entries_addr = - dict.ma_keys as usize + byteoffset + std::mem::size_of_val(&keys); - Ok(DictIterator { - process, - entries_addr, - index: 0, - kind: 0, - entries: keys.dk_nentries as usize, - values: dict.ma_values as usize, - }) + let entries_addr = dict.ma_keys as usize + byteoffset + std::mem::size_of_val(&keys); + Ok(DictIterator{process, entries_addr, index: 0, kind: 0, entries: keys.dk_nentries as usize, values: dict.ma_values as usize}) } } } } -impl<'a, P: ProcessMemory> Iterator for DictIterator<'a, P> { +impl<'a> Iterator for DictIterator<'a> { type Item = Result<(usize, usize), Error>; fn next(&mut self) -> Option { @@ -214,23 +157,14 @@ impl<'a, P: ProcessMemory> Iterator for DictIterator<'a, P> { // get the addresses of the key/value for the current index let entry = match self.kind { 0 => { - let addr = index - * std::mem::size_of::() - + self.entries_addr; - let ret = self - .process - .copy_struct::(addr); + let addr = index * std::mem::size_of::() + self.entries_addr; + let ret = self.process.copy_struct::(addr); ret.map(|entry| (entry.me_key as usize, entry.me_value as usize)) - } + }, _ => { // Python 3.11 added a PyDictUnicodeEntry , which uses the hash from the Unicode key rather than recalculate - let addr = index - * std::mem::size_of::( - ) - + self.entries_addr; - let ret = self - .process - .copy_struct::(addr); + let addr = index * std::mem::size_of::() + self.entries_addr; + let ret = self.process.copy_struct::(addr); ret.map(|entry| (entry.me_key as usize, entry.me_value as usize)) } }; @@ -242,23 +176,20 @@ impl<'a, P: ProcessMemory> Iterator for DictIterator<'a, P> { } let value = if self.values != 0 { - let valueaddr = self.values - + index - * std::mem::size_of::<*mut crate::python_bindings::v3_7_0::PyObject>( - ); + let valueaddr = self.values + index * std::mem::size_of::<* mut crate::python_bindings::v3_7_0::PyObject>(); match self.process.copy_struct(valueaddr) { Ok(addr) => addr, - Err(e) => { - return Some(Err(e.into())); - } + Err(e) => { return Some(Err(e.into())); } } } else { value }; - return Some(Ok((key, value))); + return Some(Ok((key, value))) + }, + Err(e) => { + return Some(Err(e.into())) } - Err(e) => return Some(Err(e.into())), } } @@ -266,26 +197,18 @@ impl<'a, P: ProcessMemory> Iterator for DictIterator<'a, P> { } } -pub const PY_TPFLAGS_MANAGED_DICT: usize = 1 << 4; -const PY_TPFLAGS_INT_SUBCLASS: usize = 1 << 23; -const PY_TPFLAGS_LONG_SUBCLASS: usize = 1 << 24; -const PY_TPFLAGS_LIST_SUBCLASS: usize = 1 << 25; -const PY_TPFLAGS_TUPLE_SUBCLASS: usize = 1 << 26; -const PY_TPFLAGS_BYTES_SUBCLASS: usize = 1 << 27; +pub const PY_TPFLAGS_MANAGED_DICT: usize = 1 << 4; +const PY_TPFLAGS_INT_SUBCLASS: usize = 1 << 23; +const PY_TPFLAGS_LONG_SUBCLASS: usize = 1 << 24; +const PY_TPFLAGS_LIST_SUBCLASS: usize = 1 << 25; +const PY_TPFLAGS_TUPLE_SUBCLASS: usize = 1 << 26; +const PY_TPFLAGS_BYTES_SUBCLASS: usize = 1 << 27; const PY_TPFLAGS_STRING_SUBCLASS: usize = 1 << 28; -const PY_TPFLAGS_DICT_SUBCLASS: usize = 1 << 29; +const PY_TPFLAGS_DICT_SUBCLASS: usize = 1 << 29; /// Converts a python variable in the other process to a human readable string -pub fn format_variable( - process: &P, - version: &Version, - addr: usize, - max_length: isize, -) -> Result -where - I: InterpreterState, - P: ProcessMemory, -{ +pub fn format_variable(process: &remoteprocess::Process, version: &Version, addr: usize, max_length: isize) + -> Result where I: InterpreterState { // We need at least 5 characters remaining for all this code to work, replace with an ellipsis if // we're out of space if max_length <= 5 { @@ -298,10 +221,7 @@ where // get the typename (truncating to 128 bytes if longer) let max_type_len = 128; let value_type_name = process.copy(value_type.name() as usize, max_type_len)?; - let length = value_type_name - .iter() - .position(|&x| x == 0) - .unwrap_or(max_type_len); + let length = value_type_name.iter().position(|&x| x == 0).unwrap_or(max_type_len); let value_type_name = std::str::from_utf8(&value_type_name[..length])?; let format_int = |value: i64| { @@ -319,21 +239,14 @@ where } else if flags & PY_TPFLAGS_LONG_SUBCLASS != 0 { // we don't handle arbitrary sized integer values (max is 2**60) let (value, overflowed) = copy_long(process, addr)?; - if overflowed { - if value > 0 { - "+bigint".to_owned() - } else { - "-bigint".to_owned() - } + if overflowed { + if value > 0 { "+bigint".to_owned() } else { "-bigint".to_owned() } } else { format_int(value) } - } else if flags & PY_TPFLAGS_STRING_SUBCLASS != 0 - || (version.major == 2 && (flags & PY_TPFLAGS_BYTES_SUBCLASS != 0)) - { - let value = copy_string(addr as *const I::StringObject, process)? - .replace('\'', "\\\"") - .replace('\n', "\\n"); + } else if flags & PY_TPFLAGS_STRING_SUBCLASS != 0 || + (version.major == 2 && (flags & PY_TPFLAGS_BYTES_SUBCLASS != 0)) { + let value = copy_string(addr as *const I::StringObject, process)?.replace("\"", "\\\"").replace("\n", "\\n"); if value.len() as isize >= max_length - 5 { format!("\"{}...\"", &value[..(max_length - 5) as usize]) } else { @@ -345,8 +258,8 @@ where let mut remaining = max_length - 2; for entry in DictIterator::from(process, version, addr)? { let (key, value) = entry?; - let key = format_variable::(process, version, key, remaining)?; - let value = format_variable::(process, version, value, remaining)?; + let key = format_variable::(process, version, key, remaining)?; + let value = format_variable::(process, version, value, remaining)?; remaining -= (key.len() + value.len()) as isize + 4; if remaining <= 5 { values.push("...".to_owned()); @@ -365,9 +278,8 @@ where let mut values = Vec::new(); let mut remaining = max_length - 2; for i in 0..object.size() { - let valueptr: *mut I::Object = - process.copy_struct(addr + i * std::mem::size_of::<*mut I::Object>())?; - let value = format_variable::(process, version, valueptr as usize, remaining)?; + let valueptr: *mut I::Object = process.copy_struct(addr + i * std::mem::size_of::<* mut I::Object>())?; + let value = format_variable::(process, version, valueptr as usize, remaining)?; remaining -= value.len() as isize + 2; if remaining <= 5 { values.push("...".to_owned()); @@ -382,7 +294,7 @@ where let mut remaining = max_length - 2; for i in 0..object.size() { let value_addr: *mut I::Object = process.copy_struct(object.address(addr, i))?; - let value = format_variable::(process, version, value_addr as usize, remaining)?; + let value = format_variable::(process, version, value_addr as usize, remaining)?; remaining -= value.len() as isize + 2; if remaining <= 5 { values.push("...".to_owned()); @@ -392,8 +304,7 @@ where } format!("({})", values.join(", ")) } else if value_type_name == "float" { - let value = - process.copy_pointer(addr as *const crate::python_bindings::v3_7_0::PyFloatObject)?; + let value = process.copy_pointer(addr as *const crate::python_bindings::v3_7_0::PyFloatObject)?; format!("{}", value.ob_fval) } else if value_type_name == "NoneType" { "None".to_owned() @@ -409,64 +320,39 @@ pub mod tests { // the idea here is to create various cpython interpretator structs locally // and then test out that the above code handles appropriately use super::*; - use crate::python_bindings::v3_7_0::{ - PyASCIIObject, PyBytesObject, PyUnicodeObject, PyVarObject, - }; use remoteprocess::LocalProcess; + use crate::python_bindings::v3_7_0::{PyBytesObject, PyVarObject, PyUnicodeObject, PyASCIIObject}; use std::ptr::copy_nonoverlapping; // python stores data after pybytesobject/pyasciiobject. hack by initializing a 4k buffer for testing. // TODO: get better at Rust and figure out a better solution #[allow(dead_code)] - #[repr(C)] pub struct AllocatedPyByteObject { pub base: PyBytesObject, - pub storage: [u8; 4096], + pub storage: [u8; 4096] } #[allow(dead_code)] - #[repr(C)] // Rust can optimize the layout of this struct and break our pointer arithmetic pub struct AllocatedPyASCIIObject { pub base: PyASCIIObject, - pub storage: [u8; 4096], + pub storage: [u8; 4096] } pub fn to_byteobject(bytes: &[u8]) -> AllocatedPyByteObject { let ob_size = bytes.len() as isize; - let base = PyBytesObject { - ob_base: PyVarObject { - ob_size, - ..Default::default() - }, - ..Default::default() - }; - let mut ret = AllocatedPyByteObject { - base, - storage: [0 as u8; 4096], - }; - unsafe { - copy_nonoverlapping( - bytes.as_ptr(), - ret.base.ob_sval.as_mut_ptr() as *mut u8, - bytes.len(), - ); - } + let base = PyBytesObject{ob_base: PyVarObject{ob_size, ..Default::default()}, ..Default::default()}; + let mut ret = AllocatedPyByteObject{base, storage: [0 as u8; 4096]}; + unsafe { copy_nonoverlapping(bytes.as_ptr(), ret.base.ob_sval.as_mut_ptr() as *mut u8, bytes.len()); } ret } pub fn to_asciiobject(input: &str) -> AllocatedPyASCIIObject { let bytes: Vec = input.bytes().collect(); - let mut base = PyASCIIObject { - length: bytes.len() as isize, - ..Default::default() - }; + let mut base = PyASCIIObject{length: bytes.len() as isize, ..Default::default()}; base.state.set_compact(1); base.state.set_kind(1); base.state.set_ascii(1); - let mut ret = AllocatedPyASCIIObject { - base, - storage: [0 as u8; 4096], - }; + let mut ret = AllocatedPyASCIIObject{base, storage: [0 as u8; 4096]}; unsafe { let ptr = &mut ret as *mut AllocatedPyASCIIObject as *mut u8; let dst = ptr.offset(std::mem::size_of::() as isize); @@ -480,7 +366,7 @@ pub mod tests { let original = "function_name"; let obj = to_asciiobject(original); - let unicode: &PyUnicodeObject = unsafe { std::mem::transmute(&obj.base) }; + let unicode: &PyUnicodeObject = unsafe{ std::mem::transmute(&obj.base) }; let copied = copy_string(unicode, &LocalProcess).unwrap(); assert_eq!(copied, original); } diff --git a/src/python_interpreters.rs b/src/python_interpreters.rs index cae9e85d..9efc79bf 100644 --- a/src/python_interpreters.rs +++ b/src/python_interpreters.rs @@ -7,13 +7,11 @@ pointer addresses here refer to locations in the target process memory space. This means we can't dereference them directly. */ -#![allow(clippy::unnecessary_cast)] - // these bindings are automatically generated by rust bindgen // using the generate_bindings.py script -use crate::python_bindings::{ - v2_7_15, v3_10_0, v3_11_0, v3_3_7, v3_5_5, v3_6_6, v3_7_0, v3_8_0, v3_9_5, -}; +use crate::python_bindings::{v2_7_15, v3_3_7, v3_5_5, v3_6_6, v3_7_0, v3_8_0, v3_9_5, v3_10_0, v3_11_0}; + +use std; pub trait InterpreterState { type ThreadState: ThreadState; @@ -21,7 +19,7 @@ pub trait InterpreterState { type StringObject: StringObject; type ListObject: ListObject; type TupleObject: TupleObject; - fn head(&self) -> *mut Self::ThreadState; + fn head(&self) -> * mut Self::ThreadState; fn modules(&self) -> *mut Self::Object; } @@ -29,25 +27,24 @@ pub trait ThreadState { type FrameObject: FrameObject; type InterpreterState: InterpreterState; - fn interp(&self) -> *mut Self::InterpreterState; + fn interp(&self) -> * mut Self::InterpreterState; // starting in python 3.11, there is an extra level of indirection // in getting the frame. this returns the address fn frame_address(&self) -> Option; - fn frame(&self, offset: Option) -> *mut Self::FrameObject; + fn frame(&self, offset: Option) -> * mut Self::FrameObject; fn thread_id(&self) -> u64; fn native_thread_id(&self) -> Option; - fn next(&self) -> *mut Self; + fn next(&self) -> * mut Self; } pub trait FrameObject { type CodeObject: CodeObject; - fn code(&self) -> *mut Self::CodeObject; + fn code(&self) -> * mut Self::CodeObject; fn lasti(&self) -> i32; - fn back(&self) -> *mut Self; - fn is_entry(&self) -> bool; + fn back(&self) -> * mut Self; } pub trait CodeObject { @@ -55,13 +52,13 @@ pub trait CodeObject { type BytesObject: BytesObject; type TupleObject: TupleObject; - fn name(&self) -> *mut Self::StringObject; - fn filename(&self) -> *mut Self::StringObject; - fn line_table(&self) -> *mut Self::BytesObject; + fn name(&self) -> * mut Self::StringObject; + fn filename(&self) -> * mut Self::StringObject; + fn line_table(&self) -> * mut Self::BytesObject; fn first_lineno(&self) -> i32; fn nlocals(&self) -> i32; fn argcount(&self) -> i32; - fn varnames(&self) -> *mut Self::TupleObject; + fn varnames(&self) -> * mut Self::TupleObject; fn get_line_number(&self, lasti: i32, table: &[u8]) -> i32; } @@ -91,7 +88,7 @@ pub trait ListObject { pub trait Object { type TypeObject: TypeObject; - fn ob_type(&self) -> *mut Self::TypeObject; + fn ob_type(&self) -> * mut Self::TypeObject; } pub trait TypeObject { @@ -108,111 +105,64 @@ fn offset_of(object: *const T, member: *const M) -> usize { /// (this code is identical across python versions, we are only abstracting the struct layouts here). /// String handling changes substantially between python versions, and is handled separately. macro_rules! PythonCommonImpl { - ($py: ident, $stringobject: ident) => { + ($py: ident, $stringobject: ident) => ( impl InterpreterState for $py::PyInterpreterState { type ThreadState = $py::PyThreadState; type Object = $py::PyObject; type StringObject = $py::$stringobject; type ListObject = $py::PyListObject; type TupleObject = $py::PyTupleObject; - fn head(&self) -> *mut Self::ThreadState { - self.tstate_head - } - fn modules(&self) -> *mut Self::Object { - self.modules - } + fn head(&self) -> * mut Self::ThreadState { self.tstate_head } + fn modules(&self) -> * mut Self::Object { self.modules } } impl ThreadState for $py::PyThreadState { type FrameObject = $py::PyFrameObject; type InterpreterState = $py::PyInterpreterState; - fn frame_address(&self) -> Option { - None - } - fn frame(&self, _: Option) -> *mut Self::FrameObject { - self.frame - } - fn thread_id(&self) -> u64 { - self.thread_id as u64 - } - fn native_thread_id(&self) -> Option { - None - } - fn next(&self) -> *mut Self { - self.next - } - fn interp(&self) -> *mut Self::InterpreterState { - self.interp - } + fn frame_address(&self) -> Option { None } + fn frame(&self, _: Option) -> * mut Self::FrameObject { self.frame } + fn thread_id(&self) -> u64 { self.thread_id as u64 } + fn native_thread_id(&self) -> Option { None } + fn next(&self) -> * mut Self { self.next } + fn interp(&self) -> *mut Self::InterpreterState { self.interp } } impl FrameObject for $py::PyFrameObject { type CodeObject = $py::PyCodeObject; - fn code(&self) -> *mut Self::CodeObject { - self.f_code - } - fn lasti(&self) -> i32 { - self.f_lasti as i32 - } - fn back(&self) -> *mut Self { - self.f_back - } - fn is_entry(&self) -> bool { - true - } + fn code(&self) -> * mut Self::CodeObject { self.f_code } + fn lasti(&self) -> i32 { self.f_lasti as i32 } + fn back(&self) -> * mut Self { self.f_back } } impl Object for $py::PyObject { type TypeObject = $py::PyTypeObject; - fn ob_type(&self) -> *mut Self::TypeObject { - self.ob_type as *mut Self::TypeObject - } + fn ob_type(&self) -> * mut Self::TypeObject { self.ob_type as * mut Self::TypeObject } } impl TypeObject for $py::PyTypeObject { - fn name(&self) -> *const ::std::os::raw::c_char { - self.tp_name - } - fn dictoffset(&self) -> isize { - self.tp_dictoffset - } - fn flags(&self) -> usize { - self.tp_flags as usize - } + fn name(&self) -> *const ::std::os::raw::c_char { self.tp_name } + fn dictoffset(&self) -> isize { self.tp_dictoffset } + fn flags(&self) -> usize { self.tp_flags as usize } } - }; + ) } // We can use this up until python3.10 - where code object lnotab attribute is deprecated macro_rules! PythonCodeObjectImpl { - ($py: ident, $bytesobject: ident, $stringobject: ident) => { + ($py: ident, $bytesobject: ident, $stringobject: ident) => ( impl CodeObject for $py::PyCodeObject { type BytesObject = $py::$bytesobject; type StringObject = $py::$stringobject; type TupleObject = $py::PyTupleObject; - fn name(&self) -> *mut Self::StringObject { - self.co_name as *mut Self::StringObject - } - fn filename(&self) -> *mut Self::StringObject { - self.co_filename as *mut Self::StringObject - } - fn line_table(&self) -> *mut Self::BytesObject { - self.co_lnotab as *mut Self::BytesObject - } - fn first_lineno(&self) -> i32 { - self.co_firstlineno - } - fn nlocals(&self) -> i32 { - self.co_nlocals - } - fn argcount(&self) -> i32 { - self.co_argcount - } - fn varnames(&self) -> *mut Self::TupleObject { - self.co_varnames as *mut Self::TupleObject - } + fn name(&self) -> * mut Self::StringObject { self.co_name as * mut Self::StringObject } + fn filename(&self) -> * mut Self::StringObject { self.co_filename as * mut Self::StringObject } + fn line_table(&self) -> * mut Self::BytesObject { self.co_lnotab as * mut Self::BytesObject } + fn first_lineno(&self) -> i32 { self.co_firstlineno } + fn nlocals(&self) -> i32 { self.co_nlocals } + fn argcount(&self) -> i32 { self.co_argcount } + fn varnames(&self) -> * mut Self::TupleObject { self.co_varnames as * mut Self::TupleObject } fn get_line_number(&self, lasti: i32, table: &[u8]) -> i32 { let lasti = lasti as i32; @@ -241,35 +191,27 @@ macro_rules! PythonCodeObjectImpl { line_number } } - }; + ) } // String/Byte/List/Tuple handling for Python 3.3+ macro_rules! Python3Impl { - ($py: ident) => { + ($py: ident) => ( impl BytesObject for $py::PyBytesObject { - fn size(&self) -> usize { - self.ob_base.ob_size as usize - } + fn size(&self) -> usize { self.ob_base.ob_size as usize } fn address(&self, base: usize) -> usize { base + offset_of(self, &self.ob_sval) } } impl StringObject for $py::PyUnicodeObject { - fn ascii(&self) -> bool { - self._base._base.state.ascii() != 0 - } - fn size(&self) -> usize { - self._base._base.length as usize - } - fn kind(&self) -> u32 { - self._base._base.state.kind() - } + fn ascii(&self) -> bool { self._base._base.state.ascii() != 0 } + fn size(&self) -> usize { self._base._base.length as usize } + fn kind(&self) -> u32 { self._base._base.state.kind() } fn address(&self, base: usize) -> usize { if self._base._base.state.compact() == 0 { - return unsafe { self.data.any as usize }; + return unsafe{ self.data.any as usize }; } if self._base._base.state.ascii() == 1 { @@ -282,24 +224,17 @@ macro_rules! Python3Impl { impl ListObject for $py::PyListObject { type Object = $py::PyObject; - fn size(&self) -> usize { - self.ob_base.ob_size as usize - } - fn item(&self) -> *mut *mut Self::Object { - self.ob_item - } + fn size(&self) -> usize { self.ob_base.ob_size as usize } + fn item(&self) -> *mut *mut Self::Object { self.ob_item } } impl TupleObject for $py::PyTupleObject { - fn size(&self) -> usize { - self.ob_base.ob_size as usize - } + fn size(&self) -> usize { self.ob_base.ob_size as usize } fn address(&self, base: usize, index: usize) -> usize { - base + offset_of(self, &self.ob_item) - + index * std::mem::size_of::<*mut $py::PyObject>() + base + offset_of(self, &self.ob_item) + index * std::mem::size_of::<* mut $py::PyObject>() } } - }; + ) } // Python 3.11 // Python3.11 is sufficiently different from previous versions that we can't use the macros above @@ -312,12 +247,8 @@ impl InterpreterState for v3_11_0::PyInterpreterState { type StringObject = v3_11_0::PyUnicodeObject; type ListObject = v3_11_0::PyListObject; type TupleObject = v3_11_0::PyTupleObject; - fn head(&self) -> *mut Self::ThreadState { - self.threads.head - } - fn modules(&self) -> *mut Self::Object { - self.modules - } + fn head(&self) -> * mut Self::ThreadState { self.threads.head } + fn modules(&self) -> * mut Self::Object { self.modules } } impl ThreadState for v3_11_0::PyThreadState { @@ -329,60 +260,35 @@ impl ThreadState for v3_11_0::PyThreadState { let current_frame_offset = offset_of(&cframe, &cframe.current_frame); Some(self.cframe as usize + current_frame_offset) } - fn frame(&self, addr: Option) -> *mut Self::FrameObject { - addr.unwrap() as *mut Self::FrameObject - } - fn thread_id(&self) -> u64 { - self.thread_id as u64 - } - fn native_thread_id(&self) -> Option { - Some(self.native_thread_id as u64) - } - fn next(&self) -> *mut Self { - self.next - } - fn interp(&self) -> *mut Self::InterpreterState { - self.interp - } + fn frame(&self, addr: Option) -> * mut Self::FrameObject { addr.unwrap() as * mut Self::FrameObject } + fn thread_id(&self) -> u64 { self.thread_id as u64 } + fn native_thread_id(&self) -> Option { Some(self.native_thread_id as u64) } + fn next(&self) -> * mut Self { self.next } + fn interp(&self) -> *mut Self::InterpreterState { self.interp } } impl FrameObject for v3_11_0::_PyInterpreterFrame { type CodeObject = v3_11_0::PyCodeObject; - fn code(&self) -> *mut Self::CodeObject { - self.f_code - } + fn code(&self) -> * mut Self::CodeObject { self.f_code } fn lasti(&self) -> i32 { // this returns the delta from the co_code, but we need to adjust for the // offset from co_code.co_code_adaptive. This is slightly easier to do in the // get_line_number code, so will adjust there - let co_code = self.f_code as *const _ as *const u8; - unsafe { (self.prev_instr as *const u8).offset_from(co_code) as i32 } - } - fn back(&self) -> *mut Self { - self.previous - } - fn is_entry(&self) -> bool { - self.is_entry + let co_code = self.f_code as * const _ as * const u8; + unsafe { (self.prev_instr as * const u8).offset_from(co_code) as i32} } + fn back(&self) -> * mut Self { self.previous } } impl Object for v3_11_0::PyObject { type TypeObject = v3_11_0::PyTypeObject; - fn ob_type(&self) -> *mut Self::TypeObject { - self.ob_type as *mut Self::TypeObject - } + fn ob_type(&self) -> * mut Self::TypeObject { self.ob_type as * mut Self::TypeObject } } impl TypeObject for v3_11_0::PyTypeObject { - fn name(&self) -> *const ::std::os::raw::c_char { - self.tp_name - } - fn dictoffset(&self) -> isize { - self.tp_dictoffset - } - fn flags(&self) -> usize { - self.tp_flags as usize - } + fn name(&self) -> *const ::std::os::raw::c_char { self.tp_name } + fn dictoffset(&self) -> isize { self.tp_dictoffset } + fn flags(&self) -> usize { self.tp_flags as usize } } fn read_varint(index: &mut usize, table: &[u8]) -> usize { @@ -404,7 +310,7 @@ fn read_varint(index: &mut usize, table: &[u8]) -> usize { fn read_signed_varint(index: &mut usize, table: &[u8]) -> isize { let unsigned_val = read_varint(index, table); if unsigned_val & 1 != 0 { - -((unsigned_val >> 1) as isize) + -1 * ((unsigned_val >> 1) as isize) } else { (unsigned_val >> 1) as isize } @@ -415,27 +321,13 @@ impl CodeObject for v3_11_0::PyCodeObject { type StringObject = v3_11_0::PyUnicodeObject; type TupleObject = v3_11_0::PyTupleObject; - fn name(&self) -> *mut Self::StringObject { - self.co_name as *mut Self::StringObject - } - fn filename(&self) -> *mut Self::StringObject { - self.co_filename as *mut Self::StringObject - } - fn line_table(&self) -> *mut Self::BytesObject { - self.co_linetable as *mut Self::BytesObject - } - fn first_lineno(&self) -> i32 { - self.co_firstlineno - } - fn nlocals(&self) -> i32 { - self.co_nlocals - } - fn argcount(&self) -> i32 { - self.co_argcount - } - fn varnames(&self) -> *mut Self::TupleObject { - self.co_localsplusnames as *mut Self::TupleObject - } + fn name(&self) -> * mut Self::StringObject { self.co_name as * mut Self::StringObject } + fn filename(&self) -> * mut Self::StringObject { self.co_filename as * mut Self::StringObject } + fn line_table(&self) -> * mut Self::BytesObject { self.co_linetable as * mut Self::BytesObject } + fn first_lineno(&self) -> i32 { self.co_firstlineno } + fn nlocals(&self) -> i32 { self.co_nlocals } + fn argcount(&self) -> i32 { self.co_argcount } + fn varnames(&self) -> * mut Self::TupleObject { self.co_localsplusnames as * mut Self::TupleObject } fn get_line_number(&self, lasti: i32, table: &[u8]) -> i32 { // unpack compressed table format from python 3.11 @@ -456,19 +348,21 @@ impl CodeObject for v3_11_0::PyCodeObject { bytecode_address += delta * 2; let code = (byte >> 3) & 15; let line_delta = match code { - 15 => 0, + 15 => { 0 }, 14 => { let delta = read_signed_varint(&mut index, table); read_varint(&mut index, table); // end line read_varint(&mut index, table); // start column read_varint(&mut index, table); // end column delta - } - 13 => read_signed_varint(&mut index, table), + }, + 13 => { + read_signed_varint(&mut index, table) + }, 10..=12 => { index += 2; // start column / end column (code - 10).into() - } + }, _ => { index += 1; // column 0 @@ -483,6 +377,7 @@ impl CodeObject for v3_11_0::PyCodeObject { } } + // Python 3.10 Python3Impl!(v3_10_0); PythonCommonImpl!(v3_10_0, PyUnicodeObject); @@ -492,60 +387,48 @@ impl CodeObject for v3_10_0::PyCodeObject { type StringObject = v3_10_0::PyUnicodeObject; type TupleObject = v3_10_0::PyTupleObject; - fn name(&self) -> *mut Self::StringObject { - self.co_name as *mut Self::StringObject - } - fn filename(&self) -> *mut Self::StringObject { - self.co_filename as *mut Self::StringObject - } - fn line_table(&self) -> *mut Self::BytesObject { - self.co_linetable as *mut Self::BytesObject - } - fn first_lineno(&self) -> i32 { - self.co_firstlineno - } - fn nlocals(&self) -> i32 { - self.co_nlocals - } - fn argcount(&self) -> i32 { - self.co_argcount - } - fn varnames(&self) -> *mut Self::TupleObject { - self.co_varnames as *mut Self::TupleObject - } + fn name(&self) -> * mut Self::StringObject { self.co_name as * mut Self::StringObject } + fn filename(&self) -> * mut Self::StringObject { self.co_filename as * mut Self::StringObject } + fn line_table(&self) -> * mut Self::BytesObject { self.co_linetable as * mut Self::BytesObject } + fn first_lineno(&self) -> i32 { self.co_firstlineno } + fn nlocals(&self) -> i32 { self.co_nlocals } + fn argcount(&self) -> i32 { self.co_argcount } + fn varnames(&self) -> * mut Self::TupleObject { self.co_varnames as * mut Self::TupleObject } fn get_line_number(&self, lasti: i32, table: &[u8]) -> i32 { - // in Python 3.10 we need to double the lasti instruction value here (and no I don't know why) - // https://github.com/python/cpython/blob/7b88f63e1dd4006b1a08b9c9f087dd13449ecc76/Python/ceval.c#L5999 - // Whereas in python versions up to 3.9 we didn't. - // https://github.com/python/cpython/blob/3.9/Python/ceval.c#L4713-L4714 - let lasti = 2 * lasti as i32; - - // unpack the line table. format is specified here: - // https://github.com/python/cpython/blob/3.10/Objects/lnotab_notes.txt - let size = table.len(); - let mut i = 0; - let mut line_number: i32 = self.first_lineno(); - let mut bytecode_address: i32 = 0; - while (i + 1) < size { - let delta: u8 = table[i]; - let line_delta: i8 = unsafe { std::mem::transmute(table[i + 1]) }; - i += 2; - - if line_delta == -128 { - continue; - } - - line_number += i32::from(line_delta); - bytecode_address += i32::from(delta); - if bytecode_address > lasti { - break; - } - } - - line_number + // in Python 3.10 we need to double the lasti instruction value here (and no I don't know why) + // https://github.com/python/cpython/blob/7b88f63e1dd4006b1a08b9c9f087dd13449ecc76/Python/ceval.c#L5999 + // Whereas in python versions up to 3.9 we didn't. + // https://github.com/python/cpython/blob/3.9/Python/ceval.c#L4713-L4714 + let lasti = 2 * lasti as i32; + + // unpack the line table. format is specified here: + // https://github.com/python/cpython/blob/3.10/Objects/lnotab_notes.txt + let size = table.len(); + let mut i = 0; + let mut line_number: i32 = self.first_lineno(); + let mut bytecode_address: i32 = 0; + while (i + 1) < size { + let delta: u8 = table[i]; + let line_delta: i8 = unsafe { std::mem::transmute(table[i + 1]) }; + i += 2; + + if line_delta == -128 { + continue; + } + + line_number += i32::from(line_delta); + bytecode_address += i32::from(delta); + if bytecode_address > lasti { + break; + } + } + + line_number } } + + // Python 3.9 PythonCommonImpl!(v3_9_5, PyUnicodeObject); PythonCodeObjectImpl!(v3_9_5, PyBytesObject, PyUnicodeObject); @@ -580,46 +463,27 @@ Python3Impl!(v3_3_7); PythonCommonImpl!(v2_7_15, PyStringObject); PythonCodeObjectImpl!(v2_7_15, PyStringObject, PyStringObject); impl BytesObject for v2_7_15::PyStringObject { - fn size(&self) -> usize { - self.ob_size as usize - } - fn address(&self, base: usize) -> usize { - base + offset_of(self, &self.ob_sval) - } + fn size(&self) -> usize { self.ob_size as usize } + fn address(&self, base: usize) -> usize { base + offset_of(self, &self.ob_sval) } } impl StringObject for v2_7_15::PyStringObject { - fn ascii(&self) -> bool { - true - } - fn kind(&self) -> u32 { - 1 - } - fn size(&self) -> usize { - self.ob_size as usize - } - fn address(&self, base: usize) -> usize { - base + offset_of(self, &self.ob_sval) - } + fn ascii(&self) -> bool { true } + fn kind(&self) -> u32 { 1 } + fn size(&self) -> usize { self.ob_size as usize } + fn address(&self, base: usize) -> usize { base + offset_of(self, &self.ob_sval) } } impl ListObject for v2_7_15::PyListObject { type Object = v2_7_15::PyObject; - fn size(&self) -> usize { - self.ob_size as usize - } - fn item(&self) -> *mut *mut Self::Object { - self.ob_item - } + fn size(&self) -> usize { self.ob_size as usize } + fn item(&self) -> *mut *mut Self::Object { self.ob_item } } impl TupleObject for v2_7_15::PyTupleObject { - fn size(&self) -> usize { - self.ob_size as usize - } + fn size(&self) -> usize { self.ob_size as usize } fn address(&self, base: usize, index: usize) -> usize { - base + offset_of(self, &self.ob_item) - + index * std::mem::size_of::<*mut v2_7_15::PyObject>() + base + offset_of(self, &self.ob_item) + index * std::mem::size_of::<* mut v2_7_15::PyObject>() } } @@ -630,15 +494,11 @@ mod tests { #[test] fn test_py3_11_line_numbers() { use crate::python_bindings::v3_11_0::PyCodeObject; - let code = PyCodeObject { - co_firstlineno: 4, - ..Default::default() - }; - - let table = [ - 128_u8, 0, 221, 4, 8, 132, 74, 136, 118, 209, 4, 22, 212, 4, 22, 208, 4, 22, 208, 4, - 22, 208, 4, 22, - ]; + let code = PyCodeObject {co_firstlineno:4, ..Default::default()}; + + let table = [128_u8, 0, 221, 4, 8, 132, 74, 136, 118, 209, 4, 22, 212, 4, 22, 208, 4, 22, + 208, 4, 22, 208, 4, 22]; assert_eq!(code.get_line_number(214, &table), 5); + } } diff --git a/src/python_process_info.rs b/src/python_process_info.rs deleted file mode 100644 index dd802c67..00000000 --- a/src/python_process_info.rs +++ /dev/null @@ -1,773 +0,0 @@ -use regex::Regex; -#[cfg(windows)] -use regex::RegexBuilder; -#[cfg(windows)] -use std::collections::HashMap; -use std::mem::size_of; -use std::path::Path; -use std::slice; - -use anyhow::{Context, Error, Result}; -use lazy_static::lazy_static; -use proc_maps::{get_process_maps, MapRange}; -use remoteprocess::{Pid, ProcessMemory}; - -use crate::binary_parser::{parse_binary, BinaryInfo}; -use crate::config::Config; -use crate::python_bindings::{ - pyruntime, v2_7_15, v3_10_0, v3_11_0, v3_3_7, v3_5_5, v3_6_6, v3_7_0, v3_8_0, v3_9_5, -}; -use crate::python_interpreters::{InterpreterState, ThreadState}; -use crate::stack_trace::get_stack_traces; -use crate::version::Version; - -/// Holds information about the python process: memory map layout, parsed binary info -/// for python /libpython etc. -pub struct PythonProcessInfo { - pub python_binary: Option, - // if python was compiled with './configure --enabled-shared', code/symbols will - // be in a libpython.so file instead of the executable. support that. - pub libpython_binary: Option, - pub maps: Box, - pub python_filename: std::path::PathBuf, - #[cfg(target_os = "linux")] - pub dockerized: bool, -} - -impl PythonProcessInfo { - pub fn new(process: &remoteprocess::Process) -> Result { - let filename = process - .exe() - .context("Failed to get process executable name. Check that the process is running.")?; - - #[cfg(windows)] - let filename = filename.to_lowercase(); - - #[cfg(windows)] - let is_python_bin = |pathname: &str| pathname.to_lowercase() == filename; - - #[cfg(not(windows))] - let is_python_bin = |pathname: &str| pathname == filename; - - // get virtual memory layout - let maps = get_process_maps(process.pid)?; - info!("Got virtual memory maps from pid {}:", process.pid); - for map in &maps { - debug!( - "map: {:016x}-{:016x} {}{}{} {}", - map.start(), - map.start() + map.size(), - if map.is_read() { 'r' } else { '-' }, - if map.is_write() { 'w' } else { '-' }, - if map.is_exec() { 'x' } else { '-' }, - map.filename() - .unwrap_or(&std::path::PathBuf::from("")) - .display() - ); - } - - // parse the main python binary - let (python_binary, python_filename) = { - // Get the memory address for the executable by matching against virtual memory maps - let map = maps.iter().find(|m| { - if let Some(pathname) = m.filename() { - if let Some(pathname) = pathname.to_str() { - return is_python_bin(pathname) && m.is_exec(); - } - } - false - }); - - let map = match map { - Some(map) => map, - None => { - warn!("Failed to find '{}' in virtual memory maps, falling back to first map region", filename); - // If we failed to find the executable in the virtual memory maps, just take the first file we find - // sometimes on windows get_process_exe returns stale info =( https://github.com/benfred/py-spy/issues/40 - // and on all operating systems I've tried, the exe is the first region in the maps - maps.first().ok_or_else(|| { - format_err!("Failed to get virtual memory maps from process") - })? - } - }; - - #[cfg(not(target_os = "linux"))] - let filename = std::path::PathBuf::from(filename); - - // use filename through /proc/pid/exe which works across docker namespaces and - // handles if the file was deleted - #[cfg(target_os = "linux")] - let filename = std::path::PathBuf::from(format!("/proc/{}/exe", process.pid)); - - // TODO: consistent types? u64 -> usize? for map.start etc - let python_binary = parse_binary(&filename, map.start() as u64, map.size() as u64); - - // windows symbols are stored in separate files (.pdb), load - #[cfg(windows)] - let python_binary = python_binary.and_then(|mut pb| { - get_windows_python_symbols(process.pid, &filename, map.start() as u64) - .map(|symbols| { - pb.symbols.extend(symbols); - pb - }) - .map_err(|err| err.into()) - }); - - // For OSX, need to adjust main binary symbols by subtracting _mh_execute_header - // (which we've added to by map.start already, so undo that here) - #[cfg(target_os = "macos")] - let python_binary = python_binary.map(|mut pb| { - let offset = pb.symbols["_mh_execute_header"] - map.start() as u64; - for address in pb.symbols.values_mut() { - *address -= offset; - } - - if pb.bss_addr != 0 { - pb.bss_addr -= offset; - } - pb - }); - - (python_binary, filename) - }; - - // likewise handle libpython for python versions compiled with --enabled-shared - let libpython_binary = { - let libmap = maps.iter().find(|m| { - if let Some(pathname) = m.filename() { - if let Some(pathname) = pathname.to_str() { - return is_python_lib(pathname) && m.is_exec(); - } - } - false - }); - - let mut libpython_binary: Option = None; - if let Some(libpython) = libmap { - if let Some(filename) = &libpython.filename() { - info!("Found libpython binary @ {}", filename.display()); - - // on linux the process could be running in docker, access the filename through procfs - #[cfg(target_os = "linux")] - let filename = &std::path::PathBuf::from(format!( - "/proc/{}/root{}", - process.pid, - filename.display() - )); - - #[allow(unused_mut)] - let mut parsed = - parse_binary(filename, libpython.start() as u64, libpython.size() as u64)?; - #[cfg(windows)] - parsed.symbols.extend(get_windows_python_symbols( - process.pid, - filename, - libpython.start() as u64, - )?); - libpython_binary = Some(parsed); - } - } - - // On OSX, it's possible that the Python library is a dylib loaded up from the system - // framework (like /System/Library/Frameworks/Python.framework/Versions/2.7/Python) - // In this case read in the dyld_info information and figure out the filename from there - #[cfg(target_os = "macos")] - { - if libpython_binary.is_none() { - use proc_maps::mac_maps::get_dyld_info; - let dyld_infos = get_dyld_info(process.pid)?; - - for dyld in &dyld_infos { - let segname = - unsafe { std::ffi::CStr::from_ptr(dyld.segment.segname.as_ptr()) }; - debug!( - "dyld: {:016x}-{:016x} {:10} {}", - dyld.segment.vmaddr, - dyld.segment.vmaddr + dyld.segment.vmsize, - segname.to_string_lossy(), - dyld.filename.display() - ); - } - - let python_dyld_data = dyld_infos.iter().find(|m| { - if let Some(filename) = m.filename.to_str() { - return is_python_framework(filename) - && m.segment.segname[0..7] == [95, 95, 68, 65, 84, 65, 0]; - } - false - }); - - if let Some(libpython) = python_dyld_data { - info!( - "Found libpython binary from dyld @ {}", - libpython.filename.display() - ); - - let mut binary = parse_binary( - &libpython.filename, - libpython.segment.vmaddr, - libpython.segment.vmsize, - )?; - - // TODO: bss addr offsets returned from parsing binary are wrong - // (assumes data section isn't split from text section like done here). - // BSS occurs somewhere in the data section, just scan that - // (could later tighten this up to look at segment sections too) - binary.bss_addr = libpython.segment.vmaddr; - binary.bss_size = libpython.segment.vmsize; - libpython_binary = Some(binary); - } - } - } - - libpython_binary - }; - - // If we have a libpython binary - we can tolerate failures on parsing the main python binary. - let python_binary = match libpython_binary { - None => Some(python_binary.context("Failed to parse python binary")?), - _ => python_binary.ok(), - }; - - #[cfg(target_os = "linux")] - let dockerized = is_dockerized(process.pid).unwrap_or(false); - - Ok(PythonProcessInfo { - python_binary, - libpython_binary, - maps: Box::new(maps), - python_filename, - #[cfg(target_os = "linux")] - dockerized, - }) - } - - pub fn get_symbol(&self, symbol: &str) -> Option<&u64> { - if let Some(ref pb) = self.python_binary { - if let Some(addr) = pb.symbols.get(symbol) { - info!("got symbol {} (0x{:016x}) from python binary", symbol, addr); - return Some(addr); - } - } - - if let Some(ref binary) = self.libpython_binary { - if let Some(addr) = binary.symbols.get(symbol) { - info!( - "got symbol {} (0x{:016x}) from libpython binary", - symbol, addr - ); - return Some(addr); - } - } - None - } -} - -/// Returns the version of python running in the process. -pub fn get_python_version

(python_info: &PythonProcessInfo, process: &P) -> Result -where - P: ProcessMemory, -{ - // If possible, grab the sys.version string from the processes memory (mac osx). - if let Some(&addr) = python_info.get_symbol("Py_GetVersion.version") { - info!("Getting version from symbol address"); - if let Ok(bytes) = process.copy(addr as usize, 128) { - if let Ok(version) = Version::scan_bytes(&bytes) { - return Ok(version); - } - } - } - - // otherwise get version info from scanning BSS section for sys.version string - if let Some(ref pb) = python_info.python_binary { - info!("Getting version from python binary BSS"); - let bss = process.copy(pb.bss_addr as usize, pb.bss_size as usize)?; - match Version::scan_bytes(&bss) { - Ok(version) => return Ok(version), - Err(err) => info!("Failed to get version from BSS section: {}", err), - } - } - - // try again if there is a libpython.so - if let Some(ref libpython) = python_info.libpython_binary { - info!("Getting version from libpython BSS"); - let bss = process.copy(libpython.bss_addr as usize, libpython.bss_size as usize)?; - match Version::scan_bytes(&bss) { - Ok(version) => return Ok(version), - Err(err) => info!("Failed to get version from libpython BSS section: {}", err), - } - } - - // the python_filename might have the version encoded in it (/usr/bin/python3.5 etc). - // try reading that in (will miss patch level on python, but that shouldn't matter) - info!( - "Trying to get version from path: {}", - python_info.python_filename.display() - ); - let path = Path::new(&python_info.python_filename); - if let Some(python) = path.file_name() { - if let Some(python) = python.to_str() { - if let Some(stripped_python) = python.strip_prefix("python") { - let tokens: Vec<&str> = stripped_python.split('.').collect(); - if tokens.len() >= 2 { - if let (Ok(major), Ok(minor)) = - (tokens[0].parse::(), tokens[1].parse::()) - { - return Ok(Version { - major, - minor, - patch: 0, - release_flags: "".to_owned(), - build_metadata: None, - }); - } - } - } - } - } - Err(format_err!( - "Failed to find python version from target process" - )) -} - -pub fn get_interpreter_address

( - python_info: &PythonProcessInfo, - process: &P, - version: &Version, -) -> Result -where - P: ProcessMemory, -{ - // get the address of the main PyInterpreterState object from loaded symbols if we can - // (this tends to be faster than scanning through the bss section) - match version { - Version { - major: 3, - minor: 7..=11, - .. - } => { - if let Some(&addr) = python_info.get_symbol("_PyRuntime") { - let addr = process - .copy_struct(addr as usize + pyruntime::get_interp_head_offset(version))?; - - // Make sure the interpreter addr is valid before returning - match check_interpreter_addresses(&[addr], &*python_info.maps, process, version) { - Ok(addr) => return Ok(addr), - Err(_) => { - warn!( - "Interpreter address from _PyRuntime symbol is invalid {:016x}", - addr - ); - } - }; - } - } - _ => { - if let Some(&addr) = python_info.get_symbol("interp_head") { - let addr = process.copy_struct(addr as usize)?; - match check_interpreter_addresses(&[addr], &*python_info.maps, process, version) { - Ok(addr) => return Ok(addr), - Err(_) => { - warn!( - "Interpreter address from interp_head symbol is invalid {:016x}", - addr - ); - } - }; - } - } - }; - info!("Failed to get interp_head from symbols, scanning BSS section from main binary"); - - // try scanning the BSS section of the binary for things that might be the interpreterstate - let err = if let Some(ref pb) = python_info.python_binary { - match get_interpreter_address_from_binary(pb, &*python_info.maps, process, version) { - Ok(addr) => return Ok(addr), - err => Some(err), - } - } else { - None - }; - // Before giving up, try again if there is a libpython.so - if let Some(ref lpb) = python_info.libpython_binary { - info!("Failed to get interpreter from binary BSS, scanning libpython BSS"); - match get_interpreter_address_from_binary(lpb, &*python_info.maps, process, version) { - Ok(addr) => Ok(addr), - lib_err => err.unwrap_or(lib_err), - } - } else { - err.expect("Both python and libpython are invalid.") - } -} - -fn get_interpreter_address_from_binary

( - binary: &BinaryInfo, - maps: &dyn ContainsAddr, - process: &P, - version: &Version, -) -> Result -where - P: ProcessMemory, -{ - // We're going to scan the BSS/data section for things, and try to narrowly scan things that - // look like pointers to PyinterpreterState - let bss = process.copy(binary.bss_addr as usize, binary.bss_size as usize)?; - - #[allow(clippy::cast_ptr_alignment)] - let addrs = unsafe { - slice::from_raw_parts(bss.as_ptr() as *const usize, bss.len() / size_of::()) - }; - check_interpreter_addresses(addrs, maps, process, version) -} - -// Checks whether a block of memory (from BSS/.data etc) contains pointers that are pointing -// to a valid PyInterpreterState -fn check_interpreter_addresses

( - addrs: &[usize], - maps: &dyn ContainsAddr, - process: &P, - version: &Version, -) -> Result -where - P: ProcessMemory, -{ - // This function does all the work, but needs a type of the interpreter - fn check(addrs: &[usize], maps: &dyn ContainsAddr, process: &P) -> Result - where - I: InterpreterState, - P: ProcessMemory, - { - for &addr in addrs { - if maps.contains_addr(addr) { - // this address points to valid memory. try loading it up as a PyInterpreterState - // to further check - let interp: I = match process.copy_struct(addr) { - Ok(interp) => interp, - Err(_) => continue, - }; - - // get the pythreadstate pointer from the interpreter object, and if it is also - // a valid pointer then load it up. - let threads = interp.head(); - if maps.contains_addr(threads as usize) { - // If the threadstate points back to the interpreter like we expect, then - // this is almost certainly the address of the intrepreter - let thread = match process.copy_pointer(threads) { - Ok(thread) => thread, - Err(_) => continue, - }; - - // as a final sanity check, try getting the stack_traces, and only return if this works - if thread.interp() as usize == addr - && get_stack_traces(&interp, process, 0, None).is_ok() - { - return Ok(addr); - } - } - } - } - Err(format_err!( - "Failed to find a python interpreter in the .data section" - )) - } - - // different versions have different layouts, check as appropriate - match version { - Version { - major: 2, - minor: 3..=7, - .. - } => check::(addrs, maps, process), - Version { - major: 3, minor: 3, .. - } => check::(addrs, maps, process), - Version { - major: 3, - minor: 4..=5, - .. - } => check::(addrs, maps, process), - Version { - major: 3, minor: 6, .. - } => check::(addrs, maps, process), - Version { - major: 3, minor: 7, .. - } => check::(addrs, maps, process), - Version { - major: 3, - minor: 8, - patch: 0, - .. - } => match version.release_flags.as_ref() { - "a1" | "a2" | "a3" => check::(addrs, maps, process), - _ => check::(addrs, maps, process), - }, - Version { - major: 3, minor: 8, .. - } => check::(addrs, maps, process), - Version { - major: 3, minor: 9, .. - } => check::(addrs, maps, process), - Version { - major: 3, - minor: 10, - .. - } => check::(addrs, maps, process), - Version { - major: 3, - minor: 11, - .. - } => check::(addrs, maps, process), - _ => Err(format_err!("Unsupported version of Python: {}", version)), - } -} - -pub fn get_threadstate_address( - python_info: &PythonProcessInfo, - version: &Version, - config: &Config, -) -> Result { - let threadstate_address = match version { - Version { - major: 3, - minor: 7..=11, - .. - } => match python_info.get_symbol("_PyRuntime") { - Some(&addr) => { - if let Some(offset) = pyruntime::get_tstate_current_offset(version) { - info!("Found _PyRuntime @ 0x{:016x}, getting gilstate.tstate_current from offset 0x{:x}", - addr, offset); - addr as usize + offset - } else { - error_if_gil( - config, - version, - "unknown pyruntime.gilstate.tstate_current offset", - )?; - 0 - } - } - None => { - error_if_gil(config, version, "failed to find _PyRuntime symbol")?; - 0 - } - }, - _ => match python_info.get_symbol("_PyThreadState_Current") { - Some(&addr) => { - info!("Found _PyThreadState_Current @ 0x{:016x}", addr); - addr as usize - } - None => { - error_if_gil( - config, - version, - "failed to find _PyThreadState_Current symbol", - )?; - 0 - } - }, - }; - - Ok(threadstate_address) -} - -fn error_if_gil(config: &Config, version: &Version, msg: &str) -> Result<(), Error> { - lazy_static! { - static ref WARNED: std::sync::atomic::AtomicBool = - std::sync::atomic::AtomicBool::new(false); - } - - if config.gil_only { - if !WARNED.load(std::sync::atomic::Ordering::Relaxed) { - // only print this once - eprintln!( - "Cannot detect GIL holding in version '{}' on the current platform (reason: {})", - version, msg - ); - eprintln!("Please open an issue in https://github.com/benfred/py-spy with the Python version and your platform."); - WARNED.store(true, std::sync::atomic::Ordering::Relaxed); - } - Err(format_err!( - "Cannot detect GIL holding in version '{}' on the current platform (reason: {})", - version, - msg - )) - } else { - warn!("Unable to detect GIL usage: {}", msg); - Ok(()) - } -} - -pub trait ContainsAddr { - fn contains_addr(&self, addr: usize) -> bool; -} - -impl ContainsAddr for Vec { - #[cfg(windows)] - fn contains_addr(&self, addr: usize) -> bool { - // On windows, we can't just check if a pointer is valid by looking to see if it points - // to something in the virtual memory map. Brute-force it instead - true - } - - #[cfg(not(windows))] - fn contains_addr(&self, addr: usize) -> bool { - proc_maps::maps_contain_addr(addr, self) - } -} - -#[cfg(target_os = "linux")] -fn is_dockerized(pid: Pid) -> Result { - let self_mnt = std::fs::read_link("/proc/self/ns/mnt")?; - let target_mnt = std::fs::read_link(format!("/proc/{}/ns/mnt", pid))?; - Ok(self_mnt != target_mnt) -} - -// We can't use goblin to parse external symbol files (like in a separate .pdb file) on windows, -// So use the win32 api to load up the couple of symbols we need on windows. Note: -// we still can get export's from the PE file -#[cfg(windows)] -pub fn get_windows_python_symbols( - pid: Pid, - filename: &Path, - offset: u64, -) -> std::io::Result> { - use proc_maps::win_maps::SymbolLoader; - - let handler = SymbolLoader::new(pid)?; - let _module = handler.load_module(filename)?; // need to keep this module in scope - - let mut ret = HashMap::new(); - - // currently we only need a subset of symbols, and enumerating the symbols is - // expensive (via SymEnumSymbolsW), so rather than load up all symbols like we - // do for goblin, just load the the couple we need directly. - for symbol in ["_PyThreadState_Current", "interp_head", "_PyRuntime"].iter() { - if let Ok((base, addr)) = handler.address_from_name(symbol) { - // If we have a module base (ie from PDB), need to adjust by the offset - // otherwise seems like we can take address directly - let addr = if base == 0 { - addr - } else { - offset + addr - base - }; - ret.insert(String::from(*symbol), addr); - } - } - - Ok(ret) -} - -#[cfg(any(target_os = "linux", target_os = "freebsd"))] -pub fn is_python_lib(pathname: &str) -> bool { - lazy_static! { - static ref RE: Regex = Regex::new(r"/libpython\d.\d\d?(m|d|u)?.so").unwrap(); - } - RE.is_match(pathname) -} - -#[cfg(target_os = "macos")] -pub fn is_python_lib(pathname: &str) -> bool { - lazy_static! { - static ref RE: Regex = Regex::new(r"/libpython\d.\d\d?(m|d|u)?.(dylib|so)$").unwrap(); - } - RE.is_match(pathname) || is_python_framework(pathname) -} - -#[cfg(windows)] -pub fn is_python_lib(pathname: &str) -> bool { - lazy_static! { - static ref RE: Regex = RegexBuilder::new(r"\\python\d\d\d?(m|d|u)?.dll$") - .case_insensitive(true) - .build() - .unwrap(); - } - RE.is_match(pathname) -} - -#[cfg(target_os = "macos")] -pub fn is_python_framework(pathname: &str) -> bool { - pathname.ends_with("/Python") && !pathname.contains("Python.app") -} - -#[cfg(test)] -mod tests { - use super::*; - - #[cfg(target_os = "macos")] - #[test] - fn test_is_python_lib() { - assert!(is_python_lib("~/Anaconda2/lib/libpython2.7.dylib")); - - // python lib configured with --with-pydebug (flag: d) - assert!(is_python_lib("/lib/libpython3.4d.dylib")); - - // configured --with-pymalloc (flag: m) - assert!(is_python_lib("/usr/local/lib/libpython3.8m.dylib")); - - // python2 configured with --with-wide-unicode (flag: u) - assert!(is_python_lib("./libpython2.7u.dylib")); - - assert!(!is_python_lib("/libboost_python.dylib")); - assert!(!is_python_lib("/lib/heapq.cpython-36m-darwin.dylib")); - } - - #[cfg(any(target_os = "linux", target_os = "freebsd"))] - #[test] - fn test_is_python_lib() { - // libpython bundled by pyinstaller https://github.com/benfred/py-spy/issues/42 - assert!(is_python_lib("/tmp/_MEIOqzg01/libpython2.7.so.1.0")); - - // test debug/malloc/unicode flags - assert!(is_python_lib("./libpython2.7.so")); - assert!(is_python_lib("/usr/lib/libpython3.4d.so")); - assert!(is_python_lib("/usr/local/lib/libpython3.8m.so")); - assert!(is_python_lib("/usr/lib/libpython2.7u.so")); - - // don't blindly match libraries with python in the name (boost_python etc) - assert!(!is_python_lib("/usr/lib/libboost_python.so")); - assert!(!is_python_lib( - "/usr/lib/x86_64-linux-gnu/libboost_python-py27.so.1.58.0" - )); - assert!(!is_python_lib("/usr/lib/libboost_python-py35.so")); - } - - #[cfg(windows)] - #[test] - fn test_is_python_lib() { - assert!(is_python_lib( - "C:\\Users\\test\\AppData\\Local\\Programs\\Python\\Python37\\python37.dll" - )); - // .NET host via https://github.com/pythonnet/pythonnet - assert!(is_python_lib( - "C:\\Users\\test\\AppData\\Local\\Programs\\Python\\Python37\\python37.DLL" - )); - } - - #[cfg(target_os = "macos")] - #[test] - fn test_python_frameworks() { - // homebrew v2 - assert!(!is_python_framework("/usr/local/Cellar/python@2/2.7.15_1/Frameworks/Python.framework/Versions/2.7/Resources/Python.app/Contents/MacOS/Python")); - assert!(is_python_framework( - "/usr/local/Cellar/python@2/2.7.15_1/Frameworks/Python.framework/Versions/2.7/Python" - )); - - // System python from osx 10.13.6 (high sierra) - assert!(!is_python_framework("/System/Library/Frameworks/Python.framework/Versions/2.7/Resources/Python.app/Contents/MacOS/Python")); - assert!(is_python_framework( - "/System/Library/Frameworks/Python.framework/Versions/2.7/Python" - )); - - // pyenv 3.6.6 with OSX framework enabled (https://github.com/benfred/py-spy/issues/15) - // env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install 3.6.6 - assert!(is_python_framework( - "/Users/ben/.pyenv/versions/3.6.6/Python.framework/Versions/3.6/Python" - )); - assert!(!is_python_framework("/Users/ben/.pyenv/versions/3.6.6/Python.framework/Versions/3.6/Resources/Python.app/Contents/MacOS/Python")); - - // single file pyinstaller - assert!(is_python_framework( - "/private/var/folders/3x/qy479lpd1fb2q88lc9g4d3kr0000gn/T/_MEI2Akvi8/Python" - )); - } -} diff --git a/src/python_spy.rs b/src/python_spy.rs index f5e36d68..f3ed449c 100644 --- a/src/python_spy.rs +++ b/src/python_spy.rs @@ -1,28 +1,30 @@ -#[cfg(windows)] -use regex::RegexBuilder; +use std; use std::collections::HashMap; -#[cfg(all(target_os = "linux", unwind))] +#[cfg(all(target_os="linux", unwind))] use std::collections::HashSet; -#[cfg(all(target_os = "linux", unwind))] -use std::iter::FromIterator; +use std::mem::size_of; +use std::slice; use std::path::Path; +#[cfg(all(target_os="linux", unwind))] +use std::iter::FromIterator; +use regex::Regex; +#[cfg(windows)] +use regex::RegexBuilder; -use anyhow::{Context, Error, Result}; -use remoteprocess::{Pid, Process, ProcessMemory, Tid}; +use anyhow::{Error, Result, Context}; +use lazy_static::lazy_static; +use remoteprocess::{Process, ProcessMemory, Pid, Tid}; +use proc_maps::{get_process_maps, MapRange}; -use crate::config::{Config, LockingStrategy}; + +use crate::binary_parser::{parse_binary, BinaryInfo}; +use crate::config::{Config, LockingStrategy, LineNo}; #[cfg(unwind)] use crate::native_stack_trace::NativeStack; -use crate::python_bindings::{ - v2_7_15, v3_10_0, v3_11_0, v3_3_7, v3_5_5, v3_6_6, v3_7_0, v3_8_0, v3_9_5, -}; -use crate::python_data_access::format_variable; -use crate::python_interpreters::{InterpreterState, ThreadState}; -use crate::python_process_info::{ - get_interpreter_address, get_python_version, get_threadstate_address, PythonProcessInfo, -}; +use crate::python_bindings::{pyruntime, v2_7_15, v3_3_7, v3_5_5, v3_6_6, v3_7_0, v3_8_0, v3_9_5, v3_10_0, v3_11_0}; +use crate::python_interpreters::{self, InterpreterState, ThreadState}; use crate::python_threading::thread_name_lookup; -use crate::stack_trace::{get_gil_threadid, get_stack_trace, StackTrace}; +use crate::stack_trace::{StackTrace, get_stack_traces, get_stack_trace}; use crate::version::Version; /// Lets you retrieve stack traces of a running python program @@ -40,8 +42,27 @@ pub struct PythonSpy { pub short_filenames: HashMap>, pub python_thread_ids: HashMap, pub python_thread_names: HashMap, - #[cfg(target_os = "linux")] - pub dockerized: bool, + #[cfg(target_os="linux")] + pub dockerized: bool +} + +fn error_if_gil(config: &Config, version: &Version, msg: &str) -> Result<(), Error> { + lazy_static! { + static ref WARNED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + } + + if config.gil_only { + if !WARNED.load(std::sync::atomic::Ordering::Relaxed) { + // only print this once + eprintln!("Cannot detect GIL holding in version '{}' on the current platform (reason: {})", version, msg); + eprintln!("Please open an issue in https://github.com/benfred/py-spy with the Python version and your platform."); + WARNED.store(true, std::sync::atomic::Ordering::Relaxed); + } + Err(format_err!("Cannot detect GIL holding in version '{}' on the current platform (reason: {})", version, msg)) + } else { + warn!("Unable to detect GIL usage: {}", msg); + Ok(()) + } } impl PythonSpy { @@ -56,7 +77,7 @@ impl PythonSpy { // lock the process when loading up on freebsd (rather than locking // on every memory read). Needs done after getting python process info // because procmaps also tries to attach w/ ptrace on freebsd - #[cfg(target_os = "freebsd")] + #[cfg(target_os="freebsd")] let _lock = process.lock(); let version = get_python_version(&python_info, &process)?; @@ -66,44 +87,65 @@ impl PythonSpy { info!("Found interpreter at 0x{:016x}", interpreter_address); // lets us figure out which thread has the GIL - let threadstate_address = get_threadstate_address(&python_info, &version, config)?; + let threadstate_address = match version { + Version{major: 3, minor: 7..=11, ..} => { + match python_info.get_symbol("_PyRuntime") { + Some(&addr) => { + if let Some(offset) = pyruntime::get_tstate_current_offset(&version) { + info!("Found _PyRuntime @ 0x{:016x}, getting gilstate.tstate_current from offset 0x{:x}", + addr, offset); + addr as usize + offset + } else { + error_if_gil(config, &version, "unknown pyruntime.gilstate.tstate_current offset")?; + 0 + } + }, + None => { + error_if_gil(config, &version, "failed to find _PyRuntime symbol")?; + 0 + } + } + }, + _ => { + match python_info.get_symbol("_PyThreadState_Current") { + Some(&addr) => { + info!("Found _PyThreadState_Current @ 0x{:016x}", addr); + addr as usize + }, + None => { + error_if_gil(config, &version, "failed to find _PyThreadState_Current symbol")?; + 0 + } + } + } + }; let version_string = format!("python{}.{}", version.major, version.minor); #[cfg(unwind)] let native = if config.native { - Some(NativeStack::new( - pid, - python_info.python_binary, - python_info.libpython_binary, - )?) + Some(NativeStack::new(pid, python_info.python_binary, python_info.libpython_binary)?) } else { None }; - Ok(PythonSpy { - pid, - process, - version, - interpreter_address, - threadstate_address, - python_filename: python_info.python_filename, - version_string, - #[cfg(unwind)] - native, - #[cfg(target_os = "linux")] - dockerized: python_info.dockerized, - config: config.clone(), - short_filenames: HashMap::new(), - python_thread_ids: HashMap::new(), - python_thread_names: HashMap::new(), - }) + Ok(PythonSpy{pid, process, version, interpreter_address, threadstate_address, + python_filename: python_info.python_filename, + version_string, + #[cfg(unwind)] + native, + #[cfg(target_os="linux")] + dockerized: python_info.dockerized, + config: config.clone(), + short_filenames: HashMap::new(), + python_thread_ids: HashMap::new(), + python_thread_names: HashMap::new()}) } /// Creates a PythonSpy object, retrying up to max_retries times. /// Mainly useful for the case where the process is just started and /// symbols or the python interpreter might not be loaded yet. - pub fn retry_new(pid: Pid, config: &Config, max_retries: u64) -> Result { + pub fn retry_new(pid: Pid, config: &Config, max_retries:u64) -> Result { let mut retries = 0; loop { let err = match PythonSpy::new(pid, config) { @@ -111,10 +153,10 @@ impl PythonSpy { // verify that we can load a stack trace before returning success match process.get_stack_traces() { Ok(_) => return Ok(process), - Err(err) => err, + Err(err) => err } - } - Err(err) => err, + }, + Err(err) => err }; // If we failed, retry a couple times before returning the last error @@ -131,57 +173,25 @@ impl PythonSpy { pub fn get_stack_traces(&mut self) -> Result, Error> { match self.version { // ABI for 2.3/2.4/2.5/2.6/2.7 is compatible for our purpose - Version { - major: 2, - minor: 3..=7, - .. - } => self._get_stack_traces::(), - Version { - major: 3, minor: 3, .. - } => self._get_stack_traces::(), + Version{major: 2, minor: 3..=7, ..} => self._get_stack_traces::(), + Version{major: 3, minor: 3, ..} => self._get_stack_traces::(), // ABI for 3.4 and 3.5 is the same for our purposes - Version { - major: 3, minor: 4, .. - } => self._get_stack_traces::(), - Version { - major: 3, minor: 5, .. - } => self._get_stack_traces::(), - Version { - major: 3, minor: 6, .. - } => self._get_stack_traces::(), - Version { - major: 3, minor: 7, .. - } => self._get_stack_traces::(), + Version{major: 3, minor: 4, ..} => self._get_stack_traces::(), + Version{major: 3, minor: 5, ..} => self._get_stack_traces::(), + Version{major: 3, minor: 6, ..} => self._get_stack_traces::(), + Version{major: 3, minor: 7, ..} => self._get_stack_traces::(), // v3.8.0a1 to v3.8.0a3 is compatible with 3.7 ABI, but later versions of 3.8.0 aren't - Version { - major: 3, - minor: 8, - patch: 0, - .. - } => match self.version.release_flags.as_ref() { - "a1" | "a2" | "a3" => self._get_stack_traces::(), - _ => self._get_stack_traces::(), - }, - Version { - major: 3, minor: 8, .. - } => self._get_stack_traces::(), - Version { - major: 3, minor: 9, .. - } => self._get_stack_traces::(), - Version { - major: 3, - minor: 10, - .. - } => self._get_stack_traces::(), - Version { - major: 3, - minor: 11, - .. - } => self._get_stack_traces::(), - _ => Err(format_err!( - "Unsupported version of Python: {}", - self.version - )), + Version{major: 3, minor: 8, patch: 0, ..} => { + match self.version.release_flags.as_ref() { + "a1" | "a2" | "a3" => self._get_stack_traces::(), + _ => self._get_stack_traces::() + } + } + Version{major: 3, minor: 8, ..} => self._get_stack_traces::(), + Version{major: 3, minor: 9, ..} => self._get_stack_traces::(), + Version{major: 3, minor: 10, ..} => self._get_stack_traces::(), + Version{major: 3, minor: 11, ..} => self._get_stack_traces::(), + _ => Err(format_err!("Unsupported version of Python: {}", self.version)), } } @@ -189,14 +199,9 @@ impl PythonSpy { fn _get_stack_traces(&mut self) -> Result, Error> { // Query the OS to get if each thread in the process is running or not let mut thread_activity = HashMap::new(); - if self.config.gil_only { - // Don't need to collect thread activity if we're only getting the - // GIL thread: If we're holding the GIL we're by definition active. - } else { - for thread in self.process.threads()?.iter() { - let threadid: Tid = thread.id()?; - thread_activity.insert(threadid, thread.active()?); - } + for thread in self.process.threads()?.iter() { + let threadid: Tid = thread.id()?; + thread_activity.insert(threadid, thread.active()?); } // Lock the process if appropriate. Note we have to lock AFTER getting the thread @@ -209,54 +214,35 @@ impl PythonSpy { None }; - // TODO: hoist most of this code out to stack_trace.rs, and - // then annotate the output of that with things like native stack traces etc - // have moved in gil / locals etc - let gil_thread_id = - get_gil_threadid::(self.threadstate_address, &self.process)?; + let gil_thread_id = self._get_gil_threadid::()?; // Get the python interpreter, and loop over all the python threads - let interp: I = self - .process - .copy_struct(self.interpreter_address) - .context("Failed to copy PyInterpreterState from process")?; + let interp: I = self.process.copy_struct(self.interpreter_address) + .context("Failed to copy PyInterpreterState from process")?; let mut traces = Vec::new(); let mut threads = interp.head(); while !threads.is_null() { // Get the stack trace of the python thread - let thread = self - .process - .copy_pointer(threads) - .context("Failed to copy PyThreadState")?; - threads = thread.next(); - - let python_thread_id = thread.thread_id(); - let owns_gil = python_thread_id == gil_thread_id; - - if self.config.gil_only && !owns_gil { - continue; - } - - let mut trace = get_stack_trace( - &thread, - &self.process, - self.config.dump_locals > 0, - self.config.lineno, - )?; + let thread = self.process.copy_pointer(threads).context("Failed to copy PyThreadState")?; + let mut trace = get_stack_trace(&thread, &self.process, self.config.dump_locals > 0, self.config.lineno)?; // Try getting the native thread id + let python_thread_id = thread.thread_id(); // python 3.11+ has the native thread id directly on the PyThreadState object, + // so use that if available + trace.os_thread_id = thread.native_thread_id(); + // for older versions of python, try using OS specific code to get the native - // thread id (doesn't work on freebsd, or on arm/i686 processors on linux) + // thread id (doesn' work on freebsd, or on arm/i686 processors on linux) if trace.os_thread_id.is_none() { let mut os_thread_id = self._get_os_thread_id(python_thread_id, &interp)?; // linux can see issues where pthread_ids get recycled for new OS threads, // which totally breaks the caching we were doing here. Detect this and retry if let Some(tid) = os_thread_id { - if !thread_activity.is_empty() && !thread_activity.contains_key(&tid) { + if thread_activity.len() > 0 && !thread_activity.contains_key(&tid) { info!("clearing away thread id caches, thread {} has exited", tid); self.python_thread_ids.clear(); self.python_thread_names.clear(); @@ -268,8 +254,7 @@ impl PythonSpy { } trace.thread_name = self._get_python_thread_name(python_thread_id); - trace.owns_gil = owns_gil; - trace.pid = self.process.pid; + trace.owns_gil = trace.thread_id == gil_thread_id; // Figure out if the thread is sleeping from the OS if possible trace.active = true; @@ -295,9 +280,7 @@ impl PythonSpy { { if self.config.native { if let Some(native) = self.native.as_mut() { - let thread_id = trace - .os_thread_id - .ok_or_else(|| format_err!("failed to get os threadid"))?; + let thread_id = trace.os_thread_id.ok_or_else(|| format_err!("failed to get os threadid"))?; let os_thread = remoteprocess::Thread::new(thread_id as Tid)?; trace.frames = native.merge_native_thread(&trace.frames, &os_thread)? } @@ -307,15 +290,11 @@ impl PythonSpy { for frame in &mut trace.frames { frame.short_filename = self.shorten_filename(&frame.filename); if let Some(locals) = frame.locals.as_mut() { + use crate::python_data_access::format_variable; let max_length = (128 * self.config.dump_locals) as isize; for local in locals { - let repr = format_variable::( - &self.process, - &self.version, - local.addr, - max_length, - ); - local.repr = Some(repr.unwrap_or_else(|_| "?".to_owned())); + let repr = format_variable::(&self.process, &self.version, local.addr, max_length); + local.repr = Some(repr.unwrap_or("?".to_owned())); } } } @@ -327,11 +306,7 @@ impl PythonSpy { return Err(format_err!("Max thread recursion depth reached")); } - if self.config.gil_only { - // There's only one GIL thread and we've captured it, so we can - // stop now - break; - } + threads = thread.next(); } Ok(traces) } @@ -346,31 +321,22 @@ impl PythonSpy { false } else { let frame = &frames[0]; - (frame.name == "wait" && frame.filename.ends_with("threading.py")) - || (frame.name == "select" && frame.filename.ends_with("selectors.py")) - || (frame.name == "poll" - && (frame.filename.ends_with("asyncore.py") - || frame.filename.contains("zmq") - || frame.filename.contains("gevent") - || frame.filename.contains("tornado"))) + (frame.name == "wait" && frame.filename.ends_with("threading.py")) || + (frame.name == "select" && frame.filename.ends_with("selectors.py")) || + (frame.name == "poll" && (frame.filename.ends_with("asyncore.py") || + frame.filename.contains("zmq") || + frame.filename.contains("gevent") || + frame.filename.contains("tornado"))) } } #[cfg(windows)] - fn _get_os_thread_id( - &mut self, - python_thread_id: u64, - _interp: &I, - ) -> Result, Error> { + fn _get_os_thread_id(&mut self, python_thread_id: u64, _interp: &I) -> Result, Error> { Ok(Some(python_thread_id as Tid)) } - #[cfg(target_os = "macos")] - fn _get_os_thread_id( - &mut self, - python_thread_id: u64, - _interp: &I, - ) -> Result, Error> { + #[cfg(target_os="macos")] + fn _get_os_thread_id(&mut self, python_thread_id: u64, _interp: &I) -> Result, Error> { // If we've already know this threadid, we're good if let Some(thread_id) = self.python_thread_ids.get(&python_thread_id) { return Ok(Some(*thread_id)); @@ -389,21 +355,13 @@ impl PythonSpy { Ok(None) } - #[cfg(all(target_os = "linux", not(unwind)))] - fn _get_os_thread_id( - &mut self, - _python_thread_id: u64, - _interp: &I, - ) -> Result, Error> { + #[cfg(all(target_os="linux", not(unwind)))] + fn _get_os_thread_id(&mut self, _python_thread_id: u64, _interp: &I) -> Result, Error> { Ok(None) } - #[cfg(all(target_os = "linux", unwind))] - fn _get_os_thread_id( - &mut self, - python_thread_id: u64, - interp: &I, - ) -> Result, Error> { + #[cfg(all(target_os="linux", unwind))] + fn _get_os_thread_id(&mut self, python_thread_id: u64, interp: &I) -> Result, Error> { // in nonblocking mode, we can't get the threadid reliably (method here requires reading the RBX // register which requires a ptrace attach). fallback to heuristic thread activity here if self.config.blocking == LockingStrategy::NonBlocking { @@ -424,17 +382,13 @@ impl PythonSpy { let mut all_python_threads = HashSet::new(); let mut threads = interp.head(); while !threads.is_null() { - let thread = self - .process - .copy_pointer(threads) - .context("Failed to copy PyThreadState")?; + let thread = self.process.copy_pointer(threads).context("Failed to copy PyThreadState")?; let current = thread.thread_id(); all_python_threads.insert(current); threads = thread.next(); } - let processed_os_threads: HashSet = - HashSet::from_iter(self.python_thread_ids.values().copied()); + let processed_os_threads: HashSet = HashSet::from_iter(self.python_thread_ids.values().map(|x| *x)); let unwinder = self.process.unwinder()?; @@ -445,15 +399,13 @@ impl PythonSpy { continue; } - match self._get_pthread_id(&unwinder, thread, &all_python_threads) { + match self._get_pthread_id(&unwinder, &thread, &all_python_threads) { Ok(pthread_id) => { if pthread_id != 0 { self.python_thread_ids.insert(pthread_id, threadid); } - } - Err(e) => { - warn!("Failed to get get_pthread_id for {}: {}", threadid, e); - } + }, + Err(e) => { warn!("Failed to get get_pthread_id for {}: {}", threadid, e); } }; } @@ -483,13 +435,9 @@ impl PythonSpy { Ok(None) } - #[cfg(all(target_os = "linux", unwind))] - pub fn _get_pthread_id( - &self, - unwinder: &remoteprocess::Unwinder, - thread: &remoteprocess::Thread, - threadids: &HashSet, - ) -> Result { + + #[cfg(all(target_os="linux", unwind))] + pub fn _get_pthread_id(&self, unwinder: &remoteprocess::Unwinder, thread: &remoteprocess::Thread, threadids: &HashSet) -> Result { let mut pthread_id = 0; let mut cursor = unwinder.cursor(thread)?; @@ -507,21 +455,31 @@ impl PythonSpy { Ok(pthread_id) } - #[cfg(target_os = "freebsd")] - fn _get_os_thread_id( - &mut self, - _python_thread_id: u64, - _interp: &I, - ) -> Result, Error> { + #[cfg(target_os="freebsd")] + fn _get_os_thread_id(&mut self, _python_thread_id: u64, _interp: &I) -> Result, Error> { Ok(None) } + fn _get_gil_threadid(&self) -> Result { + // figure out what thread has the GIL by inspecting _PyThreadState_Current + if self.threadstate_address > 0 { + let addr: usize = self.process.copy_struct(self.threadstate_address)?; + + // if the addr is 0, no thread is currently holding the GIL + if addr != 0 { + let threadstate: I::ThreadState = self.process.copy_struct(addr)?; + return Ok(threadstate.thread_id()); + } + } + Ok(0) + } + fn _get_python_thread_name(&mut self, python_thread_id: u64) -> Option { match self.python_thread_names.get(&python_thread_id) { Some(thread_name) => Some(thread_name.clone()), None => { - self.python_thread_names = thread_name_lookup(self).unwrap_or_default(); - self.python_thread_names.get(&python_thread_id).cloned() + self.python_thread_names = thread_name_lookup(self).unwrap_or_else(|| HashMap::new()); + self.python_thread_names.get(&python_thread_id).map(|name| name.clone()) } } } @@ -541,10 +499,10 @@ impl PythonSpy { } // on linux the process could be running in docker, access the filename through procfs - #[cfg(target_os = "linux")] + #[cfg(target_os="linux")] let filename_storage; - #[cfg(target_os = "linux")] + #[cfg(target_os="linux")] let filename = if self.dockerized { filename_storage = format!("/proc/{}/root{}", self.pid, filename); if Path::new(&filename_storage).exists() { @@ -565,14 +523,532 @@ impl PythonSpy { } } - // remove the parent prefix and convert to an optional string + // remote the parent prefix and convert to an optional string let shortened = Path::new(filename) .strip_prefix(path) .ok() .map(|p| p.to_string_lossy().to_string()); - self.short_filenames - .insert(filename.to_owned(), shortened.clone()); + self.short_filenames.insert(filename.to_owned(), shortened.clone()); shortened } } +/// Returns the version of python running in the process. +fn get_python_version(python_info: &PythonProcessInfo, process: &remoteprocess::Process) + -> Result { + // If possible, grab the sys.version string from the processes memory (mac osx). + if let Some(&addr) = python_info.get_symbol("Py_GetVersion.version") { + info!("Getting version from symbol address"); + if let Ok(bytes) = process.copy(addr as usize, 128) { + if let Ok(version) = Version::scan_bytes(&bytes) { + return Ok(version); + } + } + } + + // otherwise get version info from scanning BSS section for sys.version string + if let Some(ref pb) = python_info.python_binary { + info!("Getting version from python binary BSS"); + let bss = process.copy(pb.bss_addr as usize, + pb.bss_size as usize)?; + match Version::scan_bytes(&bss) { + Ok(version) => return Ok(version), + Err(err) => info!("Failed to get version from BSS section: {}", err) + } + } + + // try again if there is a libpython.so + if let Some(ref libpython) = python_info.libpython_binary { + info!("Getting version from libpython BSS"); + let bss = process.copy(libpython.bss_addr as usize, + libpython.bss_size as usize)?; + match Version::scan_bytes(&bss) { + Ok(version) => return Ok(version), + Err(err) => info!("Failed to get version from libpython BSS section: {}", err) + } + } + + // the python_filename might have the version encoded in it (/usr/bin/python3.5 etc). + // try reading that in (will miss patch level on python, but that shouldn't matter) + info!("Trying to get version from path: {}", python_info.python_filename.display()); + let path = Path::new(&python_info.python_filename); + if let Some(python) = path.file_name() { + if let Some(python) = python.to_str() { + if python.starts_with("python") { + let tokens: Vec<&str> = python[6..].split('.').collect(); + if tokens.len() >= 2 { + if let (Ok(major), Ok(minor)) = (tokens[0].parse::(), tokens[1].parse::()) { + return Ok(Version{major, minor, patch:0, release_flags: "".to_owned()}) + } + } + } + } + } + Err(format_err!("Failed to find python version from target process")) +} + +fn get_interpreter_address(python_info: &PythonProcessInfo, + process: &remoteprocess::Process, + version: &Version) -> Result { + // get the address of the main PyInterpreterState object from loaded symbols if we can + // (this tends to be faster than scanning through the bss section) + match version { + Version{major: 3, minor: 7..=11, ..} => { + if let Some(&addr) = python_info.get_symbol("_PyRuntime") { + let addr = process.copy_struct(addr as usize + pyruntime::get_interp_head_offset(&version))?; + + // Make sure the interpreter addr is valid before returning + match check_interpreter_addresses(&[addr], &python_info.maps, process, version) { + Ok(addr) => return Ok(addr), + Err(_) => { warn!("Interpreter address from _PyRuntime symbol is invalid {:016x}", addr); } + }; + } + }, + _ => { + if let Some(&addr) = python_info.get_symbol("interp_head") { + let addr = process.copy_struct(addr as usize)?; + match check_interpreter_addresses(&[addr], &python_info.maps, process, version) { + Ok(addr) => return Ok(addr), + Err(_) => { warn!("Interpreter address from interp_head symbol is invalid {:016x}", addr); } + }; + } + } + }; + info!("Failed to get interp_head from symbols, scanning BSS section from main binary"); + + // try scanning the BSS section of the binary for things that might be the interpreterstate + let err = + if let Some(ref pb) = python_info.python_binary { + match get_interpreter_address_from_binary(pb, &python_info.maps, process, version) { + Ok(addr) => return Ok(addr), + err => Some(err) + } + } else { + None + }; + // Before giving up, try again if there is a libpython.so + if let Some(ref lpb) = python_info.libpython_binary { + info!("Failed to get interpreter from binary BSS, scanning libpython BSS"); + match get_interpreter_address_from_binary(lpb, &python_info.maps, process, version) { + Ok(addr) => return Ok(addr), + lib_err => err.unwrap_or(lib_err) + } + } else { + err.expect("Both python and libpython are invalid.") + } +} + +fn get_interpreter_address_from_binary(binary: &BinaryInfo, + maps: &[MapRange], + process: &remoteprocess::Process, + version: &Version) -> Result { + // We're going to scan the BSS/data section for things, and try to narrowly scan things that + // look like pointers to PyinterpreterState + let bss = process.copy(binary.bss_addr as usize, binary.bss_size as usize)?; + + #[allow(clippy::cast_ptr_alignment)] + let addrs = unsafe { slice::from_raw_parts(bss.as_ptr() as *const usize, bss.len() / size_of::()) }; + check_interpreter_addresses(addrs, maps, process, version) +} + +// Checks whether a block of memory (from BSS/.data etc) contains pointers that are pointing +// to a valid PyInterpreterState +fn check_interpreter_addresses(addrs: &[usize], + maps: &[MapRange], + process: &remoteprocess::Process, + version: &Version) -> Result { + // On windows, we can't just check if a pointer is valid by looking to see if it points + // to something in the virtual memory map. Brute-force it instead + #[cfg(windows)] + fn maps_contain_addr(_: usize, _: &[MapRange]) -> bool { true } + + #[cfg(not(windows))] + use proc_maps::maps_contain_addr; + + // This function does all the work, but needs a type of the interpreter + fn check(addrs: &[usize], + maps: &[MapRange], + process: &remoteprocess::Process) -> Result + where I: python_interpreters::InterpreterState { + for &addr in addrs { + if maps_contain_addr(addr, maps) { + // this address points to valid memory. try loading it up as a PyInterpreterState + // to further check + let interp: I = match process.copy_struct(addr) { + Ok(interp) => interp, + Err(_) => continue + }; + + // get the pythreadstate pointer from the interpreter object, and if it is also + // a valid pointer then load it up. + let threads = interp.head(); + if maps_contain_addr(threads as usize, maps) { + // If the threadstate points back to the interpreter like we expect, then + // this is almost certainly the address of the intrepreter + let thread = match process.copy_pointer(threads) { + Ok(thread) => thread, + Err(_) => continue + }; + + // as a final sanity check, try getting the stack_traces, and only return if this works + if thread.interp() as usize == addr && get_stack_traces(&interp, process, LineNo::NoLine).is_ok() { + return Ok(addr); + } + } + } + } + Err(format_err!("Failed to find a python interpreter in the .data section")) + } + + // different versions have different layouts, check as appropriate + match version { + Version{major: 2, minor: 3..=7, ..} => check::(addrs, maps, process), + Version{major: 3, minor: 3, ..} => check::(addrs, maps, process), + Version{major: 3, minor: 4..=5, ..} => check::(addrs, maps, process), + Version{major: 3, minor: 6, ..} => check::(addrs, maps, process), + Version{major: 3, minor: 7, ..} => check::(addrs, maps, process), + Version{major: 3, minor: 8, patch: 0, ..} => { + match version.release_flags.as_ref() { + "a1" | "a2" | "a3" => check::(addrs, maps, process), + _ => check::(addrs, maps, process) + } + }, + Version{major: 3, minor: 8, ..} => check::(addrs, maps, process), + Version{major: 3, minor: 9, ..} => check::(addrs, maps, process), + Version{major: 3, minor: 10, ..} => check::(addrs, maps, process), + Version{major: 3, minor: 11, ..} => check::(addrs, maps, process), + _ => Err(format_err!("Unsupported version of Python: {}", version)) + } +} + +/// Holds information about the python process: memory map layout, parsed binary info +/// for python /libpython etc. +pub struct PythonProcessInfo { + python_binary: Option, + // if python was compiled with './configure --enabled-shared', code/symbols will + // be in a libpython.so file instead of the executable. support that. + libpython_binary: Option, + maps: Vec, + python_filename: std::path::PathBuf, + #[cfg(target_os="linux")] + dockerized: bool, +} + +impl PythonProcessInfo { + fn new(process: &remoteprocess::Process) -> Result { + let filename = process.exe() + .context("Failed to get process executable name. Check that the process is running.")?; + + #[cfg(windows)] + let filename = filename.to_lowercase(); + + #[cfg(windows)] + let is_python_bin = |pathname: &str| pathname.to_lowercase() == filename; + + #[cfg(not(windows))] + let is_python_bin = |pathname: &str| pathname == filename; + + // get virtual memory layout + let maps = get_process_maps(process.pid)?; + info!("Got virtual memory maps from pid {}:", process.pid); + for map in &maps { + debug!("map: {:016x}-{:016x} {}{}{} {}", map.start(), map.start() + map.size(), + if map.is_read() {'r'} else {'-'}, if map.is_write() {'w'} else {'-'}, if map.is_exec() {'x'} else {'-'}, + map.filename().unwrap_or(&std::path::PathBuf::from("")).display()); + } + + // parse the main python binary + let (python_binary, python_filename) = { + // Get the memory address for the executable by matching against virtual memory maps + let map = maps.iter() + .find(|m| { + if let Some(pathname) = m.filename() { + if let Some(pathname) = pathname.to_str() { + return is_python_bin(pathname) && m.is_exec(); + } + } + false + }); + + let map = match map { + Some(map) => map, + None => { + warn!("Failed to find '{}' in virtual memory maps, falling back to first map region", filename); + // If we failed to find the executable in the virtual memory maps, just take the first file we find + // sometimes on windows get_process_exe returns stale info =( https://github.com/benfred/py-spy/issues/40 + // and on all operating systems I've tried, the exe is the first region in the maps + &maps.first().ok_or_else(|| format_err!("Failed to get virtual memory maps from process"))? + } + }; + + let filename = std::path::PathBuf::from(filename); + + // TODO: consistent types? u64 -> usize? for map.start etc + #[allow(unused_mut)] + let python_binary = parse_binary(process.pid, &filename, map.start() as u64, map.size() as u64, true) + .and_then(|mut pb| { + // windows symbols are stored in separate files (.pdb), load + #[cfg(windows)] + { + get_windows_python_symbols(process.pid, &filename, map.start() as u64) + .map(|symbols| { pb.symbols.extend(symbols); pb }) + .map_err(|err| err.into()) + } + + // For OSX, need to adjust main binary symbols by subtracting _mh_execute_header + // (which we've added to by map.start already, so undo that here) + #[cfg(target_os = "macos")] + { + let offset = pb.symbols["_mh_execute_header"] - map.start() as u64; + for address in pb.symbols.values_mut() { + *address -= offset; + } + + if pb.bss_addr != 0 { + pb.bss_addr -= offset; + } + } + + #[cfg(not(windows))] + Ok(pb) + }); + + (python_binary, filename.clone()) + }; + + // likewise handle libpython for python versions compiled with --enabled-shared + let libpython_binary = { + let libmap = maps.iter() + .find(|m| { + if let Some(pathname) = m.filename() { + if let Some(pathname) = pathname.to_str() { + return is_python_lib(pathname) && m.is_exec(); + } + } + false + }); + + let mut libpython_binary: Option = None; + if let Some(libpython) = libmap { + if let Some(filename) = &libpython.filename() { + info!("Found libpython binary @ {}", filename.display()); + #[allow(unused_mut)] + let mut parsed = parse_binary(process.pid, filename, libpython.start() as u64, libpython.size() as u64, false)?; + #[cfg(windows)] + parsed.symbols.extend(get_windows_python_symbols(process.pid, filename, libpython.start() as u64)?); + libpython_binary = Some(parsed); + } + } + + // On OSX, it's possible that the Python library is a dylib loaded up from the system + // framework (like /System/Library/Frameworks/Python.framework/Versions/2.7/Python) + // In this case read in the dyld_info information and figure out the filename from there + #[cfg(target_os = "macos")] + { + if libpython_binary.is_none() { + use proc_maps::mac_maps::get_dyld_info; + let dyld_infos = get_dyld_info(process.pid)?; + + for dyld in &dyld_infos { + let segname = unsafe { std::ffi::CStr::from_ptr(dyld.segment.segname.as_ptr()) }; + debug!("dyld: {:016x}-{:016x} {:10} {}", + dyld.segment.vmaddr, dyld.segment.vmaddr + dyld.segment.vmsize, + segname.to_string_lossy(), dyld.filename.display()); + } + + let python_dyld_data = dyld_infos.iter() + .find(|m| { + if let Some(filename) = m.filename.to_str() { + return is_python_framework(filename) && + m.segment.segname[0..7] == [95, 95, 68, 65, 84, 65, 0]; + } + false + }); + + + if let Some(libpython) = python_dyld_data { + info!("Found libpython binary from dyld @ {}", libpython.filename.display()); + + let mut binary = parse_binary(process.pid, &libpython.filename, libpython.segment.vmaddr, libpython.segment.vmsize, false)?; + + // TODO: bss addr offsets returned from parsing binary are wrong + // (assumes data section isn't split from text section like done here). + // BSS occurs somewhere in the data section, just scan that + // (could later tighten this up to look at segment sections too) + binary.bss_addr = libpython.segment.vmaddr; + binary.bss_size = libpython.segment.vmsize; + libpython_binary = Some(binary); + } + } + } + + libpython_binary + }; + + // If we have a libpython binary - we can tolerate failures on parsing the main python binary. + let python_binary = match libpython_binary { + None => Some(python_binary.context("Failed to parse python binary")?), + _ => python_binary.ok(), + }; + + #[cfg(target_os="linux")] + let dockerized = is_dockerized(process.pid).unwrap_or(false); + + Ok(PythonProcessInfo{python_binary, libpython_binary, maps, python_filename, + #[cfg(target_os="linux")] + dockerized + }) + } + + pub fn get_symbol(&self, symbol: &str) -> Option<&u64> { + if let Some(ref pb) = self.python_binary { + if let Some(addr) = pb.symbols.get(symbol) { + info!("got symbol {} (0x{:016x}) from python binary", symbol, addr); + return Some(addr); + } + } + + if let Some(ref binary) = self.libpython_binary { + if let Some(addr) = binary.symbols.get(symbol) { + info!("got symbol {} (0x{:016x}) from libpython binary", symbol, addr); + return Some(addr); + } + } + None + } +} + +#[cfg(target_os="linux")] +fn is_dockerized(pid: Pid) -> Result { + let self_mnt = std::fs::read_link("/proc/self/ns/mnt")?; + let target_mnt = std::fs::read_link(&format!("/proc/{}/ns/mnt", pid))?; + Ok(self_mnt != target_mnt) +} + +// We can't use goblin to parse external symbol files (like in a separate .pdb file) on windows, +// So use the win32 api to load up the couple of symbols we need on windows. Note: +// we still can get export's from the PE file +#[cfg(windows)] +pub fn get_windows_python_symbols(pid: Pid, filename: &Path, offset: u64) -> std::io::Result> { + use proc_maps::win_maps::SymbolLoader; + + let handler = SymbolLoader::new(pid)?; + let _module = handler.load_module(filename)?; // need to keep this module in scope + + let mut ret = HashMap::new(); + + // currently we only need a subset of symbols, and enumerating the symbols is + // expensive (via SymEnumSymbolsW), so rather than load up all symbols like we + // do for goblin, just load the the couple we need directly. + for symbol in ["_PyThreadState_Current", "interp_head", "_PyRuntime"].iter() { + if let Ok((base, addr)) = handler.address_from_name(symbol) { + // If we have a module base (ie from PDB), need to adjust by the offset + // otherwise seems like we can take address directly + let addr = if base == 0 { addr } else { offset + addr - base }; + ret.insert(String::from(*symbol), addr); + } + } + + Ok(ret) +} + +#[cfg(any(target_os="linux", target_os="freebsd"))] +pub fn is_python_lib(pathname: &str) -> bool { + lazy_static! { + static ref RE: Regex = Regex::new(r"/libpython\d.\d\d?(m|d|u)?.so").unwrap(); + } + RE.is_match(pathname) +} + +#[cfg(target_os="macos")] +pub fn is_python_lib(pathname: &str) -> bool { + lazy_static! { + static ref RE: Regex = Regex::new(r"/libpython\d.\d\d?(m|d|u)?.(dylib|so)$").unwrap(); + } + RE.is_match(pathname) || is_python_framework(pathname) +} + +#[cfg(windows)] +pub fn is_python_lib(pathname: &str) -> bool { + lazy_static! { + static ref RE: Regex = RegexBuilder::new(r"\\python\d\d\d?(m|d|u)?.dll$").case_insensitive(true).build().unwrap(); + } + RE.is_match(pathname) +} + +#[cfg(target_os="macos")] +pub fn is_python_framework(pathname: &str) -> bool { + pathname.ends_with("/Python") && + !pathname.contains("Python.app") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[cfg(target_os="macos")] + #[test] + fn test_is_python_lib() { + assert!(is_python_lib("~/Anaconda2/lib/libpython2.7.dylib")); + + // python lib configured with --with-pydebug (flag: d) + assert!(is_python_lib("/lib/libpython3.4d.dylib")); + + // configured --with-pymalloc (flag: m) + assert!(is_python_lib("/usr/local/lib/libpython3.8m.dylib")); + + // python2 configured with --with-wide-unicode (flag: u) + assert!(is_python_lib("./libpython2.7u.dylib")); + + assert!(!is_python_lib("/libboost_python.dylib")); + assert!(!is_python_lib("/lib/heapq.cpython-36m-darwin.dylib")); + } + + #[cfg(any(target_os="linux", target_os="freebsd"))] + #[test] + fn test_is_python_lib() { + // libpython bundled by pyinstaller https://github.com/benfred/py-spy/issues/42 + assert!(is_python_lib("/tmp/_MEIOqzg01/libpython2.7.so.1.0")); + + // test debug/malloc/unicode flags + assert!(is_python_lib("./libpython2.7.so")); + assert!(is_python_lib("/usr/lib/libpython3.4d.so")); + assert!(is_python_lib("/usr/local/lib/libpython3.8m.so")); + assert!(is_python_lib("/usr/lib/libpython2.7u.so")); + + // don't blindly match libraries with python in the name (boost_python etc) + assert!(!is_python_lib("/usr/lib/libboost_python.so")); + assert!(!is_python_lib("/usr/lib/x86_64-linux-gnu/libboost_python-py27.so.1.58.0")); + assert!(!is_python_lib("/usr/lib/libboost_python-py35.so")); + + } + + #[cfg(windows)] + #[test] + fn test_is_python_lib() { + assert!(is_python_lib("C:\\Users\\test\\AppData\\Local\\Programs\\Python\\Python37\\python37.dll")); + // .NET host via https://github.com/pythonnet/pythonnet + assert!(is_python_lib("C:\\Users\\test\\AppData\\Local\\Programs\\Python\\Python37\\python37.DLL")); + } + + + #[cfg(target_os="macos")] + #[test] + fn test_python_frameworks() { + // homebrew v2 + assert!(!is_python_framework("/usr/local/Cellar/python@2/2.7.15_1/Frameworks/Python.framework/Versions/2.7/Resources/Python.app/Contents/MacOS/Python")); + assert!(is_python_framework("/usr/local/Cellar/python@2/2.7.15_1/Frameworks/Python.framework/Versions/2.7/Python")); + + // System python from osx 10.13.6 (high sierra) + assert!(!is_python_framework("/System/Library/Frameworks/Python.framework/Versions/2.7/Resources/Python.app/Contents/MacOS/Python")); + assert!(is_python_framework("/System/Library/Frameworks/Python.framework/Versions/2.7/Python")); + + // pyenv 3.6.6 with OSX framework enabled (https://github.com/benfred/py-spy/issues/15) + // env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install 3.6.6 + assert!(is_python_framework("/Users/ben/.pyenv/versions/3.6.6/Python.framework/Versions/3.6/Python")); + assert!(!is_python_framework("/Users/ben/.pyenv/versions/3.6.6/Python.framework/Versions/3.6/Resources/Python.app/Contents/MacOS/Python")); + + // single file pyinstaller + assert!(is_python_framework("/private/var/folders/3x/qy479lpd1fb2q88lc9g4d3kr0000gn/T/_MEI2Akvi8/Python")); + } +} diff --git a/src/python_threading.rs b/src/python_threading.rs index d60acd6d..b481e3df 100644 --- a/src/python_threading.rs +++ b/src/python_threading.rs @@ -2,10 +2,10 @@ use std::collections::HashMap; use anyhow::Error; -use crate::python_bindings::{v3_10_0, v3_11_0, v3_6_6, v3_7_0, v3_8_0, v3_9_5}; -use crate::python_data_access::{copy_long, copy_string, DictIterator, PY_TPFLAGS_MANAGED_DICT}; +use crate::python_bindings::{v3_6_6, v3_7_0, v3_8_0, v3_9_5, v3_10_0, v3_11_0}; use crate::python_interpreters::{InterpreterState, Object, TypeObject}; use crate::python_spy::PythonSpy; +use crate::python_data_access::{copy_string, copy_long, DictIterator, PY_TPFLAGS_MANAGED_DICT}; use crate::version::Version; @@ -13,25 +13,24 @@ use remoteprocess::ProcessMemory; /// Returns a hashmap of threadid: threadname, by inspecting the '_active' variable in the /// 'threading' module. -pub fn thread_names_from_interpreter( - interp: &I, - process: &P, - version: &Version, -) -> Result, Error> { +fn _thread_name_lookup(spy: &PythonSpy) -> Result, Error> { let mut ret = HashMap::new(); - for entry in DictIterator::from(process, version, interp.modules() as usize)? { + let process = &spy.process; + let interp: I = process.copy_struct(spy.interpreter_address)?; + for entry in DictIterator::from(process, &spy.version, interp.modules() as usize)? { let (key, value) = entry?; let module_name = copy_string(key as *const I::StringObject, process)?; if module_name == "threading" { let module: I::Object = process.copy_struct(value)?; let module_type = process.copy_pointer(module.ob_type())?; let dictptr: usize = process.copy_struct(value + module_type.dictoffset() as usize)?; - for i in DictIterator::from(process, version, dictptr)? { + for i in DictIterator::from(process, &spy.version, dictptr)? { let (key, value) = i?; let name = copy_string(key as *const I::StringObject, process)?; if name == "_active" { - for i in DictIterator::from(process, version, value)? { + + for i in DictIterator::from(process, &spy.version, value)? { let (key, value) = i?; let (threadid, _) = copy_long(process, key)?; @@ -39,17 +38,12 @@ pub fn thread_names_from_interpreter( let thread_type = process.copy_pointer(thread.ob_type())?; let dict_iter = if thread_type.flags() & PY_TPFLAGS_MANAGED_DICT != 0 { - DictIterator::from_managed_dict( - process, - version, - value, - thread.ob_type() as usize, - )? + DictIterator::from_managed_dict(process, &spy.version, value, thread.ob_type() as usize)? } else { let dict_offset = thread_type.dictoffset(); - let dict_addr = (value as isize + dict_offset) as usize; + let dict_addr =(value as isize + dict_offset) as usize; let thread_dict_addr: usize = process.copy_struct(dict_addr)?; - DictIterator::from(process, version, thread_dict_addr)? + DictIterator::from(process, &spy.version, thread_dict_addr)? }; for i in dict_iter { @@ -57,8 +51,7 @@ pub fn thread_names_from_interpreter( let varname = copy_string(key as *const I::StringObject, process)?; if varname == "_name" { - let threadname = - copy_string(value as *const I::StringObject, process)?; + let threadname = copy_string(value as *const I::StringObject, process)?; ret.insert(threadid as u64, threadname); break; } @@ -73,43 +66,18 @@ pub fn thread_names_from_interpreter( Ok(ret) } -/// Returns a hashmap of threadid: threadname, by inspecting the '_active' variable in the -/// 'threading' module. -fn _thread_name_lookup( - spy: &PythonSpy, -) -> Result, Error> { - let interp: I = spy.process.copy_struct(spy.interpreter_address)?; - thread_names_from_interpreter(&interp, &spy.process, &spy.version) -} - // try getting the threadnames, but don't sweat it if we can't. Since this relies on dictionary // processing we only handle py3.6+ right now, and this doesn't work at all if the // threading module isn't imported in the target program pub fn thread_name_lookup(process: &PythonSpy) -> Option> { let err = match process.version { - Version { - major: 3, minor: 6, .. - } => _thread_name_lookup::(process), - Version { - major: 3, minor: 7, .. - } => _thread_name_lookup::(process), - Version { - major: 3, minor: 8, .. - } => _thread_name_lookup::(process), - Version { - major: 3, minor: 9, .. - } => _thread_name_lookup::(process), - Version { - major: 3, - minor: 10, - .. - } => _thread_name_lookup::(process), - Version { - major: 3, - minor: 11, - .. - } => _thread_name_lookup::(process), - _ => return None, + Version{major: 3, minor: 6, ..} => _thread_name_lookup::(&process), + Version{major: 3, minor: 7, ..} => _thread_name_lookup::(&process), + Version{major: 3, minor: 8, ..} => _thread_name_lookup::(&process), + Version{major: 3, minor: 9, ..} => _thread_name_lookup::(&process), + Version{major: 3, minor: 10, ..} => _thread_name_lookup::(&process), + Version{major: 3, minor: 11, ..} => _thread_name_lookup::(&process), + _ => return None }; err.ok() } diff --git a/src/sampler.rs b/src/sampler.rs index db010e6c..2c6fa979 100644 --- a/src/sampler.rs +++ b/src/sampler.rs @@ -1,19 +1,18 @@ -#![allow(clippy::type_complexity)] - -use std::collections::HashMap; -use std::sync::mpsc::{self, Receiver, Sender}; -use std::sync::{Arc, Mutex}; -use std::thread; +use std::collections::{HashMap, HashSet}; +use std::path::Path; +use std::sync::mpsc::{self, Sender, Receiver}; +use std::sync::{Mutex, Arc}; use std::time::Duration; +use std::thread; use anyhow::Error; use remoteprocess::Pid; -use crate::config::Config; -use crate::python_spy::PythonSpy; -use crate::stack_trace::{ProcessInfo, StackTrace}; use crate::timer::Timer; +use crate::python_spy::PythonSpy; +use crate::config::Config; +use crate::stack_trace::{StackTrace, ProcessInfo}; use crate::version::Version; pub struct Sampler { @@ -25,7 +24,7 @@ pub struct Sampler { pub struct Sample { pub traces: Vec, pub sampling_errors: Option>, - pub late: Option, + pub late: Option } impl Sampler { @@ -40,23 +39,20 @@ impl Sampler { /// Creates a new sampler object, reading from a single process only fn new_sampler(pid: Pid, config: &Config) -> Result { let (tx, rx): (Sender, Receiver) = mpsc::channel(); - let (initialized_tx, initialized_rx): ( - Sender>, - Receiver>, - ) = mpsc::channel(); + let (initialized_tx, initialized_rx): (Sender>, Receiver>) = mpsc::channel(); let config = config.clone(); let sampling_thread = thread::spawn(move || { // We need to create this object inside the thread here since PythonSpy objects don't // have the Send trait implemented on linux let mut spy = match PythonSpy::retry_new(pid, &config, 20) { Ok(spy) => { - if initialized_tx.send(Ok(spy.version.clone())).is_err() { + if let Err(_) = initialized_tx.send(Ok(spy.version.clone())) { return; } spy - } - Err(e) => { - initialized_tx.send(Err(e)).unwrap_err(); + }, + Err(e) => { + if initialized_tx.send(Err(e)).is_err() {} return; } }; @@ -67,10 +63,7 @@ impl Sampler { Ok(traces) => traces, Err(e) => { if spy.process.exe().is_err() { - info!( - "stopped sampling pid {} because the process exited", - spy.pid - ); + info!("stopped sampling pid {} because the process exited", spy.pid); break; } sampling_errors = Some(vec![(spy.pid, e)]); @@ -79,25 +72,14 @@ impl Sampler { }; let late = sleep.err(); - if tx - .send(Sample { - traces, - sampling_errors, - late, - }) - .is_err() - { + if tx.send(Sample{traces: traces, sampling_errors, late}).is_err() { break; } } }); let version = initialized_rx.recv()??; - Ok(Sampler { - rx: Some(rx), - version: Some(version), - sampling_thread: Some(sampling_thread), - }) + Ok(Sampler{rx: Some(rx), version: Some(version), sampling_thread: Some(sampling_thread)}) } /// Creates a new sampler object that samples any python process in the @@ -108,19 +90,15 @@ impl Sampler { // Initialize a PythonSpy object per child, and build up the process tree let mut spies = HashMap::new(); let mut retries = 10; - spies.insert(pid, PythonSpyThread::new(pid, None, config)?); + spies.insert(pid, PythonSpyThread::new(pid, None, &config)?); loop { for (childpid, parentpid) in process.child_processes()? { // If we can't create the child process, don't worry about it // can happen with zombie child processes etc - match PythonSpyThread::new(childpid, Some(parentpid), config) { - Ok(spy) => { - spies.insert(childpid, spy); - } - Err(e) => { - warn!("Failed to open process {}: {}", childpid, e); - } + match PythonSpyThread::new(childpid, Some(parentpid), &config) { + Ok(spy) => { spies.insert(childpid, spy); }, + Err(e) => { warn!("Failed to open process {}: {}", childpid, e); } } } @@ -133,10 +111,7 @@ impl Sampler { // Otherwise sleep for a short time and retry retries -= 1; if retries == 0 { - return Err(format_err!( - "No python processes found in process {} or any of its subprocesses", - pid - )); + return Err(format_err!("No python processes found in process {} or any of its subprocesses", pid)); } std::thread::sleep(std::time::Duration::from_millis(100)); } @@ -147,29 +122,24 @@ impl Sampler { let monitor_spies = spies.clone(); let monitor_config = config.clone(); std::thread::spawn(move || { + let mut checked_pids = HashSet::new(); + while process.exe().is_ok() { match monitor_spies.lock() { Ok(mut spies) => { - for (childpid, parentpid) in process - .child_processes() - .expect("failed to get subprocesses") - { - if spies.contains_key(&childpid) { - continue; + for (childpid, parentpid) in process.child_processes().expect("failed to get subprocesses") { + if checked_pids.insert(childpid) == false { + continue; // Already in set } - match PythonSpyThread::new(childpid, Some(parentpid), &monitor_config) { - Ok(spy) => { - spies.insert(childpid, spy); - } - Err(e) => { - warn!("Failed to create spy for {}: {}", childpid, e); + if should_spy_process(childpid, &monitor_config) { + match PythonSpyThread::new(childpid, Some(parentpid), &monitor_config) { + Ok(spy) => {spies.insert(childpid, spy);} + Err(e) => {warn!("Failed to create spy for {}: {}", childpid, e);} } } } - } - Err(e) => { - error!("Failed to acquire lock: {}", e); - } + }, + Err(e) => { error!("Failed to acquire lock: {}", e); } } std::thread::sleep(Duration::from_millis(100)); } @@ -203,11 +173,11 @@ impl Sampler { // collect the traces from each python spy if possible for spy in spies.values_mut() { match spy.collect() { - Some(Ok(mut t)) => traces.append(&mut t), + Some(Ok(mut t)) => { traces.append(&mut t) }, Some(Err(e)) => { - let errors = sampling_errors.get_or_insert_with(Vec::new); + let errors = sampling_errors.get_or_insert_with(|| Vec::new()); errors.push((spy.process.pid, e)); - } + }, None => {} } } @@ -216,22 +186,15 @@ impl Sampler { for trace in traces.iter_mut() { let pid = trace.pid; // Annotate each trace with the process info for the current - let process = process_info - .entry(pid) - .or_insert_with(|| get_process_info(pid, &spies).map(|p| Arc::new(*p))); + let process = process_info.entry(pid).or_insert_with(|| { + get_process_info(pid, &spies).map(|p| Arc::new(*p)) + }); trace.process_info = process.clone(); } // Send the collected info back let late = sleep.err(); - if tx - .send(Sample { - traces, - sampling_errors, - late, - }) - .is_err() - { + if tx.send(Sample{traces, sampling_errors, late}).is_err() { break; } @@ -242,11 +205,7 @@ impl Sampler { } }); - Ok(Sampler { - rx: Some(rx), - version: None, - sampling_thread: Some(sampling_thread), - }) + Ok(Sampler{rx: Some(rx), version: None, sampling_thread: Some(sampling_thread)}) } } @@ -275,84 +234,61 @@ struct PythonSpyThread { notified: bool, pub process: remoteprocess::Process, pub parent: Option, - pub command_line: String, + pub command_line: String } impl PythonSpyThread { fn new(pid: Pid, parent: Option, config: &Config) -> Result { - let (initialized_tx, initialized_rx): ( - Sender>, - Receiver>, - ) = mpsc::channel(); + let (initialized_tx, initialized_rx): (Sender>, Receiver>) = mpsc::channel(); let (notify_tx, notify_rx): (Sender<()>, Receiver<()>) = mpsc::channel(); - let (sample_tx, sample_rx): ( - Sender, Error>>, - Receiver, Error>>, - ) = mpsc::channel(); + let (sample_tx, sample_rx): (Sender, Error>>, Receiver, Error>>) = mpsc::channel(); let config = config.clone(); let process = remoteprocess::Process::new(pid)?; - let command_line = process - .cmdline() - .map(|x| x.join(" ")) - .unwrap_or_else(|_| "".to_owned()); + let command_line = process.cmdline().map(|x| x.join(" ")).unwrap_or("".to_owned()); thread::spawn(move || { // We need to create this object inside the thread here since PythonSpy objects don't // have the Send trait implemented on linux let mut spy = match PythonSpy::retry_new(pid, &config, 5) { Ok(spy) => { - if initialized_tx.send(Ok(spy.version.clone())).is_err() { + if let Err(_) = initialized_tx.send(Ok(spy.version.clone())) { return; } spy - } - Err(e) => { + }, + Err(e) => { warn!("Failed to profile python from process {}: {}", pid, e); - initialized_tx.send(Err(e)).unwrap_err(); + if initialized_tx.send(Err(e)).is_err() {} return; } }; for _ in notify_rx.iter() { let result = spy.get_stack_traces(); - if result.is_err() && spy.process.exe().is_err() { - info!( - "stopped sampling pid {} because the process exited", - spy.pid - ); - break; + if let Err(_) = result { + if spy.process.exe().is_err() { + info!("stopped sampling pid {} because the process exited", spy.pid); + break; + } } if sample_tx.send(result).is_err() { break; } } }); - Ok(PythonSpyThread { - initialized_rx, - notify_tx, - sample_rx, - process, - command_line, - parent, - initialized: None, - running: false, - notified: false, - }) + Ok(PythonSpyThread{initialized_rx, notify_tx, sample_rx, process, command_line, parent, initialized: None, running: false, notified: false}) } - fn wait_initialized(&mut self) -> bool { + fn wait_initialized(&mut self) -> bool { match self.initialized_rx.recv() { Ok(status) => { self.running = status.is_ok(); self.initialized = Some(status); self.running - } + }, Err(e) => { // shouldn't happen, but will be ok if it does - warn!( - "Failed to get initialization status from PythonSpyThread: {}", - e - ); + warn!("Failed to get initialization status from PythonSpyThread: {}", e); false } } @@ -367,7 +303,7 @@ impl PythonSpyThread { self.running = status.is_ok(); self.initialized = Some(status); self.running - } + }, Err(std::sync::mpsc::TryRecvError::Empty) => false, Err(std::sync::mpsc::TryRecvError::Disconnected) => { // this *shouldn't* happen @@ -379,16 +315,12 @@ impl PythonSpyThread { fn notify(&mut self) { match self.notify_tx.send(()) { - Ok(_) => { - self.notified = true; - } - Err(_) => { - self.running = false; - } + Ok(_) => { self.notified = true; }, + Err(_) => { self.running = false; } } } - fn collect(&mut self) -> Option, Error>> { + fn collect(&mut self) -> Option, Error>> { if !self.notified { return None; } @@ -405,13 +337,35 @@ impl PythonSpyThread { fn get_process_info(pid: Pid, spies: &HashMap) -> Option> { spies.get(&pid).map(|spy| { - let parent = spy - .parent - .and_then(|parentpid| get_process_info(parentpid, spies)); - Box::new(ProcessInfo { - pid, - parent, - command_line: spy.command_line.clone(), - }) + let parent = spy.parent.and_then(|parentpid| get_process_info(parentpid, spies)); + Box::new(ProcessInfo{pid, parent, command_line: spy.command_line.clone()}) }) } + +fn get_process_name(pid: Pid) -> Result{ + let proc = remoteprocess::Process::new(pid)?; + let exe_path = proc.exe()?; + let exe_name = Path::new(&exe_path).file_stem().ok_or(anyhow!("Failed to get process name"))?; + + Ok(exe_name.to_str().unwrap().into()) +} + +// Decide if we should spy on a process, given its process id +fn should_spy_process(pid: Pid, monitor_config: &Config) -> bool { + if let Some(whitelist) = monitor_config.whitelist.as_ref() { + if whitelist.len() == 0 { + return true; + } + match get_process_name(pid) { + Ok(proc_name) => { + return whitelist.iter().any(|item|item.eq_ignore_ascii_case(&proc_name)); + } + Err(_) => { + warn!("Failed to get process name for PID: {}", pid); + return false; + } + } + } else { + return true; + } +} \ No newline at end of file diff --git a/src/speedscope.rs b/src/speedscope.rs index 3cc0725c..5ce378de 100644 --- a/src/speedscope.rs +++ b/src/speedscope.rs @@ -26,15 +26,16 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -use std::collections::HashMap; +use std::collections::{HashMap}; use std::io; use std::io::Write; use crate::stack_trace; -use remoteprocess::{Pid, Tid}; +use remoteprocess::{Tid, Pid}; -use anyhow::Error; +use anyhow::{Error}; use serde_derive::{Deserialize, Serialize}; +use serde_json; use crate::config::Config; @@ -127,51 +128,40 @@ enum ValueUnit { } impl SpeedscopeFile { - pub fn new( - samples: &HashMap<(Pid, Tid), Vec>>, - frames: &[Frame], - thread_name_map: &HashMap<(Pid, Tid), String>, - sample_rate: u64, - ) -> SpeedscopeFile { - let mut profiles: Vec = samples - .iter() - .map(|(thread_id, samples)| { - let end_value = samples.len(); - // we sample at 100 Hz, so scale the end value and weights to match the time unit - let scaled_end_value = end_value as f64 / sample_rate as f64; - let weights: Vec = samples - .iter() - .map(|_s| 1_f64 / sample_rate as f64) - .collect(); - - Profile { - profile_type: ProfileType::Sampled, - name: thread_name_map - .get(thread_id) - .map_or_else(|| "py-spy".to_string(), |x| x.clone()), - unit: ValueUnit::Seconds, - start_value: 0.0, - end_value: scaled_end_value, - samples: samples.clone(), - weights, - } - }) - .collect(); - - profiles.sort_by(|a, b| a.name.cmp(&b.name)); - - SpeedscopeFile { - // This is always the same - schema: "https://www.speedscope.app/file-format-schema.json".to_string(), - active_profile_index: None, - name: Some("py-spy profile".to_string()), - exporter: Some(format!("py-spy@{}", env!("CARGO_PKG_VERSION"))), - profiles, - shared: Shared { - frames: frames.to_owned(), - }, + pub fn new(samples: &HashMap<(Pid, Tid), Vec>>, frames: &Vec, + thread_name_map: &HashMap<(Pid, Tid), String>, sample_rate: u64) -> SpeedscopeFile { + + let mut profiles: Vec = samples.iter().map(|(thread_id, samples)| { + let end_value = samples.len(); + // we sample at 100 Hz, so scale the end value and weights to match the time unit + let scaled_end_value = end_value as f64 / sample_rate as f64; + let weights: Vec = (&samples).iter().map(|_s| 1_f64 / sample_rate as f64).collect(); + + Profile { + profile_type: ProfileType::Sampled, + name: thread_name_map.get(thread_id).map_or_else(|| "py-spy".to_string(), |x| x.clone()), + unit: ValueUnit::Seconds, + start_value: 0.0, + end_value: scaled_end_value, + samples: samples.clone(), + weights } + }).collect(); + + profiles.sort_by(|a, b| a.name.cmp(&b.name)); + + SpeedscopeFile { + // This is always the same + schema: "https://www.speedscope.app/file-format-schema.json".to_string(), + active_profile_index: None, + name: Some("py-spy profile".to_string()), + exporter: Some(format!("py-spy@{}", env!("CARGO_PKG_VERSION"))), + profiles: profiles, + shared: Shared { + frames: frames.clone() + } } + } } impl Frame { @@ -180,12 +170,8 @@ impl Frame { name: stack_frame.name.clone(), // TODO: filename? file: Some(stack_frame.filename.clone()), - line: if show_line_numbers { - Some(stack_frame.line as u32) - } else { - None - }, - col: None, + line: if show_line_numbers { Some(stack_frame.line as u32) } else { None }, + col: None } } } @@ -211,40 +197,30 @@ impl Stats { pub fn record(&mut self, stack: &stack_trace::StackTrace) -> Result<(), io::Error> { let show_line_numbers = self.config.show_line_numbers; - let mut frame_indices: Vec = stack - .frames - .iter() - .map(|frame| { - let frames = &mut self.frames; - let mut key = frame.clone(); - if !show_line_numbers { - key.line = 0; - } - *self.frame_to_index.entry(key).or_insert_with(|| { - let len = frames.len(); - frames.push(Frame::new(frame, show_line_numbers)); - len - }) + let mut frame_indices: Vec = stack.frames.iter().map(|frame| { + let frames = &mut self.frames; + let mut key = frame.clone(); + if !show_line_numbers { + key.line = 0; + } + *self.frame_to_index.entry(key).or_insert_with(|| { + let len = frames.len(); + frames.push(Frame::new(&frame, show_line_numbers)); + len }) - .collect(); + }).collect(); frame_indices.reverse(); let key = (stack.pid as Pid, stack.thread_id as Tid); - self.samples.entry(key).or_default().push(frame_indices); + self.samples.entry(key).or_insert_with(|| { + vec![] + }).push(frame_indices); let subprocesses = self.config.subprocesses; self.thread_name_map.entry(key).or_insert_with(|| { - let thread_name = stack - .thread_name - .as_ref() - .map_or_else(|| "".to_string(), |x| x.clone()); + let thread_name = stack.thread_name.as_ref().map_or_else(|| "".to_string(), |x| x.clone()); if subprocesses { - format!( - "Process {} Thread {} \"{}\"", - stack.pid, - stack.format_threadid(), - thread_name - ) + format!("Process {} Thread {} \"{}\"", stack.pid, stack.format_threadid(), thread_name) } else { format!("Thread {} \"{}\"", stack.format_threadid(), thread_name) } @@ -254,12 +230,7 @@ impl Stats { } pub fn write(&self, w: &mut dyn Write) -> Result<(), Error> { - let json = serde_json::to_string(&SpeedscopeFile::new( - &self.samples, - &self.frames, - &self.thread_name_map, - self.config.sampling_rate, - ))?; + let json = serde_json::to_string(&SpeedscopeFile::new(&self.samples, &self.frames, &self.thread_name_map, self.config.sampling_rate))?; writeln!(w, "{}", json)?; Ok(()) } @@ -273,11 +244,7 @@ mod tests { #[test] fn test_speedscope_units() { let sample_rate = 100; - let config = Config { - show_line_numbers: true, - sampling_rate: sample_rate, - ..Default::default() - }; + let config = Config{show_line_numbers: true, sampling_rate: sample_rate, ..Default::default()}; let mut stats = Stats::new(&config); let mut cursor = Cursor::new(Vec::new()); @@ -288,7 +255,6 @@ mod tests { short_filename: None, line: 0, locals: None, - is_entry: true, }; let trace = stack_trace::StackTrace { diff --git a/src/stack_trace.rs b/src/stack_trace.rs index a0d45632..68413214 100644 --- a/src/stack_trace.rs +++ b/src/stack_trace.rs @@ -1,15 +1,14 @@ +use std; use std::sync::Arc; use anyhow::{Context, Error, Result}; -use remoteprocess::{Pid, ProcessMemory}; +use remoteprocess::{ProcessMemory, Pid, Process}; use serde_derive::Serialize; -use crate::config::{Config, LineNo}; -use crate::python_data_access::{copy_bytes, copy_string}; -use crate::python_interpreters::{ - CodeObject, FrameObject, InterpreterState, ThreadState, TupleObject, -}; +use crate::python_interpreters::{InterpreterState, ThreadState, FrameObject, CodeObject, TupleObject}; +use crate::python_data_access::{copy_string, copy_bytes}; +use crate::config::LineNo; /// Call stack for a single python thread #[derive(Debug, Clone, Serialize)] @@ -29,7 +28,7 @@ pub struct StackTrace { /// The frames pub frames: Vec, /// process commandline / parent process info - pub process_info: Option>, + pub process_info: Option> } /// Information about a single function call in a stack trace @@ -47,8 +46,6 @@ pub struct Frame { pub line: i32, /// Local Variables associated with the frame pub locals: Option>, - /// If this is an entry frame. Each entry frame corresponds to one native frame. - pub is_entry: bool, } #[derive(Debug, Hash, Eq, PartialEq, Ord, PartialOrd, Clone, Serialize)] @@ -61,39 +58,20 @@ pub struct LocalVariable { #[derive(Debug, Clone, Serialize)] pub struct ProcessInfo { - pub pid: Pid, + pub pid: Pid, pub command_line: String, - pub parent: Option>, + pub parent: Option> } /// Given an InterpreterState, this function returns a vector of stack traces for each thread -pub fn get_stack_traces( - interpreter: &I, - process: &P, - threadstate_address: usize, - config: Option<&Config>, -) -> Result, Error> -where - I: InterpreterState, - P: ProcessMemory, -{ - let gil_thread_id = get_gil_threadid::(threadstate_address, process)?; - +pub fn get_stack_traces(interpreter: &I, process: &Process, lineno: LineNo) -> Result, Error> + where I: InterpreterState { + // TODO: deprecate this method let mut ret = Vec::new(); let mut threads = interpreter.head(); - - let lineno = config.map(|c| c.lineno).unwrap_or(LineNo::NoLine); - let dump_locals = config.map(|c| c.dump_locals).unwrap_or(0); - while !threads.is_null() { - let thread = process - .copy_pointer(threads) - .context("Failed to copy PyThreadState")?; - - let mut trace = get_stack_trace(&thread, process, dump_locals > 0, lineno)?; - trace.owns_gil = trace.thread_id == gil_thread_id; - - ret.push(trace); + let thread = process.copy_pointer(threads).context("Failed to copy PyThreadState")?; + ret.push(get_stack_trace(&thread, process, false, lineno)?); // This seems to happen occasionally when scanning BSS addresses for valid interpreters if ret.len() > 4096 { return Err(format_err!("Max thread recursion depth reached")); @@ -104,16 +82,8 @@ where } /// Gets a stack trace for an individual thread -pub fn get_stack_trace( - thread: &T, - process: &P, - copy_locals: bool, - lineno: LineNo, -) -> Result -where - T: ThreadState, - P: ProcessMemory, -{ +pub fn get_stack_trace(thread: &T, process: &Process, copy_locals: bool, lineno: LineNo) -> Result + where T: ThreadState { // TODO: just return frames here? everything else probably should be returned out of scope let mut frames = Vec::new(); @@ -125,19 +95,15 @@ where let mut frame_ptr = thread.frame(frame_address); while !frame_ptr.is_null() { - let frame = process - .copy_pointer(frame_ptr) - .context("Failed to copy PyFrameObject")?; - let code = process - .copy_pointer(frame.code()) - .context("Failed to copy PyCodeObject")?; + let frame = process.copy_pointer(frame_ptr).context("Failed to copy PyFrameObject")?; + let code = process.copy_pointer(frame.code()).context("Failed to copy PyCodeObject")?; let filename = copy_string(code.filename(), process).context("Failed to copy filename")?; let name = copy_string(code.name(), process).context("Failed to copy function name")?; let line = match lineno { LineNo::NoLine => 0, - LineNo::First => code.first_lineno(), + LineNo::FirstLineNo => code.first_lineno(), LineNo::LastInstruction => match get_line_number(&code, frame.lasti(), process) { Ok(line) => line, Err(e) => { @@ -145,13 +111,10 @@ where // can happen in extreme cases (https://github.com/benfred/py-spy/issues/164) // Rather than fail set the linenumber to 0. This is used by the native extensions // to indicate that we can't load a line number and it should be handled gracefully - warn!( - "Failed to get line number from {}.{}: {}", - filename, name, e - ); + warn!("Failed to get line number from {}.{}: {}", filename, name, e); 0 } - }, + } }; let locals = if copy_locals { @@ -160,17 +123,7 @@ where None }; - let is_entry = frame.is_entry(); - - frames.push(Frame { - name, - filename, - line, - short_filename: None, - module: None, - locals, - is_entry, - }); + frames.push(Frame{name, filename, line, short_filename: None, module: None, locals}); if frames.len() > 4096 { return Err(format_err!("Max frame recursion depth reached")); } @@ -178,16 +131,7 @@ where frame_ptr = frame.back(); } - Ok(StackTrace { - pid: 0, - frames, - thread_id: thread.thread_id(), - thread_name: None, - owns_gil: false, - active: true, - os_thread_id: thread.native_thread_id(), - process_info: None, - }) + Ok(StackTrace{pid: process.pid, frames, thread_id: thread.thread_id(), thread_name: None, owns_gil: false, active: true, os_thread_id: None, process_info: None}) } impl StackTrace { @@ -201,35 +145,27 @@ impl StackTrace { pub fn format_threadid(&self) -> String { // native threadids in osx are kinda useless, use the pthread id instead - #[cfg(target_os = "macos")] + #[cfg(target_os="macos")] return format!("{:#X}", self.thread_id); // otherwise use the native threadid if given - #[cfg(not(target_os = "macos"))] + #[cfg(not(target_os="macos"))] match self.os_thread_id { Some(tid) => format!("{}", tid), - None => format!("{:#X}", self.thread_id), + None => format!("{:#X}", self.thread_id) } } } /// Returns the line number from a PyCodeObject (given the lasti index from a PyFrameObject) -fn get_line_number( - code: &C, - lasti: i32, - process: &P, -) -> Result { - let table = - copy_bytes(code.line_table(), process).context("Failed to copy line number table")?; +fn get_line_number(code: &C, lasti: i32, process: &P) -> Result { + let table = copy_bytes(code.line_table(), process).context("Failed to copy line number table")?; Ok(code.get_line_number(lasti, &table)) } -fn get_locals( - code: &C, - frameptr: *const F, - frame: &F, - process: &P, -) -> Result, Error> { + +fn get_locals(code: &C, frameptr: *const F, frame: &F, process: &P) + -> Result, Error> { let local_count = code.nlocals() as usize; let argcount = code.argcount() as usize; let varnames = process.copy_pointer(code.varnames())?; @@ -240,69 +176,38 @@ fn get_locals( let mut ret = Vec::new(); for i in 0..local_count { - let nameptr: *const C::StringObject = - process.copy_struct(varnames.address(code.varnames() as usize, i))?; + let nameptr: *const C::StringObject = process.copy_struct(varnames.address(code.varnames() as usize, i))?; let name = copy_string(nameptr, process)?; let addr: usize = process.copy_struct(locals_addr + i * ptr_size)?; if addr == 0 { continue; } - ret.push(LocalVariable { - name, - addr, - arg: i < argcount, - repr: None, - }); + ret.push(LocalVariable{name, addr, arg: i < argcount, repr: None}); } Ok(ret) } -pub fn get_gil_threadid( - threadstate_address: usize, - process: &P, -) -> Result { - // figure out what thread has the GIL by inspecting _PyThreadState_Current - if threadstate_address > 0 { - let addr: usize = process.copy_struct(threadstate_address)?; - - // if the addr is 0, no thread is currently holding the GIL - if addr != 0 { - let threadstate: I::ThreadState = process.copy_struct(addr)?; - return Ok(threadstate.thread_id()); - } - } - Ok(0) -} - impl ProcessInfo { pub fn to_frame(&self) -> Frame { - Frame { - name: format!("process {}:\"{}\"", self.pid, self.command_line), + Frame{name: format!("process {}:\"{}\"", self.pid, self.command_line), filename: String::from(""), - module: None, - short_filename: None, - line: 0, - locals: None, - is_entry: true, - } + module: None, short_filename: None, line: 0, locals: None} } } #[cfg(test)] mod tests { use super::*; - use crate::python_bindings::v3_7_0::PyCodeObject; - use crate::python_data_access::tests::to_byteobject; use remoteprocess::LocalProcess; + use crate::python_bindings::v3_7_0::{PyCodeObject}; + use crate::python_data_access::tests::to_byteobject; #[test] fn test_get_line_number() { let mut lnotab = to_byteobject(&[0u8, 1, 10, 1, 8, 1, 4, 1]); - let code = PyCodeObject { - co_firstlineno: 3, - co_lnotab: &mut lnotab.base.ob_base.ob_base, - ..Default::default() - }; + let code = PyCodeObject{co_firstlineno: 3, + co_lnotab: &mut lnotab.base.ob_base.ob_base, + ..Default::default()}; let lineno = get_line_number(&code, 30, &LocalProcess).unwrap(); assert_eq!(lineno, 7); } diff --git a/src/timer.rs b/src/timer.rs index 8a86e8d4..af7ecbf9 100644 --- a/src/timer.rs +++ b/src/timer.rs @@ -1,8 +1,9 @@ -use std::time::{Duration, Instant}; +use std::time::{Instant, Duration}; #[cfg(windows)] use winapi::um::timeapi; -use rand_distr::{Distribution, Exp}; +use rand; +use rand_distr::{Exp, Distribution}; /// Timer is an iterator that sleeps an appropriate amount of time between iterations /// so that we can sample the process a certain number of times a second. @@ -24,16 +25,10 @@ impl Timer { // https://randomascii.wordpress.com/2013/07/08/windows-timer-resolution-megawatts-wasted/ // and http://www.belshe.com/2010/06/04/chrome-cranking-up-the-clock/ #[cfg(windows)] - unsafe { - timeapi::timeBeginPeriod(1); - } + unsafe { timeapi::timeBeginPeriod(1); } let start = Instant::now(); - Timer { - start, - desired: Duration::from_secs(0), - exp: Exp::new(rate).unwrap(), - } + Timer{start, desired: Duration::from_secs(0), exp: Exp::new(rate).unwrap()} } } @@ -65,8 +60,6 @@ impl Iterator for Timer { impl Drop for Timer { fn drop(&mut self) { #[cfg(windows)] - unsafe { - timeapi::timeEndPeriod(1); - } + unsafe { timeapi::timeEndPeriod(1); } } } diff --git a/src/utils.rs b/src/utils.rs index 1d8778bb..f9674fa7 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -11,9 +11,9 @@ pub fn resolve_filename(filename: &str, modulename: &str) -> Option { let module = Path::new(modulename); if let Some(parent) = module.parent() { if let Some(name) = path.file_name() { - let temp = parent.join(name); + let temp = parent.join(name); if temp.exists() { - return Some(temp.to_string_lossy().to_string()); + return Some(temp.to_string_lossy().to_owned().to_string()) } } } diff --git a/src/version.rs b/src/version.rs index 946febc7..8b079d26 100644 --- a/src/version.rs +++ b/src/version.rs @@ -1,39 +1,32 @@ use lazy_static::lazy_static; use regex::bytes::Regex; +use std; + +use anyhow::{Error}; -use anyhow::Error; #[derive(Debug, PartialEq, Eq, Clone)] pub struct Version { pub major: u64, pub minor: u64, pub patch: u64, - pub release_flags: String, - pub build_metadata: Option, + pub release_flags: String } impl Version { pub fn scan_bytes(data: &[u8]) -> Result { lazy_static! { - static ref RE: Regex = Regex::new( - r"((2|3)\.(3|4|5|6|7|8|9|10|11)\.(\d{1,2}))((a|b|c|rc)\d{1,2})?(\+(?:[0-9a-z-]+(?:[.][0-9a-z-]+)*)?)? (.{1,64})" - ) - .unwrap(); + static ref RE: Regex = Regex::new(r"((2|3)\.(3|4|5|6|7|8|9|10|11)\.(\d{1,2}))((a|b|c|rc)\d{1,2})?\+? (.{1,64})").unwrap(); } if let Some(cap) = RE.captures_iter(data).next() { let release = match cap.get(5) { - Some(x) => std::str::from_utf8(x.as_bytes())?, - None => "", + Some(x) => { std::str::from_utf8(x.as_bytes())? }, + None => "" }; let major = std::str::from_utf8(&cap[2])?.parse::()?; let minor = std::str::from_utf8(&cap[3])?.parse::()?; let patch = std::str::from_utf8(&cap[4])?.parse::()?; - let build_metadata = if let Some(s) = cap.get(7) { - Some(std::str::from_utf8(&s.as_bytes()[1..])?.to_owned()) - } else { - None - }; let version = std::str::from_utf8(&cap[0])?; info!("Found matching version string '{}'", version); @@ -48,13 +41,7 @@ impl Version { } } - return Ok(Version { - major, - minor, - patch, - release_flags: release.to_owned(), - build_metadata, - }); + return Ok(Version{major, minor, patch, release_flags:release.to_owned()}); } Err(format_err!("failed to find version string")) } @@ -62,15 +49,7 @@ impl Version { impl std::fmt::Display for Version { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!( - f, - "{}.{}.{}{}", - self.major, self.minor, self.patch, self.release_flags - )?; - if let Some(build_metadata) = &self.build_metadata { - write!(f, "+{}", build_metadata,)? - } - Ok(()) + write!(f, "{}.{}.{}{}", self.major, self.minor, self.patch, self.release_flags) } } @@ -80,62 +59,18 @@ mod tests { #[test] fn test_find_version() { let version = Version::scan_bytes(b"2.7.10 (default, Oct 6 2017, 22:29:07)").unwrap(); - assert_eq!( - version, - Version { - major: 2, - minor: 7, - patch: 10, - release_flags: "".to_owned(), - build_metadata: None, - } - ); - - let version = Version::scan_bytes( - b"3.6.3 |Anaconda custom (64-bit)| (default, Oct 6 2017, 12:04:38)", - ) - .unwrap(); - assert_eq!( - version, - Version { - major: 3, - minor: 6, - patch: 3, - release_flags: "".to_owned(), - build_metadata: None, - } - ); - - let version = - Version::scan_bytes(b"Python 3.7.0rc1 (v3.7.0rc1:dfad352267, Jul 20 2018, 13:27:54)") - .unwrap(); - assert_eq!( - version, - Version { - major: 3, - minor: 7, - patch: 0, - release_flags: "rc1".to_owned(), - build_metadata: None, - } - ); - - let version = - Version::scan_bytes(b"Python 3.10.0rc1 (tags/v3.10.0rc1, Aug 28 2021, 18:25:40)") - .unwrap(); - assert_eq!( - version, - Version { - major: 3, - minor: 10, - patch: 0, - release_flags: "rc1".to_owned(), - build_metadata: None, - } - ); + assert_eq!(version, Version{major: 2, minor: 7, patch: 10, release_flags: "".to_owned()}); - let version = - Version::scan_bytes(b"1.7.0rc1 (v1.7.0rc1:dfad352267, Jul 20 2018, 13:27:54)"); + let version = Version::scan_bytes(b"3.6.3 |Anaconda custom (64-bit)| (default, Oct 6 2017, 12:04:38)").unwrap(); + assert_eq!(version, Version{major: 3, minor: 6, patch: 3, release_flags: "".to_owned()}); + + let version = Version::scan_bytes(b"Python 3.7.0rc1 (v3.7.0rc1:dfad352267, Jul 20 2018, 13:27:54)").unwrap(); + assert_eq!(version, Version{major: 3, minor: 7, patch: 0, release_flags: "rc1".to_owned()}); + + let version = Version::scan_bytes(b"Python 3.10.0rc1 (tags/v3.10.0rc1, Aug 28 2021, 18:25:40)").unwrap(); + assert_eq!(version, Version{major: 3, minor: 10, patch: 0, release_flags: "rc1".to_owned()}); + + let version = Version::scan_bytes(b"1.7.0rc1 (v1.7.0rc1:dfad352267, Jul 20 2018, 13:27:54)"); assert!(version.is_err(), "don't match unsupported "); let version = Version::scan_bytes(b"3.7 10 "); @@ -146,51 +81,6 @@ mod tests { // v2.7.15+ is a valid version string apparently: https://github.com/benfred/py-spy/issues/81 let version = Version::scan_bytes(b"2.7.15+ (default, Oct 2 2018, 22:12:08)").unwrap(); - assert_eq!( - version, - Version { - major: 2, - minor: 7, - patch: 15, - release_flags: "".to_owned(), - build_metadata: Some("".to_owned()), - } - ); - - let version = Version::scan_bytes(b"2.7.10+dcba (default)").unwrap(); - assert_eq!( - version, - Version { - major: 2, - minor: 7, - patch: 10, - release_flags: "".to_owned(), - build_metadata: Some("dcba".to_owned()), - } - ); - - let version = Version::scan_bytes(b"2.7.10+5-4.abcd (default)").unwrap(); - assert_eq!( - version, - Version { - major: 2, - minor: 7, - patch: 10, - release_flags: "".to_owned(), - build_metadata: Some("5-4.abcd".to_owned()), - } - ); - - let version = Version::scan_bytes(b"2.8.5+cinder (default)").unwrap(); - assert_eq!( - version, - Version { - major: 2, - minor: 8, - patch: 5, - release_flags: "".to_owned(), - build_metadata: Some("cinder".to_owned()), - } - ); + assert_eq!(version, Version{major: 2, minor: 7, patch: 15, release_flags: "".to_owned()}); } } diff --git a/tests/integration_test.rs b/tests/integration_test.rs index 9d7c36cc..233d0b66 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -1,6 +1,6 @@ extern crate py_spy; -use py_spy::{Config, Pid, PythonSpy}; use std::collections::HashSet; +use py_spy::{Config, PythonSpy, Pid}; struct ScriptRunner { #[allow(dead_code)] @@ -9,16 +9,11 @@ struct ScriptRunner { impl ScriptRunner { fn new(process_name: &str, filename: &str) -> ScriptRunner { - let child = std::process::Command::new(process_name) - .arg(filename) - .spawn() - .unwrap(); - ScriptRunner { child } + let child = std::process::Command::new(process_name).arg(filename).spawn().unwrap(); + ScriptRunner{child} } - fn id(&self) -> Pid { - self.child.id() as _ - } + fn id(&self) -> Pid { self.child.id() as _ } } impl Drop for ScriptRunner { @@ -32,7 +27,7 @@ impl Drop for ScriptRunner { struct TestRunner { #[allow(dead_code)] child: ScriptRunner, - spy: PythonSpy, + spy: PythonSpy } impl TestRunner { @@ -40,13 +35,13 @@ impl TestRunner { let child = ScriptRunner::new("python", filename); std::thread::sleep(std::time::Duration::from_millis(400)); let spy = PythonSpy::retry_new(child.id(), &config, 20).unwrap(); - TestRunner { child, spy } + TestRunner{child, spy} } } #[test] fn test_busy_loop() { - #[cfg(target_os = "macos")] + #[cfg(target_os="macos")] { // We need root permissions here to run this on OSX if unsafe { libc::geteuid() } != 0 { @@ -70,10 +65,7 @@ fn test_thread_reuse() { // and this caused errors on native unwind (since the native thread had // exited). Test that this works with a simple script that creates // a couple short lived threads, and then profiling with native enabled - let config = Config { - native: true, - ..Default::default() - }; + let config = Config{native: true, ..Default::default()}; let mut runner = TestRunner::new(config, "./tests/scripts/thread_reuse.py"); let mut errors = 0; @@ -94,7 +86,7 @@ fn test_thread_reuse() { #[test] fn test_long_sleep() { - #[cfg(target_os = "macos")] + #[cfg(target_os="macos")] { // We need root permissions here to run this on OSX if unsafe { libc::geteuid() } != 0 { @@ -110,24 +102,18 @@ fn test_long_sleep() { // Make sure the stack trace is what we expect assert_eq!(trace.frames[0].name, "longsleep"); - assert_eq!( - trace.frames[0].short_filename, - Some("longsleep.py".to_owned()) - ); + assert_eq!(trace.frames[0].short_filename, Some("longsleep.py".to_owned())); assert_eq!(trace.frames[0].line, 5); assert_eq!(trace.frames[1].name, ""); assert_eq!(trace.frames[1].line, 9); - assert_eq!( - trace.frames[1].short_filename, - Some("longsleep.py".to_owned()) - ); + assert_eq!(trace.frames[1].short_filename, Some("longsleep.py".to_owned())); assert!(!traces[0].owns_gil); // we should reliably be able to detect the thread is sleeping on osx/windows // linux+freebsd is trickier - #[cfg(any(target_os = "macos", target_os = "windows"))] + #[cfg(any(target_os="macos", target_os="windows"))] assert!(!traces[0].active); } @@ -168,7 +154,7 @@ fn test_thread_names() { #[test] fn test_recursive() { - #[cfg(target_os = "macos")] + #[cfg(target_os="macos")] { // We need root permissions here to run this on OSX if unsafe { libc::geteuid() } != 0 { @@ -178,7 +164,7 @@ fn test_recursive() { // there used to be a problem where the top-level functions being returned // weren't actually entry points: https://github.com/benfred/py-spy/issues/56 - // This was fixed by locking the process while we are profiling it. Test that + // This was fixed by locking the process while we are profling it. Test that // the fix works by generating some samples from a program that would exhibit // this behaviour let mut runner = TestRunner::new(Config::default(), "./tests/scripts/recursive.py"); @@ -190,7 +176,7 @@ fn test_recursive() { assert!(trace.frames.len() <= 22); - let top_level_frame = &trace.frames[trace.frames.len() - 1]; + let top_level_frame = &trace.frames[trace.frames.len()-1]; assert_eq!(top_level_frame.name, ""); assert!((top_level_frame.line == 8) || (top_level_frame.line == 7)); @@ -200,7 +186,7 @@ fn test_recursive() { #[test] fn test_unicode() { - #[cfg(target_os = "macos")] + #[cfg(target_os="macos")] { if unsafe { libc::geteuid() } != 0 { return; @@ -213,25 +199,19 @@ fn test_unicode() { let trace = &traces[0]; assert_eq!(trace.frames[0].name, "function1"); - assert_eq!( - trace.frames[0].short_filename, - Some("unicode💩.py".to_owned()) - ); + assert_eq!(trace.frames[0].short_filename, Some("unicode💩.py".to_owned())); assert_eq!(trace.frames[0].line, 6); assert_eq!(trace.frames[1].name, ""); assert_eq!(trace.frames[1].line, 9); - assert_eq!( - trace.frames[1].short_filename, - Some("unicode💩.py".to_owned()) - ); + assert_eq!(trace.frames[1].short_filename, Some("unicode💩.py".to_owned())); assert!(!traces[0].owns_gil); } #[test] fn test_local_vars() { - #[cfg(target_os = "macos")] + #[cfg(target_os="macos")] { // We need root permissions here to run this on OSX if unsafe { libc::geteuid() } != 0 { @@ -239,10 +219,7 @@ fn test_local_vars() { } } - let config = Config { - dump_locals: 1, - ..Default::default() - }; + let config = Config{dump_locals: 1, ..Default::default()}; let mut runner = TestRunner::new(config, "./tests/scripts/local_vars.py"); let traces = runner.spy.get_stack_traces().unwrap(); @@ -300,17 +277,14 @@ fn test_local_vars() { // we only support dictionary lookup on python 3.6+ right now if runner.spy.version.major == 3 && runner.spy.version.minor >= 6 { - assert_eq!( - local5.repr, - Some("{\"a\": False, \"b\": (1, 2, 3)}".to_owned()) - ); + assert_eq!(local5.repr, Some("{\"a\": False, \"b\": (1, 2, 3)}".to_owned())); } } -#[cfg(not(target_os = "freebsd"))] +#[cfg(not(target_os="freebsd"))] #[test] fn test_subprocesses() { - #[cfg(target_os = "macos")] + #[cfg(target_os="macos")] { // We need root permissions here to run this on OSX if unsafe { libc::geteuid() } != 0 { @@ -322,10 +296,7 @@ fn test_subprocesses() { // was in a zombie state. Verify that this works now let process = ScriptRunner::new("python", "./tests/scripts/subprocesses.py"); std::thread::sleep(std::time::Duration::from_millis(1000)); - let config = Config { - subprocesses: true, - ..Default::default() - }; + let config = Config{subprocesses: true, ..Default::default()}; let sampler = py_spy::sampler::Sampler::new(process.id(), &config).unwrap(); std::thread::sleep(std::time::Duration::from_millis(1000)); @@ -347,10 +318,10 @@ fn test_subprocesses() { } } -#[cfg(not(target_os = "freebsd"))] +#[cfg(not(target_os="freebsd"))] #[test] fn test_subprocesses_zombiechild() { - #[cfg(target_os = "macos")] + #[cfg(target_os="macos")] { // We need root permissions here to run this on OSX if unsafe { libc::geteuid() } != 0 { @@ -362,26 +333,20 @@ fn test_subprocesses_zombiechild() { // was in a zombie state. Verify that this works now let process = ScriptRunner::new("python", "./tests/scripts/subprocesses_zombie_child.py"); std::thread::sleep(std::time::Duration::from_millis(200)); - let config = Config { - subprocesses: true, - ..Default::default() - }; + let config = Config{subprocesses: true, ..Default::default()}; let _sampler = py_spy::sampler::Sampler::new(process.id(), &config).unwrap(); } #[test] fn test_negative_linenumber_increment() { - #[cfg(target_os = "macos")] + #[cfg(target_os="macos")] { // We need root permissions here to run this on OSX if unsafe { libc::geteuid() } != 0 { return; } } - let mut runner = TestRunner::new( - Config::default(), - "./tests/scripts/negative_linenumber_offsets.py", - ); + let mut runner = TestRunner::new(Config::default(), "./tests/scripts/negative_linenumber_offsets.py"); let traces = runner.spy.get_stack_traces().unwrap(); assert_eq!(traces.len(), 1); @@ -395,25 +360,22 @@ fn test_negative_linenumber_increment() { assert!(trace.frames[1].line >= 5 && trace.frames[0].line <= 10); assert_eq!(trace.frames[2].name, ""); assert_eq!(trace.frames[2].line, 13) - } + }, 2 => { assert_eq!(trace.frames[0].name, "f"); assert!(trace.frames[0].line >= 5 && trace.frames[0].line <= 10); assert_eq!(trace.frames[1].name, ""); assert_eq!(trace.frames[1].line, 13); - } - _ => panic!("Unknown python major version"), + }, + _ => panic!("Unknown python major version") } } -#[cfg(target_os = "linux")] +#[cfg(target_os="linux")] #[test] fn test_delayed_subprocess() { let process = ScriptRunner::new("bash", "./tests/scripts/delayed_launch.sh"); - let config = Config { - subprocesses: true, - ..Default::default() - }; + let config = Config{subprocesses: true, ..Default::default()}; let sampler = py_spy::sampler::Sampler::new(process.id(), &config).unwrap(); for sample in sampler { // should have one trace from the subprocess