[CI] remove peft & fix torch was upgraded by optimum (#1272)

* remove peft

* fix torch version

Author: CSY-ModelCloud

.github/workflows/unit_tests.yml

@@ -562,12 +562,11 @@ jobs:
           uv pip install dist/*.whl -i http://${{ needs.check-vm.outputs.ip }}/simple/ --trusted-host ${{ needs.check-vm.outputs.ip }} --extra-index-url https://pypi.org/simple
 
           if [ "${{ matrix.test_script }}" == "test_transformers" ]; then
-            echo "===== install peft from git ====="
-            uv pip install -U git+https://github.com/huggingface/peft.git -i http://${{ needs.check-vm.outputs.ip }}/simple/ --trusted-host ${{ needs.check-vm.outputs.ip }}
             echo "===== install optimum from git ====="
             uv pip install -U git+https://github.com/huggingface/optimum.git -i http://${{ needs.check-vm.outputs.ip }}/simple/ --trusted-host ${{ needs.check-vm.outputs.ip }}
             echo "===== install transformers from git ====="
             uv pip install -U git+https://github.com/huggingface/transformers.git -i http://${{ needs.check-vm.outputs.ip }}/simple/ --trusted-host ${{ needs.check-vm.outputs.ip }}
+            uv pip install torch==2.5.1 # fix optimum will install torch 2.6.0
           fi
 
           if [[ "${{ matrix.test_script }}" == "test_sglang" ]]; then