Merge pull request #380 from binance/modularize_connectors

Modularize Connectors

Author: alplabin

.github/workflows/ci.yaml

@@ -0,0 +1,69 @@
+name: Python Connectors CI
+
+on:
+  push:
+    branches: 
+      - master
+
+jobs:
+  detect-clients:
+    runs-on: ubuntu-latest
+    outputs:
+      clients: ${{ steps.detect.outputs.clients }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Detect all clients
+        id: detect
+        run: |
+          ALL_CLIENTS=$(ls -d clients/*/ | xargs -n 1 basename | jq -R -s -c 'split("\n")[:-1]')
+          echo "Detected clients: $ALL_CLIENTS"
+          echo "clients=$ALL_CLIENTS" >> $GITHUB_ENV
+          echo "::set-output name=clients::$ALL_CLIENTS"
+
+  build:
+    needs: detect-clients
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        client: ${{ fromJson(needs.detect-clients.outputs.clients) }}
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        poetry-version: ["latest", "main", "1.8.4"]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install poetry
+        uses: abatilo/actions-poetry@v3
+        with:
+          poetry-version: ${{ matrix.poetry-version }}
+
+      - name: Build ${{ matrix.client }} client
+        run: |
+          if [ "${{ matrix.client }}" == "common" ]; then
+            cd common
+          else
+            cd clients/${{ matrix.client }}
+          fi
+
+          # Install dependencies
+          poetry install
+
+          # Format
+          poetry run black .
+
+          # Lint
+          poetry run ruff check --fix .
+
+          # Test
+          poetry run pytest tests
+
+          # Build
+          poetry build

.github/workflows/pythonpackage.yml

@@ -0,0 +1,192 @@
+name: Sync Connectors with Public Repo
+
+on:
+  # push:
+    # branches:
+      # - master
+  workflow_dispatch:
+
+jobs:
+  detect-targets:
+    runs-on: self-hosted
+    outputs:
+      modified_targets: ${{ steps.filter.outputs.modified_targets }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+
+      - name: Detect modified targets
+        id: filter
+        run: |
+          MODIFIED_TARGETS=()
+
+          # Determine base commit: use HEAD^ if it exists, otherwise use the first commit
+          if git rev-parse --verify HEAD^ >/dev/null 2>&1; then
+            BASE_COMMIT="HEAD^"
+          else
+            BASE_COMMIT=$(git rev-list --max-parents=0 HEAD)
+          fi
+
+          # Check common directory
+          if ! git diff --quiet "$BASE_COMMIT" HEAD -- "common"; then
+            echo "Changes detected in common"
+            MODIFIED_TARGETS+=("common")
+          fi
+
+          # Loop through each client directory in "clients/"
+          for client in clients/*; do
+            CLIENT_NAME=$(basename "$client")
+            # Check for changes in the client directory between BASE_COMMIT and HEAD
+            if ! git diff --quiet "$BASE_COMMIT" HEAD -- "$client"; then
+              echo "Changes detected in $CLIENT_NAME"
+              MODIFIED_TARGETS+=("$CLIENT_NAME")
+            fi
+          done
+
+          # Detect any change whose top-level folder isn’t clients/, common/, .github/
+          if git diff --name-only "$BASE_COMMIT" HEAD \
+              | grep -Ev '^(clients|common|\.github)/' \
+              | grep -q .; then
+            echo "Changes detected in repo root"
+            MODIFIED_TARGETS+=("root")
+          fi
+
+          # Convert the array of modified targets into a JSON array
+          MODIFIED_TARGETS_JSON=$(printf '%s\n' "${MODIFIED_TARGETS[@]}" | jq -R -s -c 'split("\n") | map(select(. != ""))')
+
+          echo "modified_targets: $MODIFIED_TARGETS_JSON"
+          echo "modified_targets=$MODIFIED_TARGETS_JSON" >> $GITHUB_ENV
+          echo "::set-output name=modified_targets::$MODIFIED_TARGETS_JSON"
+
+  sync-connectors:
+    runs-on: self-hosted
+    needs: detect-targets
+    if: ${{ needs.detect-targets.outputs.modified_targets != '[]' }}
+    strategy:
+      matrix:
+        target: ${{ fromJson(needs.detect-targets.outputs.modified_targets) }}
+    permissions:
+      contents: write
+      pull-requests: write
+      repository-projects: write
+
+    steps:
+      - name: Checkout source repository
+        uses: actions/checkout@v3
+
+      - name: Determine directory path
+        id: get_dir_path
+        run: |
+          if [[ "${{ matrix.target }}" == "root" ]]; then
+            DIR_PATH="."
+          elif [[ "${{ matrix.target }}" == "common" ]]; then
+            DIR_PATH="${{ matrix.target }}"
+          else
+            DIR_PATH="clients/${{ matrix.target }}"
+          fi
+          echo "DIR_PATH=$DIR_PATH" >> $GITHUB_ENV
+
+      - name: Determine version for ${{ matrix.target }}
+        if: ${{ matrix.target != 'root' }}
+        id: get_version
+        run: |
+          VERSION=$(jq -r .version ${{ env.DIR_PATH }}/pyproject.toml)
+          echo "VERSION=$VERSION" >> $GITHUB_ENV
+          echo "Detected version: $VERSION"
+
+      - name: Extract latest changes from CHANGELOG
+        if: ${{ matrix.target != 'root' }}
+        id: get_changelog
+        run: |
+          VERSION=${{ env.VERSION }}
+
+          CHANGELOG_CONTENT=$(awk -v version="## ${VERSION}" '/^## / {p=0} $0 ~ version {p=1} p' ${{ env.DIR_PATH }}/CHANGELOG.md | tail -n +2)
+          ESCAPED_CHANGELOG=$(echo "$CHANGELOG_CONTENT" | sed 's/`/\\`/g')
+          echo "CHANGELOG<<EOF" >> $GITHUB_ENV
+          echo "$ESCAPED_CHANGELOG" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+
+      - name: Sync ${{ matrix.target }} changes to public repo 
+        env:
+          GH_PAT: ${{ secrets.GH_PAT }}
+        id: check_changes
+        run: |
+          TARGET_REPO="binance/binance-connector-python"
+          DEFAULT_BRANCH=$(git symbolic-ref refs/remotes/origin/HEAD | sed 's@^refs/remotes/origin/@@')
+
+          # Configure git
+          git config --global user.name "github-actions"
+          git config --global user.email "github-actions@github.com"
+
+          git clone https://x-access-token:${GH_PAT}@github.com/binance/binance-connector-python.git public-repo
+
+          cd public-repo
+          git fetch origin ${DEFAULT_BRANCH}
+
+          if [[ "${{ matrix.target }}" == "root" ]]; then
+            BRANCH="root-update-${GITHUB_RUN_NUMBER}"
+            COMMIT_MSG="Update root-level docs and configs"
+            PR_TITLE="Update root-level documentation and configuration"
+            PR_BODY="Apply recent updates to root-level documentation and configuration files."
+          else
+            BRANCH="rc-${{ matrix.target }}-v${{ env.VERSION }}"
+            COMMIT_MSG="Release ${{ matrix.target }} v${{ env.VERSION }}"
+            PR_TITLE="Release ${{ matrix.target }} v${{ env.VERSION }}"
+            if [ -n "${{ env.CHANGELOG }}" ]; then
+              PR_BODY="${{ env.CHANGELOG }}"
+            else
+              PR_BODY="Release v${{ env.VERSION }}."
+            fi
+          fi
+
+          # Compare directories
+          if ! diff -qr --exclude=".git" --exclude="dist" --exclude="poetry.lock" ../${{ env.DIR_PATH }} ${{ env.DIR_PATH }} > /dev/null; then
+            echo "Changes detected in ${{ matrix.target }}"
+          else
+            echo "No changes detected in ${{ matrix.target }}. Exiting..."
+            exit 0
+          fi
+
+          if [[ "${{ matrix.target }}" == "root" ]]; then
+            rsync -avh --delete \
+              --exclude='.git' \
+              --exclude='.github' \
+              --exclude='poetry.lock' \
+              --exclude='clients' \
+              --exclude='common' \
+              --exclude='public-repo' \
+              ../ .
+          else
+            rsync -avh --delete ../${{ env.DIR_PATH }}/ ${{ env.DIR_PATH }}/
+          fi
+
+          # Create new branch
+          git checkout -b "${BRANCH}"
+
+          git add .
+          git commit -m "${COMMIT_MSG}"
+          git push origin "${BRANCH}"
+
+          # Authenticate with GitHub CLI
+          echo "${GH_PAT}" | gh auth login --with-token
+
+          # Create a Pull Request
+          PR_URL=$(gh pr create \
+            --repo "$TARGET_REPO" \
+            --base ${DEFAULT_BRANCH} \
+            --head "${BRANCH}" \
+            --title "${PR_TITLE}" \
+            --body "${PR_BODY}")
+
+          # Extract PR number from URL
+          PR_NUMBER=$(echo "$PR_URL" | grep -oE '[0-9]+$')
+
+          # Mark the PR as ready for review
+          gh pr ready "$PR_NUMBER"
+
+          echo "Pull Request created and is ready to be reviewed: #$PR_NUMBER"
+
+          # Add reviewers
+          gh pr edit "$PR_NUMBER" --add-reviewer binance/solutions-managers

.github/workflows/release.yml

@@ -6,6 +6,7 @@ __pycache__
 .tox
 .python-version
 .DS_Store
+poetry.lock
 
 # Environments
 .env
@@ -19,4 +20,4 @@ venv/
 # document build
 docs/build/
 
-examples/*.ini
+examples/*.ini