Merge branch 'master' into removeWeb3

Aniket-Engg · web-flow · commit 04b0bbba7efc · 2025-11-03T21:34:26.000+05:30
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -49,6 +49,11 @@ parameters:
     type: boolean
     default: false
 
+  # Timings fetch window (fewer = faster API calls)
+  timings_limit:
+    type: integer
+    default: 1
+
   # Rerun-failed configuration
   rerun_failed_history:
     type: integer
@@ -154,10 +159,78 @@ jobs:
             set -e
             tar -cf remix-dist.tar dist/apps/remix-ide dist/apps/remix-ide-e2e dist/apps/remix-ide/assets/js/soljson dist/libs 2>/dev/null || true
             gzip -1 remix-dist.tar
+      - run:
+          name: Fetch CircleCI timings for shard planning (best effort)
+          command: |
+            if [ -n "${CIRCLECI_TOKEN:-}" ]; then
+              echo "Fetching timings for branch ${CIRCLE_BRANCH:-all}..."
+              yarn ci:timings --slug gh/remix-project-org/remix-project --workflow web --branch "${CIRCLE_BRANCH:-}" --limit << pipeline.parameters.timings_limit >> --json timings-current.json || true
+              if [ ! -s timings-current.json ] || [ "$(jq -r '.files | length' timings-current.json 2>/dev/null || echo 0)" -eq 0 ]; then
+                echo "No timings for current branch; falling back to master"
+                yarn ci:timings --slug gh/remix-project-org/remix-project --workflow web --branch master --limit << pipeline.parameters.timings_limit >> --json timings-current.json || true
+              fi
+            else
+              echo "CIRCLECI_TOKEN not set; shard planning will use median defaults."
+            fi
+      - run:
+          name: Precompute and cache E2E shard splits
+          command: |
+            set -e
+            PARALLEL=<< pipeline.parameters.e2e_parallelism >>
+            COMMIT_SHA=${CIRCLE_SHA1:-unknown}
+            CACHE_PREFIX="reports/shards/cache/${COMMIT_SHA}-shards${PARALLEL}"
+            mkdir -p "$CACHE_PREFIX" reports/shards
+            echo "Enumerating enabled test basenames from dist..."
+            TEST_NAMES=$(find dist/apps/remix-ide-e2e/src/tests -type f \( -name "*.test.js" -o -name "*.spec.js" \) -print0 \
+              | xargs -0 grep -IL "@disabled" \
+              | xargs -I {} basename {} \
+              | sed 's/\.js$//' \
+              | grep -v metamask)
+            # Persist full test list for visibility
+            printf '%s\n' "$TEST_NAMES" > reports/shards/all-tests.txt
+            COUNT=$(printf '%s\n' "$TEST_NAMES" | wc -l | awk '{print $1}')
+            echo "Found $COUNT enabled tests. Planning for $PARALLEL shard(s)."
+            # Generate a manifest (index 0 is arbitrary; manifest includes all bins)
+            printf '%s\n' "$TEST_NAMES" | node scripts/plan-shards.js \
+              --shards "$PARALLEL" \
+              --index 0 \
+              --timings timings-current.json \
+              --manifest-out "${CACHE_PREFIX}/manifest.json" \
+              --verbose > /dev/null || true
+            # Generate per-shard file lists deterministically
+            for i in $(seq 0 $((PARALLEL-1))); do
+              OUT=$(printf '%s\n' "$TEST_NAMES" | node scripts/plan-shards.js --shards "$PARALLEL" --index "$i" --timings timings-current.json)
+              printf '%s\n' "$OUT" > "${CACHE_PREFIX}/files-$i.txt"
+            done
+            echo "Precomputed shard files saved under $CACHE_PREFIX"
+      - run:
+          name: Generate shard overview (human + JSON)
+          command: |
+            # Build overview.txt and overview.json next to manifest.json
+            PARALLEL=<< pipeline.parameters.e2e_parallelism >>
+            COMMIT_SHA=${CIRCLE_SHA1:-unknown}
+            CACHE_PREFIX="reports/shards/cache/${COMMIT_SHA}-shards${PARALLEL}"
+            MANIFEST_PATH="${CACHE_PREFIX}/manifest.json"
+            echo "Looking for manifest at: $MANIFEST_PATH"
+            if [ -f "$MANIFEST_PATH" ]; then
+              node scripts/generate-shard-overview.js "$CACHE_PREFIX" timings-current.json || true
+            else
+              echo "No manifest found at $MANIFEST_PATH; skipping overview generation."
+              ls -la "$(dirname "$MANIFEST_PATH")" || true
+            fi
+      - save_cache:
+          key: v1-shards-{{ .Environment.CIRCLE_SHA1 }}-<< pipeline.parameters.e2e_parallelism >>
+          paths:
+            - reports/shards/cache
       - persist_to_workspace:
           root: .
           paths:
             - remix-dist.tar.gz
+            - reports/shards
+      - store_artifacts:
+          path: ./reports/shards
+      - store_artifacts:
+          path: ./timings-current.json
 
   build-plugin:
     docker:
@@ -283,6 +356,9 @@ jobs:
       - restore_cache:
           keys:
             - v1-deps-{{ checksum "yarn.lock" }}
+      - restore_cache:
+          keys:
+            - v1-shards-{{ .Environment.CIRCLE_SHA1 }}-<< pipeline.parameters.e2e_parallelism >>
       - run: yarn
       - run:
           name: Unpack dist from workspace
@@ -307,12 +383,12 @@ jobs:
                 command: |
                   if [ -n "${CIRCLECI_TOKEN:-}" ]; then
                     echo "Fetching timings for branch ${CIRCLE_BRANCH:-all}..."
-                    yarn ci:timings --slug gh/remix-project-org/remix-project --workflow web --branch "${CIRCLE_BRANCH:-}" --limit 30 --json timings-current.json || true
+                    yarn ci:timings --slug gh/remix-project-org/remix-project --workflow web --branch "${CIRCLE_BRANCH:-}" --limit << pipeline.parameters.timings_limit >> --json timings-current.json || true
                     
                     # If no timings found for this branch (new branch), fall back to master
                     if [ ! -s timings-current.json ] || [ "$(jq -r '.files | length' timings-current.json 2>/dev/null || echo 0)" -eq 0 ]; then
                       echo "⚠️  No timings found for branch ${CIRCLE_BRANCH:-current}, falling back to master..."
-                      yarn ci:timings --slug gh/remix-project-org/remix-project --workflow web --branch master --limit 30 --json timings-current.json || true
+                      yarn ci:timings --slug gh/remix-project-org/remix-project --workflow web --branch master --limit << pipeline.parameters.timings_limit >> --json timings-current.json || true
                     fi
                   else
                     echo "CIRCLECI_TOKEN not set; skipping timings fetch. Self-split will fall back to median weights."
@@ -359,12 +435,12 @@ jobs:
           command: |
             if [ -n "${CIRCLECI_TOKEN:-}" ]; then
               echo "Fetching timings for branch ${CIRCLE_BRANCH:-all}..."
-              yarn ci:timings --slug gh/remix-project-org/remix-project --workflow web --branch "${CIRCLE_BRANCH:-}" --limit 30 --json timings-current.json || true
+              yarn ci:timings --slug gh/remix-project-org/remix-project --workflow web --branch "${CIRCLE_BRANCH:-}" --limit << pipeline.parameters.timings_limit >> --json timings-current.json || true
               
               # If no timings found for this branch (new branch), fall back to master
               if [ ! -s timings-current.json ] || [ "$(jq -r '.files | length' timings-current.json 2>/dev/null || echo 0)" -eq 0 ]; then
                 echo "⚠️  No timings found for branch ${CIRCLE_BRANCH:-current}, falling back to master..."
-                yarn ci:timings --slug gh/remix-project-org/remix-project --workflow web --branch master --limit 30 --json timings-current.json || true
+                yarn ci:timings --slug gh/remix-project-org/remix-project --workflow web --branch master --limit << pipeline.parameters.timings_limit >> --json timings-current.json || true
               fi
             else
               echo "CIRCLECI_TOKEN not set; planning will use median default weights."
@@ -726,6 +802,7 @@ workflows:
               scriptparameter: [""]
               shard_index: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
               vertical_mode: [true]
+              skip_timings: [true]
       - tests-passed:
           requires:
             - lint
diff --git a/apps/remix-ide/ci/browser_test.sh b/apps/remix-ide/ci/browser_test.sh
@@ -18,22 +18,51 @@ SELF_SPLIT=${SELF_SPLIT:-0}
 TIMINGS_JSON=${TIMINGS_JSON:-timings-current.json}
 E2E_RETRIES=${E2E_RETRIES:-0} # number of retries on failure per test (0 = no retry)
 
+# Cache key components for deterministic shard reuse (e.g., on CircleCI reruns)
+# - Keyed by commit SHA and shard count; includes shard index per file
+COMMIT_SHA=${CIRCLE_SHA1:-$(git rev-parse --short=12 HEAD 2>/dev/null || echo unknown)}
+SHARD_CACHE_DIR=${SHARD_CACHE_DIR:-reports/shards/cache}
+SHARD_CACHE_KEY="${COMMIT_SHA}-shards${PARALLEL_TOTAL}"
+SHARD_CACHE_PREFIX="${SHARD_CACHE_DIR}/${SHARD_CACHE_KEY}"
+SHARD_CACHE_FILE="${SHARD_CACHE_PREFIX}/files-${PARALLEL_INDEX}.txt"
+SHARD_MANIFEST_FILE="${SHARD_CACHE_PREFIX}/manifest.json"
+
 # Build the list of enabled test files
 BASE_FILES=$(find dist/apps/remix-ide-e2e/src/tests -type f \( -name "*.test.js" -o -name "*.spec.js" \) -print0 \
   | xargs -0 grep -IL "@disabled" \
   | xargs -I {} basename {} \
   | sed 's/\.js$//' \
   | grep -v 'metamask')
 
-if [ "$SELF_SPLIT" = "1" ]; then
-  echo "==> Using self shard planner (shards=$PARALLEL_TOTAL index=$PARALLEL_INDEX)"
-  echo "ENV: CIRCLE_BRANCH=${CIRCLE_BRANCH:-local} CIRCLE_NODE_TOTAL=$PARALLEL_TOTAL CIRCLE_NODE_INDEX=$PARALLEL_INDEX E2E_RETRIES=$E2E_RETRIES"
-  mkdir -p reports/shards
-  TESTFILES=$(printf '%s\n' "$BASE_FILES" | node scripts/plan-shards.js --shards "$PARALLEL_TOTAL" --index "$PARALLEL_INDEX" --timings "$TIMINGS_JSON" --verbose --manifest-out reports/shards/manifest-$PARALLEL_INDEX.json)
+# Try cache first (works for both SELF_SPLIT and CircleCI split modes)
+mkdir -p "$SHARD_CACHE_PREFIX" reports/shards
+if [ -f "$SHARD_CACHE_FILE" ]; then
+  echo "==> Using cached shard selection: key=$SHARD_CACHE_KEY index=$PARALLEL_INDEX"
+  TESTFILES=$(cat "$SHARD_CACHE_FILE")
 else
-  echo "==> Using CircleCI timings split"
-  mkdir -p reports/shards
-  TESTFILES=$(printf '%s\n' "$BASE_FILES" | circleci tests split --split-by=timings)
+  if [ "$SELF_SPLIT" = "1" ]; then
+    echo "==> Using self shard planner (shards=$PARALLEL_TOTAL index=$PARALLEL_INDEX)"
+    echo "ENV: CIRCLE_BRANCH=${CIRCLE_BRANCH:-local} CIRCLE_NODE_TOTAL=$PARALLEL_TOTAL CIRCLE_NODE_INDEX=$PARALLEL_INDEX E2E_RETRIES=$E2E_RETRIES"
+
+    # Optional: precompute and cache all shard files in one go when requested
+    if [ "${SHARD_PRECOMPUTE_ALL:-0}" = "1" ] && [ ! -f "$SHARD_MANIFEST_FILE" ]; then
+      echo "==> Precomputing all shard splits and caching (key=$SHARD_CACHE_KEY)"
+      for i in $(seq 0 $((PARALLEL_TOTAL-1))); do
+        OUT=$(printf '%s\n' "$BASE_FILES" | node scripts/plan-shards.js --shards "$PARALLEL_TOTAL" --index "$i" --timings "$TIMINGS_JSON" --manifest-out "$SHARD_MANIFEST_FILE")
+        printf '%s\n' "$OUT" > "${SHARD_CACHE_PREFIX}/files-$i.txt"
+      done
+      TESTFILES=$(cat "$SHARD_CACHE_FILE")
+    else
+      TESTFILES=$(printf '%s\n' "$BASE_FILES" | node scripts/plan-shards.js --shards "$PARALLEL_TOTAL" --index "$PARALLEL_INDEX" --timings "$TIMINGS_JSON" --verbose --manifest-out "$SHARD_MANIFEST_FILE")
+      # Cache the freshly computed list for this index
+      printf '%s\n' "$TESTFILES" > "$SHARD_CACHE_FILE"
+    fi
+  else
+    echo "==> Using CircleCI timings split"
+    TESTFILES=$(printf '%s\n' "$BASE_FILES" | circleci tests split --split-by=timings)
+    # Cache the selection so reruns reuse the same split
+    printf '%s\n' "$TESTFILES" > "$SHARD_CACHE_FILE"
+  fi
 fi
 
 printf '%s\n' "$TESTFILES" > reports/shards/files-$PARALLEL_INDEX.txt
