Merge pull request #6476 from remix-project-org/feat/nx-cloud/setup

CI Refactor Optimize Money Saving

Author: bunsenstraat

.circleci/config.yml

@@ -1 +1 @@
-v20
+v20.19.0

.nvmrc

@@ -86,6 +86,15 @@ yarn run serve:production
 ```
 Production build will be served by default to `http://localhost:8080/` or `http://127.0.0.1:8080/`
 
+## Nx Cloud caching
+
+This repo uses Nx Cloud to speed up builds and keep CI deterministic via remote caching.
+
+- Configuration: `nx.json` uses the Nx Cloud runner and reads the token from the `NX_CLOUD_ACCESS_TOKEN` environment variable.
+- CI: CircleCI jobs automatically use `--cloud` when the token is present; for forked PRs (no secrets), they fall back to local-only caching. Build logs are stored under `logs/nx-build.log`.
+- Verifying locally: run the same target twice; the second run should print “Nx read the output from the cache”. Example: `nx run remix-ide:build` and run it again.
+- Insights: View cache analytics and run details at https://nx.app (links appear in Nx output when the token is configured).
+
 ## Docker:
 
 Prerequisites: 

README.md

@@ -43,9 +43,9 @@ module.exports = composePlugins(withNx(), (config) => {
   // add public path
   config.output.publicPath = '/'
 
-  // set filename
-  config.output.filename = `[name].plugin-contract-verification.${versionData.timestamp}.js`
-  config.output.chunkFilename = `[name].plugin-contract-verification.${versionData.timestamp}.js`
+  // set deterministic filenames for better caching
+  config.output.filename = `[name].plugin-contract-verification.[contenthash].js`
+  config.output.chunkFilename = `[name].plugin-contract-verification.[contenthash].js`
 
   // add copy & provide plugin
   config.plugins.push(

apps/contract-verification/webpack.config.js

@@ -9,7 +9,6 @@
       "executor": "@nrwl/webpack:webpack",
       "outputs": ["{options.outputPath}"],
       "defaultConfiguration": "development",
-      "dependsOn": ["install"],
       "options": {
         "compiler": "babel",
         "outputPath": "dist/apps/quick-dapp",
@@ -40,6 +39,7 @@
     },
     "install": {
       "executor": "nx:run-commands",
+      "cache": false,
       "options": {
         "commands": [
           "cd apps/quick-dapp && yarn"

apps/quick-dapp/project.json

@@ -9,7 +9,6 @@
       "executor": "@nrwl/webpack:webpack",
       "outputs": ["{options.outputPath}"],
       "defaultConfiguration": "development",
-      "dependsOn": ["install"],
       "options": {
         "compiler": "babel",
         "outputPath": "dist/apps/remix-dapp",
@@ -39,6 +38,7 @@
     },
     "install": {
       "executor": "nx:run-commands",
+      "cache": false,
       "options": {
         "commands": [
           "cd apps/remix-dapp && yarn"

apps/remix-dapp/project.json

@@ -39,6 +39,7 @@ module.exports = {
       .click('*[data-id="treeViewLitreeViewItemcontracts/2_Owner.sol"]')
       .clickLaunchIcon('solidity')
       .click('*[id="compileBtn"]')
+      .saveScreenshot('./reports/screenshots/compile2_Owner.png')
       .pause(10000)
       .clickLaunchIcon('solidityStaticAnalysis')
       .useXpath()
@@ -57,10 +58,12 @@ module.exports = {
       .pause(1000)
       .waitForElementVisible('span#ssaRemixtab')
       .click('span#ssaRemixtab')
-      .assert.containsText('span#ssaRemixtab > *[data-id="RemixStaticAnalysisErrorCount', '388')
+      .waitForElementContainsText('span#ssaRemixtab > *[data-id="RemixStaticAnalysisErrorCount', '384')
+  
       .click('label[id="headingshowLibWarnings"]')
       .pause(1000)
-      .assert.containsText('span#ssaRemixtab > *[data-id="RemixStaticAnalysisErrorCount', '3')
+      .waitForElementContainsText('span#ssaRemixtab > *[data-id="RemixStaticAnalysisErrorCount', '3')
+    
       .end()
   }
 }

apps/remix-ide-e2e/src/tests/staticAnalysis.test.ts

@@ -4,24 +4,71 @@ set -e
 
 
 
-BUILD_ID=${CIRCLE_BUILD_NUM:-${TRAVIS_JOB_NUMBER}}
+BUILD_ID=${CIRCLE_BUILD_NUM:-local}
 echo "$BUILD_ID"
 TEST_EXITCODE=0
-npx ganache &
-npx http-server -p 9090 --cors='*' ./node_modules &
-yarn run serve:production &
+npx ganache > /dev/null 2>&1 &
+npx http-server -p 9090 --cors='*' ./node_modules > /dev/null 2>&1 &
+yarn run serve:production > /dev/null 2>&1 &
 sleep 5
 
-# grep -IRiL "@disabled" "dist/apps/remix-ide-e2e/src/tests" | grep "\.spec\|\.test" | xargs -I {} basename {} .test.js | grep -E "\b[${2}]"
-# TESTFILES=$(grep -IRiL "@disabled" "dist/apps/remix-ide-e2e/src/tests" | grep "\.spec\|\.test" | xargs -I {} basename {} .test.js | grep -E "\b[$2]" | circleci tests split --split-by=timings )
-node apps/remix-ide/ci/splice_tests.js $2 $3
-TESTFILES=$(node apps/remix-ide/ci/splice_tests.js $2 $3 | grep -v 'metamask' | circleci tests split --split-by=timings)
+PARALLEL_TOTAL=${CIRCLE_NODE_TOTAL:-1}
+PARALLEL_INDEX=${CIRCLE_NODE_INDEX:-0}
+SELF_SPLIT=${SELF_SPLIT:-0}
+TIMINGS_JSON=${TIMINGS_JSON:-timings-current.json}
+E2E_RETRIES=${E2E_RETRIES:-0} # number of retries on failure per test (0 = no retry)
+
+# Build the list of enabled test files
+BASE_FILES=$(find dist/apps/remix-ide-e2e/src/tests -type f \( -name "*.test.js" -o -name "*.spec.js" \) -print0 \
+  | xargs -0 grep -IL "@disabled" \
+  | xargs -I {} basename {} \
+  | sed 's/\.js$//' \
+  | grep -v 'metamask')
+
+if [ "$SELF_SPLIT" = "1" ]; then
+  echo "==> Using self shard planner (shards=$PARALLEL_TOTAL index=$PARALLEL_INDEX)"
+  echo "ENV: CIRCLE_BRANCH=${CIRCLE_BRANCH:-local} CIRCLE_NODE_TOTAL=$PARALLEL_TOTAL CIRCLE_NODE_INDEX=$PARALLEL_INDEX E2E_RETRIES=$E2E_RETRIES"
+  mkdir -p reports/shards
+  TESTFILES=$(printf '%s\n' "$BASE_FILES" | node scripts/plan-shards.js --shards "$PARALLEL_TOTAL" --index "$PARALLEL_INDEX" --timings "$TIMINGS_JSON" --verbose --manifest-out reports/shards/manifest-$PARALLEL_INDEX.json)
+else
+  echo "==> Using CircleCI timings split"
+  mkdir -p reports/shards
+  TESTFILES=$(printf '%s\n' "$BASE_FILES" | circleci tests split --split-by=timings)
+fi
+
+printf '%s\n' "$TESTFILES" > reports/shards/files-$PARALLEL_INDEX.txt
+COUNT=$(printf '%s\n' "$TESTFILES" | wc -l | awk '{print $1}')
+echo "==> Shard $PARALLEL_INDEX selected $COUNT test files"
+echo "==> Preview (first 20):"
+printf '%s\n' "$TESTFILES" | head -n 20
+echo "==> Preview (last 10):"
+printf '%s\n' "$TESTFILES" | tail -n 10
+echo "==> Full list (for grepability):"
+printf '%s\n' "$TESTFILES"
+
+# If this batch includes remixd (slither) tests, prepare pip3/slither toolchain on-demand
+printf '%s\n' "$TESTFILES" | ./apps/remix-ide/ci/setup_slither_if_needed.sh
 for TESTFILE in $TESTFILES; do
-    if ! npx nightwatch --config dist/apps/remix-ide-e2e/nightwatch-${1}.js dist/apps/remix-ide-e2e/src/tests/${TESTFILE}.js --env=$1; then
-      if ! npx nightwatch --config dist/apps/remix-ide-e2e/nightwatch-${1}.js dist/apps/remix-ide-e2e/src/tests/${TESTFILE}.js --env=$1; then
+    echo "Running test: ${TESTFILE}.js (retries on fail: $E2E_RETRIES)"
+    attempt=0
+    while true; do
+      if npx nightwatch --config dist/apps/remix-ide-e2e/nightwatch-${1}.js dist/apps/remix-ide-e2e/src/tests/${TESTFILE}.js --env=$1; then
+        # success
+        break
+      fi
+      # failure
+      if [ "$attempt" -lt "$E2E_RETRIES" ]; then
+        attempt=$((attempt+1))
+        echo "Retrying ${TESTFILE}.js (attempt $attempt of $E2E_RETRIES)"
+        continue
+      else
         TEST_EXITCODE=1
         break
       fi
+    done
+    # Stop the shard loop on permanent failure
+    if [ "$TEST_EXITCODE" -eq 1 ]; then
+      break
     fi
 done
 

apps/remix-ide/.travis.yml

@@ -2,7 +2,7 @@
 
 set -e
 
-BUILD_ID=${CIRCLE_BUILD_NUM:-${TRAVIS_JOB_NUMBER}}
+BUILD_ID=${CIRCLE_BUILD_NUM:-local}
 echo "$BUILD_ID"
 TEST_EXITCODE=0
 
@@ -14,6 +14,8 @@ yarn run serve:production &
 sleep 5
 
 TESTFILES=$(grep -IRiL "\'@disabled\': \?true" "dist/apps/remix-ide-e2e/src/tests" | grep $1 | sort | circleci tests split )
+# Prepare slither toolchain if remixd tests are present
+printf '%s\n' "$TESTFILES" | ./apps/remix-ide/ci/setup_slither_if_needed.sh
 for TESTFILE in $TESTFILES; do
     npx nightwatch --config dist/apps/remix-ide-e2e/nightwatch-chrome.js $TESTFILE --env=chrome  || TEST_EXITCODE=1
 done