Merge pull request #200 from umihico/feat/restart-updating-chrome

umihico · web-flow · commit a192e95c8c10 · 2023-10-16T21:04:48.000+09:00
Restart updating chrome with GoogleChromeLabs API
diff --git a/.github/workflows/auto-update.yml b/.github/workflows/auto-update.yml
@@ -25,52 +25,26 @@ jobs:
           SHA256_DIGEST=$(docker inspect public.ecr.aws/lambda/python:latest | jq -r '.[0].RepoDigests[0] | split(":") | .[1]' )
           echo "SHA256_DIGEST=${SHA256_DIGEST}" >> $GITHUB_OUTPUT
       - name: Note Chromium versions
-        id: chromium-versions
+        id: chrome-versions
         run: |
-          echo "CHANNEL=stable" >> $GITHUB_OUTPUT
-          echo "MAJOR_VERSION=114" >> $GITHUB_OUTPUT
-          echo "POSITION=1135561" >> $GITHUB_OUTPUT
-          exit 0
-          # This way of getting Chromium version is not working anymore. https://chromedriver.chromium.org/downloads
-          WHOLE_JSON=$(curl https://omahaproxy.appspot.com/all.json)
-          JSON=$(echo $WHOLE_JSON | jq -r '.[] | select(.os=="linux") | .versions[] | select(.channel=="stable")')
-          CHANNEL="stable"
-          POSITION=$(echo $JSON | jq -r '.branch_base_position')
-          if (( POSITION < 900000 )); then
-              JSON=$(echo $WHOLE_JSON | jq -r '.[] | select(.os=="linux") | .versions[] | select(.channel=="beta")')
-              CHANNEL="beta"
-              POSITION=$(echo $JSON | jq -r '.branch_base_position')
-              echo "CHANNEL=${CHANNEL}" >> $GITHUB_OUTPUT
-          fi
-
-          MAJOR_VERSION=$(echo $JSON | jq -r '.version | split(".") | .[0]')
+          WHOLE_JSON=$(curl https://googlechromelabs.github.io/chrome-for-testing/last-known-good-versions-with-downloads.json)
+          CHANNEL="Stable"
+          REVISION=$(echo $WHOLE_JSON | jq -r '.channels.Stable.revision')
+          echo "REVISION=${REVISION}" >> $GITHUB_OUTPUT
+          CHROME_VERSION=$(echo $WHOLE_JSON | jq -r '.channels.Stable.version')
+          echo "CHROME_VERSION=${CHROME_VERSION}" >> $GITHUB_OUTPUT
+          MAJOR_VERSION=$(echo $WHOLE_JSON | jq -r '.channels.Stable.version | split(".") | .[0]')
           echo "MAJOR_VERSION=${MAJOR_VERSION}" >> $GITHUB_OUTPUT
-
-          for _ in {1..100}; do
-              DOWNLOAD_URL="https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F${POSITION}%2Fchrome-linux.zip?alt=media"
-              curl -I $DOWNLOAD_URL | head -1 | grep -q 404 || break
-              POSITION=$(($POSITION-1))
-              sleep 1
-          done
-
-          echo "POSITION=${POSITION}" >> $GITHUB_OUTPUT
-      - name: Note chromedriver version
-        id: chromedriver-version
-        run: |
-          MAJOR_VERSION=${{ steps.chromium-versions.outputs.MAJOR_VERSION }}
-          DRIVER_VERSION=$(curl "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_${MAJOR_VERSION}")
-          echo "DRIVER_VERSION=${DRIVER_VERSION}" >> $GITHUB_OUTPUT
       - name: Update Selenium
         id: selenium-version
         run: |
           echo "SELENIUM_VERSION=$(curl -s https://pypi.org/pypi/selenium/json | jq -r .info.version)" >> $GITHUB_OUTPUT
       - name: Update Dockerfile
         run: |
           SHA256_DIGEST=${{ steps.docker-image-digest.outputs.SHA256_DIGEST }}
-          POSITION=${{ steps.chromium-versions.outputs.POSITION }}
-          DRIVER_VERSION=${{ steps.chromedriver-version.outputs.DRIVER_VERSION }}
+          CHROME_VERSION=${{ steps.chrome-versions.outputs.CHROME_VERSION }}
           SELENIUM_VERSION=${{ steps.selenium-version.outputs.SELENIUM_VERSION }}
-          sed -r "s/public.ecr.aws\/lambda\/python[:@a-z0-9]+/public.ecr.aws\/lambda\/python\@sha256\:${SHA256_DIGEST}/g; s/chromedriver.storage.googleapis.com\/[0-9.]+/chromedriver.storage.googleapis.com\/${DRIVER_VERSION}/g; s/Linux_x64%2F[0-9]+%2Fchrome-linux.zip/Linux_x64%2F${POSITION}%2Fchrome-linux.zip/g; s/selenium==[0-9\.]*/selenium==${SELENIUM_VERSION}/g" -i Dockerfile
+          sed -r "s/public.ecr.aws\/lambda\/python[:@a-z0-9]+/public.ecr.aws\/lambda\/python\@sha256\:${SHA256_DIGEST}/g; s/chrome-for-testing\/[0-9.]+/chrome-for-testing\/${CHROME_VERSION}/g; s/selenium==[0-9\.]*/selenium==${SELENIUM_VERSION}/g" -i Dockerfile
       - name: Deploy
         run: sls deploy
         env:
@@ -104,11 +78,10 @@ jobs:
         run: cat /tmp/scraping-result.txt | grep -q "This domain is for use in illustrative examples in documents"
       - name: Update README
         run: |
-          CHROME_VERSION=${{ steps.chrome-version.outputs.CHROME_VERSION }}
-          DRIVER_VERSION=${{ steps.chromedriver-version.outputs.DRIVER_VERSION }}
+          CHROME_VERSION=${{ steps.chrome-versions.outputs.CHROME_VERSION }}
           SELENIUM_VERSION=${{ steps.selenium-version.outputs.SELENIUM_VERSION }}
           PYTHON_VERSION=${{ steps.python-version.outputs.PYTHON_VERSION }}
-          sed -r "s/- chromium [0-9.]+/- chromium ${CHROME_VERSION}/g; s/- chromedriver [0-9.]+/- chromedriver ${DRIVER_VERSION}/g; s/- selenium [0-9.]+/- selenium ${SELENIUM_VERSION}/g; s/- Python [0-9.]+/- Python ${PYTHON_VERSION}/g" -i README.md
+          sed -r "s/- chromium [0-9.]+/- chromium ${CHROME_VERSION}/g; s/- chromedriver [0-9.]+/- chromedriver ${CHROME_VERSION}/g; s/- selenium [0-9.]+/- selenium ${SELENIUM_VERSION}/g; s/- Python [0-9.]+/- Python ${PYTHON_VERSION}/g" -i README.md
       - name: Detect changes
         id: detect-changes
         run: |
@@ -129,11 +102,10 @@ jobs:
           git checkout -b $BRANCH
           git add Dockerfile README.md
           echo "SHA256_DIGEST=${{ steps.docker-image-digest.outputs.SHA256_DIGEST }}" > /tmp/body-text.txt
-          echo "POSITION=${{ steps.chromium-versions.outputs.POSITION }}" >> /tmp/body-text.txt
-          echo "DRIVER_VERSION=${{ steps.chromedriver-version.outputs.DRIVER_VERSION }}" >> /tmp/body-text.txt
+          echo "REVISION=${{ steps.chrome-versions.outputs.REVISION }}" >> /tmp/body-text.txt
           echo "PYTHON_VERSION=${{ steps.python-version.outputs.PYTHON_VERSION }}" >> /tmp/body-text.txt
           echo "SELENIUM_VERSION=${{ steps.selenium-version.outputs.SELENIUM_VERSION }}" >> /tmp/body-text.txt
-          echo -e "CHROME_VERSION=${{ steps.chrome-version.outputs.CHROME_VERSION }}\n\n" >> /tmp/body-text.txt
+          echo -e "CHROME_VERSION=${{ steps.chrome-versions.outputs.CHROME_VERSION }}\n\n" >> /tmp/body-text.txt
           git diff --staged >> /tmp/body-text.txt
           git commit -m "${TITLE}"
           git push --set-upstream origin $BRANCH
@@ -151,7 +123,7 @@ jobs:
           echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
           PYTHON_VERSION=${{ steps.python-version.outputs.PYTHON_VERSION }}
           SELENIUM_VERSION=${{ steps.selenium-version.outputs.SELENIUM_VERSION }}
-          CHROME_VERSION=${{ steps.chrome-version.outputs.CHROME_VERSION }}
+          CHROME_VERSION=${{ steps.chrome-versions.outputs.CHROME_VERSION }}
           MAJOR_PYTHON_VERSION=$(echo $PYTHON_VERSION | cut -d "." -f 1)
           MINOR_PYTHON_VERSION=$(echo $PYTHON_VERSION | cut -d "." -f 2)
           docker image tag serverless-docker-selenium-lambda-prod:img umihico/aws-lambda-selenium-python:latest
diff --git a/Dockerfile b/Dockerfile
@@ -1,9 +1,9 @@
 FROM public.ecr.aws/lambda/python@sha256:0b8d0fa373384edace5d97e7f96e835fe43f065045a0bcb09e0dbc20fc33d351 as build
 RUN yum install -y unzip && \
-    curl -Lo "/tmp/chromedriver.zip" "https://chromedriver.storage.googleapis.com/114.0.5735.90/chromedriver_linux64.zip" && \
-    curl -Lo "/tmp/chrome-linux.zip" "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F1135561%2Fchrome-linux.zip?alt=media" && \
-    unzip /tmp/chromedriver.zip -d /opt/ && \
-    unzip /tmp/chrome-linux.zip -d /opt/
+    curl -Lo "/tmp/chromedriver-linux64.zip" "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/117.0.5938.92/linux64/chromedriver-linux64.zip" && \
+    curl -Lo "/tmp/chrome-linux64.zip" "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/117.0.5938.92/linux64/chrome-linux64.zip" && \
+    unzip /tmp/chromedriver-linux64.zip -d /opt/ && \
+    unzip /tmp/chrome-linux64.zip -d /opt/
 
 FROM public.ecr.aws/lambda/python@sha256:0b8d0fa373384edace5d97e7f96e835fe43f065045a0bcb09e0dbc20fc33d351
 RUN yum install atk cups-libs gtk3 libXcomposite alsa-lib \
diff --git a/README.md b/README.md
@@ -3,11 +3,6 @@
 ![badge](https://github.com/umihico/docker-selenium-lambda/actions/workflows/demo-test.yml/badge.svg)
 ![badge](https://github.com/umihico/docker-selenium-lambda/actions/workflows/auto-update.yml/badge.svg)
 
-> :warning: **Caution!**
->
-> Updating Chromium and Chromedriver versions to v115 or later is postponed in this repository. I'll try to fix later.
-> See https://chromedriver.chromium.org/downloads
-
 This is minimum demo of headless chrome and selenium on container image on AWS Lambda
 
 This image goes with these versions. [These are automatically updated and tested everyday.](https://github.com/umihico/docker-selenium-lambda/actions)
diff --git a/main.py b/main.py
@@ -8,7 +8,7 @@ def handler(event=None, context=None):
     service = webdriver.ChromeService("/opt/chromedriver")
 
     options.binary_location = '/opt/chrome/chrome'
-    options.add_argument('--headless')
+    options.add_argument("--headless=new")
     options.add_argument('--no-sandbox')
     options.add_argument("--disable-gpu")
     options.add_argument("--window-size=1280x1696")
