Fix: install missing packages

.github/workflows/build-citus-community-nightlies.yml

@@ -10,9 +10,6 @@ env:
   DOCKERHUB_USER_NAME: ${{ secrets.DOCKERHUB_USER_NAME }}
   DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
 on:
-  push:
-    branches:
-      - "**"
   schedule:
     - cron: "30 1 * * *"
   workflow_dispatch:

.github/workflows/build-package-test.yml

@@ -12,8 +12,8 @@ env:
   CURRENT_BRANCH: ${GITHUB_REF##*/}
   TEST: true
 on:
-  push:
-    branches: "**"
+  pull_request:
+    types: [opened, reopened,synchronize]
   workflow_dispatch:
 
 concurrency:
@@ -40,61 +40,67 @@ jobs:
       fail-fast: false
       matrix:
         TARGET_PLATFORM:
-          - centos,7
-          - oraclelinux,8
-          - oraclelinux,7
-          - almalinux,8
-          - almalinux,9
+          - centos-7
+          - oraclelinux-8
+          - oraclelinux-7
+          - almalinux-8
+          - almalinux-9
         POSTGRES_VERSION:
-          - 11
-          - 12
-          - 13
-          - 14
-          - 15
-          - 16
+          - pg11
+          - pg12
+          - pg13
+          - pg14
+          - pg15
+          - pg16
+        exclude:
+          - TARGET_PLATFORM: centos-7
+            POSTGRES_VERSION: pg16
+          - TARGET_PLATFORM: oraclelinux-7
+            POSTGRES_VERSION: pg16
         include:
-          - TARGET_PLATFORM: debian,buster
-          - TARGET_PLATFORM: debian,bullseye
-          - TARGET_PLATFORM: debian,bookworm
-          # removing temporarily since postgres 16 packages does not exist for ubuntu bionic
-          # - TARGET_PLATFORM: ubuntu,bionic
-          - TARGET_PLATFORM: ubuntu,focal
-          - TARGET_PLATFORM: ubuntu,jammy
-          - TARGET_PLATFORM: ubuntu,kinetic
+          - TARGET_PLATFORM: debian-buster
+            POSTGRES_VERSION: all
+          - TARGET_PLATFORM: debian-bullseye
+            POSTGRES_VERSION: all
+          - TARGET_PLATFORM: debian-bookworm
+            POSTGRES_VERSION: all
+          - TARGET_PLATFORM: ubuntu-bionic
+            POSTGRES_VERSION: all
+          - TARGET_PLATFORM: ubuntu-focal
+            POSTGRES_VERSION: all
+          - TARGET_PLATFORM: ubuntu-jammy
+            POSTGRES_VERSION: all
+          - TARGET_PLATFORM: ubuntu-kinetic
+            POSTGRES_VERSION: all
           - TARGET_PLATFORM: pgxn
+            POSTGRES_VERSION: all
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
 
-      - name: Install package dependencies
-        run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources
+      - name: 'Login to GitHub Container Registry'
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{github.actor}}
+          password: ${{secrets.GITHUB_TOKEN}}
 
-      - name: Install wheel for el/8
-        if: matrix.TARGET_PLATFORM  == 'centos,8'
-        run: python -m pip install wheel
-
-      - name: Build image for the target platform
-        run: |
-          git checkout -- dockerfiles
-          ./update_image
-        env:
-          TARGET_PLATFORM: ${{ matrix.TARGET_PLATFORM }}
-          POSTGRES_VERSION: ${{ matrix.POSTGRES_VERSION }}
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          file: dockerfiles/${{ matrix.TARGET_PLATFORM }}-${{ matrix.POSTGRES_VERSION }}/Dockerfile
+          pull: true
+          push: true
+          tags: ghcr.io/citusdata/packaging-test:${{ matrix.TARGET_PLATFORM }}-${{ matrix.POSTGRES_VERSION }}
+          # cache-from: type=gha
+          # cache-to: type=gha,mode=max
 
-      - name: Clone tools repo for test
-        run: git clone -b v0.8.27 --depth=1  https://github.com/citusdata/tools.git tools
+      # - name: Clone tools repo for test
+      #   run: git clone -b v0.8.27 --depth=1  https://github.com/citusdata/tools.git tools
 
-      - name: Execute packaging tests
-        run: |
-          python -m pip install -r tools/packaging_automation/requirements.txt
-          python -m pytest -q tools/packaging_automation/tests/test_citus_package.py -k 'test_build_packages'
-        env:
-          PACKAGING_IMAGE_PLATFORM: "${{matrix.TARGET_PLATFORM}}"
-
-      - name: Push images
-        run: |
-          export CURRENT_BRANCH=${GITHUB_REF##*/}
-          ./ci/push_images
-        env:
-          TARGET_PLATFORM: ${{ matrix.TARGET_PLATFORM }}
-          POSTGRES_VERSION: ${{ matrix.POSTGRES_VERSION }}
+      # - name: Execute packaging tests
+      #   run: |
+      #     python -m pip install -r tools/packaging_automation/requirements.txt
+      #     python -m pytest -q tools/packaging_automation/tests/test_citus_package.py -k 'test_build_packages'
+      #   env:
+      #     PACKAGING_IMAGE_PLATFORM: "${{matrix.TARGET_PLATFORM}}"

.github/workflows/build-package.yml

@@ -13,7 +13,7 @@ env:
   TEST: false
 on:
   push:
-    branches: "**"
+    branches: "develop"
   workflow_dispatch:
 
 concurrency:

.github/workflows/image-health-check.yml

@@ -12,8 +12,6 @@ env:
   CURRENT_BRANCH: "${GITHUB_REF##*/}"
   TEST: false
 on:
-  push:
-    branches: "**"
   schedule:
     - cron: "30 2 * * *"
   workflow_dispatch:
@@ -82,13 +80,9 @@ jobs:
         env:
           TARGET_PLATFORM: ${{ matrix.TARGET_PLATFORM }}
           POSTGRES_VERSION: ${{ matrix.POSTGRES_VERSION }}
-
-      - name: Clone tools repo for test
-        run: git clone -b v0.8.27 --depth=1  https://github.com/citusdata/tools.git tools
-
+      # TODO we should probably fold this into the image building thing, to verify if images are built correctly before pushing
       - name: Execute packaging tests
         run: |
-          python -m pip install -r tools/packaging_automation/requirements.txt
           python -m pytest -q tools/packaging_automation/tests/test_citus_package.py -k 'test_build_packages'
         env:
           PACKAGING_IMAGE_PLATFORM: "${{matrix.TARGET_PLATFORM}}"

dockerfiles/almalinux-8-pg11/Dockerfile

@@ -75,7 +75,6 @@ RUN if [[ almalinux   == oraclelinux ]] && [[ 8 == 7 ]]; then yum install -y wge
 
 # install build tools and PostgreSQL development files
 RUN ( yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm  \
-    && ( [[ 11  != 16 ]] || sed -i '/\[pgdg16-updates-testing\]/{n;n;n;s/.*/enabled=1/}' /etc/yum.repos.d/pgdg-redhat-all.repo ) \
     && [[ -z "epel-release" ]] || yum install -y epel-release) \
     && yum groupinstall -y 'Development Tools' \
     && yum install -y \
@@ -98,6 +97,8 @@ RUN ( yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-8
         tar \
         libzstd \
         libzstd-devel \
+        python3-pip \
+        python3-devel \
         llvm-toolset ccache spectool curl \
     && ( [[ 8 != 8 ]] || dnf -qy module disable postgresql ) \
     && yum install -y postgresql11-server postgresql11-devel \
@@ -171,4 +172,10 @@ ENV PGVERSION=11 \
 COPY scripts /scripts
 VOLUME /packages
 
+# add citus tools
+RUN git clone -b v0.8.27 --depth=1  https://github.com/citusdata/tools.git tools
+RUN export PATH="$HOME/.pyenv/bin:$PATH" \
+    && bash -l -c "$(pyenv init -) && pyenv virtualenv ${PYTHON_VERSION} packaging_env && pyenv activate packaging_env && pip install -r tools/packaging_automation/requirements.txt" \
+    && ([[ 8 != 7 ]] || bash -l -c "$(pyenv init -) && pyenv activate packaging_env && pip uninstall -y urllib3 && pip install 'urllib3<2'")
+
 ENTRYPOINT ["/scripts/fetch_and_build_rpm"]

dockerfiles/almalinux-8-pg12/Dockerfile

@@ -75,7 +75,6 @@ RUN if [[ almalinux   == oraclelinux ]] && [[ 8 == 7 ]]; then yum install -y wge
 
 # install build tools and PostgreSQL development files
 RUN ( yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm  \
-    && ( [[ 12  != 16 ]] || sed -i '/\[pgdg16-updates-testing\]/{n;n;n;s/.*/enabled=1/}' /etc/yum.repos.d/pgdg-redhat-all.repo ) \
     && [[ -z "epel-release" ]] || yum install -y epel-release) \
     && yum groupinstall -y 'Development Tools' \
     && yum install -y \
@@ -98,6 +97,8 @@ RUN ( yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-8
         tar \
         libzstd \
         libzstd-devel \
+        python3-pip \
+        python3-devel \
         llvm-toolset ccache spectool curl \
     && ( [[ 8 != 8 ]] || dnf -qy module disable postgresql ) \
     && yum install -y postgresql12-server postgresql12-devel \
@@ -171,4 +172,10 @@ ENV PGVERSION=12 \
 COPY scripts /scripts
 VOLUME /packages
 
+# add citus tools
+RUN git clone -b v0.8.27 --depth=1  https://github.com/citusdata/tools.git tools
+RUN export PATH="$HOME/.pyenv/bin:$PATH" \
+    && bash -l -c "$(pyenv init -) && pyenv virtualenv ${PYTHON_VERSION} packaging_env && pyenv activate packaging_env && pip install -r tools/packaging_automation/requirements.txt" \
+    && ([[ 8 != 7 ]] || bash -l -c "$(pyenv init -) && pyenv activate packaging_env && pip uninstall -y urllib3 && pip install 'urllib3<2'")
+
 ENTRYPOINT ["/scripts/fetch_and_build_rpm"]

dockerfiles/almalinux-8-pg13/Dockerfile

@@ -75,7 +75,6 @@ RUN if [[ almalinux   == oraclelinux ]] && [[ 8 == 7 ]]; then yum install -y wge
 
 # install build tools and PostgreSQL development files
 RUN ( yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm  \
-    && ( [[ 13  != 16 ]] || sed -i '/\[pgdg16-updates-testing\]/{n;n;n;s/.*/enabled=1/}' /etc/yum.repos.d/pgdg-redhat-all.repo ) \
     && [[ -z "epel-release" ]] || yum install -y epel-release) \
     && yum groupinstall -y 'Development Tools' \
     && yum install -y \
@@ -98,6 +97,8 @@ RUN ( yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-8
         tar \
         libzstd \
         libzstd-devel \
+        python3-pip \
+        python3-devel \
         llvm-toolset ccache spectool curl \
     && ( [[ 8 != 8 ]] || dnf -qy module disable postgresql ) \
     && yum install -y postgresql13-server postgresql13-devel \
@@ -171,4 +172,10 @@ ENV PGVERSION=13 \
 COPY scripts /scripts
 VOLUME /packages
 
+# add citus tools
+RUN git clone -b v0.8.27 --depth=1  https://github.com/citusdata/tools.git tools
+RUN export PATH="$HOME/.pyenv/bin:$PATH" \
+    && bash -l -c "$(pyenv init -) && pyenv virtualenv ${PYTHON_VERSION} packaging_env && pyenv activate packaging_env && pip install -r tools/packaging_automation/requirements.txt" \
+    && ([[ 8 != 7 ]] || bash -l -c "$(pyenv init -) && pyenv activate packaging_env && pip uninstall -y urllib3 && pip install 'urllib3<2'")
+
 ENTRYPOINT ["/scripts/fetch_and_build_rpm"]

dockerfiles/almalinux-8-pg14/Dockerfile

@@ -75,7 +75,6 @@ RUN if [[ almalinux   == oraclelinux ]] && [[ 8 == 7 ]]; then yum install -y wge
 
 # install build tools and PostgreSQL development files
 RUN ( yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm  \
-    && ( [[ 14  != 16 ]] || sed -i '/\[pgdg16-updates-testing\]/{n;n;n;s/.*/enabled=1/}' /etc/yum.repos.d/pgdg-redhat-all.repo ) \
     && [[ -z "epel-release" ]] || yum install -y epel-release) \
     && yum groupinstall -y 'Development Tools' \
     && yum install -y \
@@ -98,6 +97,8 @@ RUN ( yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-8
         tar \
         libzstd \
         libzstd-devel \
+        python3-pip \
+        python3-devel \
         llvm-toolset ccache spectool curl \
     && ( [[ 8 != 8 ]] || dnf -qy module disable postgresql ) \
     && yum install -y postgresql14-server postgresql14-devel \
@@ -171,4 +172,10 @@ ENV PGVERSION=14 \
 COPY scripts /scripts
 VOLUME /packages
 
+# add citus tools
+RUN git clone -b v0.8.27 --depth=1  https://github.com/citusdata/tools.git tools
+RUN export PATH="$HOME/.pyenv/bin:$PATH" \
+    && bash -l -c "$(pyenv init -) && pyenv virtualenv ${PYTHON_VERSION} packaging_env && pyenv activate packaging_env && pip install -r tools/packaging_automation/requirements.txt" \
+    && ([[ 8 != 7 ]] || bash -l -c "$(pyenv init -) && pyenv activate packaging_env && pip uninstall -y urllib3 && pip install 'urllib3<2'")
+
 ENTRYPOINT ["/scripts/fetch_and_build_rpm"]

dockerfiles/almalinux-8-pg15/Dockerfile

@@ -75,7 +75,6 @@ RUN if [[ almalinux   == oraclelinux ]] && [[ 8 == 7 ]]; then yum install -y wge
 
 # install build tools and PostgreSQL development files
 RUN ( yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm  \
-    && ( [[ 15  != 16 ]] || sed -i '/\[pgdg16-updates-testing\]/{n;n;n;s/.*/enabled=1/}' /etc/yum.repos.d/pgdg-redhat-all.repo ) \
     && [[ -z "epel-release" ]] || yum install -y epel-release) \
     && yum groupinstall -y 'Development Tools' \
     && yum install -y \
@@ -98,6 +97,8 @@ RUN ( yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-8
         tar \
         libzstd \
         libzstd-devel \
+        python3-pip \
+        python3-devel \
         llvm-toolset ccache spectool curl \
     && ( [[ 8 != 8 ]] || dnf -qy module disable postgresql ) \
     && yum install -y postgresql15-server postgresql15-devel \
@@ -171,4 +172,10 @@ ENV PGVERSION=15 \
 COPY scripts /scripts
 VOLUME /packages
 
+# add citus tools
+RUN git clone -b v0.8.27 --depth=1  https://github.com/citusdata/tools.git tools
+RUN export PATH="$HOME/.pyenv/bin:$PATH" \
+    && bash -l -c "$(pyenv init -) && pyenv virtualenv ${PYTHON_VERSION} packaging_env && pyenv activate packaging_env && pip install -r tools/packaging_automation/requirements.txt" \
+    && ([[ 8 != 7 ]] || bash -l -c "$(pyenv init -) && pyenv activate packaging_env && pip uninstall -y urllib3 && pip install 'urllib3<2'")
+
 ENTRYPOINT ["/scripts/fetch_and_build_rpm"]

dockerfiles/almalinux-8-pg16/Dockerfile

@@ -75,7 +75,6 @@ RUN if [[ almalinux   == oraclelinux ]] && [[ 8 == 7 ]]; then yum install -y wge
 
 # install build tools and PostgreSQL development files
 RUN ( yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm  \
-    && ( [[ 16  != 16 ]] || sed -i '/\[pgdg16-updates-testing\]/{n;n;n;s/.*/enabled=1/}' /etc/yum.repos.d/pgdg-redhat-all.repo ) \
     && [[ -z "epel-release" ]] || yum install -y epel-release) \
     && yum groupinstall -y 'Development Tools' \
     && yum install -y \
@@ -98,6 +97,8 @@ RUN ( yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-8
         tar \
         libzstd \
         libzstd-devel \
+        python3-pip \
+        python3-devel \
         llvm-toolset ccache spectool curl \
     && ( [[ 8 != 8 ]] || dnf -qy module disable postgresql ) \
     && yum install -y postgresql16-server postgresql16-devel \
@@ -171,4 +172,10 @@ ENV PGVERSION=16 \
 COPY scripts /scripts
 VOLUME /packages
 
+# add citus tools
+RUN git clone -b v0.8.27 --depth=1  https://github.com/citusdata/tools.git tools
+RUN export PATH="$HOME/.pyenv/bin:$PATH" \
+    && bash -l -c "$(pyenv init -) && pyenv virtualenv ${PYTHON_VERSION} packaging_env && pyenv activate packaging_env && pip install -r tools/packaging_automation/requirements.txt" \
+    && ([[ 8 != 7 ]] || bash -l -c "$(pyenv init -) && pyenv activate packaging_env && pip uninstall -y urllib3 && pip install 'urllib3<2'")
+
 ENTRYPOINT ["/scripts/fetch_and_build_rpm"]

dockerfiles/almalinux-9-pg11/Dockerfile

@@ -75,7 +75,6 @@ RUN if [[ almalinux   == oraclelinux ]] && [[ 9 == 7 ]]; then yum install -y wge
 
 # install build tools and PostgreSQL development files
 RUN ( yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-9-x86_64/pgdg-redhat-repo-latest.noarch.rpm  \
-    && ( [[ 11  != 16 ]] || sed -i '/\[pgdg16-updates-testing\]/{n;n;n;s/.*/enabled=1/}' /etc/yum.repos.d/pgdg-redhat-all.repo ) \
     && [[ -z "epel-release" ]] || yum install -y epel-release) \
     && yum groupinstall -y 'Development Tools' \
     && yum install -y \
@@ -98,6 +97,8 @@ RUN ( yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-9
         tar \
         libzstd \
         libzstd-devel \
+        python3-pip \
+        python3-devel \
         llvm-toolset ccache rpmdevtools krb5-devel \
     && ( [[ 9 != 8 ]] || dnf -qy module disable postgresql ) \
     && yum install -y postgresql11-server postgresql11-devel \
@@ -171,4 +172,10 @@ ENV PGVERSION=11 \
 COPY scripts /scripts
 VOLUME /packages
 
+# add citus tools
+RUN git clone -b v0.8.27 --depth=1  https://github.com/citusdata/tools.git tools
+RUN export PATH="$HOME/.pyenv/bin:$PATH" \
+    && bash -l -c "$(pyenv init -) && pyenv virtualenv ${PYTHON_VERSION} packaging_env && pyenv activate packaging_env && pip install -r tools/packaging_automation/requirements.txt" \
+    && ([[ 9 != 7 ]] || bash -l -c "$(pyenv init -) && pyenv activate packaging_env && pip uninstall -y urllib3 && pip install 'urllib3<2'")
+
 ENTRYPOINT ["/scripts/fetch_and_build_rpm"]

dockerfiles/almalinux-9-pg12/Dockerfile

@@ -75,7 +75,6 @@ RUN if [[ almalinux   == oraclelinux ]] && [[ 9 == 7 ]]; then yum install -y wge
 
 # install build tools and PostgreSQL development files
 RUN ( yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-9-x86_64/pgdg-redhat-repo-latest.noarch.rpm  \
-    && ( [[ 12  != 16 ]] || sed -i '/\[pgdg16-updates-testing\]/{n;n;n;s/.*/enabled=1/}' /etc/yum.repos.d/pgdg-redhat-all.repo ) \
     && [[ -z "epel-release" ]] || yum install -y epel-release) \
     && yum groupinstall -y 'Development Tools' \
     && yum install -y \
@@ -98,6 +97,8 @@ RUN ( yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-9
         tar \
         libzstd \
         libzstd-devel \
+        python3-pip \
+        python3-devel \
         llvm-toolset ccache rpmdevtools krb5-devel \
     && ( [[ 9 != 8 ]] || dnf -qy module disable postgresql ) \
     && yum install -y postgresql12-server postgresql12-devel \
@@ -171,4 +172,10 @@ ENV PGVERSION=12 \
 COPY scripts /scripts
 VOLUME /packages
 
+# add citus tools
+RUN git clone -b v0.8.27 --depth=1  https://github.com/citusdata/tools.git tools
+RUN export PATH="$HOME/.pyenv/bin:$PATH" \
+    && bash -l -c "$(pyenv init -) && pyenv virtualenv ${PYTHON_VERSION} packaging_env && pyenv activate packaging_env && pip install -r tools/packaging_automation/requirements.txt" \
+    && ([[ 9 != 7 ]] || bash -l -c "$(pyenv init -) && pyenv activate packaging_env && pip uninstall -y urllib3 && pip install 'urllib3<2'")
+
 ENTRYPOINT ["/scripts/fetch_and_build_rpm"]