Overhaul to github actions workflows

.github/ignore-notebooks.txt

@@ -0,0 +1,5 @@
+01_crewai_langgraph_redis
+doc2cache_llama3_1
+semantic_caching_gemini
+01_collaborative_filtering
+05_nvidia_ai_rag_redis

.github/workflows/nightly-test.yml

@@ -0,0 +1,103 @@
+name: Tests - Nightly Run
+
+on:
+  schedule:
+    - cron: "0 3 * * *"  # 3 AM UTC nightly
+  workflow_dispatch:
+
+env:
+  PYTHON_VERSION: "3.11"
+
+jobs:
+  # ---------------------------------------------------------
+  # 1) Gather all notebooks (except skip-list)
+  # ---------------------------------------------------------
+  gather_all_notebooks:
+    runs-on: ubuntu-latest
+    outputs:
+      notebooks: ${{ steps.get_nbs.outputs.notebooks }}
+    steps:
+      - uses: actions/checkout@v2
+
+      - id: get_nbs
+        run: |
+          # 1) Read ignore patterns from .github/ignore-notebooks.txt
+          IGNORE_LIST=()
+          while IFS= read -r skip_nb || [ -n "$skip_nb" ]; do
+            # Skip empty lines or comment lines
+            [[ -z "$skip_nb" || "$skip_nb" =~ ^# ]] && continue
+            IGNORE_LIST+=("$skip_nb")
+          done < .github/ignore-notebooks.txt
+      
+          # 2) Find all .ipynb in python-recipes (or your path)
+          NBS=$(find python-recipes -name '*.ipynb')
+
+          # 3) Filter out notebooks that match anything in IGNORE_LIST
+          FILTERED_NBS=()
+          for nb in $NBS; do
+            skip=false
+            for ignore_nb in "${IGNORE_LIST[@]}"; do
+              if [[ "$nb" == *"$ignore_nb"* ]]; then
+                skip=true
+                break
+              fi
+            done
+
+            if [ "$skip" = false ]; then
+              FILTERED_NBS+=("$nb")
+            fi
+          done
+
+          # 4) Convert the final array to compact JSON for GitHub Actions
+          NB_JSON=$(printf '%s\n' "${FILTERED_NBS[@]}" \
+            | jq -R . \
+            | jq -s -c .)
+
+          # 5) Default to an empty array if there's nothing left
+          if [ -z "$NB_JSON" ] || [ "$NB_JSON" = "[]" ]; then
+            NB_JSON="[]"
+          fi
+
+          echo "All valid notebooks: $NB_JSON"
+          echo "notebooks=$NB_JSON" >> $GITHUB_OUTPUT
+
+  # ---------------------------------------------------------
+  # 2) Test all notebooks in parallel
+  # ---------------------------------------------------------
+  test_all_notebooks:
+    needs: gather_all_notebooks
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        notebook: ${{ fromJson(needs.gather_all_notebooks.outputs.notebooks) }}
+
+    services:
+      redis:
+        image: redis/redis-stack-server:latest
+        ports:
+          - 6379:6379
+
+    steps:
+      - uses: actions/checkout@v2
+
+      # Setup Python
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Create and activate venv
+        run: |
+          python -m venv venv
+          source venv/bin/activate
+          pip install --upgrade pip setuptools wheel
+          pip install pytest nbval
+
+      - name: Test notebook
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
+        run: |
+          echo "Testing notebook: ${{ matrix.notebook }}"
+          source venv/bin/activate
+          pytest --nbval-lax --disable-warnings "${{ matrix.notebook }}"

.github/workflows/test.yml

@@ -1,55 +1,110 @@
-name: Test Suite
+name: Tests - PR/Push
 
 on:
-  pull_request:
-    branches:
-      - main
   push:
-    branches:
-      - main
-  schedule:
-    - cron: '0 0 * * 1' # Runs every Monday
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+env:
+  PYTHON_VERSION: "3.11"
 
 jobs:
-  test:
-    name: Python ${{ matrix.python-version }} - ${{ matrix.connection }} [redis-stack ${{matrix.redis-stack-version}}]
+  # ---------------------------------------------------------
+  # 1) Gather the changed notebooks to produce a matrix list
+  # ---------------------------------------------------------
+  gather_notebooks:
     runs-on: ubuntu-latest
+    outputs:
+      notebooks: ${{ steps.get_nbs.outputs.notebooks }}
+    steps:
+      - uses: actions/checkout@v2
+
+      - id: get_nbs
+        run: |
+          # Compare this commit/PR to 'main' and list changed .ipynb files
+          git fetch --depth=1 origin main
+          CHANGED_NOTEBOOKS=$(git diff --name-only origin/main | grep '\.ipynb$' || true)
+
+          # 1) Read ignore patterns from .github/ignore-notebooks.txt
+          IGNORE_LIST=()
+          while IFS= read -r skip_nb || [ -n "$skip_nb" ]; do
+            # Skip empty lines or comment lines
+            [[ -z "$skip_nb" || "$skip_nb" =~ ^# ]] && continue
+            IGNORE_LIST+=("$skip_nb")
+          done < .github/ignore-notebooks.txt
+
+          # 2) Filter out notebooks in CHANGED_NOTEBOOKS that match ignore patterns
+          FILTERED_NBS=()
+          for nb in $CHANGED_NOTEBOOKS; do
+            skip=false
+
+            # Check if in ignore list
+            for ignore_nb in "${IGNORE_LIST[@]}"; do
+              # Partial match:
+              if [[ "$nb" == *"$ignore_nb"* ]]; then
+                skip=true
+                break
+              fi
+            done
+
+            if [ "$skip" = false ]; then
+              FILTERED_NBS+=("$nb")
+            fi
+          done
+
+          # 3) Build a single-line JSON array
+          NB_JSON=$(printf '%s\n' "${FILTERED_NBS[@]}" \
+            | jq -R . \
+            | jq -s -c .)
+
+          # 4) Fallback to an empty array if there's nothing left
+          if [ -z "$NB_JSON" ] || [ "$NB_JSON" = "[]" ]; then
+            NB_JSON="[]"
+          fi
+
+          echo "All valid notebooks: $NB_JSON"
+
+          # 5) Write to $GITHUB_OUTPUT (modern approach instead of ::set-output)
+          echo "notebooks=$NB_JSON" >> $GITHUB_OUTPUT
 
+  # ---------------------------------------------------------
+  # 2) Test each changed notebook in parallel
+  # ---------------------------------------------------------
+  test_notebooks:
+    needs: gather_notebooks
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.11]
-        connection: ['plain']
-        redis-stack-version: ['latest']
+        notebook: ${{ fromJson(needs.gather_notebooks.outputs.notebooks) }}
 
     services:
       redis:
-        image: redis/redis-stack-server:${{matrix.redis-stack-version}}
+        image: redis/redis-stack-server:latest
         ports:
           - 6379:6379
 
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python-version }}
-        cache: 'pip'
-
-    - name: Install dependencies
-      run: |
-        pip install --no-cache-dir -r requirements.txt
-
-    - name: Set Redis version
-      run: |
-        echo "REDIS_VERSION=${{ matrix.redis-stack-version }}" >> $GITHUB_ENV
-
-    - name: Run notebooks
-      if: matrix.connection == 'plain' && matrix.redis-stack-version == 'latest'
-      env:
-        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-        LLAMA_CLOUD_API_KEY: ${{ secrets.LLAMA_CLOUD_API_KEY }}
-        GCP_REGION: ${{ secrets.GCP_REGION }}
-        GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
-      run: |
-        pytest --verbose --nbval-lax python-recipes/RAG/ python-recipes/vector-search python-recipes/redis-intro python-recipes/recommendation-systems python-recipes/agents python-recipes/computer-vision --ignore python-recipes/agents/01_crewai_langgraph_redis.ipynb --ignore python-recipes/RAG/05_nvidia_ai_rag_redis.ipynb --ignore python-recipes/semantic-cache/doc2cache_llama3_1.ipynb
+      - uses: actions/checkout@v2
+
+      # Setup Python
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Create and activate venv
+        run: |
+          python -m venv venv
+          source venv/bin/activate
+          pip install --upgrade pip setuptools wheel
+          pip install pytest nbval
+
+      - name: Test notebook
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
+        run: |
+          echo "Testing notebook: ${{ matrix.notebook }}"
+          source venv/bin/activate
+          pytest --nbval-lax --disable-warnings "${{ matrix.notebook }}"

README.md

@@ -128,28 +128,13 @@ Redis integrates with many different players in the AI ecosystem. Here's a curat
 | [DocArray](https://docs.docarray.org/user_guide/storing/index_redis/) | DocArray Integration of Redis as a VectorDB by Jina AI |
 
 
-## Content
-- [Vector Similarity Search: From Basics to Production](https://mlops.community/vector-similarity-search-from-basics-to-production/) - Introductory blog post to VSS and Redis as a VectorDB.
-- [Improving RAG quality with RAGAs](https://redis.io/blog/get-better-rag-responses-with-ragas/)
-- [Level-up RAG with RedisVL](https://redis.io/blog/level-up-rag-apps-with-redis-vector-library/)
+# Other Helpful Resources
+
 - [Vector Databases and Large Language Models](https://youtu.be/GJDN8u3Y-T4) - Talk given at LLMs in Production Part 1 by Sam Partee.
+- [Level-up RAG with RedisVL](https://redis.io/blog/level-up-rag-apps-with-redis-vector-library/)
+- [Improving RAG quality with RAGAs](https://redis.io/blog/get-better-rag-responses-with-ragas/)
 - [Vector Databases and AI-powered Search Talk](https://www.youtube.com/watch?v=g2bNHLeKlAg) - Video "Vector Databases and AI-powered Search" given by Sam Partee at SDSC 2023.
-- [Real-Time Product Recommendations](https://jina.ai/news/real-time-product-recommendation-using-redis-and-docarray/) - Content-based recsys design with Redis and DocArray.
-- [NVIDIA Recsys with Redis](https://developer.nvidia.com/blog/offline-to-online-feature-storage-for-real-time-recommendation-systems-with-nvidia-merlin/)
-- [LabLab AI Redis Tech Page](https://lablab.ai/tech/redis)
-- [Storing and querying for embeddings with Redis](https://blog.baeke.info/2023/03/21/storing-and-querying-for-embeddings-with-redis/)
-- [Building Intelligent Apps with Redis Vector Similarity Search](https://redis.com/blog/build-intelligent-apps-redis-vector-similarity-search/)
-- [RedisDays Trading Signals](https://www.youtube.com/watch?v=_Lrbesg4DhY) - Video "Using AI to Reveal Trading Signals Buried in Corporate Filings".
-
-## Benchmarks
+- [NVIDIA RecSys with Redis](https://developer.nvidia.com/blog/offline-to-online-feature-storage-for-real-time-recommendation-systems-with-nvidia-merlin/)
 - [Benchmarking results for vector databases](https://redis.io/blog/benchmarking-results-for-vector-databases/) - Benchmarking results for vector databases, including Redis and 7 other Vector Database players.
-- [ANN Benchmarks](https://ann-benchmarks.com) - Standard ANN Benchmarks site. *Only using single Redis OSS instance/client.*
-
-## Docs
-- [Redis Vector Library Docs](https://redisvl.com)
-- [Redis Vector Database QuickStart](https://redis.io/docs/get-started/vector-database/)
-- [Redis Vector Similarity Docs](https://redis.io/docs/interact/search-and-query/advanced-concepts/vectors/) - Official Redis literature for Vector Similarity Search.
-- [Redis-py Search Docs](https://redis.readthedocs.io/en/latest/redismodules.html#redisearch-commands) - Redis-py client library docs for RediSearch.
-- [Redis-py General Docs](https://redis.readthedocs.io/en/latest/) - Redis-py client library documentation.
-- [Redis Stack](https://redis.io/docs/stack/) - Redis Stack documentation.
-- [Redis Clients](https://redis.io/docs/clients/) - Redis client list.
+- [Redis Vector Library Docs](https://docs.redisvl.com)
+- [Redis Vector Search API Docs](https://redis.io/docs/interact/search-and-query/advanced-concepts/vectors/) - Official Redis literature for Vector Similarity Search.

contributing.md

@@ -11,17 +11,6 @@ Open a PR with your addition. We expect the following standards:
 3. New additions should be added to the bottom of the list (unless otherwise noted).
 4. New additions should not contain any profanity or offensive language.
 
-### What it takes to get a Star
-
-When reviewing the PR, we will determine whether a new entry gets a star!
-
-Examples that:
-- are well-documented and easy to follow
-- pertain to a new or creative use case
-- follow good coding/writing hygiene
-
-will be considered for getting a special star ⭐.
-
 ## Updating your Pull Request
 
 Sometimes, a maintainer will ask you to edit your Pull Request before it is included. This is normally due to spelling errors or because your PR didn't match the list format.