Merge pull request #952 from bact/add-test-suites

Specify a limited test suite

Author: bact

.github/workflows/unittest.yml

@@ -1,4 +1,4 @@
-name: Unit test and code coverage
+name: Unit test and coverage
 
 on:
   push:
@@ -18,9 +18,14 @@ jobs:
       fail-fast: false
       matrix:
         os: ["macos-latest", "ubuntu-latest", "windows-latest"]
-        python-version: ["3.10"]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
 
     runs-on: ${{ matrix.os }}
+    env:
+      PYICU_WIN_VER: 2.14
+      INSTALL_PYICU_WIN: false
+      INSTALL_TORCH: false
+      INSTALL_FULL_DEPS: false
 
     steps:
     - name: Checkout
@@ -32,8 +37,11 @@ jobs:
         cache: "pip"
     - name: Install build tools
       run: |
-        python -m pip install --upgrade "pip<24.1" "setuptools==73.0.1"
-        python -m pip install coverage coveralls
+        pip install --upgrade "pip<24.1" "setuptools>=65.0.2,<=73.0.1"
+        pip install coverage coveralls
+      # pip<24.1 because https://github.com/omry/omegaconf/pull/1195
+      # setuptools>=65.0.2 because https://github.com/pypa/setuptools/commit/d03da04e024ad4289342077eef6de40013630a44#diff-9ea6e1e3dde6d4a7e08c7c88eceed69ca745d0d2c779f8f85219b22266efff7fR1
+      # setuptools<=73.0.1 because https://github.com/pypa/setuptools/issues/4620
     - name: Install ICU (macOS)
       if: startsWith(matrix.os, 'macos-')
       run: |
@@ -43,26 +51,38 @@ jobs:
         ICU_VER=$(pkg-config --modversion icu-i18n)
         echo "ICU_VER=${ICU_VER}"
         echo "ICU_VER=${ICU_VER}" >> "${GITHUB_ENV}"
-    - name: Install ICU (Windows)
-      if: startsWith(matrix.os, 'windows-')
+    - name: Install PyICU (Windows)
+      if: startsWith(matrix.os, 'windows-') && env.INSTALL_PYICU_WIN == 'true'
+      shell: powershell
       run: |
-        python -m pip install "https://github.com/cgohlke/pyicu-build/releases/download/v2.14/PyICU-2.14-cp310-cp310-win_amd64.whl"
-      # if needed, get pip wheel link from https://github.com/cgohlke/pyicu-build/releases
+        $PYTHON_WIN_VER = "${{ matrix.python-version }}"
+        $CP_VER = "cp" + $PYTHON_WIN_VER.Replace(".", "")
+        $WHEEL_URL = "https://github.com/cgohlke/pyicu-build/releases/download/v${{ env.PYICU_WIN_VER }}/PyICU-${{ env.PYICU_WIN_VER }}-${CP_VER}-${CP_VER}-win_amd64.whl"
+        pip install "$WHEEL_URL"
+      # Get wheel URL from https://github.com/cgohlke/pyicu-build/releases
     - name: Install PyTorch
+      if: env.INSTALL_TORCH == 'true'
       run: pip install torch
-      # if needed, get pip wheel link from http://download.pytorch.org/whl/torch/
-    # - name: Install dependencies
-    #   env:
-    #     SKLEARN_ALLOW_DEPRECATED_SKLEARN_PACKAGE_INSTALL: True
-    #   run: |
-    #     python -m pip install -r docker_requirements.txt
+      # If torch for the platform is not available in PyPI, use this command:
+      # pip install "<torch_wheel_url>"
+      # Get wheel URL from http://download.pytorch.org/whl/torch/
+    - name: Install dependencies
+      if: env.INSTALL_FULL_DEPS == 'true'
+      env:
+        SKLEARN_ALLOW_DEPRECATED_SKLEARN_PACKAGE_INSTALL: True
+      run: pip install -r docker_requirements.txt
     - name: Install PyThaiNLP
-      run: |
-        python -m pip install .
-    - name: Test
+      run: pip install .
+      # Use the command below, if you want to install a small set of external
+      # packages, which includes numpy, pyicu, python-crfsuite, and requests:
+      # pip install .[compact]
+    - name: Unit test and code coverage
+      run: coverage run -m unittest tests
+      # Use 'unittest tests' instead of 'unittest discover' to avoid loading
+      # tests with external imports.
+      # Test cases loaded is defined in __init__.py in the tests directory.
+    - name: Coverage report
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         COVERALLS_SERVICE_NAME: github
-      run: |
-        coveralls
-#        coverage run -m unittest discover
+      run: coveralls

pythainlp/ancient/aksonhan.py

@@ -1,11 +1,10 @@
 # -*- coding: utf-8 -*-
 # SPDX-FileCopyrightText: 2016-2024 PyThaiNLP Project
 # SPDX-License-Identifier: Apache-2.0
-from pythainlp.util import Trie
 from pythainlp import thai_consonants, thai_tonemarks
-from pythainlp.tokenize import Tokenizer
 from pythainlp.corpus import thai_orst_words
-
+from pythainlp.tokenize import Tokenizer
+from pythainlp.util import Trie
 
 _dict_aksonhan = {}
 for i in list(thai_consonants):

pythainlp/augment/lm/fasttext.py

@@ -3,8 +3,10 @@
 # SPDX-License-Identifier: Apache-2.0
 import itertools
 from typing import List, Tuple
+
 from gensim.models.fasttext import FastText as FastText_gensim
 from gensim.models.keyedvectors import KeyedVectors
+
 from pythainlp.tokenize import word_tokenize
 
 

pythainlp/augment/lm/phayathaibert.py

@@ -2,21 +2,20 @@
 # SPDX-FileCopyrightText: 2016-2024 PyThaiNLP Project
 # SPDX-License-Identifier: Apache-2.0
 
-from typing import List
 import random
 import re
+from typing import List
 
 from pythainlp.phayathaibert.core import ThaiTextProcessor
 
-
 _MODEL_NAME = "clicknext/phayathaibert"
 
 
 class ThaiTextAugmenter:
     def __init__(self) -> None:
         from transformers import (
-            AutoTokenizer,
             AutoModelForMaskedLM,
+            AutoTokenizer,
             pipeline,
         )
 

pythainlp/augment/word2vec/__init__.py

@@ -8,5 +8,5 @@
 __all__ = ["Word2VecAug", "Thai2fitAug", "LTW2VAug"]
 
 from pythainlp.augment.word2vec.core import Word2VecAug
-from pythainlp.augment.word2vec.thai2fit import Thai2fitAug
 from pythainlp.augment.word2vec.ltw2v import LTW2VAug
+from pythainlp.augment.word2vec.thai2fit import Thai2fitAug

pythainlp/augment/word2vec/bpemb_wv.py

@@ -2,6 +2,7 @@
 # SPDX-FileCopyrightText: 2016-2024 PyThaiNLP Project
 # SPDX-License-Identifier: Apache-2.0
 from typing import List, Tuple
+
 from pythainlp.augment.word2vec.core import Word2VecAug
 
 

pythainlp/augment/word2vec/core.py

@@ -1,8 +1,8 @@
 # -*- coding: utf-8 -*-
 # SPDX-FileCopyrightText: 2016-2024 PyThaiNLP Project
 # SPDX-License-Identifier: Apache-2.0
-from typing import List, Tuple
 import itertools
+from typing import List, Tuple
 
 
 class Word2VecAug:

pythainlp/augment/word2vec/ltw2v.py

@@ -2,6 +2,7 @@
 # SPDX-FileCopyrightText: 2016-2024 PyThaiNLP Project
 # SPDX-License-Identifier: Apache-2.0
 from typing import List, Tuple
+
 from pythainlp.augment.word2vec.core import Word2VecAug
 from pythainlp.corpus import get_corpus_path
 from pythainlp.tokenize import word_tokenize

pythainlp/augment/word2vec/thai2fit.py

@@ -2,6 +2,7 @@
 # SPDX-FileCopyrightText: 2016-2024 PyThaiNLP Project
 # SPDX-License-Identifier: Apache-2.0
 from typing import List, Tuple
+
 from pythainlp.augment.word2vec.core import Word2VecAug
 from pythainlp.corpus import get_corpus_path
 from pythainlp.tokenize import THAI2FIT_TOKENIZER

pythainlp/augment/wordnet.py

@@ -9,15 +9,15 @@
     "postype2wordnet",
 ]
 
-from collections import OrderedDict
 import itertools
+from collections import OrderedDict
 from typing import List
 
 from nltk.corpus import wordnet as wn
+
 from pythainlp.corpus import wordnet
-from pythainlp.tokenize import word_tokenize
 from pythainlp.tag import pos_tag
-
+from pythainlp.tokenize import word_tokenize
 
 orchid = {
     "": "",