Added testing to the new Windows CI

.github/workflows/OCV-PR-Windows.yaml

@@ -46,9 +46,9 @@ jobs:
       - branch_eval
     strategy:
       # NOTE: Debugging
-      fail-fast: true
-      # fail-fast: false
-      max-parallel: 3
+      # fail-fast: true
+      fail-fast: false
+      max-parallel: 2
       matrix:
         arch: [ x86, x64, arm64 ]
         config: [ base ]
@@ -126,7 +126,7 @@ jobs:
         source_branch: "${{ github.event.repository.name == 'ci-gha-workflow' && '' || github.head_ref }}"
         gitcache: '${{ env.GIT_CACHE }}'
         workdir: '${{ github.workspace }}'
-      
+
     - name: Update extra dnn models
       if: ${{ matrix.arch != 'arm64' }}
       timeout-minutes: 60
@@ -152,4 +152,29 @@ jobs:
         workdir: '${{ github.workspace }}'
         builddir: 'build-contrib'
         generator: '${{ matrix.vs }}'
-        options: '-A ${{ matrix.cmake_arch }} -DOPENCV_EXTRA_MODULES_PATH=opencv_contrib/modules ${{ env.CMAKE_OPT }}'
+        options: '-A ${{ matrix.cmake_arch }} -DCMAKE_BUILD_TYPE=Release -DOPENCV_EXTRA_MODULES_PATH=opencv_contrib/modules ${{ env.CMAKE_OPT }}'
+
+    - name: Run OpenCV tests
+      uses: ./run-tests
+      env:
+        OPENCV_TEST_DATA_PATH: '${{ github.workspace }}/opencv_extra/testdata'
+        OPENCV_TEST_REQUIRE_DATA: 1
+        OPENCV_TEST_CHECK_OPTIONAL_DATA: 1
+        PYTHONPATH: '${{ github.workspace }}/${{ env.MAIN_BUILD_DIR }}/python_loader:$PYTHONPATH'
+      with:
+        workdir: '${{ github.workspace }}'
+        builddir: '${{ env.MAIN_BUILD_DIR }}'
+        logdir: '${{ github.workspace }}'
+        plan: "test-plan-win-${{ matrix.branch }}.json"
+        suite: "[ ${{ (github.event.repository.name == 'opencv_contrib') && '''contrib'''  || '''main''' }} ]"
+        enable_python: "false"
+        enable_java: "false"
+        filter: "[ 'windows-common' ]"
+        suffix: '${{ matrix.arch }}_${{ matrix.branch }}'
+        timeout: 15
+
+    - if: ${{ always() && env.WARNINGS == '1' }}
+      name: Warnings check
+      run: |
+        echo "::error Warnings have been found!"
+        exit 1

configure-and-build/action.yml

@@ -32,14 +32,22 @@ runs:
     - working-directory: ${{ inputs.workdir }}
       shell: bash
       run: |
+        set -x
         echo "::group::Build"
-        cmake --build '${{ inputs.builddir }}' 2>&1 | tee '${{ inputs.builddir }}/log.txt'
+        if [ "$RUNNER_OS" == "Windows" ]; then
+          cfg="--config Release"
+        fi
+        cmake --build '${{ inputs.builddir }}' ${cfg} 2>&1 | tee '${{ inputs.builddir }}/log.txt'
         echo "::endgroup::"
 
     - shell: bash
       run: |
+        pyexe=python3
+        if [ "$RUNNER_OS" == "Windows" ]; then
+          pyexe=python
+        fi
         set +e
-        python3 ./scripts/warnings-handling.py \
+        ${pyexe} ./scripts/warnings-handling.py \
           '${{ inputs.workdir }}/${{ inputs.builddir }}/log.txt'
         if [ $? -ne 0 ]; then
           echo "WARNINGS=1" >> $GITHUB_ENV

run-tests/action.yml

@@ -13,8 +13,10 @@ inputs:
     description: "Test plan (relative to scripts subfolder)"
   suite:
     description: "Suite in test plan (gtest) (JSON list)"
+    default: "default"
   filter:
     description: "Filter(s) in test plan (gtest) (JSON list)"
+    default: "default"
   timeout:
     description: "One test timeout (min)"
     default: "10"
@@ -40,15 +42,21 @@ runs:
     shell: bash
     run: |
       echo "### Test summary:" >> $GITHUB_STEP_SUMMARY
-      python3 scripts/runner.py \
-        '--workdir=${{ inputs.workdir }}/${{ inputs.builddir }}' \
+      pyexe=python3
+      if [ "$RUNNER_OS" == "Windows" ]; then
+        pyexe=python
+        opt="--exesuffix=.exe --bindir=bin/Release"
+      fi
+      ${pyexe} scripts/runner.py \
         '--bindir=bin' \
+        '--workdir=${{ inputs.workdir }}/${{ inputs.builddir }}' \
         '--logdir=${{ inputs.logdir }}' \
         '--plan=scripts/${{ inputs.plan }}' \
         '--timeout=${{ inputs.timeout }}' \
-        --suite=${{ join(fromJSON(inputs.suite), ' --suite=') }} \
-        --filter=${{ join(fromJSON(inputs.filter), ' --filter=') }} \
         '--options=${{ inputs.options }}' \
+        --filter=${{ join(fromJSON(inputs.filter), ' --filter=') }} \
+        --suite=${{ join(fromJSON(inputs.suite), ' --suite=') }} \
+        ${opt} \
         "--summary=$GITHUB_STEP_SUMMARY"
 
   # Logs
@@ -68,16 +76,24 @@ runs:
     working-directory: ${{ inputs.workdir }}/${{ inputs.builddir }}
     run: |
       echo "::group::Python test"
-      python3 ../opencv/modules/python/test/test.py --repo ../opencv -v
+      pyexe=python3
+      if [ "$RUNNER_OS" == "Windows" ]; then
+        pyexe=python
+      fi
+      ${pyexe} ../opencv/modules/python/test/test.py --repo ../opencv -v
       echo "::endgroup::"
 
   - if: ${{ always() && fromJSON(inputs.enable_python) }}
     shell: bash
     working-directory: ${{ inputs.workdir }}/${{ inputs.builddir }}
     run: |
       echo "::group::Python app test"
+      pyexe=python3
+      if [ "$RUNNER_OS" == "Windows" ]; then
+        pyexe=python
+      fi
       if [ -f "../opencv/apps/python_app_test.py" ]; then
-        python3 "../opencv/apps/python_app_test.py" --repo ../opencv -v
+        ${pyexe} "../opencv/apps/python_app_test.py" --repo ../opencv -v
       fi
       echo "::endgroup::"
 
@@ -89,7 +105,11 @@ runs:
     working-directory: ${{ inputs.workdir }}/${{ inputs.builddir }}
     run: |
       echo "::group::Java test"
-      ${{ inputs.wrapper }} python3 ../opencv/modules/ts/misc/run.py . -a -t java
+      pyexe=python3
+      if [ "$RUNNER_OS" == "Windows" ]; then
+        pyexe=python
+      fi
+      ${{ inputs.wrapper }} ${pyexe} ../opencv/modules/ts/misc/run.py . -a -t java
       echo "::endgroup::"
 
   # Logs

scripts/runner.py

@@ -141,6 +141,7 @@ async def run_one(name, cmd, logname, env, args):
     parser.add_argument("--bindir", default="bin", type=Path, help="Directory with binaries (relative to workdir)")
     parser.add_argument("--verbose", action='store_true', help="Output all lines")
     parser.add_argument("--summary", type=Path, help="Path to summary file to generate")
+    parser.add_argument("--exesuffix", type=str, help="Suffix for executables (e.g. '.exe' or 'd.exe')")
     args = parser.parse_args()
 
     if not args.logdir.is_absolute():
@@ -199,6 +200,8 @@ async def run_one(name, cmd, logname, env, args):
             extra_args.append("--gtest_filter=*:-{}".format(":".join(filter)))
 
         actual_exe = args.workdir / args.bindir / Path(actual_exe)
+        if args.exesuffix:
+            actual_exe = actual_exe.with_suffix(args.exesuffix)
         if len(wrap) == 0 and (not actual_exe.exists() or not actual_exe.is_file()):
             print("Executable not found: {}".format(actual_exe))
             res = -3

scripts/test-plan-4.x.json

@@ -2,7 +2,7 @@
     "suites": {
         "short": [ "opencv_test_core", "opencv_test_imgproc", "opencv_test_photo" ],
 
-        "linux": [
+        "main": [
             "opencv_test_calib3d",
             "opencv_test_core",
             "opencv_test_dnn",
@@ -32,7 +32,7 @@
             "opencv_perf_videoio"
         ],
 
-        "linux-contrib": [
+        "contrib": [
             "opencv_test_bgsegm",
             "opencv_test_bioinspired",
             "opencv_test_dnn_superres",

scripts/test-plan-win-4.x.json

@@ -0,0 +1,104 @@
+{
+    "suites": {
+        "short": [ "opencv_test_core", "opencv_test_imgproc", "opencv_test_photo" ],
+
+        "main": [
+            "opencv_test_calib3d",
+            "opencv_test_core",
+            "opencv_test_dnn",
+            "opencv_test_features2d",
+            "opencv_test_flann",
+            "opencv_test_gapi",
+            "opencv_test_highgui",
+            "opencv_test_imgcodecs",
+            "opencv_test_imgproc",
+            "opencv_test_ml",
+            "opencv_test_objdetect",
+            "opencv_test_photo",
+            "opencv_test_stitching",
+            "opencv_test_video",
+            "opencv_test_videoio",
+            "opencv_perf_calib3d",
+            "opencv_perf_core",
+            "opencv_perf_dnn",
+            "opencv_perf_features2d",
+            "opencv_perf_gapi",
+            "opencv_perf_imgcodecs",
+            "opencv_perf_imgproc",
+            "opencv_perf_objdetect",
+            "opencv_perf_photo",
+            "opencv_perf_stitching",
+            "opencv_perf_video",
+            "opencv_perf_videoio"
+        ],
+
+        "contrib": [
+            "opencv_test_bgsegm",
+            "opencv_test_bioinspired",
+            "opencv_test_dnn_superres",
+            "opencv_test_face",
+            "opencv_test_fuzzy",
+            "opencv_test_hdf",
+            "opencv_test_img_aug",
+            "opencv_test_img_hash",
+            "opencv_test_intensity_transform",
+            "opencv_test_line_descriptor",
+            "opencv_test_mcc",
+            "opencv_test_optflow",
+            "opencv_test_phase_unwrapping",
+            "opencv_test_quality",
+            "opencv_test_rapid",
+            "opencv_test_reg",
+            "opencv_test_rgbd",
+            "opencv_test_saliency",
+            "opencv_test_sfm",
+            "opencv_test_shape",
+            "opencv_test_stereo",
+            "opencv_test_structured_light",
+            "opencv_test_superres",
+            "opencv_test_text",
+            "opencv_test_tracking",
+            "opencv_test_videostab",
+            "opencv_test_wechat_qrcode",
+            "opencv_test_xfeatures2d",
+            "opencv_test_ximgproc",
+            "opencv_test_xphoto",
+            "opencv_perf_bioinspired",
+            "opencv_perf_dnn_superres",
+            "opencv_perf_features2d",
+            "opencv_perf_line_descriptor",
+            "opencv_perf_optflow",
+            "opencv_perf_reg",
+            "opencv_perf_rgbd",
+            "opencv_perf_stereo",
+            "opencv_perf_superres",
+            "opencv_perf_tracking",
+            "opencv_perf_xfeatures2d",
+            "opencv_perf_ximgproc",
+            "opencv_perf_xphoto"
+        ]
+    },
+    "filters": {
+        "windows-common": {
+            "test_core": [
+                "Samples.findFile"
+            ],
+            "test_videoio": [
+                "videoio/videocapture_acceleration.read/yuv420p_libvpxxvp9_mp4_MSMF_NONE_MAT"
+            ],
+            "perf_dnn": [
+                "Utils_blobFromImages.HWC_TO_NCHW/6"
+            ]
+        }
+    },
+    "options": {
+        "default": {
+            "args": {
+                "test": "--skip_unstable=1",
+                "perf": "--perf_impl=plain --perf_min_samples=1 --perf_force_samples=1 --perf_verify_sanity --skip_unstable=1"
+            },
+            "exe": {},
+            "env": {}
+        }
+    }
+}