GH-46734: [C++] Arrow Flight SQL ODBC layer

.github/workflows/cpp.yml

@@ -310,7 +310,8 @@ jobs:
       ARROW_DATASET: ON
       ARROW_FLIGHT: ON
       ARROW_FLIGHT_SQL: ON
-      ARROW_FLIGHT_SQL_ODBC: OFF
+      ARROW_FLIGHT_SQL_ODBC: ON
+      ARROW_FLIGHT_SQL_ODBC_INSTALLER: ON
       ARROW_GANDIVA: ON
       ARROW_GCS: ON
       ARROW_HDFS: OFF
@@ -389,6 +390,10 @@ jobs:
           PIPX_BASE_PYTHON: ${{ steps.python-install.outputs.python-path }}
         run: |
           ci/scripts/install_gcs_testbench.sh default
+      - name: Register Flight SQL ODBC Driver
+        shell: cmd
+        run: |
+          call "cpp\src\arrow\flight\sql\odbc\tests\install_odbc.cmd" ${{ github.workspace }}\build\cpp\%ARROW_BUILD_TYPE%\libarrow_flight_sql_odbc.dll
       - name: Test
         shell: msys2 {0}
         run: |

.github/workflows/cpp_extra.yml

@@ -341,8 +341,11 @@ jobs:
     if: >-
       needs.check-labels.outputs.force == 'true' ||
       contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') ||
-      contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++')
+      contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++') ||
+      contains(join(github.event.pull_request.changed_files, ' '), 'cpp/src/arrow/flight/sql/odbc/')
     timeout-minutes: 240
+    permissions:
+      packages: write
     env:
       ARROW_BUILD_SHARED: ON
       ARROW_BUILD_STATIC: OFF
@@ -351,10 +354,10 @@ jobs:
       ARROW_DEPENDENCY_SOURCE: VCPKG
       ARROW_FLIGHT_SQL_ODBC: ON
       ARROW_FLIGHT_SQL_ODBC_INSTALLER: ON
-      ARROW_SIMD_LEVEL: AVX2
+      ARROW_HOME: /usr
       CMAKE_GENERATOR: Ninja
       CMAKE_INSTALL_PREFIX: /usr
-      VCPKG_BINARY_SOURCES: 'clear;nuget,GitHub,readwrite'
+      VCPKG_BINARY_SOURCES: 'clear;nugettimeout,600;nuget,GitHub,readwrite'
       VCPKG_DEFAULT_TRIPLET: x64-windows
     steps:
       - name: Disable Crash Dialogs
@@ -373,10 +376,6 @@ jobs:
       - name: Download Timezone Database
         shell: bash
         run: ci/scripts/download_tz_database.sh
-      - name: Install cmake
-        shell: bash
-        run: |
-          ci/scripts/install_cmake.sh 4.1.2 /usr
       - name: Install ccache
         shell: bash
         run: |

.gitignore

@@ -32,6 +32,7 @@ dependency-reduced-pom.xml
 MANIFEST
 compile_commands.json
 build.ninja
+build*/
 
 # Generated Visual Studio files
 *.vcxproj

ci/scripts/cpp_build.sh

@@ -261,7 +261,7 @@ else
     -DCMAKE_BUILD_TYPE=${ARROW_BUILD_TYPE:-debug} \
     -DCMAKE_VERBOSE_MAKEFILE=${CMAKE_VERBOSE_MAKEFILE:-OFF} \
     -DCMAKE_C_FLAGS="${CFLAGS:-}" \
-    -DCMAKE_CXX_FLAGS="${CXXFLAGS:-}" \
+    -DCMAKE_CXX_FLAGS="${CXXFLAGS:-} -I${ODBC_INCLUDE_DIR:-} -L${ODBC_LIB_DIR:-}" \
     -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD:-20}" \
     -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR:-lib} \
     -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}} \
@@ -272,6 +272,7 @@ else
     -DgRPC_SOURCE=${gRPC_SOURCE:-} \
     -DGTest_SOURCE=${GTest_SOURCE:-} \
     -Dlz4_SOURCE=${lz4_SOURCE:-} \
+    -DODBC_INCLUDE_DIR="${ODBC_INCLUDE_DIR:-}" \
     -Dopentelemetry-cpp_SOURCE=${opentelemetry_cpp_SOURCE:-} \
     -DORC_SOURCE=${ORC_SOURCE:-} \
     -DPARQUET_BUILD_EXAMPLES=${PARQUET_BUILD_EXAMPLES:-OFF} \

ci/scripts/cpp_test.sh

@@ -61,6 +61,7 @@ case "$(uname)" in
     n_jobs=$(sysctl -n hw.ncpu)
     # TODO: https://github.com/apache/arrow/issues/40410
     exclude_tests+=("arrow-s3fs-test")
+    exclude_tests+=("arrow-flight-sql-odbc-test")
     ;;
   MINGW*)
     n_jobs=${NUMBER_OF_PROCESSORS:-1}

cpp/Brewfile

@@ -28,6 +28,7 @@ brew "git"
 brew "glog"
 brew "googletest"
 brew "grpc"
+brew "libiodbc"
 brew "llvm"
 brew "lz4"
 brew "mimalloc"

cpp/CMakePresets.json

@@ -315,6 +315,17 @@
       "displayName": "Debug build with tests and Flight SQL",
       "cacheVariables": {}
     },
+    {
+      "name": "ninja-debug-flight-sql-odbc",
+      "inherits": [
+        "features-flight-sql",
+        "base-debug"
+      ],
+      "displayName": "Debug build with tests and Flight SQL ODBC",
+      "cacheVariables": {
+        "ARROW_FLIGHT_SQL_ODBC": "ON"
+      }
+    },
     {
       "name": "ninja-debug-gandiva",
       "inherits": [
@@ -511,6 +522,17 @@
       "displayName": "Release build with Flight SQL",
       "cacheVariables": {}
     },
+    {
+      "name": "ninja-release-flight-sql-odbc",
+      "inherits": [
+        "features-flight-sql",
+        "base-release"
+      ],
+      "displayName": "Release build with Flight SQL ODBC",
+      "cacheVariables": {
+        "ARROW_FLIGHT_SQL_ODBC": "ON"
+      }
+    },
     {
       "name": "ninja-release-gandiva",
       "inherits": [

cpp/cmake_modules/DefineOptions.cmake

@@ -107,8 +107,8 @@ macro(tsort_bool_option_dependencies)
 endmacro()
 
 macro(resolve_option_dependencies)
-  # Arrow Flight SQL ODBC is available only for Windows for now.
-  if(NOT WIN32)
+  # Arrow Flight SQL ODBC is available only for Windows and macOS for now.
+  if(NOT WIN32 AND NOT APPLE)
     set(ARROW_FLIGHT_SQL_ODBC OFF)
   endif()
   if(MSVC_TOOLCHAIN)

cpp/src/arrow/flight/sql/odbc/README

@@ -0,0 +1,83 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+## Steps to Register the 64-bit Apache Arrow ODBC driver on Windows
+
+After the build succeeds, the ODBC DLL will be located in 
+`build\debug\Debug` for a debug build and `build\release\Release` for a release build.
+
+1. Open Power Shell as administrator.
+
+2. Register your ODBC DLL:
+    Need to replace <path\to\repo> with actual path to repository in the commands.
+
+    i. `cd to repo.`
+    ii. `cd <path\to\repo>`
+    iii. Run script to register your ODBC DLL as Apache Arrow Flight SQL ODBC Driver
+    `.\cpp\src\arrow\flight\sql\odbc\tests\install_odbc.cmd <path\to\repo>\cpp\build\< release | debug >\< Release | Debug>\arrow_flight_sql_odbc.dll`
+    Example command for reference:
+    `.\cpp\src\arrow\flight\sql\odbc\tests\install_odbc.cmd C:\path\to\arrow\cpp\build\release\Release\arrow_flight_sql_odbc.dll`
+
+If the registration is successful, then Apache Arrow Flight SQL ODBC Driver 
+should show as an available ODBC driver in the x64 ODBC Driver Manager.
+
+## Steps to Generate Windows Installer
+1. Build with  `ARROW_FLIGHT_SQL_ODBC=ON` and `ARROW_FLIGHT_SQL_ODBC_INSTALLER=ON`.
+2. `cd` to `build` folder.
+3. Run `cpack`. 
+
+If the generation is successful, you will find `Apache Arrow Flight SQL ODBC-<version>-win64.msi` generated under the `build` folder.
+
+## Steps to Register the 64-bit Apache Arrow ODBC driver on macOS
+
+After the build succeeds, the ODBC DYLIB will be located in 
+`build\debug` for a debug build and `build\release` for a release build.
+
+1. Open terminal shell.
+
+2. Register your ODBC DYLIB:
+    Need to replace <path\to\repo> with actual path to repository in the commands.
+
+    i. `cd to repo.`
+    ii. `cd <path\to\repo>`
+    iii. Give script permission to execute
+    `chmod +x cpp/src/arrow/flight/sql/odbc/install/mac/install_odbc.sh`
+    iv. Run script with `sudo` to register your ODBC DYLIB as Apache Arrow Flight SQL ODBC Driver
+    `sudo cpp/src/arrow/flight/sql/odbc/install/mac/install_odbc.sh <path\to\repo>/cpp/build/< release | debug >/libarrow_flight_sql_odbc.dylib`
+    Example command for reference:
+    `sudo cpp/src/arrow/flight/sql/odbc/install/mac/install_odbc.sh /path/to/arrow/cpp/build/release/libarrow_flight_sql_odbc.dylib`
+
+If the registration is successful, then Apache Arrow Flight SQL ODBC Driver 
+should be shown at `~/Library/ODBC/odbcinst.ini`
+
+## Steps to Enable Logging
+Arrow Flight SQL ODBC driver uses Arrow's internal logging framework. By default, the log messages are printed to the terminal.
+1. Set environment variable `ARROW_ODBC_LOG_LEVEL` to any of the following valid values to enable logging. If `ARROW_ODBC_LOG_LEVEL` is set to a non-empty string that does not match any of the following values, `DEBUG` level is used by default. 
+
+The characters are case-insensitive.
+- TRACE
+- DEBUG 
+- INFO
+- WARNING
+- ERROR 
+- FATAL
+
+The Windows ODBC driver currently does not support writing log files. `ARROW_USE_GLOG` is required to write log files, and `ARROW_USE_GLOG` is disabled on Windows platform since plasma using `glog` is not fully tested on windows.
+
+Note: GH-47670 running more than 1 tests with logging enabled is not fully supported. 

cpp/src/arrow/flight/sql/odbc/entry_points.cc

@@ -83,6 +83,18 @@ SQLRETURN SQL_API SQLGetDiagRec(SQLSMALLINT handle_type, SQLHANDLE handle,
       buffer_length, text_length_ptr);
 }
 
+#if defined(__APPLE__)
+// macOS ODBC Driver Manager doesn't map SQLError to SQLGetDiagRec, so we need to
+// implement SQLError for macOS.
+// on Windows, SQLError mapping implemented by Driver Manager is preferred.
+SQLRETURN SQL_API SQLError(SQLHENV env, SQLHDBC conn, SQLHSTMT stmt, SQLWCHAR* sql_state,
+                           SQLINTEGER* native_error_ptr, SQLWCHAR* message_text,
+                           SQLSMALLINT buffer_length, SQLSMALLINT* text_length_ptr) {
+  return arrow::flight::sql::odbc::SQLError(env, conn, stmt, sql_state, native_error_ptr,
+                                            message_text, buffer_length, text_length_ptr);
+}
+#endif  // __APPLE__
+
 SQLRETURN SQL_API SQLGetEnvAttr(SQLHENV env, SQLINTEGER attr, SQLPOINTER value_ptr,
                                 SQLINTEGER buffer_len, SQLINTEGER* str_len_ptr) {
   return arrow::flight::sql::odbc::SQLGetEnvAttr(env, attr, value_ptr, buffer_len,

cpp/src/arrow/flight/sql/odbc/install/mac/install_odbc.sh

@@ -0,0 +1,76 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Used by macOS ODBC installer script and macOS ODBC testing
+
+ODBC_64BIT="$1"
+
+if [[ -z "$ODBC_64BIT" ]]; then
+  echo "error: 64-bit driver is not specified. Call format: install_odbc abs_path_to_64_bit_driver"
+  exit 1
+fi
+
+if [ ! -f $ODBC_64BIT ]; then
+    echo "64-bit driver can not be found: $ODBC_64BIT"
+	  echo "Call format: install_odbc abs_path_to_64_bit_driver"
+    exit 1
+fi
+
+USER_ODBCINST_FILE="$HOME/Library/ODBC/odbcinst.ini"
+DRIVER_NAME="Apache Arrow Flight SQL ODBC Driver"
+DSN_NAME="Apache Arrow Flight SQL ODBC DSN"
+
+mkdir -p $HOME/Library/ODBC
+
+touch "$USER_ODBCINST_FILE"
+
+# Admin privilege is needed to add ODBC driver registration
+if [ $EUID -ne 0 ]; then 
+    echo "Please run this script with sudo"
+    exit 1
+fi
+
+if grep -q "^\[$DRIVER_NAME\]" "$USER_ODBCINST_FILE"; then
+  echo "Driver [$DRIVER_NAME] already exists in odbcinst.ini"
+else
+  echo "Adding [$DRIVER_NAME] to odbcinst.ini..."
+  echo "
+[$DRIVER_NAME]
+Description=An ODBC Driver for Apache Arrow Flight SQL
+Driver=$ODBC_64BIT
+" >> "$USER_ODBCINST_FILE"
+fi
+
+# Check if [ODBC Drivers] section exists
+if grep -q '^\[ODBC Drivers\]' "$USER_ODBCINST_FILE"; then
+  # Section exists: check if driver entry exists
+  if ! grep -q "^${DRIVER_NAME}=" "$USER_ODBCINST_FILE"; then
+    # Driver entry does not exist, add under [ODBC Drivers]
+    sed -i '' "/^\[ODBC Drivers\]/a\\
+${DRIVER_NAME}=Installed
+" "$USER_ODBCINST_FILE"
+  fi
+else
+  # Section doesn't exist, append both section and driver entry at end
+  {
+    echo ""
+    echo "[ODBC Drivers]"
+    echo "${DRIVER_NAME}=Installed"
+  } >> "$USER_ODBCINST_FILE"
+fi