Merge remote-tracking branch 'origin/main' into saumya/datetimeoffset-executemany

Author: gargsaumya

.github/workflows/pr-code-coverage.yml

@@ -94,24 +94,35 @@ jobs:
               labelToAdd = 'pr-size: large';
             }
 
-            // Remove existing size labels if any
+            // Get existing labels
             const existingLabels = pr.labels.map(l => l.name);
             const sizeLabels = ['pr-size: small', 'pr-size: medium', 'pr-size: large'];
-            for (const label of existingLabels) {
-              if (sizeLabels.includes(label)) {
+            
+            // Find current size label (if any)
+            const currentSizeLabel = existingLabels.find(label => sizeLabels.includes(label));
+            
+            // Only make changes if the label needs to be updated
+            if (currentSizeLabel !== labelToAdd) {
+              console.log(`Current size label: ${currentSizeLabel || 'none'}`);
+              console.log(`Required size label: ${labelToAdd} (Total changes: ${totalChanges})`);
+              
+              // Remove existing size label if different from required
+              if (currentSizeLabel) {
+                console.log(`Removing outdated label: ${currentSizeLabel}`);
                 await github.rest.issues.removeLabel({
                   ...context.repo,
                   issue_number: pr.number,
-                  name: label,
+                  name: currentSizeLabel,
                 });
               }
-            }
 
-            // Add new size label
-            await github.rest.issues.addLabels({
-              ...context.repo,
-              issue_number: pr.number,
-              labels: [labelToAdd],
-            });
-
-            console.log(`Added label: ${labelToAdd} (Total changes: ${totalChanges})`);
+              // Add new size label
+              console.log(`Adding new label: ${labelToAdd}`);
+              await github.rest.issues.addLabels({
+                ...context.repo,
+                issue_number: pr.number,
+                labels: [labelToAdd],
+              });
+            } else {
+              console.log(`Label already correct: ${labelToAdd} (Total changes: ${totalChanges}) - no changes needed`);
+            }

.github/workflows/pr-format-check.yml

@@ -39,15 +39,12 @@ PyBind11 provides:
 
 We are currently in **Public Preview**.
 
-## What's new in v0.11.0
-
-- **Database Metadata & Introspection:** Added comprehensive `getInfo()` method and extensive catalog APIs (`SQLGetTypeInfo`, `SQLProcedures`, `SQLForeignKeys`, `SQLColumns`) for complete database schema discovery and introspection capabilities.
-- **Advanced Parameter Management:** Implemented `setinputsizes()` with SQL type constants and enhanced parameter validation through the `SQLTypes` class for precise type control in parameterized queries.
-- **Large Data Streaming Enhancements:** Extended streaming support to VARBINARY(MAX) and VARCHAR(MAX) across all fetch operations (`fetchone`, `fetchmany`, `fetchall`) with improved chunked retrieval for efficient memory usage.
-- **Output Data Conversion System:** Introduced flexible output converter framework with `add_output_converter()`, `remove_output_converter()`, and related methods for customizable data transformations during result fetching.
-- **Connection-Level Execute:** Added direct `execute()` method to Connection class for streamlined query execution without explicit cursor management.
-- **Financial Data Type Support:** Comprehensive support for SQL Server MONEY and SMALLMONEY types with proper boundary value handling and decimal precision.
-- **Enhanced Configuration APIs:** Added connection timeout control, decimal separator configuration (`getDecimalSeperator()`, `setDecimalSeperator()`), and improved global variable management.
+## What's new in v0.12.0
+
+- **Complex Data Type Support:** Added native support for DATETIMEOFFSET and UNIQUEIDENTIFIER data types with full round-trip handling, enabling seamless integration with Python's timezone-aware `datetime` objects and `uuid.UUID` types.
+- **Support for monetary or currency values data types:** Extended MONEY and SMALLMONEY support to `executemany` operations with proper NULL handling and decimal conversion for improved bulk financial data processing.
+- **Improved Database Metadata API:** Added `getinfo()` method with enhanced ODBC metadata retrieval, allowing users to query driver/data source information using ODBC info types.
+- **Data Processing Optimizations:** Removed aggressive datetime parsing to prevent incorrect type conversions and improve data integrity across diverse datetime formats and string data.
 
 For more information, please visit the project link on Github: https://github.com/microsoft/mssql-python
 

PyPI_Description.md

@@ -7,6 +7,44 @@ trigger:
       - main
 
 jobs:
+- job: CodeQLAnalysis
+  displayName: 'CodeQL Security Analysis'
+  pool:
+    vmImage: 'ubuntu-latest'
+  
+  steps:
+  - script: |
+      sudo apt-get update
+      sudo apt-get install -y build-essential cmake curl git python3 python3-pip python3-dev python3-venv unixodbc-dev
+    displayName: 'Install build dependencies for CodeQL'
+
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '3.13'
+      addToPath: true
+    displayName: 'Use Python 3.13 for CodeQL'
+
+  - script: |
+      python -m pip install --upgrade pip
+      pip install -r requirements.txt
+    displayName: 'Install Python dependencies for CodeQL'
+
+  - task: CodeQL3000Init@0
+    inputs:
+      Enabled: true
+    displayName: 'Initialize CodeQL'
+
+  # Build the C++ extension for CodeQL analysis
+  - script: |
+      cd mssql_python/pybind
+      chmod +x build.sh
+      ./build.sh
+    displayName: 'Build C++ extension for CodeQL analysis'
+
+  - task: CodeQL3000Finalize@0
+    condition: always()
+    displayName: 'Finalize CodeQL'
+
 - job: PytestOnWindows
   displayName: 'Windows x64'
   pool:
@@ -72,11 +110,11 @@ jobs:
       testResultsFiles: '**/test-results.xml'
       testRunTitle: 'Publish test results'
 
-  - task: PublishCodeCoverageResults@1
-    inputs:
-      codeCoverageTool: 'Cobertura'
-      summaryFileLocation: 'coverage.xml'
-    displayName: 'Publish code coverage results'
+  # - task: PublishCodeCoverageResults@1
+  #   inputs:
+  #     codeCoverageTool: 'Cobertura'
+  #     summaryFileLocation: 'coverage.xml'
+  #   displayName: 'Publish code coverage results'
 
 - job: PytestOnMacOS
   displayName: 'macOS x86_64'
@@ -1477,3 +1515,79 @@ jobs:
     inputs:
       testResultsFiles: '**/test-results-alpine-arm64.xml'
       testRunTitle: 'Publish pytest results on Alpine ARM64'
+
+- job: CodeCoverageReport
+  displayName: 'Full Code Coverage Report in Ubuntu x86_64'
+  pool:
+    vmImage: 'ubuntu-latest'
+
+  steps:
+  - script: |
+      # Install build dependencies
+      sudo apt-get update
+      sudo apt-get install -y cmake gcc g++ lcov unixodbc-dev llvm clang
+    displayName: 'Install build dependencies'
+
+  - script: |
+      # Start SQL Server container
+      docker pull mcr.microsoft.com/mssql/server:2022-latest
+      docker run \
+        --name sqlserver \
+        -e ACCEPT_EULA=Y \
+        -e MSSQL_SA_PASSWORD="$(DB_PASSWORD)" \
+        -p 1433:1433 \
+        -d mcr.microsoft.com/mssql/server:2022-latest
+
+      # Wait until SQL Server is ready
+      for i in {1..30}; do
+        docker exec sqlserver \
+          /opt/mssql-tools18/bin/sqlcmd \
+          -S localhost \
+          -U SA \
+          -P "$(DB_PASSWORD)" \
+          -C -Q "SELECT 1" && break
+        sleep 2
+      done
+    displayName: 'Start SQL Server container'
+    env:
+      DB_PASSWORD: $(DB_PASSWORD)
+
+  - script: |
+      # Install Python dependencies
+      python -m pip install --upgrade pip
+      pip install -r requirements.txt
+      pip install coverage-lcov lcov-cobertura
+    displayName: 'Install Python dependencies'
+
+  - script: |
+      # Build pybind bindings with coverage instrumentation
+      cd mssql_python/pybind
+      ./build.sh codecov
+    displayName: 'Build pybind bindings with coverage'
+
+  - script: |
+      # Generate unified coverage (Python + C++)
+      chmod +x ./generate_codecov.sh
+      ./generate_codecov.sh
+
+      # Convert unified LCOV to Cobertura XML for ADO reporting
+      lcov_cobertura total.info --output unified-coverage/coverage.xml
+    displayName: 'Generate unified coverage (Python + C++)'
+    env:
+      DB_CONNECTION_STRING: 'Driver=ODBC Driver 18 for SQL Server;Server=tcp:127.0.0.1,1433;Database=master;Uid=SA;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes'
+      DB_PASSWORD: $(DB_PASSWORD)
+
+  - task: PublishTestResults@2
+    condition: succeededOrFailed()
+    inputs:
+      testResultsFiles: '**/test-results.xml'
+      testRunTitle: 'Publish pytest results with unified coverage'
+
+  - task: PublishCodeCoverageResults@2
+    condition: succeededOrFailed()
+    inputs:
+      codeCoverageTool: Cobertura
+      summaryFileLocation: 'unified-coverage/coverage.xml'
+      reportDirectory: 'unified-coverage'
+      failIfCoverageEmpty: true
+    displayName: 'Publish unified code coverage results'

eng/pipelines/pr-validation-pipeline.yml

@@ -0,0 +1,103 @@
+#!/bin/bash
+set -euo pipefail
+
+echo "==================================="
+echo "[STEP 1] Installing dependencies"
+echo "==================================="
+
+# Update package list
+sudo apt-get update
+
+# Install LLVM (for llvm-profdata, llvm-cov)
+if ! command -v llvm-profdata &>/dev/null; then
+    echo "[ACTION] Installing LLVM via apt"
+    sudo apt-get install -y llvm
+fi
+
+# Install lcov (provides lcov + genhtml)
+if ! command -v genhtml &>/dev/null; then
+    echo "[ACTION] Installing lcov via apt"
+    sudo apt-get install -y lcov
+fi
+
+# Install Python plugin for LCOV export
+if ! python -m pip show coverage-lcov &>/dev/null; then
+    echo "[ACTION] Installing coverage-lcov via pip"
+    python -m pip install coverage-lcov
+fi
+
+# Install LCOV → Cobertura converter (for ADO)
+if ! python -m pip show lcov-cobertura &>/dev/null; then
+    echo "[ACTION] Installing lcov-cobertura via pip"
+    python -m pip install lcov-cobertura
+fi
+
+echo "==================================="
+echo "[STEP 2] Running pytest with Python coverage"
+echo "==================================="
+
+# Cleanup old coverage
+rm -f .coverage coverage.xml python-coverage.info cpp-coverage.info total.info
+rm -rf htmlcov unified-coverage
+
+# Run pytest with Python coverage (XML + HTML output)
+python -m pytest -v \
+  --junitxml=test-results.xml \
+  --cov=mssql_python \
+  --cov-report=xml:coverage.xml \
+  --cov-report=html \
+  --capture=tee-sys \
+  --cache-clear
+
+# Convert Python coverage to LCOV format (restrict to repo only)
+echo "[ACTION] Converting Python coverage to LCOV"
+coverage lcov -o python-coverage.info --include="mssql_python/*"
+
+echo "==================================="
+echo "[STEP 3] Processing C++ coverage (Clang/LLVM)"
+echo "==================================="
+
+# Merge raw profile data from pybind runs
+if [ ! -f default.profraw ]; then
+    echo "[ERROR] default.profraw not found. Did you build with -fprofile-instr-generate?"
+    exit 1
+fi
+
+llvm-profdata merge -sparse default.profraw -o default.profdata
+
+# Find the pybind .so file (Linux build)
+PYBIND_SO=$(find mssql_python -name "*.so" | head -n 1)
+if [ -z "$PYBIND_SO" ]; then
+    echo "[ERROR] Could not find pybind .so"
+    exit 1
+fi
+
+echo "[INFO] Using pybind module: $PYBIND_SO"
+
+# Export C++ coverage, excluding Python headers, pybind11, and system includes
+llvm-cov export "$PYBIND_SO" \
+  -instr-profile=default.profdata \
+  -ignore-filename-regex='(python3\.[0-9]+|cpython|pybind11|/usr/include/|/usr/lib/)' \
+  --skip-functions \
+  -format=lcov > cpp-coverage.info
+
+echo "==================================="
+echo "[STEP 4] Merging Python + C++ coverage"
+echo "==================================="
+
+# Merge LCOV reports (ignore inconsistencies in Python LCOV export)
+lcov -a python-coverage.info -a cpp-coverage.info -o total.info \
+  --ignore-errors inconsistent,corrupt
+
+# Normalize paths so everything starts from mssql_python/
+echo "[ACTION] Normalizing paths in LCOV report"
+sed -i "s|$(pwd)/||g" total.info
+
+# Generate full HTML report
+genhtml total.info \
+  --output-directory unified-coverage \
+  --quiet \
+  --title "Unified Coverage Report"
+
+# Generate Cobertura XML (for Azure DevOps Code Coverage tab)
+lcov_cobertura total.info --output coverage.xml

generate_codecov.sh

@@ -140,7 +140,7 @@ def getDecimalSeparator():
 from .logging_config import setup_logging, get_logger
 
 # Constants
-from .constants import ConstantsDDBC
+from .constants import ConstantsDDBC, GetInfoConstants
 
 # Export specific constants for setencoding()
 SQL_CHAR = ConstantsDDBC.SQL_CHAR.value
@@ -205,3 +205,55 @@ def _custom_setattr(name, value):
 SQL_DATE = ConstantsDDBC.SQL_DATE.value
 SQL_TIME = ConstantsDDBC.SQL_TIME.value
 SQL_TIMESTAMP = ConstantsDDBC.SQL_TIMESTAMP.value
+
+# Export GetInfo constants at module level
+# Driver and database information
+SQL_DRIVER_NAME = GetInfoConstants.SQL_DRIVER_NAME.value
+SQL_DRIVER_VER = GetInfoConstants.SQL_DRIVER_VER.value
+SQL_DRIVER_ODBC_VER = GetInfoConstants.SQL_DRIVER_ODBC_VER.value
+SQL_DATA_SOURCE_NAME = GetInfoConstants.SQL_DATA_SOURCE_NAME.value
+SQL_DATABASE_NAME = GetInfoConstants.SQL_DATABASE_NAME.value
+SQL_SERVER_NAME = GetInfoConstants.SQL_SERVER_NAME.value
+SQL_USER_NAME = GetInfoConstants.SQL_USER_NAME.value
+
+# SQL conformance and support
+SQL_SQL_CONFORMANCE = GetInfoConstants.SQL_SQL_CONFORMANCE.value
+SQL_KEYWORDS = GetInfoConstants.SQL_KEYWORDS.value
+SQL_IDENTIFIER_QUOTE_CHAR = GetInfoConstants.SQL_IDENTIFIER_QUOTE_CHAR.value
+SQL_SEARCH_PATTERN_ESCAPE = GetInfoConstants.SQL_SEARCH_PATTERN_ESCAPE.value
+
+# Catalog and schema support
+SQL_CATALOG_TERM = GetInfoConstants.SQL_CATALOG_TERM.value
+SQL_SCHEMA_TERM = GetInfoConstants.SQL_SCHEMA_TERM.value
+SQL_TABLE_TERM = GetInfoConstants.SQL_TABLE_TERM.value
+SQL_PROCEDURE_TERM = GetInfoConstants.SQL_PROCEDURE_TERM.value
+
+# Transaction support
+SQL_TXN_CAPABLE = GetInfoConstants.SQL_TXN_CAPABLE.value
+SQL_DEFAULT_TXN_ISOLATION = GetInfoConstants.SQL_DEFAULT_TXN_ISOLATION.value
+
+# Data type support
+SQL_NUMERIC_FUNCTIONS = GetInfoConstants.SQL_NUMERIC_FUNCTIONS.value
+SQL_STRING_FUNCTIONS = GetInfoConstants.SQL_STRING_FUNCTIONS.value
+SQL_DATETIME_FUNCTIONS = GetInfoConstants.SQL_DATETIME_FUNCTIONS.value
+
+# Limits
+SQL_MAX_COLUMN_NAME_LEN = GetInfoConstants.SQL_MAX_COLUMN_NAME_LEN.value
+SQL_MAX_TABLE_NAME_LEN = GetInfoConstants.SQL_MAX_TABLE_NAME_LEN.value
+SQL_MAX_SCHEMA_NAME_LEN = GetInfoConstants.SQL_MAX_SCHEMA_NAME_LEN.value
+SQL_MAX_CATALOG_NAME_LEN = GetInfoConstants.SQL_MAX_CATALOG_NAME_LEN.value
+SQL_MAX_IDENTIFIER_LEN = GetInfoConstants.SQL_MAX_IDENTIFIER_LEN.value
+
+# Also provide a function to get all constants
+def get_info_constants():
+    """
+    Returns a dictionary of all available GetInfo constants.
+    
+    This provides all SQLGetInfo constants that can be used with the Connection.getinfo() method
+    to retrieve metadata about the database server and driver.
+    
+    Returns:
+        dict: Dictionary mapping constant names to their integer values
+    """
+    return {name: member.value for name, member in GetInfoConstants.__members__.items()}
+