PERF: Optimize execute() hot path: soft reset, prepare caching, and guarded diagnostics

benchmarks/perf-benchmarking.py

@@ -49,6 +49,10 @@ def _init_conn_strings():
         CONN_STR_PYODBC = f"Driver={{ODBC Driver 18 for SQL Server}};{CONN_STR}"
     else:
         CONN_STR_PYODBC = CONN_STR
+        # mssql-python manages its own driver — strip Driver= from its connection string
+        import re
+        CONN_STR = re.sub(r"Driver=[^;]*;?", "", CONN_STR, flags=re.IGNORECASE).strip(";")
+
 
 
 class BenchmarkResult:

mssql_python/cursor.py

@@ -12,6 +12,7 @@
 # pylint: disable=too-many-lines  # Large file due to comprehensive DB-API 2.0 implementation
 
 import decimal
+import logging
 import uuid
 import datetime
 import warnings
@@ -140,6 +141,7 @@ def __init__(self, connection: "Connection", timeout: int = 0) -> None:
             False
         ]  # Indicates if last_executed_stmt was prepared by ddbc shim.
         # Is a list instead of a bool coz bools in Python are immutable.
+        self._cached_param_types = None  # Cached ParamInfo list for prepare-cache reuse
 
         # Initialize attributes that may be defined later to avoid pylint warnings
         # Note: _original_fetch* methods are not initialized here as they need to be
@@ -747,6 +749,18 @@ def _reset_cursor(self) -> None:
 
         # Reinitialize the statement handle
         self._initialize_cursor()
+        self.is_stmt_prepared = [False]
+        self._cached_param_types = None
+
+    def _soft_reset_cursor(self) -> None:
+        """Lightweight reset: close cursor and unbind params without freeing the HSTMT.
+
+        Preserves the prepared statement plan on the server so repeated
+        executions of the same SQL skip SQLPrepare entirely.
+        """
+        if self.hstmt:
+            ddbc_bindings.DDBCSQLResetStmt(self.hstmt)
+        self._clear_rownumber()
 
     def close(self) -> None:
         """
@@ -1363,8 +1377,10 @@ def execute(  # pylint: disable=too-many-locals,too-many-branches,too-many-state
 
         self._check_closed()  # Check if the cursor is closed
         if reset_cursor:
-            logger.debug("execute: Resetting cursor state")
-            self._reset_cursor()
+            if self.hstmt:
+                self._soft_reset_cursor()
+            else:
+                self._reset_cursor()
         else:
             # Close just the ODBC cursor (not the statement handle) so the
             # prepared plan can be reused.  SQLFreeStmt(SQL_CLOSE) releases
@@ -1409,20 +1425,24 @@ def execute(  # pylint: disable=too-many-locals,too-many-branches,too-many-state
             # Check if single parameter is a nested container that should be unwrapped
             # e.g., execute("SELECT ?", (value,)) vs execute("SELECT ?, ?", ((1, 2),))
             if isinstance(parameters, tuple) and len(parameters) == 1:
-                # Could be either (value,) for single param or ((tuple),) for nested
-                # Check if it's a nested container
                 if isinstance(parameters[0], (tuple, list, dict)):
                     actual_params = parameters[0]
                 else:
                     actual_params = parameters
             else:
                 actual_params = parameters
 
-            # Convert parameters based on detected style
-            operation, converted_params = detect_and_convert_parameters(operation, actual_params)
-
-            # Convert back to list format expected by the binding code
-            parameters = list(converted_params)
+            # Skip detect_and_convert_parameters when re-executing the same SQL —
+            # the parameter style (qmark vs pyformat) won't change between calls.
+            if operation == self.last_executed_stmt and isinstance(actual_params, (tuple, list)):
+                parameters = (
+                    list(actual_params) if not isinstance(actual_params, list) else actual_params
+                )
+            else:
+                operation, converted_params = detect_and_convert_parameters(
+                    operation, actual_params
+                )
+                parameters = list(converted_params)
         else:
             parameters = []
 
@@ -1450,35 +1470,36 @@ def execute(  # pylint: disable=too-many-locals,too-many-branches,too-many-state
                 paraminfo = self._create_parameter_types_list(param, param_info, parameters, i)
                 parameters_type.append(paraminfo)
 
-        # TODO: Use a more sophisticated string compare that handles redundant spaces etc.
-        #       Also consider storing last query's hash instead of full query string. This will help
-        #       in low-memory conditions
-        #       (Ex: huge number of parallel queries with huge query string sizes)
-        if operation != self.last_executed_stmt:
-            # Executing a new statement. Reset is_stmt_prepared to false
+        # Prepare caching: skip SQLPrepare when re-executing the same SQL
+        # with parameters. The HSTMT is reused via _soft_reset_cursor, so the
+        # server-side plan from the previous SQLPrepare is still valid.
+        same_sql = parameters and operation == self.last_executed_stmt and self.is_stmt_prepared[0]
+        if not same_sql:
             self.is_stmt_prepared = [False]
+        effective_use_prepare = use_prepare and not same_sql
 
-        for i, param in enumerate(parameters):
-            logger.debug(
-                """Parameter number: %s, Parameter: %s,
-                Param Python Type: %s, ParamInfo: %s, %s, %s, %s, %s""",
-                i + 1,
-                param,
-                str(type(param)),
-                parameters_type[i].paramSQLType,
-                parameters_type[i].paramCType,
-                parameters_type[i].columnSize,
-                parameters_type[i].decimalDigits,
-                parameters_type[i].inputOutputType,
-            )
+        if logger.isEnabledFor(logging.DEBUG):
+            for i, param in enumerate(parameters):
+                logger.debug(
+                    """Parameter number: %s, Parameter: %s,
+                    Param Python Type: %s, ParamInfo: %s, %s, %s, %s, %s""",
+                    i + 1,
+                    param,
+                    str(type(param)),
+                    parameters_type[i].paramSQLType,
+                    parameters_type[i].paramCType,
+                    parameters_type[i].columnSize,
+                    parameters_type[i].decimalDigits,
+                    parameters_type[i].inputOutputType,
+                )
 
         ret = ddbc_bindings.DDBCSQLExecute(
             self.hstmt,
             operation,
             parameters,
             parameters_type,
             self.is_stmt_prepared,
-            use_prepare,
+            effective_use_prepare,
             encoding_settings,
         )
         # Check return code
@@ -1491,8 +1512,8 @@ def execute(  # pylint: disable=too-many-locals,too-many-branches,too-many-state
             self._reset_cursor()
             raise
 
-        # Capture any diagnostic messages (SQL_SUCCESS_WITH_INFO, etc.)
-        if self.hstmt:
+        # Capture diagnostic messages only when the driver signalled info.
+        if ret == ddbc_sql_const.SQL_SUCCESS_WITH_INFO.value and self.hstmt:
             self.messages.extend(ddbc_bindings.DDBCSQLGetAllDiagRecords(self.hstmt))
 
         self.last_executed_stmt = operation

mssql_python/pybind/ddbc_bindings.cpp

@@ -1378,6 +1378,25 @@ void SqlHandle::close_cursor() {
     }
 }
 
+SQLRETURN SQLResetStmt_wrap(SqlHandlePtr statementHandle) {
+    if (!SQLFreeStmt_ptr) {
+        DriverLoader::getInstance().loadDriver();
+    }
+    SQLHANDLE hStmt = statementHandle->get();
+    if (!hStmt) {
+        return SQL_INVALID_HANDLE;
+    }
+
+    SQLRETURN rc = SQLFreeStmt_ptr(hStmt, SQL_CLOSE);
+    if (SQL_SUCCEEDED(rc)) {
+        rc = SQLFreeStmt_ptr(hStmt, SQL_RESET_PARAMS);
+    }
+    if (SQL_SUCCEEDED(rc) && SQLSetStmtAttr_ptr) {
+        SQLSetStmtAttr_ptr(hStmt, SQL_ATTR_PARAMSET_SIZE, (SQLPOINTER)1, 0);
+    }
+    return rc;
+}
+
 SQLRETURN SQLGetTypeInfo_Wrapper(SqlHandlePtr StatementHandle, SQLSMALLINT DataType) {
     if (!SQLGetTypeInfo_ptr) {
         ThrowStdException("SQLGetTypeInfo function not loaded");
@@ -5800,6 +5819,7 @@ PYBIND11_MODULE(ddbc_bindings, m) {
           py::arg("wcharEncoding") = "utf-16le");
     m.def("DDBCSQLFetchArrowBatch", &FetchArrowBatch_wrap, "Fetch an arrow batch of given length from the result set");
     m.def("DDBCSQLFreeHandle", &SQLFreeHandle_wrap, "Free a handle");
+    m.def("DDBCSQLResetStmt", &SQLResetStmt_wrap, "Close cursor and unbind params without freeing HSTMT");
     m.def("DDBCSQLCheckError", &SQLCheckError_Wrap, "Check for driver errors");
     m.def("DDBCSQLGetAllDiagRecords", &SQLGetAllDiagRecords,
           "Get all diagnostic records for a handle", py::arg("handle"));