From a23518b12679650c7c0749a5bac7e96bb1e9c7fa Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 3 Dec 2025 13:21:31 -0600 Subject: [PATCH 01/16] PYTHON-5528 Add exponential backoff to operation retry loop for server overloaded errors (cherry picked from commit 95948d3def875f1efb4d912ab8f507f518604ef5) --- .evergreen/resync-specs.sh | 3 + test/asynchronous/test_client_backpressure.py | 59 + .../backpressure-retry-loop.json | 3376 ++++++++++++++++ .../backpressure-retry-max-attempts.json | 3413 +++++++++++++++++ tools/synchro.py | 1 + 5 files changed, 6852 insertions(+) create mode 100644 test/asynchronous/test_client_backpressure.py create mode 100644 test/client-backpressure/backpressure-retry-loop.json create mode 100644 test/client-backpressure/backpressure-retry-max-attempts.json diff --git a/.evergreen/resync-specs.sh b/.evergreen/resync-specs.sh index d2bd89c781..4bb9c86304 100755 --- a/.evergreen/resync-specs.sh +++ b/.evergreen/resync-specs.sh @@ -94,6 +94,9 @@ do change-streams|change_streams) cpjson change-streams/tests/ change_streams/ ;; + client-backpressure|client_backpressure) + cpjson client-backpressure/tests client-backpressure + ;; client-side-encryption|csfle|fle) cpjson client-side-encryption/tests/ client-side-encryption/spec cpjson client-side-encryption/corpus/ client-side-encryption/corpus diff --git a/test/asynchronous/test_client_backpressure.py b/test/asynchronous/test_client_backpressure.py new file mode 100644 index 0000000000..669be87aca --- /dev/null +++ b/test/asynchronous/test_client_backpressure.py @@ -0,0 +1,59 @@ +# Copyright 2025-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import os +import sys + +sys.path[0:0] = [""] + +from test.asynchronous import ( + AsyncIntegrationTest, + async_client_context, + unittest, +) +from test.asynchronous.unified_format import generate_test_classes +from test.utils_shared import EventListener, OvertCommandListener + +_IS_SYNC = False + + +class AsyncTestClientBackpressure(AsyncIntegrationTest): + listener: EventListener + + @classmethod + def setUpClass(cls) -> None: + cls.listener = OvertCommandListener() + + @async_client_context.require_connection + async def asyncSetUp(self) -> None: + await super().asyncSetUp() + self.listener.reset() + self.client = await self.async_rs_or_single_client( + event_listeners=[self.listener], retryWrites=False + ) + + +_TEST_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "client-backpressure") + + +globals().update( + generate_test_classes( + _TEST_PATH, + module=__name__, + ) +) + +if __name__ == "__main__": + unittest.main() diff --git a/test/client-backpressure/backpressure-retry-loop.json b/test/client-backpressure/backpressure-retry-loop.json new file mode 100644 index 0000000000..b121d9bd3d --- /dev/null +++ b/test/client-backpressure/backpressure-retry-loop.json @@ -0,0 +1,3376 @@ +{ + "description": "tests that operations respect overload backoff retry loop", + "schemaVersion": "1.3", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ] + } + }, + { + "client": { + "id": "failPointClient", + "useMultipleMongoses": false + } + }, + { + "database": { + "id": "utilDb", + "client": "failPointClient", + "databaseName": "retryable-writes-tests" + } + }, + { + "collection": { + "id": "utilCollection", + "database": "utilDb", + "collectionName": "coll" + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "retryable-writes-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "coll" + } + } + ], + "initialData": [ + { + "collectionName": "coll", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ] + } + ], + "tests": [ + { + "description": "client.listDatabases retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listDatabases" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "listDatabases", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandSucceededEvent": { + "commandName": "listDatabases" + } + } + ] + } + ] + }, + { + "description": "client.listDatabaseNames retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listDatabases" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "listDatabaseNames" + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandSucceededEvent": { + "commandName": "listDatabases" + } + } + ] + } + ] + }, + { + "description": "client.createChangeStream retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + }, + "saveResultAsEntity": "changeStream" + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "client.clientBulkWrite retries using operation loop", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "retryable-writes-tests.coll", + "document": { + "_id": 8, + "x": 88 + } + } + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandSucceededEvent": { + "commandName": "bulkWrite" + } + } + ] + } + ] + }, + { + "description": "database.aggregate retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$listLocalSessions": {} + }, + { + "$limit": 1 + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "database.listCollections retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listCollections" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "listCollections", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandSucceededEvent": { + "commandName": "listCollections" + } + } + ] + } + ] + }, + { + "description": "database.listCollectionNames retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listCollections" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "listCollectionNames", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandSucceededEvent": { + "commandName": "listCollections" + } + } + ] + } + ] + }, + { + "description": "database.runCommand retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "ping" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "runCommand", + "arguments": { + "command": { + "ping": 1 + }, + "commandName": "ping" + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandSucceededEvent": { + "commandName": "ping" + } + } + ] + } + ] + }, + { + "description": "database.createChangeStream retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + }, + "saveResultAsEntity": "changeStream" + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.aggregate retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "aggregate", + "arguments": { + "pipeline": [] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.countDocuments retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "countDocuments", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.estimatedDocumentCount retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "count" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "estimatedDocumentCount" + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandSucceededEvent": { + "commandName": "count" + } + } + ] + } + ] + }, + { + "description": "collection.distinct retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "distinct" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "distinct", + "arguments": { + "fieldName": "x", + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandSucceededEvent": { + "commandName": "distinct" + } + } + ] + } + ] + }, + { + "description": "collection.find retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "find", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandSucceededEvent": { + "commandName": "find" + } + } + ] + } + ] + }, + { + "description": "collection.findOne retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOne", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandSucceededEvent": { + "commandName": "find" + } + } + ] + } + ] + }, + { + "description": "collection.listIndexes retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "listIndexes" + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "listIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.listIndexNames retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "listIndexNames" + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "listIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.createChangeStream retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + }, + "saveResultAsEntity": "changeStream" + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.insertOne retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "insertOne", + "arguments": { + "document": { + "_id": 2, + "x": 22 + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandSucceededEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.insertMany retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "insertMany", + "arguments": { + "documents": [ + { + "_id": 2, + "x": 22 + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandSucceededEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.deleteOne retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "delete" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "deleteOne", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandSucceededEvent": { + "commandName": "delete" + } + } + ] + } + ] + }, + { + "description": "collection.deleteMany retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "delete" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandSucceededEvent": { + "commandName": "delete" + } + } + ] + } + ] + }, + { + "description": "collection.replaceOne retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "replaceOne", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandSucceededEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.updateOne retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "updateOne", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandSucceededEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.updateMany retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "updateMany", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandSucceededEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndDelete retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndDelete", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandSucceededEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndReplace retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndReplace", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandSucceededEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndUpdate retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndUpdate", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandSucceededEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.bulkWrite retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "bulkWrite", + "arguments": { + "requests": [ + { + "insertOne": { + "document": { + "_id": 2, + "x": 22 + } + } + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandSucceededEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.createIndex retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "createIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "createIndex", + "arguments": { + "keys": { + "x": 11 + }, + "name": "x_11" + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "createIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.dropIndex retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "object": "utilCollection", + "name": "createIndex", + "arguments": { + "keys": { + "x": 11 + }, + "name": "x_11" + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "dropIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "dropIndex", + "arguments": { + "name": "x_11" + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "dropIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.dropIndexes retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "dropIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "dropIndexes" + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "dropIndexes" + } + } + ] + } + ] + } + ] +} diff --git a/test/client-backpressure/backpressure-retry-max-attempts.json b/test/client-backpressure/backpressure-retry-max-attempts.json new file mode 100644 index 0000000000..a499aa490b --- /dev/null +++ b/test/client-backpressure/backpressure-retry-max-attempts.json @@ -0,0 +1,3413 @@ +{ + "description": "tests that operations retry at most maxAttempts=5 times", + "schemaVersion": "1.3", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ] + } + }, + { + "client": { + "id": "failPointClient", + "useMultipleMongoses": false + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "retryable-writes-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "coll" + } + } + ], + "initialData": [ + { + "collectionName": "coll", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ] + } + ], + "tests": [ + { + "description": "client.listDatabases retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "listDatabases" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "listDatabases", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + } + ] + } + ] + }, + { + "description": "client.listDatabaseNames retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "listDatabases" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "listDatabaseNames", + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + } + ] + } + ] + }, + { + "description": "client.createChangeStream retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "client.clientBulkWrite retries at most maxAttempts=5 times", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "bulkWrite" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "retryable-writes-tests.coll", + "document": { + "_id": 8, + "x": 88 + } + } + } + ] + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + } + ] + } + ] + }, + { + "description": "database.aggregate retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$listLocalSessions": {} + }, + { + "$limit": 1 + } + ] + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "database.listCollections retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "listCollections" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "listCollections", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + } + ] + } + ] + }, + { + "description": "database.listCollectionNames retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "listCollections" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "listCollectionNames", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + } + ] + } + ] + }, + { + "description": "database.runCommand retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "ping" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "runCommand", + "arguments": { + "command": { + "ping": 1 + }, + "commandName": "ping" + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + } + ] + } + ] + }, + { + "description": "database.createChangeStream retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.aggregate retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "aggregate", + "arguments": { + "pipeline": [] + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.countDocuments retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "countDocuments", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.estimatedDocumentCount retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "count" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "estimatedDocumentCount", + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + } + ] + } + ] + }, + { + "description": "collection.distinct retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "distinct" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "distinct", + "arguments": { + "fieldName": "x", + "filter": {} + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + } + ] + } + ] + }, + { + "description": "collection.find retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "find", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + } + ] + } + ] + }, + { + "description": "collection.findOne retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOne", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + } + ] + } + ] + }, + { + "description": "collection.listIndexes retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "listIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "listIndexes", + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.listIndexNames retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "listIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "listIndexNames", + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.createChangeStream retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.insertOne retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "insertOne", + "arguments": { + "document": { + "_id": 2, + "x": 22 + } + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.insertMany retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "insertMany", + "arguments": { + "documents": [ + { + "_id": 2, + "x": 22 + } + ] + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.deleteOne retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "delete" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "deleteOne", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + } + ] + } + ] + }, + { + "description": "collection.deleteMany retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "delete" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "deleteMany", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + } + ] + } + ] + }, + { + "description": "collection.replaceOne retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "replaceOne", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.updateOne retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "updateOne", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.updateMany retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "updateMany", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndDelete retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndDelete", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndReplace retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndReplace", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndUpdate retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndUpdate", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.bulkWrite retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "bulkWrite", + "arguments": { + "requests": [ + { + "insertOne": { + "document": { + "_id": 2, + "x": 22 + } + } + } + ] + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.createIndex retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "createIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "createIndex", + "arguments": { + "keys": { + "x": 11 + }, + "name": "x_11" + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.dropIndex retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "dropIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "dropIndex", + "arguments": { + "name": "x_11" + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.dropIndexes retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "dropIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "dropIndexes", + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + } + ] + } + ] + } + ] +} diff --git a/tools/synchro.py b/tools/synchro.py index 661d8988cb..c87b4e8139 100644 --- a/tools/synchro.py +++ b/tools/synchro.py @@ -212,6 +212,7 @@ def async_only_test(f: str) -> bool: "test_backpressure.py", "test_change_stream.py", "test_client.py", + "test_client_backpressure.py", "test_client_bulk_write.py", "test_client_context.py", "test_client_metadata.py", From 9c6eeed5f768b4ad3ffe6cb605d8456c8a3c0ea1 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 3 Dec 2025 13:22:10 -0600 Subject: [PATCH 02/16] add async test (cherry picked from commit ebf65519f12d160da6d2088c22f6c69fad5a37d0) --- test/test_client_backpressure.py | 57 ++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 test/test_client_backpressure.py diff --git a/test/test_client_backpressure.py b/test/test_client_backpressure.py new file mode 100644 index 0000000000..5492f5f271 --- /dev/null +++ b/test/test_client_backpressure.py @@ -0,0 +1,57 @@ +# Copyright 2025-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import os +import sys + +sys.path[0:0] = [""] + +from test import ( + IntegrationTest, + client_context, + unittest, +) +from test.unified_format import generate_test_classes +from test.utils_shared import EventListener, OvertCommandListener + +_IS_SYNC = True + + +class TestClientBackpressure(IntegrationTest): + listener: EventListener + + @classmethod + def setUpClass(cls) -> None: + cls.listener = OvertCommandListener() + + @client_context.require_connection + def setUp(self) -> None: + super().setUp() + self.listener.reset() + self.client = self.rs_or_single_client(event_listeners=[self.listener], retryWrites=False) + + +_TEST_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "client-backpressure") + + +globals().update( + generate_test_classes( + _TEST_PATH, + module=__name__, + ) +) + +if __name__ == "__main__": + unittest.main() From f83e16bc31f71fc15d75ac60f91a96c08cf09650 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 3 Dec 2025 14:05:58 -0600 Subject: [PATCH 03/16] fix test setup --- test/asynchronous/test_client_backpressure.py | 8 ++++++-- test/test_client_backpressure.py | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/test/asynchronous/test_client_backpressure.py b/test/asynchronous/test_client_backpressure.py index 669be87aca..183b03fdb1 100644 --- a/test/asynchronous/test_client_backpressure.py +++ b/test/asynchronous/test_client_backpressure.py @@ -14,6 +14,7 @@ from __future__ import annotations import os +import pathlib import sys sys.path[0:0] = [""] @@ -45,8 +46,11 @@ async def asyncSetUp(self) -> None: ) -_TEST_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "client-backpressure") - +# Location of JSON test specifications. +if _IS_SYNC: + _TEST_PATH = os.path.join(pathlib.Path(__file__).resolve().parent, "client-backpressure") +else: + _TEST_PATH = os.path.join(pathlib.Path(__file__).resolve().parent.parent, "client-backpressure") globals().update( generate_test_classes( diff --git a/test/test_client_backpressure.py b/test/test_client_backpressure.py index 5492f5f271..2dd9187ebb 100644 --- a/test/test_client_backpressure.py +++ b/test/test_client_backpressure.py @@ -14,6 +14,7 @@ from __future__ import annotations import os +import pathlib import sys sys.path[0:0] = [""] @@ -43,8 +44,11 @@ def setUp(self) -> None: self.client = self.rs_or_single_client(event_listeners=[self.listener], retryWrites=False) -_TEST_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "client-backpressure") - +# Location of JSON test specifications. +if _IS_SYNC: + _TEST_PATH = os.path.join(pathlib.Path(__file__).resolve().parent, "client-backpressure") +else: + _TEST_PATH = os.path.join(pathlib.Path(__file__).resolve().parent.parent, "client-backpressure") globals().update( generate_test_classes( From 6333244098ccedd3cf7ae272d80c20195e2df0da Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 3 Dec 2025 14:59:34 -0600 Subject: [PATCH 04/16] implement handshake prose test --- pymongo/asynchronous/pool.py | 1 + pymongo/synchronous/pool.py | 1 + test/asynchronous/test_client_metadata.py | 13 +++++++++++++ test/test_client_metadata.py | 13 +++++++++++++ 4 files changed, 28 insertions(+) diff --git a/pymongo/asynchronous/pool.py b/pymongo/asynchronous/pool.py index 2c9c70cd46..ea41da27ec 100644 --- a/pymongo/asynchronous/pool.py +++ b/pymongo/asynchronous/pool.py @@ -269,6 +269,7 @@ async def _hello( cmd = self.hello_cmd() performing_handshake = not self.performed_handshake awaitable = False + cmd["backpressure"] = True if performing_handshake: self.performed_handshake = True cmd["client"] = self.opts.metadata diff --git a/pymongo/synchronous/pool.py b/pymongo/synchronous/pool.py index 9e50fa5903..c833cd8a46 100644 --- a/pymongo/synchronous/pool.py +++ b/pymongo/synchronous/pool.py @@ -269,6 +269,7 @@ def _hello( cmd = self.hello_cmd() performing_handshake = not self.performed_handshake awaitable = False + cmd["backpressure"] = True if performing_handshake: self.performed_handshake = True cmd["client"] = self.opts.metadata diff --git a/test/asynchronous/test_client_metadata.py b/test/asynchronous/test_client_metadata.py index 2f175cceed..a0fdb1c269 100644 --- a/test/asynchronous/test_client_metadata.py +++ b/test/asynchronous/test_client_metadata.py @@ -227,6 +227,19 @@ async def test_duplicate_driver_name_no_op(self): # add same metadata again await self.check_metadata_added(client, "Framework", None, None) + async def test_handshake_documents_include_backpressure(self): + # Create a `MongoClient` that is configured to record all handshake documents sent to the server as a part of + # connection establishment. + client = await self.async_rs_or_single_client("mongodb://" + self.server.address_string) + + # Send a `ping` command to the server and verify that the command succeeds. This ensure that a connection is + # established on all topologies. Note: MockupDB only supports standalone servers. + await client.admin.command("ping") + + # Assert that for every handshake document intercepted: + # the document has a field `backpressure` whose value is `true`. + self.assertEqual(self.handshake_req["backpressure"], True) + if __name__ == "__main__": unittest.main() diff --git a/test/test_client_metadata.py b/test/test_client_metadata.py index a94c5aa25e..8132378936 100644 --- a/test/test_client_metadata.py +++ b/test/test_client_metadata.py @@ -227,6 +227,19 @@ def test_duplicate_driver_name_no_op(self): # add same metadata again self.check_metadata_added(client, "Framework", None, None) + def test_handshake_documents_include_backpressure(self): + # Create a `MongoClient` that is configured to record all handshake documents sent to the server as a part of + # connection establishment. + client = self.rs_or_single_client("mongodb://" + self.server.address_string) + + # Send a `ping` command to the server and verify that the command succeeds. This ensure that a connection is + # established on all topologies. Note: MockupDB only supports standalone servers. + client.admin.command("ping") + + # Assert that for every handshake document intercepted: + # the document has a field `backpressure` whose value is `true`. + self.assertEqual(self.handshake_req["backpressure"], True) + if __name__ == "__main__": unittest.main() From 46440748e4b0ccb78303dc11bad9815ac3a19915 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 3 Dec 2025 16:55:24 -0600 Subject: [PATCH 05/16] update unified test and add prose test --- test/asynchronous/test_client_backpressure.py | 53 ++- .../backpressure-retry-loop.json | 384 ------------------ test/test_client_backpressure.py | 55 ++- 3 files changed, 106 insertions(+), 386 deletions(-) diff --git a/test/asynchronous/test_client_backpressure.py b/test/asynchronous/test_client_backpressure.py index 183b03fdb1..db76e797e2 100644 --- a/test/asynchronous/test_client_backpressure.py +++ b/test/asynchronous/test_client_backpressure.py @@ -16,6 +16,10 @@ import os import pathlib import sys +from time import perf_counter +from unittest.mock import patch + +from pymongo.errors import OperationFailure sys.path[0:0] = [""] @@ -41,10 +45,57 @@ def setUpClass(cls) -> None: async def asyncSetUp(self) -> None: await super().asyncSetUp() self.listener.reset() + self.app_name = self.__class__.__name__.lower() self.client = await self.async_rs_or_single_client( - event_listeners=[self.listener], retryWrites=False + event_listeners=[self.listener], retryWrites=False, appName=self.app_name ) + @patch("random.random") + async def test_01_operation_retry_uses_exponential_backoff(self, random_func): + # Drivers should test that retries do not occur immediately when a SystemOverloadedError is encountered. + + # 1. let `client` be a `MongoClient` + client = self.client + + # 2. let `collection` be a collection + collection = client.test.test + + # 3. Now, run transactions without backoff: + + # a. Configure the random number generator used for jitter to always return `0` -- this effectively disables backoff. + random_func.return_value = 0 + + # b. Configure the following failPoint: + fail_point = dict( + mode="alwaysOn", + data=dict( + failCommands=["insert"], + errorCode=2, + errorLabels=["SystemOverloadedError", "RetryableError"], + appName=self.app_name, + ), + ) + async with self.fail_point(fail_point): + # c. Execute the following command. Expect that the command errors. Measure the duration of the command execution. + start0 = perf_counter() + with self.assertRaises(OperationFailure): + await collection.insert_one({"a": 1}) + end0 = perf_counter() + + # d. Configure the random number generator used for jitter to always return `1`. + random_func.return_value = 1 + + # e. Execute step c again. + start1 = perf_counter() + with self.assertRaises(OperationFailure): + await collection.insert_one({"a": 1}) + end1 = perf_counter() + + # f. Compare the two time between the two runs. + # The sum of 5 backoffs is 3.1 seconds. There is a 1-second window to account for potential variance between the two + # runs. + self.assertTrue(abs((end1 - start1) - (end0 - start0 + 3.1)) < 1) + # Location of JSON test specifications. if _IS_SYNC: diff --git a/test/client-backpressure/backpressure-retry-loop.json b/test/client-backpressure/backpressure-retry-loop.json index b121d9bd3d..20bdfe3a69 100644 --- a/test/client-backpressure/backpressure-retry-loop.json +++ b/test/client-backpressure/backpressure-retry-loop.json @@ -85,18 +85,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -186,18 +174,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -284,18 +260,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -391,18 +355,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -502,18 +454,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -610,18 +550,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -711,18 +639,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -812,18 +728,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -916,18 +820,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1018,18 +910,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1119,18 +999,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1220,18 +1088,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1318,18 +1174,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1420,18 +1264,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1521,18 +1353,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1622,18 +1442,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1720,18 +1528,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1818,18 +1614,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1920,18 +1704,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2024,18 +1796,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2130,18 +1890,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2231,18 +1979,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2332,18 +2068,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2436,18 +2160,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2542,18 +2254,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2648,18 +2348,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2749,18 +2437,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2853,18 +2529,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2959,18 +2623,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -3069,18 +2721,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -3173,18 +2813,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "object": "utilCollection", "name": "createIndex", @@ -3284,18 +2912,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", diff --git a/test/test_client_backpressure.py b/test/test_client_backpressure.py index 2dd9187ebb..55f34a77fb 100644 --- a/test/test_client_backpressure.py +++ b/test/test_client_backpressure.py @@ -16,6 +16,10 @@ import os import pathlib import sys +from time import perf_counter +from unittest.mock import patch + +from pymongo.errors import OperationFailure sys.path[0:0] = [""] @@ -41,7 +45,56 @@ def setUpClass(cls) -> None: def setUp(self) -> None: super().setUp() self.listener.reset() - self.client = self.rs_or_single_client(event_listeners=[self.listener], retryWrites=False) + self.app_name = self.__class__.__name__.lower() + self.client = self.rs_or_single_client( + event_listeners=[self.listener], retryWrites=False, appName=self.app_name + ) + + @patch("random.random") + def test_01_operation_retry_uses_exponential_backoff(self, random_func): + # Drivers should test that retries do not occur immediately when a SystemOverloadedError is encountered. + + # 1. let `client` be a `MongoClient` + client = self.client + + # 2. let `collection` be a collection + collection = client.test.test + + # 3. Now, run transactions without backoff: + + # a. Configure the random number generator used for jitter to always return `0` -- this effectively disables backoff. + random_func.return_value = 0 + + # b. Configure the following failPoint: + fail_point = dict( + mode="alwaysOn", + data=dict( + failCommands=["insert"], + errorCode=2, + errorLabels=["SystemOverloadedError", "RetryableError"], + appName=self.app_name, + ), + ) + with self.fail_point(fail_point): + # c. Execute the following command. Expect that the command errors. Measure the duration of the command execution. + start0 = perf_counter() + with self.assertRaises(OperationFailure): + collection.insert_one({"a": 1}) + end0 = perf_counter() + + # d. Configure the random number generator used for jitter to always return `1`. + random_func.return_value = 1 + + # e. Execute step c again. + start1 = perf_counter() + with self.assertRaises(OperationFailure): + collection.insert_one({"a": 1}) + end1 = perf_counter() + + # f. Compare the two time between the two runs. + # The sum of 5 backoffs is 3.1 seconds. There is a 1-second window to account for potential variance between the two + # runs. + self.assertTrue(abs((end1 - start1) - (end0 - start0 + 3.1)) < 1) # Location of JSON test specifications. From 4ba159c2c8b2a2ae63685ce21de3d37d45c58072 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 4 Dec 2025 06:16:01 -0600 Subject: [PATCH 06/16] fix overload retry for clientBulk --- pymongo/asynchronous/client_bulk.py | 14 +++++++++++++- pymongo/synchronous/client_bulk.py | 14 +++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/pymongo/asynchronous/client_bulk.py b/pymongo/asynchronous/client_bulk.py index 151942c8a8..508b8e41c3 100644 --- a/pymongo/asynchronous/client_bulk.py +++ b/pymongo/asynchronous/client_bulk.py @@ -563,9 +563,21 @@ async def _execute_command( error, ConnectionFailure ) and not isinstance(error, (NotPrimaryError, WaitQueueTimeoutError)) + retryable_label_error = ( + hasattr(error, "details") + and isinstance(error.details, dict) + and "errorLabels" in error.details + and isinstance(error.details["errorLabels"], list) + and "RetryableError" in error.details["errorLabels"] + ) + # Synthesize the full bulk result without modifying the # current one because this write operation may be retried. - if retryable and (retryable_top_level_error or retryable_network_error): + if retryable and ( + retryable_top_level_error + or retryable_network_error + or retryable_label_error + ): full = copy.deepcopy(full_result) _merge_command(self.ops, self.idx_offset, full, result) _throw_client_bulk_write_exception(full, self.verbose_results) diff --git a/pymongo/synchronous/client_bulk.py b/pymongo/synchronous/client_bulk.py index a606d028e1..e8167bcedc 100644 --- a/pymongo/synchronous/client_bulk.py +++ b/pymongo/synchronous/client_bulk.py @@ -561,9 +561,21 @@ def _execute_command( error, ConnectionFailure ) and not isinstance(error, (NotPrimaryError, WaitQueueTimeoutError)) + retryable_label_error = ( + hasattr(error, "details") + and isinstance(error.details, dict) + and "errorLabels" in error.details + and isinstance(error.details["errorLabels"], list) + and "RetryableError" in error.details["errorLabels"] + ) + # Synthesize the full bulk result without modifying the # current one because this write operation may be retried. - if retryable and (retryable_top_level_error or retryable_network_error): + if retryable and ( + retryable_top_level_error + or retryable_network_error + or retryable_label_error + ): full = copy.deepcopy(full_result) _merge_command(self.ops, self.idx_offset, full, result) _throw_client_bulk_write_exception(full, self.verbose_results) From 13338814a3a403579f1d7bb482981efa002417dc Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 4 Dec 2025 09:14:01 -0600 Subject: [PATCH 07/16] fix prose test --- test/asynchronous/test_client_backpressure.py | 1 + test/test_client_backpressure.py | 1 + 2 files changed, 2 insertions(+) diff --git a/test/asynchronous/test_client_backpressure.py b/test/asynchronous/test_client_backpressure.py index db76e797e2..b66497d9ee 100644 --- a/test/asynchronous/test_client_backpressure.py +++ b/test/asynchronous/test_client_backpressure.py @@ -51,6 +51,7 @@ async def asyncSetUp(self) -> None: ) @patch("random.random") + @async_client_context.require_failCommand_appName async def test_01_operation_retry_uses_exponential_backoff(self, random_func): # Drivers should test that retries do not occur immediately when a SystemOverloadedError is encountered. diff --git a/test/test_client_backpressure.py b/test/test_client_backpressure.py index 55f34a77fb..b6776d53ba 100644 --- a/test/test_client_backpressure.py +++ b/test/test_client_backpressure.py @@ -51,6 +51,7 @@ def setUp(self) -> None: ) @patch("random.random") + @client_context.require_failCommand_appName def test_01_operation_retry_uses_exponential_backoff(self, random_func): # Drivers should test that retries do not occur immediately when a SystemOverloadedError is encountered. From 9a88851f34573bc1ed8c38613532eebb7ff7d774 Mon Sep 17 00:00:00 2001 From: Kevin Albertson Date: Thu, 4 Dec 2025 11:58:10 -0500 Subject: [PATCH 08/16] PYTHON-5664 extract using `tar` command (#2636) (cherry picked from commit 3093a7c7cbd048a84a7014c03a6e295955397eb0) --- .evergreen/scripts/setup_tests.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 7908836f50..939423ffcc 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -1,12 +1,10 @@ from __future__ import annotations import base64 -import io import os import platform import shutil import stat -import tarfile from pathlib import Path from urllib import request @@ -117,9 +115,10 @@ def setup_libmongocrypt(): LOGGER.info(f"Fetching {url}...") with request.urlopen(request.Request(url), timeout=15.0) as response: # noqa: S310 if response.status == 200: - fileobj = io.BytesIO(response.read()) - with tarfile.open("libmongocrypt.tar.gz", fileobj=fileobj) as fid: - fid.extractall(Path.cwd() / "libmongocrypt") + with Path("libmongocrypt.tar.gz").open("wb") as f: + f.write(response.read()) + Path("libmongocrypt").mkdir() + run_command("tar -xzf libmongocrypt.tar.gz -C libmongocrypt") LOGGER.info(f"Fetching {url}... done.") run_command("ls -la libmongocrypt") From 9e85611b8b84227de8134c382c8657b02390fc5e Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 4 Dec 2025 12:21:48 -0600 Subject: [PATCH 09/16] update justfile from master --- justfile | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/justfile b/justfile index 7ac5bd33ff..92bdee5be3 100644 --- a/justfile +++ b/justfile @@ -1,7 +1,5 @@ # See https://just.systems/man/en/ for instructions set shell := ["bash", "-c"] -# Do not modify the lock file when running justfile commands. -export UV_FROZEN := "1" # Commonly used command segments. typing_run := "uv run --group typing --extra aws --extra encryption --extra ocsp --extra snappy --extra test --extra zstd" @@ -16,7 +14,7 @@ default: [private] resync: - @uv sync --quiet --frozen + @uv sync --quiet install: bash .evergreen/scripts/setup-dev-env.sh @@ -50,12 +48,12 @@ typing-pyright: && resync {{typing_run}} pyright -p strict_pyrightconfig.json test/test_typing_strict.py [group('lint')] -lint: && resync - uv run pre-commit run --all-files +lint *args="": && resync + uvx pre-commit run --all-files {{args}} [group('lint')] -lint-manual: && resync - uv run pre-commit run --all-files --hook-stage manual +lint-manual *args="": && resync + uvx pre-commit run --all-files --hook-stage manual {{args}} [group('test')] test *args="-v --durations=5 --maxfail=10": && resync @@ -73,6 +71,10 @@ setup-tests *args="": teardown-tests: bash .evergreen/scripts/teardown-tests.sh +[group('test')] +integration-tests: + bash integration_tests/run.sh + [group('server')] run-server *args="": bash .evergreen/scripts/run-server.sh {{args}} From 77dc3b711e696627b73f25ff22e47498f697c0c7 Mon Sep 17 00:00:00 2001 From: Casey Clements Date: Mon, 1 Dec 2025 15:15:09 -0500 Subject: [PATCH 10/16] PYTHON-5656: Fixes broken link to aggregation pipeline docs. (#2627) (cherry picked from commit 8bf826339110605c27e5f7338c075b3fdb26949e) --- pymongo/asynchronous/database.py | 2 +- pymongo/synchronous/database.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pymongo/asynchronous/database.py b/pymongo/asynchronous/database.py index 9b84869315..5e34e52e69 100644 --- a/pymongo/asynchronous/database.py +++ b/pymongo/asynchronous/database.py @@ -701,7 +701,7 @@ async def aggregate( .. versionadded:: 3.9 .. _aggregation pipeline: - https://mongodb.com/docs/manual/reference/operator/aggregation-pipeline + https://www.mongodb.com/docs/manual/core/aggregation-pipeline/ .. _aggregate command: https://mongodb.com/docs/manual/reference/command/aggregate diff --git a/pymongo/synchronous/database.py b/pymongo/synchronous/database.py index 6877854f46..27cc102d05 100644 --- a/pymongo/synchronous/database.py +++ b/pymongo/synchronous/database.py @@ -701,7 +701,7 @@ def aggregate( .. versionadded:: 3.9 .. _aggregation pipeline: - https://mongodb.com/docs/manual/reference/operator/aggregation-pipeline + https://www.mongodb.com/docs/manual/core/aggregation-pipeline/ .. _aggregate command: https://mongodb.com/docs/manual/reference/command/aggregate From 5dd20a1bcac1b55dabecd907e6819009e0ebcdef Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 4 Dec 2025 13:53:10 -0600 Subject: [PATCH 11/16] Deprioritize previous servers for overload retry attempts --- pymongo/asynchronous/collection.py | 86 ++++++++++++++++++---------- pymongo/asynchronous/database.py | 54 +++++++++-------- pymongo/asynchronous/helpers.py | 29 ---------- pymongo/asynchronous/mongo_client.py | 18 +++--- pymongo/synchronous/collection.py | 82 +++++++++++++++++--------- pymongo/synchronous/database.py | 50 ++++++++++------ pymongo/synchronous/helpers.py | 29 ---------- pymongo/synchronous/mongo_client.py | 18 +++--- 8 files changed, 188 insertions(+), 178 deletions(-) diff --git a/pymongo/asynchronous/collection.py b/pymongo/asynchronous/collection.py index c6cb69af25..60755dbe89 100644 --- a/pymongo/asynchronous/collection.py +++ b/pymongo/asynchronous/collection.py @@ -20,7 +20,6 @@ from typing import ( TYPE_CHECKING, Any, - AsyncContextManager, Callable, Coroutine, Generic, @@ -58,7 +57,6 @@ AsyncCursor, AsyncRawBatchCursor, ) -from pymongo.asynchronous.helpers import _retry_overload from pymongo.collation import validate_collation_or_none from pymongo.common import _ecoc_coll_name, _esc_coll_name from pymongo.errors import ( @@ -573,11 +571,6 @@ async def watch( await change_stream._initialize_cursor() return change_stream - async def _conn_for_writes( - self, session: Optional[AsyncClientSession], operation: str - ) -> AsyncContextManager[AsyncConnection]: - return await self._database.client._conn_for_writes(session, operation) - async def _command( self, conn: AsyncConnection, @@ -654,7 +647,10 @@ async def _create_helper( if "size" in options: options["size"] = float(options["size"]) cmd.update(options) - async with await self._conn_for_writes(session, operation=_Op.CREATE) as conn: + + async def inner( + session: Optional[AsyncClientSession], conn: AsyncConnection, _retryable_write: bool + ) -> None: if qev2_required and conn.max_wire_version < 21: raise ConfigurationError( "Driver support of Queryable Encryption is incompatible with server. " @@ -671,6 +667,8 @@ async def _create_helper( session=session, ) + await self.database.client._retryable_write(False, inner, session, _Op.CREATE) + async def _create( self, options: MutableMapping[str, Any], @@ -2229,7 +2227,6 @@ async def create_indexes( return await self._create_indexes(indexes, session, **kwargs) @_csot.apply - @_retry_overload async def _create_indexes( self, indexes: Sequence[IndexModel], session: Optional[AsyncClientSession], **kwargs: Any ) -> list[str]: @@ -2243,7 +2240,10 @@ async def _create_indexes( command (like maxTimeMS) can be passed as keyword arguments. """ names = [] - async with await self._conn_for_writes(session, operation=_Op.CREATE_INDEXES) as conn: + + async def inner( + session: Optional[AsyncClientSession], conn: AsyncConnection, _retryable_write: bool + ) -> list[str]: supports_quorum = conn.max_wire_version >= 9 def gen_indexes() -> Iterator[Mapping[str, Any]]: @@ -2272,7 +2272,11 @@ def gen_indexes() -> Iterator[Mapping[str, Any]]: write_concern=self._write_concern_for(session), session=session, ) - return names + return names + + return await self.database.client._retryable_write( + False, inner, session, _Op.CREATE_INDEXES + ) async def create_index( self, @@ -2474,7 +2478,6 @@ async def drop_index( await self._drop_index(index_or_name, session, comment, **kwargs) @_csot.apply - @_retry_overload async def _drop_index( self, index_or_name: _IndexKeyHint, @@ -2493,7 +2496,10 @@ async def _drop_index( cmd.update(kwargs) if comment is not None: cmd["comment"] = comment - async with await self._conn_for_writes(session, operation=_Op.DROP_INDEXES) as conn: + + async def inner( + session: Optional[AsyncClientSession], conn: AsyncConnection, _retryable_write: bool + ) -> None: await self._command( conn, cmd, @@ -2503,6 +2509,8 @@ async def _drop_index( session=session, ) + await self.database.client._retryable_write(False, inner, session, _Op.DROP_INDEXES) + async def list_indexes( self, session: Optional[AsyncClientSession] = None, @@ -2766,17 +2774,22 @@ def gen_indexes() -> Iterator[Mapping[str, Any]]: cmd = {"createSearchIndexes": self.name, "indexes": list(gen_indexes())} cmd.update(kwargs) - async with await self._conn_for_writes( - session, operation=_Op.CREATE_SEARCH_INDEXES - ) as conn: + async def inner( + session: Optional[AsyncClientSession], conn: AsyncConnection, _retryable_write: bool + ) -> list[str]: resp = await self._command( conn, cmd, read_preference=ReadPreference.PRIMARY, codec_options=_UNICODE_REPLACE_CODEC_OPTIONS, + session=session, ) return [index["name"] for index in resp["indexesCreated"]] + return self.database.client._retryable_write( + False, inner, session, _Op.CREATE_SEARCH_INDEXES + ) + async def drop_search_index( self, name: str, @@ -2802,15 +2815,21 @@ async def drop_search_index( cmd.update(kwargs) if comment is not None: cmd["comment"] = comment - async with await self._conn_for_writes(session, operation=_Op.DROP_SEARCH_INDEXES) as conn: + + async def inner( + session: Optional[AsyncClientSession], conn: AsyncConnection, _retryable_write: bool + ) -> None: await self._command( conn, cmd, read_preference=ReadPreference.PRIMARY, allowable_errors=["ns not found", 26], codec_options=_UNICODE_REPLACE_CODEC_OPTIONS, + session=session, ) + return self.database.client._retryable_write(False, inner, session, _Op.DROP_SEARCH_INDEXES) + async def update_search_index( self, name: str, @@ -2838,15 +2857,21 @@ async def update_search_index( cmd.update(kwargs) if comment is not None: cmd["comment"] = comment - async with await self._conn_for_writes(session, operation=_Op.UPDATE_SEARCH_INDEX) as conn: + + async def inner( + session: Optional[AsyncClientSession], conn: AsyncConnection, _retryable_write: bool + ) -> None: await self._command( conn, cmd, read_preference=ReadPreference.PRIMARY, allowable_errors=["ns not found", 26], codec_options=_UNICODE_REPLACE_CODEC_OPTIONS, + session=session, ) + return self.database.client._retryable_write(False, inner, session, _Op.UPDATE_SEARCH_INDEX) + async def options( self, session: Optional[AsyncClientSession] = None, @@ -3075,7 +3100,6 @@ async def aggregate_raw_batches( ) @_csot.apply - @_retry_overload async def rename( self, new_name: str, @@ -3127,17 +3151,21 @@ async def rename( if comment is not None: cmd["comment"] = comment write_concern = self._write_concern_for_cmd(cmd, session) + client = self._database.client - async with await self._conn_for_writes(session, operation=_Op.RENAME) as conn: - async with self._database.client._tmp_session(session) as s: - return await conn.command( - "admin", - cmd, - write_concern=write_concern, - parse_write_concern_error=True, - session=s, - client=self._database.client, - ) + async def inner( + session: Optional[AsyncClientSession], conn: AsyncConnection, _retryable_write: bool + ) -> MutableMapping[str, Any]: + return await conn.command( + "admin", + cmd, + write_concern=write_concern, + parse_write_concern_error=True, + session=session, + client=client, + ) + + return client._retryable_write(False, inner, session, _Op.RENAME) async def distinct( self, diff --git a/pymongo/asynchronous/database.py b/pymongo/asynchronous/database.py index 5e34e52e69..7ad9017f70 100644 --- a/pymongo/asynchronous/database.py +++ b/pymongo/asynchronous/database.py @@ -38,7 +38,6 @@ from pymongo.asynchronous.change_stream import AsyncDatabaseChangeStream from pymongo.asynchronous.collection import AsyncCollection from pymongo.asynchronous.command_cursor import AsyncCommandCursor -from pymongo.asynchronous.helpers import _retry_overload from pymongo.common import _ecoc_coll_name, _esc_coll_name from pymongo.database_shared import _check_name, _CodecDocumentType from pymongo.errors import CollectionInvalid, InvalidOperation @@ -479,7 +478,6 @@ async def watch( return change_stream @_csot.apply - @_retry_overload async def create_collection( self, name: str, @@ -822,7 +820,6 @@ async def command( ... @_csot.apply - @_retry_overload async def command( self, command: Union[str, MutableMapping[str, Any]], @@ -935,14 +932,15 @@ async def command( if read_preference is None: read_preference = (session and session._txn_read_preference()) or ReadPreference.PRIMARY - async with await self._client._conn_for_reads( - read_preference, session, operation=command_name - ) as ( - connection, - read_preference, - ): + + async def inner( + session: Optional[AsyncClientSession], + _server: Server, + conn: AsyncConnection, + read_preference: _ServerMode, + ) -> Union[dict[str, Any], _CodecDocumentType]: return await self._command( - connection, + conn, command, value, check, @@ -953,8 +951,11 @@ async def command( **kwargs, ) + return await self._client._retryable_read( + inner, read_preference, session, command_name, None, False + ) + @_csot.apply - @_retry_overload async def cursor_command( self, command: Union[str, MutableMapping[str, Any]], @@ -1021,17 +1022,17 @@ async def cursor_command( async with self._client._tmp_session(session) as tmp_session: opts = codec_options or DEFAULT_CODEC_OPTIONS - if read_preference is None: read_preference = ( tmp_session and tmp_session._txn_read_preference() ) or ReadPreference.PRIMARY - async with await self._client._conn_for_reads( - read_preference, tmp_session, command_name - ) as ( - conn, - read_preference, - ): + + async def inner( + session: Optional[AsyncClientSession], + _server: Server, + conn: AsyncConnection, + read_preference: _ServerMode, + ) -> AsyncCommandCursor[_DocumentType]: response = await self._command( conn, command, @@ -1040,7 +1041,7 @@ async def cursor_command( None, read_preference, opts, - session=tmp_session, + session=session, **kwargs, ) coll = self.get_collection("$cmd", read_preference=read_preference) @@ -1050,7 +1051,7 @@ async def cursor_command( response["cursor"], conn.address, max_await_time_ms=max_await_time_ms, - session=tmp_session, + session=session, comment=comment, ) await cmd_cursor._maybe_pin_connection(conn) @@ -1058,6 +1059,10 @@ async def cursor_command( else: raise InvalidOperation("Command does not return a cursor.") + return await self.client._retryable_read( + inner, read_preference, tmp_session, command_name, False + ) + async def _retryable_read_command( self, command: Union[str, MutableMapping[str, Any]], @@ -1259,9 +1264,11 @@ async def _drop_helper( if comment is not None: command["comment"] = comment - async with await self._client._conn_for_writes(session, operation=_Op.DROP) as connection: + async def inner( + session: Optional[AsyncClientSession], conn: AsyncConnection, _retryable_write: bool + ) -> dict[str, Any]: return await self._command( - connection, + conn, command, allowable_errors=["ns not found", 26], write_concern=self._write_concern_for(session), @@ -1269,8 +1276,9 @@ async def _drop_helper( session=session, ) + return await self.client._retryable_write(False, inner, session, _Op.DROP) + @_csot.apply - @_retry_overload async def drop_collection( self, name_or_collection: Union[str, AsyncCollection[_DocumentTypeArg]], diff --git a/pymongo/asynchronous/helpers.py b/pymongo/asynchronous/helpers.py index 9dc87ad285..c008f64a63 100644 --- a/pymongo/asynchronous/helpers.py +++ b/pymongo/asynchronous/helpers.py @@ -30,7 +30,6 @@ from pymongo import _csot from pymongo.errors import ( OperationFailure, - PyMongoError, ) from pymongo.helpers_shared import _REAUTHENTICATION_REQUIRED_CODE from pymongo.lock import _async_create_lock @@ -164,34 +163,6 @@ async def should_retry(self, attempt: int, delay: float) -> bool: return True -def _retry_overload(func: F) -> F: - @functools.wraps(func) - async def inner(self: Any, *args: Any, **kwargs: Any) -> Any: - retry_policy = self._retry_policy - attempt = 0 - while True: - try: - res = await func(self, *args, **kwargs) - await retry_policy.record_success(retry=attempt > 0) - return res - except PyMongoError as exc: - if not exc.has_error_label("RetryableError"): - raise - attempt += 1 - delay = 0 - if exc.has_error_label("SystemOverloadedError"): - delay = retry_policy.backoff(attempt) - if not await retry_policy.should_retry(attempt, delay): - raise - - # Implement exponential backoff on retry. - if delay: - await asyncio.sleep(delay) - continue - - return cast(F, inner) - - async def _getaddrinfo( host: Any, port: Any, **kwargs: Any ) -> list[ diff --git a/pymongo/asynchronous/mongo_client.py b/pymongo/asynchronous/mongo_client.py index 486e00ae46..7e9b5c9a9d 100644 --- a/pymongo/asynchronous/mongo_client.py +++ b/pymongo/asynchronous/mongo_client.py @@ -69,7 +69,6 @@ from pymongo.asynchronous.client_session import _EmptyServerSession from pymongo.asynchronous.command_cursor import AsyncCommandCursor from pymongo.asynchronous.helpers import ( - _retry_overload, _RetryPolicy, _TokenBucket, ) @@ -2403,7 +2402,6 @@ async def list_database_names( return [doc["name"] async for doc in res] @_csot.apply - @_retry_overload async def drop_database( self, name_or_database: Union[str, database.AsyncDatabase[_DocumentTypeArg]], @@ -2446,15 +2444,13 @@ async def drop_database( f"name_or_database must be an instance of str or a AsyncDatabase, not {type(name)}" ) - async with await self._conn_for_writes(session, operation=_Op.DROP_DATABASE) as conn: - await self[name]._command( - conn, - {"dropDatabase": 1, "comment": comment}, - read_preference=ReadPreference.PRIMARY, - write_concern=self._write_concern_for(session), - parse_write_concern_error=True, - session=session, - ) + return await self[name].command( + {"dropDatabase": 1, "comment": comment}, + read_preference=ReadPreference.PRIMARY, + write_concern=self._write_concern_for(session), + parse_write_concern_error=True, + session=session, + ) @_csot.apply async def bulk_write( diff --git a/pymongo/synchronous/collection.py b/pymongo/synchronous/collection.py index 9ee8e63947..6444e75ad8 100644 --- a/pymongo/synchronous/collection.py +++ b/pymongo/synchronous/collection.py @@ -21,7 +21,6 @@ TYPE_CHECKING, Any, Callable, - ContextManager, Generic, Iterable, Iterator, @@ -89,7 +88,6 @@ Cursor, RawBatchCursor, ) -from pymongo.synchronous.helpers import _retry_overload from pymongo.typings import _CollationIn, _DocumentType, _DocumentTypeArg, _Pipeline from pymongo.write_concern import DEFAULT_WRITE_CONCERN, WriteConcern, validate_boolean @@ -574,11 +572,6 @@ def watch( change_stream._initialize_cursor() return change_stream - def _conn_for_writes( - self, session: Optional[ClientSession], operation: str - ) -> ContextManager[Connection]: - return self._database.client._conn_for_writes(session, operation) - def _command( self, conn: Connection, @@ -655,7 +648,10 @@ def _create_helper( if "size" in options: options["size"] = float(options["size"]) cmd.update(options) - with self._conn_for_writes(session, operation=_Op.CREATE) as conn: + + def inner( + session: Optional[ClientSession], conn: Connection, _retryable_write: bool + ) -> None: if qev2_required and conn.max_wire_version < 21: raise ConfigurationError( "Driver support of Queryable Encryption is incompatible with server. " @@ -672,6 +668,8 @@ def _create_helper( session=session, ) + self.database.client._retryable_write(False, inner, session, _Op.CREATE) + def _create( self, options: MutableMapping[str, Any], @@ -2226,7 +2224,6 @@ def create_indexes( return self._create_indexes(indexes, session, **kwargs) @_csot.apply - @_retry_overload def _create_indexes( self, indexes: Sequence[IndexModel], session: Optional[ClientSession], **kwargs: Any ) -> list[str]: @@ -2240,7 +2237,10 @@ def _create_indexes( command (like maxTimeMS) can be passed as keyword arguments. """ names = [] - with self._conn_for_writes(session, operation=_Op.CREATE_INDEXES) as conn: + + def inner( + session: Optional[ClientSession], conn: Connection, _retryable_write: bool + ) -> list[str]: supports_quorum = conn.max_wire_version >= 9 def gen_indexes() -> Iterator[Mapping[str, Any]]: @@ -2269,7 +2269,9 @@ def gen_indexes() -> Iterator[Mapping[str, Any]]: write_concern=self._write_concern_for(session), session=session, ) - return names + return names + + return self.database.client._retryable_write(False, inner, session, _Op.CREATE_INDEXES) def create_index( self, @@ -2471,7 +2473,6 @@ def drop_index( self._drop_index(index_or_name, session, comment, **kwargs) @_csot.apply - @_retry_overload def _drop_index( self, index_or_name: _IndexKeyHint, @@ -2490,7 +2491,10 @@ def _drop_index( cmd.update(kwargs) if comment is not None: cmd["comment"] = comment - with self._conn_for_writes(session, operation=_Op.DROP_INDEXES) as conn: + + def inner( + session: Optional[ClientSession], conn: Connection, _retryable_write: bool + ) -> None: self._command( conn, cmd, @@ -2500,6 +2504,8 @@ def _drop_index( session=session, ) + self.database.client._retryable_write(False, inner, session, _Op.DROP_INDEXES) + def list_indexes( self, session: Optional[ClientSession] = None, @@ -2763,15 +2769,22 @@ def gen_indexes() -> Iterator[Mapping[str, Any]]: cmd = {"createSearchIndexes": self.name, "indexes": list(gen_indexes())} cmd.update(kwargs) - with self._conn_for_writes(session, operation=_Op.CREATE_SEARCH_INDEXES) as conn: + def inner( + session: Optional[ClientSession], conn: Connection, _retryable_write: bool + ) -> list[str]: resp = self._command( conn, cmd, read_preference=ReadPreference.PRIMARY, codec_options=_UNICODE_REPLACE_CODEC_OPTIONS, + session=session, ) return [index["name"] for index in resp["indexesCreated"]] + return self.database.client._retryable_write( + False, inner, session, _Op.CREATE_SEARCH_INDEXES + ) + def drop_search_index( self, name: str, @@ -2797,15 +2810,21 @@ def drop_search_index( cmd.update(kwargs) if comment is not None: cmd["comment"] = comment - with self._conn_for_writes(session, operation=_Op.DROP_SEARCH_INDEXES) as conn: + + def inner( + session: Optional[ClientSession], conn: Connection, _retryable_write: bool + ) -> None: self._command( conn, cmd, read_preference=ReadPreference.PRIMARY, allowable_errors=["ns not found", 26], codec_options=_UNICODE_REPLACE_CODEC_OPTIONS, + session=session, ) + return self.database.client._retryable_write(False, inner, session, _Op.DROP_SEARCH_INDEXES) + def update_search_index( self, name: str, @@ -2833,15 +2852,21 @@ def update_search_index( cmd.update(kwargs) if comment is not None: cmd["comment"] = comment - with self._conn_for_writes(session, operation=_Op.UPDATE_SEARCH_INDEX) as conn: + + def inner( + session: Optional[ClientSession], conn: Connection, _retryable_write: bool + ) -> None: self._command( conn, cmd, read_preference=ReadPreference.PRIMARY, allowable_errors=["ns not found", 26], codec_options=_UNICODE_REPLACE_CODEC_OPTIONS, + session=session, ) + return self.database.client._retryable_write(False, inner, session, _Op.UPDATE_SEARCH_INDEX) + def options( self, session: Optional[ClientSession] = None, @@ -3068,7 +3093,6 @@ def aggregate_raw_batches( ) @_csot.apply - @_retry_overload def rename( self, new_name: str, @@ -3120,17 +3144,21 @@ def rename( if comment is not None: cmd["comment"] = comment write_concern = self._write_concern_for_cmd(cmd, session) + client = self._database.client - with self._conn_for_writes(session, operation=_Op.RENAME) as conn: - with self._database.client._tmp_session(session) as s: - return conn.command( - "admin", - cmd, - write_concern=write_concern, - parse_write_concern_error=True, - session=s, - client=self._database.client, - ) + def inner( + session: Optional[ClientSession], conn: Connection, _retryable_write: bool + ) -> MutableMapping[str, Any]: + return conn.command( + "admin", + cmd, + write_concern=write_concern, + parse_write_concern_error=True, + session=session, + client=client, + ) + + return client._retryable_write(False, inner, session, _Op.RENAME) def distinct( self, diff --git a/pymongo/synchronous/database.py b/pymongo/synchronous/database.py index 27cc102d05..4fa8f4f1b4 100644 --- a/pymongo/synchronous/database.py +++ b/pymongo/synchronous/database.py @@ -43,7 +43,6 @@ from pymongo.synchronous.change_stream import DatabaseChangeStream from pymongo.synchronous.collection import Collection from pymongo.synchronous.command_cursor import CommandCursor -from pymongo.synchronous.helpers import _retry_overload from pymongo.typings import _CollationIn, _DocumentType, _DocumentTypeArg, _Pipeline if TYPE_CHECKING: @@ -479,7 +478,6 @@ def watch( return change_stream @_csot.apply - @_retry_overload def create_collection( self, name: str, @@ -822,7 +820,6 @@ def command( ... @_csot.apply - @_retry_overload def command( self, command: Union[str, MutableMapping[str, Any]], @@ -935,12 +932,15 @@ def command( if read_preference is None: read_preference = (session and session._txn_read_preference()) or ReadPreference.PRIMARY - with self._client._conn_for_reads(read_preference, session, operation=command_name) as ( - connection, - read_preference, - ): + + def inner( + session: Optional[ClientSession], + _server: Server, + conn: Connection, + read_preference: _ServerMode, + ) -> Union[dict[str, Any], _CodecDocumentType]: return self._command( - connection, + conn, command, value, check, @@ -951,8 +951,11 @@ def command( **kwargs, ) + return self._client._retryable_read( + inner, read_preference, session, command_name, None, False + ) + @_csot.apply - @_retry_overload def cursor_command( self, command: Union[str, MutableMapping[str, Any]], @@ -1019,15 +1022,17 @@ def cursor_command( with self._client._tmp_session(session) as tmp_session: opts = codec_options or DEFAULT_CODEC_OPTIONS - if read_preference is None: read_preference = ( tmp_session and tmp_session._txn_read_preference() ) or ReadPreference.PRIMARY - with self._client._conn_for_reads(read_preference, tmp_session, command_name) as ( - conn, - read_preference, - ): + + def inner( + session: Optional[ClientSession], + _server: Server, + conn: Connection, + read_preference: _ServerMode, + ) -> CommandCursor[_DocumentType]: response = self._command( conn, command, @@ -1036,7 +1041,7 @@ def cursor_command( None, read_preference, opts, - session=tmp_session, + session=session, **kwargs, ) coll = self.get_collection("$cmd", read_preference=read_preference) @@ -1046,7 +1051,7 @@ def cursor_command( response["cursor"], conn.address, max_await_time_ms=max_await_time_ms, - session=tmp_session, + session=session, comment=comment, ) cmd_cursor._maybe_pin_connection(conn) @@ -1054,6 +1059,10 @@ def cursor_command( else: raise InvalidOperation("Command does not return a cursor.") + return self.client._retryable_read( + inner, read_preference, tmp_session, command_name, False + ) + def _retryable_read_command( self, command: Union[str, MutableMapping[str, Any]], @@ -1252,9 +1261,11 @@ def _drop_helper( if comment is not None: command["comment"] = comment - with self._client._conn_for_writes(session, operation=_Op.DROP) as connection: + def inner( + session: Optional[ClientSession], conn: Connection, _retryable_write: bool + ) -> dict[str, Any]: return self._command( - connection, + conn, command, allowable_errors=["ns not found", 26], write_concern=self._write_concern_for(session), @@ -1262,8 +1273,9 @@ def _drop_helper( session=session, ) + return self.client._retryable_write(False, inner, session, _Op.DROP) + @_csot.apply - @_retry_overload def drop_collection( self, name_or_collection: Union[str, Collection[_DocumentTypeArg]], diff --git a/pymongo/synchronous/helpers.py b/pymongo/synchronous/helpers.py index 2c57b7a597..c093b379ea 100644 --- a/pymongo/synchronous/helpers.py +++ b/pymongo/synchronous/helpers.py @@ -30,7 +30,6 @@ from pymongo import _csot from pymongo.errors import ( OperationFailure, - PyMongoError, ) from pymongo.helpers_shared import _REAUTHENTICATION_REQUIRED_CODE from pymongo.lock import _create_lock @@ -164,34 +163,6 @@ def should_retry(self, attempt: int, delay: float) -> bool: return True -def _retry_overload(func: F) -> F: - @functools.wraps(func) - def inner(self: Any, *args: Any, **kwargs: Any) -> Any: - retry_policy = self._retry_policy - attempt = 0 - while True: - try: - res = func(self, *args, **kwargs) - retry_policy.record_success(retry=attempt > 0) - return res - except PyMongoError as exc: - if not exc.has_error_label("RetryableError"): - raise - attempt += 1 - delay = 0 - if exc.has_error_label("SystemOverloadedError"): - delay = retry_policy.backoff(attempt) - if not retry_policy.should_retry(attempt, delay): - raise - - # Implement exponential backoff on retry. - if delay: - time.sleep(delay) - continue - - return cast(F, inner) - - def _getaddrinfo( host: Any, port: Any, **kwargs: Any ) -> list[ diff --git a/pymongo/synchronous/mongo_client.py b/pymongo/synchronous/mongo_client.py index 30b8c4fc61..acf5e6b98f 100644 --- a/pymongo/synchronous/mongo_client.py +++ b/pymongo/synchronous/mongo_client.py @@ -112,7 +112,6 @@ from pymongo.synchronous.client_session import _EmptyServerSession from pymongo.synchronous.command_cursor import CommandCursor from pymongo.synchronous.helpers import ( - _retry_overload, _RetryPolicy, _TokenBucket, ) @@ -2393,7 +2392,6 @@ def list_database_names( return [doc["name"] for doc in res] @_csot.apply - @_retry_overload def drop_database( self, name_or_database: Union[str, database.Database[_DocumentTypeArg]], @@ -2436,15 +2434,13 @@ def drop_database( f"name_or_database must be an instance of str or a Database, not {type(name)}" ) - with self._conn_for_writes(session, operation=_Op.DROP_DATABASE) as conn: - self[name]._command( - conn, - {"dropDatabase": 1, "comment": comment}, - read_preference=ReadPreference.PRIMARY, - write_concern=self._write_concern_for(session), - parse_write_concern_error=True, - session=session, - ) + return self[name].command( + {"dropDatabase": 1, "comment": comment}, + read_preference=ReadPreference.PRIMARY, + write_concern=self._write_concern_for(session), + parse_write_concern_error=True, + session=session, + ) @_csot.apply def bulk_write( From 49f605a9efb185dce0ba1925c6777f565ffd5937 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 4 Dec 2025 14:42:16 -0600 Subject: [PATCH 12/16] return type fixes --- pymongo/asynchronous/collection.py | 8 ++++---- pymongo/asynchronous/database.py | 2 +- pymongo/asynchronous/mongo_client.py | 2 +- pymongo/synchronous/collection.py | 4 ++-- pymongo/synchronous/database.py | 2 +- pymongo/synchronous/mongo_client.py | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pymongo/asynchronous/collection.py b/pymongo/asynchronous/collection.py index 60755dbe89..0eeff7a043 100644 --- a/pymongo/asynchronous/collection.py +++ b/pymongo/asynchronous/collection.py @@ -2786,7 +2786,7 @@ async def inner( ) return [index["name"] for index in resp["indexesCreated"]] - return self.database.client._retryable_write( + return await self.database.client._retryable_write( False, inner, session, _Op.CREATE_SEARCH_INDEXES ) @@ -2828,7 +2828,7 @@ async def inner( session=session, ) - return self.database.client._retryable_write(False, inner, session, _Op.DROP_SEARCH_INDEXES) + await self.database.client._retryable_write(False, inner, session, _Op.DROP_SEARCH_INDEXES) async def update_search_index( self, @@ -2870,7 +2870,7 @@ async def inner( session=session, ) - return self.database.client._retryable_write(False, inner, session, _Op.UPDATE_SEARCH_INDEX) + await self.database.client._retryable_write(False, inner, session, _Op.UPDATE_SEARCH_INDEX) async def options( self, @@ -3165,7 +3165,7 @@ async def inner( client=client, ) - return client._retryable_write(False, inner, session, _Op.RENAME) + return await client._retryable_write(False, inner, session, _Op.RENAME) async def distinct( self, diff --git a/pymongo/asynchronous/database.py b/pymongo/asynchronous/database.py index 7ad9017f70..3af77ffe15 100644 --- a/pymongo/asynchronous/database.py +++ b/pymongo/asynchronous/database.py @@ -1060,7 +1060,7 @@ async def inner( raise InvalidOperation("Command does not return a cursor.") return await self.client._retryable_read( - inner, read_preference, tmp_session, command_name, False + inner, read_preference, tmp_session, command_name, None, False ) async def _retryable_read_command( diff --git a/pymongo/asynchronous/mongo_client.py b/pymongo/asynchronous/mongo_client.py index 7e9b5c9a9d..d3e8e64ee9 100644 --- a/pymongo/asynchronous/mongo_client.py +++ b/pymongo/asynchronous/mongo_client.py @@ -2444,7 +2444,7 @@ async def drop_database( f"name_or_database must be an instance of str or a AsyncDatabase, not {type(name)}" ) - return await self[name].command( + await self[name].command( {"dropDatabase": 1, "comment": comment}, read_preference=ReadPreference.PRIMARY, write_concern=self._write_concern_for(session), diff --git a/pymongo/synchronous/collection.py b/pymongo/synchronous/collection.py index 6444e75ad8..f7da050f4c 100644 --- a/pymongo/synchronous/collection.py +++ b/pymongo/synchronous/collection.py @@ -2823,7 +2823,7 @@ def inner( session=session, ) - return self.database.client._retryable_write(False, inner, session, _Op.DROP_SEARCH_INDEXES) + self.database.client._retryable_write(False, inner, session, _Op.DROP_SEARCH_INDEXES) def update_search_index( self, @@ -2865,7 +2865,7 @@ def inner( session=session, ) - return self.database.client._retryable_write(False, inner, session, _Op.UPDATE_SEARCH_INDEX) + self.database.client._retryable_write(False, inner, session, _Op.UPDATE_SEARCH_INDEX) def options( self, diff --git a/pymongo/synchronous/database.py b/pymongo/synchronous/database.py index 4fa8f4f1b4..60a55276c6 100644 --- a/pymongo/synchronous/database.py +++ b/pymongo/synchronous/database.py @@ -1060,7 +1060,7 @@ def inner( raise InvalidOperation("Command does not return a cursor.") return self.client._retryable_read( - inner, read_preference, tmp_session, command_name, False + inner, read_preference, tmp_session, command_name, None, False ) def _retryable_read_command( diff --git a/pymongo/synchronous/mongo_client.py b/pymongo/synchronous/mongo_client.py index acf5e6b98f..a9723fd087 100644 --- a/pymongo/synchronous/mongo_client.py +++ b/pymongo/synchronous/mongo_client.py @@ -2434,7 +2434,7 @@ def drop_database( f"name_or_database must be an instance of str or a Database, not {type(name)}" ) - return self[name].command( + self[name].command( {"dropDatabase": 1, "comment": comment}, read_preference=ReadPreference.PRIMARY, write_concern=self._write_concern_for(session), From d676283073d5d1257173ef7245735f9357a107b0 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 16 Dec 2025 07:00:59 -0600 Subject: [PATCH 13/16] update retry logic --- pymongo/asynchronous/mongo_client.py | 13 ++++++++----- pymongo/synchronous/mongo_client.py | 13 ++++++++----- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/pymongo/asynchronous/mongo_client.py b/pymongo/asynchronous/mongo_client.py index d3e8e64ee9..effc1f1c12 100644 --- a/pymongo/asynchronous/mongo_client.py +++ b/pymongo/asynchronous/mongo_client.py @@ -2796,7 +2796,9 @@ async def run(self) -> T: if isinstance(exc, (ConnectionFailure, OperationFailure)): # ConnectionFailures do not supply a code property exc_code = getattr(exc, "code", None) - always_retryable = exc.has_error_label("RetryableError") + always_retryable = exc.has_error_label( + "RetryableError" + ) and exc.has_error_label("SystemOverloadedError") overloaded = exc.has_error_label("SystemOverloadedError") if not always_retryable and ( self._is_not_eligible_for_retry() @@ -2819,7 +2821,9 @@ async def run(self) -> T: ): exc_to_check = exc.error retryable_write_label = exc_to_check.has_error_label("RetryableWriteError") - always_retryable = exc_to_check.has_error_label("RetryableError") + always_retryable = exc_to_check.has_error_label( + "RetryableError" + ) and exc.has_error_label("SystemOverloadedError") overloaded = exc_to_check.has_error_label("SystemOverloadedError") if not self._retryable and not always_retryable: raise @@ -2847,15 +2851,14 @@ async def run(self) -> T: self._deprioritized_servers.append(self._server) self._always_retryable = always_retryable - if always_retryable: + if overloaded: delay = self._retry_policy.backoff(self._attempt_number) if overloaded else 0 if not await self._retry_policy.should_retry(self._attempt_number, delay): if exc_to_check.has_error_label("NoWritesPerformed") and self._last_error: raise self._last_error from exc else: raise - if overloaded: - await asyncio.sleep(delay) + await asyncio.sleep(delay) def _is_not_eligible_for_retry(self) -> bool: """Checks if the exchange is not eligible for retry""" diff --git a/pymongo/synchronous/mongo_client.py b/pymongo/synchronous/mongo_client.py index a9723fd087..290caa5a16 100644 --- a/pymongo/synchronous/mongo_client.py +++ b/pymongo/synchronous/mongo_client.py @@ -2786,7 +2786,9 @@ def run(self) -> T: if isinstance(exc, (ConnectionFailure, OperationFailure)): # ConnectionFailures do not supply a code property exc_code = getattr(exc, "code", None) - always_retryable = exc.has_error_label("RetryableError") + always_retryable = exc.has_error_label( + "RetryableError" + ) and exc.has_error_label("SystemOverloadedError") overloaded = exc.has_error_label("SystemOverloadedError") if not always_retryable and ( self._is_not_eligible_for_retry() @@ -2809,7 +2811,9 @@ def run(self) -> T: ): exc_to_check = exc.error retryable_write_label = exc_to_check.has_error_label("RetryableWriteError") - always_retryable = exc_to_check.has_error_label("RetryableError") + always_retryable = exc_to_check.has_error_label( + "RetryableError" + ) and exc.has_error_label("SystemOverloadedError") overloaded = exc_to_check.has_error_label("SystemOverloadedError") if not self._retryable and not always_retryable: raise @@ -2837,15 +2841,14 @@ def run(self) -> T: self._deprioritized_servers.append(self._server) self._always_retryable = always_retryable - if always_retryable: + if overloaded: delay = self._retry_policy.backoff(self._attempt_number) if overloaded else 0 if not self._retry_policy.should_retry(self._attempt_number, delay): if exc_to_check.has_error_label("NoWritesPerformed") and self._last_error: raise self._last_error from exc else: raise - if overloaded: - time.sleep(delay) + time.sleep(delay) def _is_not_eligible_for_retry(self) -> bool: """Checks if the exchange is not eligible for retry""" From 2a2251fb20676547add16ebec7eb779ffce137a1 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 16 Dec 2025 07:31:27 -0600 Subject: [PATCH 14/16] clean up tests --- test/asynchronous/test_backpressure.py | 230 ------------------ test/asynchronous/test_client_backpressure.py | 207 +++++++++++++++- test/test_backpressure.py | 230 ------------------ test/test_client_backpressure.py | 207 +++++++++++++++- tools/synchro.py | 1 - 5 files changed, 410 insertions(+), 465 deletions(-) delete mode 100644 test/asynchronous/test_backpressure.py delete mode 100644 test/test_backpressure.py diff --git a/test/asynchronous/test_backpressure.py b/test/asynchronous/test_backpressure.py deleted file mode 100644 index 11f8edde67..0000000000 --- a/test/asynchronous/test_backpressure.py +++ /dev/null @@ -1,230 +0,0 @@ -# Copyright 2025-present MongoDB, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Test Client Backpressure spec.""" -from __future__ import annotations - -import asyncio -import sys - -import pymongo - -sys.path[0:0] = [""] - -from test.asynchronous import ( - AsyncIntegrationTest, - AsyncPyMongoTestCase, - async_client_context, - unittest, -) - -from pymongo.asynchronous import helpers -from pymongo.asynchronous.helpers import _MAX_RETRIES, _RetryPolicy, _TokenBucket -from pymongo.errors import PyMongoError - -_IS_SYNC = False - -# Mock an system overload error. -mock_overload_error = { - "configureFailPoint": "failCommand", - "mode": {"times": 1}, - "data": { - "failCommands": ["find", "insert", "update"], - "errorCode": 462, # IngressRequestRateLimitExceeded - "errorLabels": ["RetryableError"], - }, -} - - -class TestBackpressure(AsyncIntegrationTest): - RUN_ON_LOAD_BALANCER = True - - @async_client_context.require_failCommand_appName - async def test_retry_overload_error_command(self): - await self.db.t.insert_one({"x": 1}) - - # Ensure command is retried on overload error. - fail_many = mock_overload_error.copy() - fail_many["mode"] = {"times": _MAX_RETRIES} - async with self.fail_point(fail_many): - await self.db.command("find", "t") - - # Ensure command stops retrying after _MAX_RETRIES. - fail_too_many = mock_overload_error.copy() - fail_too_many["mode"] = {"times": _MAX_RETRIES + 1} - async with self.fail_point(fail_too_many): - with self.assertRaises(PyMongoError) as error: - await self.db.command("find", "t") - - self.assertIn("RetryableError", str(error.exception)) - - @async_client_context.require_failCommand_appName - async def test_retry_overload_error_find(self): - await self.db.t.insert_one({"x": 1}) - - # Ensure command is retried on overload error. - fail_many = mock_overload_error.copy() - fail_many["mode"] = {"times": _MAX_RETRIES} - async with self.fail_point(fail_many): - await self.db.t.find_one() - - # Ensure command stops retrying after _MAX_RETRIES. - fail_too_many = mock_overload_error.copy() - fail_too_many["mode"] = {"times": _MAX_RETRIES + 1} - async with self.fail_point(fail_too_many): - with self.assertRaises(PyMongoError) as error: - await self.db.t.find_one() - - self.assertIn("RetryableError", str(error.exception)) - - @async_client_context.require_failCommand_appName - async def test_retry_overload_error_insert_one(self): - await self.db.t.insert_one({"x": 1}) - - # Ensure command is retried on overload error. - fail_many = mock_overload_error.copy() - fail_many["mode"] = {"times": _MAX_RETRIES} - async with self.fail_point(fail_many): - await self.db.t.find_one() - - # Ensure command stops retrying after _MAX_RETRIES. - fail_too_many = mock_overload_error.copy() - fail_too_many["mode"] = {"times": _MAX_RETRIES + 1} - async with self.fail_point(fail_too_many): - with self.assertRaises(PyMongoError) as error: - await self.db.t.find_one() - - self.assertIn("RetryableError", str(error.exception)) - - @async_client_context.require_failCommand_appName - async def test_retry_overload_error_update_many(self): - # Even though update_many is not a retryable write operation, it will - # still be retried via the "RetryableError" error label. - await self.db.t.insert_one({"x": 1}) - - # Ensure command is retried on overload error. - fail_many = mock_overload_error.copy() - fail_many["mode"] = {"times": _MAX_RETRIES} - async with self.fail_point(fail_many): - await self.db.t.update_many({}, {"$set": {"x": 2}}) - - # Ensure command stops retrying after _MAX_RETRIES. - fail_too_many = mock_overload_error.copy() - fail_too_many["mode"] = {"times": _MAX_RETRIES + 1} - async with self.fail_point(fail_too_many): - with self.assertRaises(PyMongoError) as error: - await self.db.t.update_many({}, {"$set": {"x": 2}}) - - self.assertIn("RetryableError", str(error.exception)) - - @async_client_context.require_failCommand_appName - async def test_retry_overload_error_getMore(self): - coll = self.db.t - await coll.insert_many([{"x": 1} for _ in range(10)]) - - # Ensure command is retried on overload error. - fail_many = { - "configureFailPoint": "failCommand", - "mode": {"times": _MAX_RETRIES}, - "data": { - "failCommands": ["getMore"], - "errorCode": 462, # IngressRequestRateLimitExceeded - "errorLabels": ["RetryableError"], - }, - } - cursor = coll.find(batch_size=2) - await cursor.next() - async with self.fail_point(fail_many): - await cursor.to_list() - - # Ensure command stops retrying after _MAX_RETRIES. - fail_too_many = fail_many.copy() - fail_too_many["mode"] = {"times": _MAX_RETRIES + 1} - cursor = coll.find(batch_size=2) - await cursor.next() - async with self.fail_point(fail_too_many): - with self.assertRaises(PyMongoError) as error: - await cursor.to_list() - - self.assertIn("RetryableError", str(error.exception)) - - @async_client_context.require_failCommand_appName - async def test_limit_retry_command(self): - client = await self.async_rs_or_single_client() - client._retry_policy.token_bucket.tokens = 1 - db = client.pymongo_test - await db.t.insert_one({"x": 1}) - - # Ensure command is retried once overload error. - fail_many = mock_overload_error.copy() - fail_many["mode"] = {"times": 1} - async with self.fail_point(fail_many): - await db.command("find", "t") - - # Ensure command stops retrying when there are no tokens left. - fail_too_many = mock_overload_error.copy() - fail_too_many["mode"] = {"times": 2} - async with self.fail_point(fail_too_many): - with self.assertRaises(PyMongoError) as error: - await db.command("find", "t") - - self.assertIn("RetryableError", str(error.exception)) - - -class TestRetryPolicy(AsyncPyMongoTestCase): - async def test_retry_policy(self): - capacity = 10 - retry_policy = _RetryPolicy(_TokenBucket(capacity=capacity)) - self.assertEqual(retry_policy.attempts, helpers._MAX_RETRIES) - self.assertEqual(retry_policy.backoff_initial, helpers._BACKOFF_INITIAL) - self.assertEqual(retry_policy.backoff_max, helpers._BACKOFF_MAX) - for i in range(1, helpers._MAX_RETRIES + 1): - self.assertTrue(await retry_policy.should_retry(i, 0)) - self.assertFalse(await retry_policy.should_retry(helpers._MAX_RETRIES + 1, 0)) - for i in range(capacity - helpers._MAX_RETRIES): - self.assertTrue(await retry_policy.should_retry(1, 0)) - # No tokens left, should not retry. - self.assertFalse(await retry_policy.should_retry(1, 0)) - self.assertEqual(retry_policy.token_bucket.tokens, 0) - - # record_success should generate tokens. - for _ in range(int(2 / helpers.DEFAULT_RETRY_TOKEN_RETURN)): - await retry_policy.record_success(retry=False) - self.assertAlmostEqual(retry_policy.token_bucket.tokens, 2) - for i in range(2): - self.assertTrue(await retry_policy.should_retry(1, 0)) - self.assertFalse(await retry_policy.should_retry(1, 0)) - - # Recording a successful retry should return 1 additional token. - await retry_policy.record_success(retry=True) - self.assertAlmostEqual( - retry_policy.token_bucket.tokens, 1 + helpers.DEFAULT_RETRY_TOKEN_RETURN - ) - self.assertTrue(await retry_policy.should_retry(1, 0)) - self.assertFalse(await retry_policy.should_retry(1, 0)) - self.assertAlmostEqual(retry_policy.token_bucket.tokens, helpers.DEFAULT_RETRY_TOKEN_RETURN) - - async def test_retry_policy_csot(self): - retry_policy = _RetryPolicy(_TokenBucket()) - self.assertTrue(await retry_policy.should_retry(1, 0.5)) - with pymongo.timeout(0.5): - self.assertTrue(await retry_policy.should_retry(1, 0)) - self.assertTrue(await retry_policy.should_retry(1, 0.1)) - # Would exceed the timeout, should not retry. - self.assertFalse(await retry_policy.should_retry(1, 1.0)) - self.assertTrue(await retry_policy.should_retry(1, 1.0)) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/asynchronous/test_client_backpressure.py b/test/asynchronous/test_client_backpressure.py index b66497d9ee..c82d84e181 100644 --- a/test/asynchronous/test_client_backpressure.py +++ b/test/asynchronous/test_client_backpressure.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +"""Test Client Backpressure spec.""" from __future__ import annotations import os @@ -19,21 +21,222 @@ from time import perf_counter from unittest.mock import patch -from pymongo.errors import OperationFailure - sys.path[0:0] = [""] from test.asynchronous import ( AsyncIntegrationTest, + AsyncPyMongoTestCase, async_client_context, unittest, ) from test.asynchronous.unified_format import generate_test_classes from test.utils_shared import EventListener, OvertCommandListener +import pymongo +from pymongo.asynchronous import helpers +from pymongo.asynchronous.helpers import _MAX_RETRIES, _RetryPolicy, _TokenBucket +from pymongo.errors import OperationFailure, PyMongoError + _IS_SYNC = False +# Mock an system overload error. +mock_overload_error = { + "configureFailPoint": "failCommand", + "mode": {"times": 1}, + "data": { + "failCommands": ["find", "insert", "update"], + "errorCode": 462, # IngressRequestRateLimitExceeded + "errorLabels": ["RetryableError", "SystemOverloadedError"], + }, +} + + +class TestBackpressure(AsyncIntegrationTest): + RUN_ON_LOAD_BALANCER = True + + @async_client_context.require_failCommand_appName + async def test_retry_overload_error_command(self): + await self.db.t.insert_one({"x": 1}) + + # Ensure command is retried on overload error. + fail_many = mock_overload_error.copy() + fail_many["mode"] = {"times": _MAX_RETRIES} + async with self.fail_point(fail_many): + await self.db.command("find", "t") + + # Ensure command stops retrying after _MAX_RETRIES. + fail_too_many = mock_overload_error.copy() + fail_too_many["mode"] = {"times": _MAX_RETRIES + 1} + async with self.fail_point(fail_too_many): + with self.assertRaises(PyMongoError) as error: + await self.db.command("find", "t") + + self.assertIn("RetryableError", str(error.exception)) + self.assertIn("SystemOverloadedError", str(error.exception)) + + @async_client_context.require_failCommand_appName + async def test_retry_overload_error_find(self): + await self.db.t.insert_one({"x": 1}) + + # Ensure command is retried on overload error. + fail_many = mock_overload_error.copy() + fail_many["mode"] = {"times": _MAX_RETRIES} + async with self.fail_point(fail_many): + await self.db.t.find_one() + + # Ensure command stops retrying after _MAX_RETRIES. + fail_too_many = mock_overload_error.copy() + fail_too_many["mode"] = {"times": _MAX_RETRIES + 1} + async with self.fail_point(fail_too_many): + with self.assertRaises(PyMongoError) as error: + await self.db.t.find_one() + + self.assertIn("RetryableError", str(error.exception)) + self.assertIn("SystemOverloadedError", str(error.exception)) + + @async_client_context.require_failCommand_appName + async def test_retry_overload_error_insert_one(self): + await self.db.t.insert_one({"x": 1}) + + # Ensure command is retried on overload error. + fail_many = mock_overload_error.copy() + fail_many["mode"] = {"times": _MAX_RETRIES} + async with self.fail_point(fail_many): + await self.db.t.find_one() + + # Ensure command stops retrying after _MAX_RETRIES. + fail_too_many = mock_overload_error.copy() + fail_too_many["mode"] = {"times": _MAX_RETRIES + 1} + async with self.fail_point(fail_too_many): + with self.assertRaises(PyMongoError) as error: + await self.db.t.find_one() + + self.assertIn("RetryableError", str(error.exception)) + self.assertIn("SystemOverloadedError", str(error.exception)) + + @async_client_context.require_failCommand_appName + async def test_retry_overload_error_update_many(self): + # Even though update_many is not a retryable write operation, it will + # still be retried via the "RetryableError" error label. + await self.db.t.insert_one({"x": 1}) + + # Ensure command is retried on overload error. + fail_many = mock_overload_error.copy() + fail_many["mode"] = {"times": _MAX_RETRIES} + async with self.fail_point(fail_many): + await self.db.t.update_many({}, {"$set": {"x": 2}}) + + # Ensure command stops retrying after _MAX_RETRIES. + fail_too_many = mock_overload_error.copy() + fail_too_many["mode"] = {"times": _MAX_RETRIES + 1} + async with self.fail_point(fail_too_many): + with self.assertRaises(PyMongoError) as error: + await self.db.t.update_many({}, {"$set": {"x": 2}}) + + self.assertIn("RetryableError", str(error.exception)) + self.assertIn("SystemOverloadedError", str(error.exception)) + + @async_client_context.require_failCommand_appName + async def test_retry_overload_error_getMore(self): + coll = self.db.t + await coll.insert_many([{"x": 1} for _ in range(10)]) + + # Ensure command is retried on overload error. + fail_many = { + "configureFailPoint": "failCommand", + "mode": {"times": _MAX_RETRIES}, + "data": { + "failCommands": ["getMore"], + "errorCode": 462, # IngressRequestRateLimitExceeded + "errorLabels": ["RetryableError", "SystemOverloadedError"], + }, + } + cursor = coll.find(batch_size=2) + await cursor.next() + async with self.fail_point(fail_many): + await cursor.to_list() + + # Ensure command stops retrying after _MAX_RETRIES. + fail_too_many = fail_many.copy() + fail_too_many["mode"] = {"times": _MAX_RETRIES + 1} + cursor = coll.find(batch_size=2) + await cursor.next() + async with self.fail_point(fail_too_many): + with self.assertRaises(PyMongoError) as error: + await cursor.to_list() + + self.assertIn("RetryableError", str(error.exception)) + self.assertIn("SystemOverloadedError", str(error.exception)) + + @async_client_context.require_failCommand_appName + async def test_limit_retry_command(self): + client = await self.async_rs_or_single_client() + client._retry_policy.token_bucket.tokens = 1 + db = client.pymongo_test + await db.t.insert_one({"x": 1}) + + # Ensure command is retried once overload error. + fail_many = mock_overload_error.copy() + fail_many["mode"] = {"times": 1} + async with self.fail_point(fail_many): + await db.command("find", "t") + + # Ensure command stops retrying when there are no tokens left. + fail_too_many = mock_overload_error.copy() + fail_too_many["mode"] = {"times": 2} + async with self.fail_point(fail_too_many): + with self.assertRaises(PyMongoError) as error: + await db.command("find", "t") + + self.assertIn("RetryableError", str(error.exception)) + self.assertIn("SystemOverloadedError", str(error.exception)) + + +class TestRetryPolicy(AsyncPyMongoTestCase): + async def test_retry_policy(self): + capacity = 10 + retry_policy = _RetryPolicy(_TokenBucket(capacity=capacity)) + self.assertEqual(retry_policy.attempts, helpers._MAX_RETRIES) + self.assertEqual(retry_policy.backoff_initial, helpers._BACKOFF_INITIAL) + self.assertEqual(retry_policy.backoff_max, helpers._BACKOFF_MAX) + for i in range(1, helpers._MAX_RETRIES + 1): + self.assertTrue(await retry_policy.should_retry(i, 0)) + self.assertFalse(await retry_policy.should_retry(helpers._MAX_RETRIES + 1, 0)) + for i in range(capacity - helpers._MAX_RETRIES): + self.assertTrue(await retry_policy.should_retry(1, 0)) + # No tokens left, should not retry. + self.assertFalse(await retry_policy.should_retry(1, 0)) + self.assertEqual(retry_policy.token_bucket.tokens, 0) + + # record_success should generate tokens. + for _ in range(int(2 / helpers.DEFAULT_RETRY_TOKEN_RETURN)): + await retry_policy.record_success(retry=False) + self.assertAlmostEqual(retry_policy.token_bucket.tokens, 2) + for i in range(2): + self.assertTrue(await retry_policy.should_retry(1, 0)) + self.assertFalse(await retry_policy.should_retry(1, 0)) + + # Recording a successful retry should return 1 additional token. + await retry_policy.record_success(retry=True) + self.assertAlmostEqual( + retry_policy.token_bucket.tokens, 1 + helpers.DEFAULT_RETRY_TOKEN_RETURN + ) + self.assertTrue(await retry_policy.should_retry(1, 0)) + self.assertFalse(await retry_policy.should_retry(1, 0)) + self.assertAlmostEqual(retry_policy.token_bucket.tokens, helpers.DEFAULT_RETRY_TOKEN_RETURN) + + async def test_retry_policy_csot(self): + retry_policy = _RetryPolicy(_TokenBucket()) + self.assertTrue(await retry_policy.should_retry(1, 0.5)) + with pymongo.timeout(0.5): + self.assertTrue(await retry_policy.should_retry(1, 0)) + self.assertTrue(await retry_policy.should_retry(1, 0.1)) + # Would exceed the timeout, should not retry. + self.assertFalse(await retry_policy.should_retry(1, 1.0)) + self.assertTrue(await retry_policy.should_retry(1, 1.0)) + +# Prose tests. class AsyncTestClientBackpressure(AsyncIntegrationTest): listener: EventListener diff --git a/test/test_backpressure.py b/test/test_backpressure.py deleted file mode 100644 index fac1d6236d..0000000000 --- a/test/test_backpressure.py +++ /dev/null @@ -1,230 +0,0 @@ -# Copyright 2025-present MongoDB, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Test Client Backpressure spec.""" -from __future__ import annotations - -import asyncio -import sys - -import pymongo - -sys.path[0:0] = [""] - -from test import ( - IntegrationTest, - PyMongoTestCase, - client_context, - unittest, -) - -from pymongo.errors import PyMongoError -from pymongo.synchronous import helpers -from pymongo.synchronous.helpers import _MAX_RETRIES, _RetryPolicy, _TokenBucket - -_IS_SYNC = True - -# Mock an system overload error. -mock_overload_error = { - "configureFailPoint": "failCommand", - "mode": {"times": 1}, - "data": { - "failCommands": ["find", "insert", "update"], - "errorCode": 462, # IngressRequestRateLimitExceeded - "errorLabels": ["RetryableError"], - }, -} - - -class TestBackpressure(IntegrationTest): - RUN_ON_LOAD_BALANCER = True - - @client_context.require_failCommand_appName - def test_retry_overload_error_command(self): - self.db.t.insert_one({"x": 1}) - - # Ensure command is retried on overload error. - fail_many = mock_overload_error.copy() - fail_many["mode"] = {"times": _MAX_RETRIES} - with self.fail_point(fail_many): - self.db.command("find", "t") - - # Ensure command stops retrying after _MAX_RETRIES. - fail_too_many = mock_overload_error.copy() - fail_too_many["mode"] = {"times": _MAX_RETRIES + 1} - with self.fail_point(fail_too_many): - with self.assertRaises(PyMongoError) as error: - self.db.command("find", "t") - - self.assertIn("RetryableError", str(error.exception)) - - @client_context.require_failCommand_appName - def test_retry_overload_error_find(self): - self.db.t.insert_one({"x": 1}) - - # Ensure command is retried on overload error. - fail_many = mock_overload_error.copy() - fail_many["mode"] = {"times": _MAX_RETRIES} - with self.fail_point(fail_many): - self.db.t.find_one() - - # Ensure command stops retrying after _MAX_RETRIES. - fail_too_many = mock_overload_error.copy() - fail_too_many["mode"] = {"times": _MAX_RETRIES + 1} - with self.fail_point(fail_too_many): - with self.assertRaises(PyMongoError) as error: - self.db.t.find_one() - - self.assertIn("RetryableError", str(error.exception)) - - @client_context.require_failCommand_appName - def test_retry_overload_error_insert_one(self): - self.db.t.insert_one({"x": 1}) - - # Ensure command is retried on overload error. - fail_many = mock_overload_error.copy() - fail_many["mode"] = {"times": _MAX_RETRIES} - with self.fail_point(fail_many): - self.db.t.find_one() - - # Ensure command stops retrying after _MAX_RETRIES. - fail_too_many = mock_overload_error.copy() - fail_too_many["mode"] = {"times": _MAX_RETRIES + 1} - with self.fail_point(fail_too_many): - with self.assertRaises(PyMongoError) as error: - self.db.t.find_one() - - self.assertIn("RetryableError", str(error.exception)) - - @client_context.require_failCommand_appName - def test_retry_overload_error_update_many(self): - # Even though update_many is not a retryable write operation, it will - # still be retried via the "RetryableError" error label. - self.db.t.insert_one({"x": 1}) - - # Ensure command is retried on overload error. - fail_many = mock_overload_error.copy() - fail_many["mode"] = {"times": _MAX_RETRIES} - with self.fail_point(fail_many): - self.db.t.update_many({}, {"$set": {"x": 2}}) - - # Ensure command stops retrying after _MAX_RETRIES. - fail_too_many = mock_overload_error.copy() - fail_too_many["mode"] = {"times": _MAX_RETRIES + 1} - with self.fail_point(fail_too_many): - with self.assertRaises(PyMongoError) as error: - self.db.t.update_many({}, {"$set": {"x": 2}}) - - self.assertIn("RetryableError", str(error.exception)) - - @client_context.require_failCommand_appName - def test_retry_overload_error_getMore(self): - coll = self.db.t - coll.insert_many([{"x": 1} for _ in range(10)]) - - # Ensure command is retried on overload error. - fail_many = { - "configureFailPoint": "failCommand", - "mode": {"times": _MAX_RETRIES}, - "data": { - "failCommands": ["getMore"], - "errorCode": 462, # IngressRequestRateLimitExceeded - "errorLabels": ["RetryableError"], - }, - } - cursor = coll.find(batch_size=2) - cursor.next() - with self.fail_point(fail_many): - cursor.to_list() - - # Ensure command stops retrying after _MAX_RETRIES. - fail_too_many = fail_many.copy() - fail_too_many["mode"] = {"times": _MAX_RETRIES + 1} - cursor = coll.find(batch_size=2) - cursor.next() - with self.fail_point(fail_too_many): - with self.assertRaises(PyMongoError) as error: - cursor.to_list() - - self.assertIn("RetryableError", str(error.exception)) - - @client_context.require_failCommand_appName - def test_limit_retry_command(self): - client = self.rs_or_single_client() - client._retry_policy.token_bucket.tokens = 1 - db = client.pymongo_test - db.t.insert_one({"x": 1}) - - # Ensure command is retried once overload error. - fail_many = mock_overload_error.copy() - fail_many["mode"] = {"times": 1} - with self.fail_point(fail_many): - db.command("find", "t") - - # Ensure command stops retrying when there are no tokens left. - fail_too_many = mock_overload_error.copy() - fail_too_many["mode"] = {"times": 2} - with self.fail_point(fail_too_many): - with self.assertRaises(PyMongoError) as error: - db.command("find", "t") - - self.assertIn("RetryableError", str(error.exception)) - - -class TestRetryPolicy(PyMongoTestCase): - def test_retry_policy(self): - capacity = 10 - retry_policy = _RetryPolicy(_TokenBucket(capacity=capacity)) - self.assertEqual(retry_policy.attempts, helpers._MAX_RETRIES) - self.assertEqual(retry_policy.backoff_initial, helpers._BACKOFF_INITIAL) - self.assertEqual(retry_policy.backoff_max, helpers._BACKOFF_MAX) - for i in range(1, helpers._MAX_RETRIES + 1): - self.assertTrue(retry_policy.should_retry(i, 0)) - self.assertFalse(retry_policy.should_retry(helpers._MAX_RETRIES + 1, 0)) - for i in range(capacity - helpers._MAX_RETRIES): - self.assertTrue(retry_policy.should_retry(1, 0)) - # No tokens left, should not retry. - self.assertFalse(retry_policy.should_retry(1, 0)) - self.assertEqual(retry_policy.token_bucket.tokens, 0) - - # record_success should generate tokens. - for _ in range(int(2 / helpers.DEFAULT_RETRY_TOKEN_RETURN)): - retry_policy.record_success(retry=False) - self.assertAlmostEqual(retry_policy.token_bucket.tokens, 2) - for i in range(2): - self.assertTrue(retry_policy.should_retry(1, 0)) - self.assertFalse(retry_policy.should_retry(1, 0)) - - # Recording a successful retry should return 1 additional token. - retry_policy.record_success(retry=True) - self.assertAlmostEqual( - retry_policy.token_bucket.tokens, 1 + helpers.DEFAULT_RETRY_TOKEN_RETURN - ) - self.assertTrue(retry_policy.should_retry(1, 0)) - self.assertFalse(retry_policy.should_retry(1, 0)) - self.assertAlmostEqual(retry_policy.token_bucket.tokens, helpers.DEFAULT_RETRY_TOKEN_RETURN) - - def test_retry_policy_csot(self): - retry_policy = _RetryPolicy(_TokenBucket()) - self.assertTrue(retry_policy.should_retry(1, 0.5)) - with pymongo.timeout(0.5): - self.assertTrue(retry_policy.should_retry(1, 0)) - self.assertTrue(retry_policy.should_retry(1, 0.1)) - # Would exceed the timeout, should not retry. - self.assertFalse(retry_policy.should_retry(1, 1.0)) - self.assertTrue(retry_policy.should_retry(1, 1.0)) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/test_client_backpressure.py b/test/test_client_backpressure.py index b6776d53ba..40ea5eb8e1 100644 --- a/test/test_client_backpressure.py +++ b/test/test_client_backpressure.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +"""Test Client Backpressure spec.""" from __future__ import annotations import os @@ -19,21 +21,222 @@ from time import perf_counter from unittest.mock import patch -from pymongo.errors import OperationFailure - sys.path[0:0] = [""] from test import ( IntegrationTest, + PyMongoTestCase, client_context, unittest, ) from test.unified_format import generate_test_classes from test.utils_shared import EventListener, OvertCommandListener +import pymongo +from pymongo.errors import OperationFailure, PyMongoError +from pymongo.synchronous import helpers +from pymongo.synchronous.helpers import _MAX_RETRIES, _RetryPolicy, _TokenBucket + _IS_SYNC = True +# Mock an system overload error. +mock_overload_error = { + "configureFailPoint": "failCommand", + "mode": {"times": 1}, + "data": { + "failCommands": ["find", "insert", "update"], + "errorCode": 462, # IngressRequestRateLimitExceeded + "errorLabels": ["RetryableError", "SystemOverloadedError"], + }, +} + + +class TestBackpressure(IntegrationTest): + RUN_ON_LOAD_BALANCER = True + + @client_context.require_failCommand_appName + def test_retry_overload_error_command(self): + self.db.t.insert_one({"x": 1}) + + # Ensure command is retried on overload error. + fail_many = mock_overload_error.copy() + fail_many["mode"] = {"times": _MAX_RETRIES} + with self.fail_point(fail_many): + self.db.command("find", "t") + + # Ensure command stops retrying after _MAX_RETRIES. + fail_too_many = mock_overload_error.copy() + fail_too_many["mode"] = {"times": _MAX_RETRIES + 1} + with self.fail_point(fail_too_many): + with self.assertRaises(PyMongoError) as error: + self.db.command("find", "t") + + self.assertIn("RetryableError", str(error.exception)) + self.assertIn("SystemOverloadedError", str(error.exception)) + + @client_context.require_failCommand_appName + def test_retry_overload_error_find(self): + self.db.t.insert_one({"x": 1}) + + # Ensure command is retried on overload error. + fail_many = mock_overload_error.copy() + fail_many["mode"] = {"times": _MAX_RETRIES} + with self.fail_point(fail_many): + self.db.t.find_one() + + # Ensure command stops retrying after _MAX_RETRIES. + fail_too_many = mock_overload_error.copy() + fail_too_many["mode"] = {"times": _MAX_RETRIES + 1} + with self.fail_point(fail_too_many): + with self.assertRaises(PyMongoError) as error: + self.db.t.find_one() + + self.assertIn("RetryableError", str(error.exception)) + self.assertIn("SystemOverloadedError", str(error.exception)) + + @client_context.require_failCommand_appName + def test_retry_overload_error_insert_one(self): + self.db.t.insert_one({"x": 1}) + + # Ensure command is retried on overload error. + fail_many = mock_overload_error.copy() + fail_many["mode"] = {"times": _MAX_RETRIES} + with self.fail_point(fail_many): + self.db.t.find_one() + + # Ensure command stops retrying after _MAX_RETRIES. + fail_too_many = mock_overload_error.copy() + fail_too_many["mode"] = {"times": _MAX_RETRIES + 1} + with self.fail_point(fail_too_many): + with self.assertRaises(PyMongoError) as error: + self.db.t.find_one() + + self.assertIn("RetryableError", str(error.exception)) + self.assertIn("SystemOverloadedError", str(error.exception)) + + @client_context.require_failCommand_appName + def test_retry_overload_error_update_many(self): + # Even though update_many is not a retryable write operation, it will + # still be retried via the "RetryableError" error label. + self.db.t.insert_one({"x": 1}) + + # Ensure command is retried on overload error. + fail_many = mock_overload_error.copy() + fail_many["mode"] = {"times": _MAX_RETRIES} + with self.fail_point(fail_many): + self.db.t.update_many({}, {"$set": {"x": 2}}) + + # Ensure command stops retrying after _MAX_RETRIES. + fail_too_many = mock_overload_error.copy() + fail_too_many["mode"] = {"times": _MAX_RETRIES + 1} + with self.fail_point(fail_too_many): + with self.assertRaises(PyMongoError) as error: + self.db.t.update_many({}, {"$set": {"x": 2}}) + + self.assertIn("RetryableError", str(error.exception)) + self.assertIn("SystemOverloadedError", str(error.exception)) + + @client_context.require_failCommand_appName + def test_retry_overload_error_getMore(self): + coll = self.db.t + coll.insert_many([{"x": 1} for _ in range(10)]) + + # Ensure command is retried on overload error. + fail_many = { + "configureFailPoint": "failCommand", + "mode": {"times": _MAX_RETRIES}, + "data": { + "failCommands": ["getMore"], + "errorCode": 462, # IngressRequestRateLimitExceeded + "errorLabels": ["RetryableError", "SystemOverloadedError"], + }, + } + cursor = coll.find(batch_size=2) + cursor.next() + with self.fail_point(fail_many): + cursor.to_list() + + # Ensure command stops retrying after _MAX_RETRIES. + fail_too_many = fail_many.copy() + fail_too_many["mode"] = {"times": _MAX_RETRIES + 1} + cursor = coll.find(batch_size=2) + cursor.next() + with self.fail_point(fail_too_many): + with self.assertRaises(PyMongoError) as error: + cursor.to_list() + + self.assertIn("RetryableError", str(error.exception)) + self.assertIn("SystemOverloadedError", str(error.exception)) + + @client_context.require_failCommand_appName + def test_limit_retry_command(self): + client = self.rs_or_single_client() + client._retry_policy.token_bucket.tokens = 1 + db = client.pymongo_test + db.t.insert_one({"x": 1}) + + # Ensure command is retried once overload error. + fail_many = mock_overload_error.copy() + fail_many["mode"] = {"times": 1} + with self.fail_point(fail_many): + db.command("find", "t") + + # Ensure command stops retrying when there are no tokens left. + fail_too_many = mock_overload_error.copy() + fail_too_many["mode"] = {"times": 2} + with self.fail_point(fail_too_many): + with self.assertRaises(PyMongoError) as error: + db.command("find", "t") + + self.assertIn("RetryableError", str(error.exception)) + self.assertIn("SystemOverloadedError", str(error.exception)) + + +class TestRetryPolicy(PyMongoTestCase): + def test_retry_policy(self): + capacity = 10 + retry_policy = _RetryPolicy(_TokenBucket(capacity=capacity)) + self.assertEqual(retry_policy.attempts, helpers._MAX_RETRIES) + self.assertEqual(retry_policy.backoff_initial, helpers._BACKOFF_INITIAL) + self.assertEqual(retry_policy.backoff_max, helpers._BACKOFF_MAX) + for i in range(1, helpers._MAX_RETRIES + 1): + self.assertTrue(retry_policy.should_retry(i, 0)) + self.assertFalse(retry_policy.should_retry(helpers._MAX_RETRIES + 1, 0)) + for i in range(capacity - helpers._MAX_RETRIES): + self.assertTrue(retry_policy.should_retry(1, 0)) + # No tokens left, should not retry. + self.assertFalse(retry_policy.should_retry(1, 0)) + self.assertEqual(retry_policy.token_bucket.tokens, 0) + + # record_success should generate tokens. + for _ in range(int(2 / helpers.DEFAULT_RETRY_TOKEN_RETURN)): + retry_policy.record_success(retry=False) + self.assertAlmostEqual(retry_policy.token_bucket.tokens, 2) + for i in range(2): + self.assertTrue(retry_policy.should_retry(1, 0)) + self.assertFalse(retry_policy.should_retry(1, 0)) + + # Recording a successful retry should return 1 additional token. + retry_policy.record_success(retry=True) + self.assertAlmostEqual( + retry_policy.token_bucket.tokens, 1 + helpers.DEFAULT_RETRY_TOKEN_RETURN + ) + self.assertTrue(retry_policy.should_retry(1, 0)) + self.assertFalse(retry_policy.should_retry(1, 0)) + self.assertAlmostEqual(retry_policy.token_bucket.tokens, helpers.DEFAULT_RETRY_TOKEN_RETURN) + + def test_retry_policy_csot(self): + retry_policy = _RetryPolicy(_TokenBucket()) + self.assertTrue(retry_policy.should_retry(1, 0.5)) + with pymongo.timeout(0.5): + self.assertTrue(retry_policy.should_retry(1, 0)) + self.assertTrue(retry_policy.should_retry(1, 0.1)) + # Would exceed the timeout, should not retry. + self.assertFalse(retry_policy.should_retry(1, 1.0)) + self.assertTrue(retry_policy.should_retry(1, 1.0)) + +# Prose tests. class TestClientBackpressure(IntegrationTest): listener: EventListener diff --git a/tools/synchro.py b/tools/synchro.py index c87b4e8139..91820644e3 100644 --- a/tools/synchro.py +++ b/tools/synchro.py @@ -209,7 +209,6 @@ def async_only_test(f: str) -> bool: "test_auth_oidc.py", "test_auth_spec.py", "test_bulk.py", - "test_backpressure.py", "test_change_stream.py", "test_client.py", "test_client_backpressure.py", From a653b2d5b9b725b08b0e904fa463d2c936d2ea30 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 16 Dec 2025 10:49:25 -0600 Subject: [PATCH 15/16] update client bulkwrite logic --- pymongo/asynchronous/client_bulk.py | 1 + pymongo/synchronous/client_bulk.py | 1 + 2 files changed, 2 insertions(+) diff --git a/pymongo/asynchronous/client_bulk.py b/pymongo/asynchronous/client_bulk.py index 508b8e41c3..bda374e9b9 100644 --- a/pymongo/asynchronous/client_bulk.py +++ b/pymongo/asynchronous/client_bulk.py @@ -569,6 +569,7 @@ async def _execute_command( and "errorLabels" in error.details and isinstance(error.details["errorLabels"], list) and "RetryableError" in error.details["errorLabels"] + and "SystemOverloadedError" in error.details["errorLabels"] ) # Synthesize the full bulk result without modifying the diff --git a/pymongo/synchronous/client_bulk.py b/pymongo/synchronous/client_bulk.py index e8167bcedc..30f32488ec 100644 --- a/pymongo/synchronous/client_bulk.py +++ b/pymongo/synchronous/client_bulk.py @@ -567,6 +567,7 @@ def _execute_command( and "errorLabels" in error.details and isinstance(error.details["errorLabels"], list) and "RetryableError" in error.details["errorLabels"] + and "SystemOverloadedError" in error.details["errorLabels"] ) # Synthesize the full bulk result without modifying the From 14e401b318547d0b3cec0a4ea630a64237ddaf4b Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 16 Dec 2025 11:59:51 -0600 Subject: [PATCH 16/16] fix client bulk write handling --- pymongo/asynchronous/mongo_client.py | 2 +- pymongo/synchronous/mongo_client.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pymongo/asynchronous/mongo_client.py b/pymongo/asynchronous/mongo_client.py index effc1f1c12..0d4e766ce7 100644 --- a/pymongo/asynchronous/mongo_client.py +++ b/pymongo/asynchronous/mongo_client.py @@ -2823,7 +2823,7 @@ async def run(self) -> T: retryable_write_label = exc_to_check.has_error_label("RetryableWriteError") always_retryable = exc_to_check.has_error_label( "RetryableError" - ) and exc.has_error_label("SystemOverloadedError") + ) and exc_to_check.has_error_label("SystemOverloadedError") overloaded = exc_to_check.has_error_label("SystemOverloadedError") if not self._retryable and not always_retryable: raise diff --git a/pymongo/synchronous/mongo_client.py b/pymongo/synchronous/mongo_client.py index 290caa5a16..908bec114e 100644 --- a/pymongo/synchronous/mongo_client.py +++ b/pymongo/synchronous/mongo_client.py @@ -2813,7 +2813,7 @@ def run(self) -> T: retryable_write_label = exc_to_check.has_error_label("RetryableWriteError") always_retryable = exc_to_check.has_error_label( "RetryableError" - ) and exc.has_error_label("SystemOverloadedError") + ) and exc_to_check.has_error_label("SystemOverloadedError") overloaded = exc_to_check.has_error_label("SystemOverloadedError") if not self._retryable and not always_retryable: raise