Fix RUBY-2004 Indicated attempt not always correct (#1568)

* RUBY-2004 Use do-end syntax for expect blocks

* RUBY-2004 Refactor for generality

* RUBY-2004 Indicate whether modern or legacy retry logic is used

* RUBY-2004 Organize more

* RUBY-2004 Fix incorrect operation attempt numbering

* RUBY-2004 Speed up the tests

* RUBY-2004 Test retryable writes

* RUBY-2004 Indicate when retries are disabled

* RUBY-2004 Explicitly request modern behavior because some test confgurations use legacy retries

* RUBY-2004 Need wired tiger for modern retries

* RUBY-2004 Misc cleanup

Author: p-mongo

lib/mongo/retryable.rb

@@ -119,7 +119,12 @@ def read_with_retry(session = nil, server_selector = nil, &block)
         legacy_read_with_retry(session, server_selector, &block)
       else
         server = select_server(cluster, server_selector, session)
-        yield server
+        begin
+          yield server
+        rescue Error::SocketError, Error::SocketTimeoutError, Error::OperationFailure => e
+          e.add_note('retries disabled')
+          raise e
+        end
       end
     end
 
@@ -214,12 +219,14 @@ def write_with_retry(session, write_concern, ending_transaction = false, &block)
       begin
         yield(server, txn_num, false)
       rescue Error::SocketError, Error::SocketTimeoutError => e
+        e.add_note('modern retry')
         e.add_note("attempt 1")
         if session.in_transaction? && !ending_transaction
           raise e
         end
         retry_write(e, session, txn_num, &block)
       rescue Error::OperationFailure => e
+        e.add_note('modern retry')
         e.add_note("attempt 1")
         if e.unsupported_retryable_write?
           raise_unsupported_error(e)
@@ -250,7 +257,12 @@ def write_with_retry(session, write_concern, ending_transaction = false, &block)
     def nro_write_with_retry(session, write_concern, &block)
       if session && session.client.options[:retry_writes]
         server = select_server(cluster, ServerSelector.primary, session)
-        yield server
+        begin
+          yield server
+        rescue Error::SocketError, Error::SocketTimeoutError, Error::OperationFailure => e
+          e.add_note('retries disabled')
+          raise e
+        end
       else
         legacy_write_with_retry(nil, session, &block)
       end
@@ -280,7 +292,8 @@ def legacy_write_with_retry(server = nil, session = nil)
         server ||= select_server(cluster, ServerSelector.primary, session)
         yield server
       rescue Error::OperationFailure => e
-        e.add_note("attempt #{attempt + 1}")
+        e.add_note('legacy retry')
+        e.add_note("attempt #{attempt}")
         server = nil
         if attempt > client.max_write_retries
           raise e
@@ -298,18 +311,19 @@ def legacy_write_with_retry(server = nil, session = nil)
     private
 
     def modern_read_with_retry(session, server_selector, &block)
-      attempt = 0
       server = select_server(cluster, server_selector, session)
       begin
         yield server
       rescue Error::SocketError, Error::SocketTimeoutError => e
-        e.add_note("attempt #{attempt + 1}")
+        e.add_note('modern retry')
+        e.add_note("attempt 1")
         if session.in_transaction?
           raise e
         end
         retry_read(e, server_selector, session, &block)
       rescue Error::OperationFailure => e
-        e.add_note("attempt #{attempt + 1}")
+        e.add_note('modern retry')
+        e.add_note("attempt 1")
         if session.in_transaction? || !e.write_retryable?
           raise e
         end
@@ -324,15 +338,17 @@ def legacy_read_with_retry(session, server_selector)
         attempt += 1
         yield server
       rescue Error::SocketError, Error::SocketTimeoutError => e
-        e.add_note("attempt #{attempt + 1}")
+        e.add_note('legacy retry')
+        e.add_note("attempt #{attempt}")
         if attempt > client.max_read_retries || (session && session.in_transaction?)
           raise e
         end
         log_retry(e, message: 'Legacy read retry')
         server = select_server(cluster, server_selector, session)
         retry
       rescue Error::OperationFailure => e
-        e.add_note("attempt #{attempt + 1}")
+        e.add_note('legacy retry')
+        e.add_note("attempt #{attempt}")
         if e.retryable? && !(session && session.in_transaction?)
           if attempt > client.max_read_retries
             raise e
@@ -375,16 +391,19 @@ def retry_read(original_error, server_selector, session, &block)
       begin
         yield server, true
       rescue Error::SocketError, Error::SocketTimeoutError => e
+        e.add_note('modern retry')
         e.add_note("attempt 2")
         raise e
       rescue Error::OperationFailure => e
+        e.add_note('modern retry')
         unless e.write_retryable?
           original_error.add_note("later retry failed: #{e.class}: #{e}")
           raise original_error
         end
         e.add_note("attempt 2")
         raise e
       rescue => e
+        e.add_note('modern retry')
         original_error.add_note("later retry failed: #{e.class}: #{e}")
         raise original_error
       end
@@ -398,15 +417,19 @@ def retry_write(original_error, session, txn_num, &block)
       # server unknown). Here we just need to wait for server selection.
       server = select_server(cluster, ServerSelector.primary, session)
       unless server.retry_writes?
+        # Do not need to add "modern retry" here, it should already be on
+        # the first exception.
         original_error.add_note('did not retry because server selected for retry does not supoprt retryable writes')
         raise original_error
       end
       log_retry(original_error, message: 'Write retry')
       yield(server, txn_num, true)
     rescue Error::SocketError, Error::SocketTimeoutError => e
+      e.add_note('modern retry')
       e.add_note('attempt 2')
       raise e
     rescue Error::OperationFailure => e
+      e.add_note('modern retry')
       if e.write_retryable?
         e.add_note('attempt 2')
         raise e
@@ -415,6 +438,8 @@ def retry_write(original_error, session, txn_num, &block)
         raise original_error
       end
     rescue => e
+      # Do not need to add "modern retry" here, it should already be on
+      # the first exception.
       original_error.add_note("later retry failed: #{e.class}: #{e}")
       raise original_error
     end