Enforce status checking after Valid() returns false for IteratorWrapper (#11975)

Summary:
... when compiled with ASSERT_STATUS_CHECKED = 1.

The main change is in iterator_wrapper.h. The remaining changes are just fixing existing unit tests. Adding this check to IteratorWrapper gives a good coverage as the class is used in many places, including child iterators under merging iterator, merging iterator under DB iter, file_iter under level iterator, etc. This change can catch the bug fixed in https://github.com/facebook/rocksdb/issues/11782.

Future follow up: enable `ASSERT_STATUS_CHECKED=1` for stress test and for DEBUG_LEVEL=0.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/11975

Test Plan:
* `ASSERT_STATUS_CHECKED=1 DEBUG_LEVEL=2 make -j32 J=32 check`
* I tried to run stress test with `ASSERT_STATUS_CHECKED=1`, but there are a lot of existing stress code that ignore status checking, and fail without the change in this PR. So defer that to a follow up task.

Reviewed By: ajkr

Differential Revision: D50383790

Pulled By: cbi42

fbshipit-source-id: 1a28ce0f5fdf1890f93400b26b3b1b3a287624ce

db/blob/db_blob_basic_test.cc

@@ -168,6 +168,7 @@ TEST_F(DBBlobBasicTest, IterateBlobsFromCache) {
       ASSERT_EQ(iter->value().ToString(), blobs[i]);
       ++i;
     }
+    ASSERT_OK(iter->status());
     ASSERT_EQ(i, num_blobs);
     ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_ADD), 0);
   }
@@ -203,6 +204,7 @@ TEST_F(DBBlobBasicTest, IterateBlobsFromCache) {
       ASSERT_EQ(iter->value().ToString(), blobs[i]);
       ++i;
     }
+    ASSERT_OK(iter->status());
     ASSERT_EQ(i, num_blobs);
     ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_ADD),
               num_blobs);
@@ -224,6 +226,7 @@ TEST_F(DBBlobBasicTest, IterateBlobsFromCache) {
       ASSERT_EQ(iter->value().ToString(), blobs[i]);
       ++i;
     }
+    ASSERT_OK(iter->status());
     ASSERT_EQ(i, num_blobs);
     ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_ADD), 0);
   }
@@ -2123,6 +2126,7 @@ TEST_F(DBBlobWithTimestampTest, IterateBlobs) {
                        /*key_is_internal*/ false);
       iter->Prev();
     }
+    ASSERT_OK(iter->status());
   }
 
   // Backward iteration, then reverse to forward.
@@ -2169,6 +2173,7 @@ TEST_F(DBBlobWithTimestampTest, IterateBlobs) {
         iter->Next();
       }
     }
+    ASSERT_OK(iter->status());
   }
 
   // Backward iterating multiple versions of the same key, get in this order:
@@ -2187,6 +2192,7 @@ TEST_F(DBBlobWithTimestampTest, IterateBlobs) {
         iter->Prev();
       }
     }
+    ASSERT_OK(iter->status());
   }
 
   int upper_bound_idx = num_blobs - 2;
@@ -2209,6 +2215,7 @@ TEST_F(DBBlobWithTimestampTest, IterateBlobs) {
         iter->Next();
       }
     }
+    ASSERT_OK(iter->status());
   }
 
   // Backward iteration with upper and lower bound.
@@ -2224,6 +2231,7 @@ TEST_F(DBBlobWithTimestampTest, IterateBlobs) {
         iter->Prev();
       }
     }
+    ASSERT_OK(iter->status());
   }
 }
 

db/comparator_db_test.cc

@@ -165,6 +165,7 @@ void DoRandomIteraratorTest(DB* db, std::vector<std::string> source_strings,
     AssertItersEqual(iter.get(), result_iter.get());
     is_valid = iter->Valid();
   }
+  ASSERT_OK(iter->status());
 }
 
 class DoubleComparator : public Comparator {

db/db_basic_test.cc

@@ -138,6 +138,7 @@ TEST_F(DBBasicTest, ReadOnlyDB) {
       ASSERT_OK(iter->status());
       ++count;
     }
+    ASSERT_OK(iter->status());
     // Always expect two keys: "foo" and "bar"
     ASSERT_EQ(count, 2);
   };

db/db_block_cache_test.cc

@@ -389,6 +389,7 @@ TEST_F(DBBlockCacheTest, FillCacheAndIterateDB) {
   while (iter->Valid()) {
     iter->Next();
   }
+  ASSERT_OK(iter->status());
   delete iter;
   iter = nullptr;
 }

db/db_bloom_filter_test.cc

@@ -1905,6 +1905,7 @@ TEST_F(DBBloomFilterTest, PrefixExtractorWithFilter2) {
   for (iter->Seek("zzzzz_AAAA"); iter->Valid(); iter->Next()) {
     iter_res.emplace_back(iter->value().ToString());
   }
+  ASSERT_OK(iter->status());
 
   std::vector<std::string> expected_res = {"val1", "val2", "val3", "val4"};
   ASSERT_EQ(iter_res, expected_res);

db/db_compaction_filter_test.cc

@@ -805,6 +805,7 @@ TEST_F(DBTestCompactionFilter, CompactionFilterIgnoreSnapshot) {
       count++;
       iter->Next();
     }
+    ASSERT_OK(iter->status());
     ASSERT_EQ(count, 6);
     read_options.snapshot = nullptr;
     std::unique_ptr<Iterator> iter1(db_->NewIterator(read_options));
@@ -815,6 +816,7 @@ TEST_F(DBTestCompactionFilter, CompactionFilterIgnoreSnapshot) {
       count++;
       iter1->Next();
     }
+    ASSERT_OK(iter1->status());
     // We have deleted 10 keys from 40 using the compaction filter
     //  Keys 6-9 before the snapshot and 100-105 after the snapshot
     ASSERT_EQ(count, 30);

db/db_compaction_test.cc

@@ -2673,6 +2673,7 @@ TEST_P(DBCompactionTestWithParam, ConvertCompactionStyle) {
     keys_in_db.append(iter->key().ToString());
     keys_in_db.push_back(',');
   }
+  ASSERT_OK(iter->status());
   delete iter;
 
   std::string expected_keys;

db/db_flush_test.cc

@@ -1377,6 +1377,7 @@ TEST_F(DBFlushTest, MemPurgeDeleteAndDeleteRange) {
         ASSERT_EQ(value, NOT_FOUND);
       count++;
     }
+    ASSERT_OK(iter->status());
 
     // Expected count here is 3: KEY3, KEY4, KEY5.
     ASSERT_EQ(count, EXPECTED_COUNT_FORLOOP);

db/db_iter_test.cc

@@ -275,6 +275,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
 
     db_iter->Next();
     ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
   }
   // Test to check the SeekToLast() with iterate_upper_bound not set
   {
@@ -1415,6 +1416,7 @@ TEST_F(DBIteratorTest, DBIterator1) {
   ASSERT_EQ(db_iter->key().ToString(), "b");
   db_iter->Next();
   ASSERT_FALSE(db_iter->Valid());
+  ASSERT_OK(db_iter->status());
 }
 
 TEST_F(DBIteratorTest, DBIterator2) {
@@ -1528,6 +1530,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
     ASSERT_EQ(db_iter->value().ToString(), "merge_1");
     db_iter->Prev();
     ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
   }
 
   {
@@ -1552,6 +1555,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
     ASSERT_EQ(db_iter->value().ToString(), "merge_1,merge_2");
     db_iter->Prev();
     ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
   }
 
   {
@@ -1576,6 +1580,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
     ASSERT_EQ(db_iter->value().ToString(), "merge_1,merge_2,merge_3");
     db_iter->Prev();
     ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
   }
 
   {
@@ -1600,6 +1605,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
     ASSERT_EQ(db_iter->value().ToString(), "put_1");
     db_iter->Prev();
     ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
   }
 
   {
@@ -1624,6 +1630,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
     ASSERT_EQ(db_iter->value().ToString(), "put_1,merge_4");
     db_iter->Prev();
     ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
   }
 
   {
@@ -1648,6 +1655,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
     ASSERT_EQ(db_iter->value().ToString(), "put_1,merge_4,merge_5");
     db_iter->Prev();
     ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
   }
 
   {
@@ -1672,6 +1680,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
     ASSERT_EQ(db_iter->value().ToString(), "put_1,merge_4,merge_5,merge_6");
     db_iter->Prev();
     ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
   }
 
   {
@@ -1726,6 +1735,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
     ASSERT_EQ(db_iter->value().ToString(), "merge_1");
     db_iter->Prev();
     ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
   }
 
   {
@@ -1750,6 +1760,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
     ASSERT_EQ(db_iter->value().ToString(), "merge_1,merge_2");
     db_iter->Prev();
     ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
   }
 
   {
@@ -1774,6 +1785,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
     ASSERT_EQ(db_iter->value().ToString(), "merge_1,merge_2,merge_3");
     db_iter->Prev();
     ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
   }
 
   {
@@ -1794,6 +1806,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
         nullptr /* read_callback */));
     db_iter->SeekToLast();
     ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
   }
 
   {
@@ -1818,6 +1831,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
     ASSERT_EQ(db_iter->value().ToString(), "merge_4");
     db_iter->Prev();
     ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
   }
 
   {
@@ -1842,6 +1856,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
     ASSERT_EQ(db_iter->value().ToString(), "merge_4,merge_5");
     db_iter->Prev();
     ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
   }
 
   {
@@ -1866,6 +1881,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
     ASSERT_EQ(db_iter->value().ToString(), "merge_4,merge_5,merge_6");
     db_iter->Prev();
     ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
   }
 }
 
@@ -1910,6 +1926,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
     ASSERT_EQ(db_iter->value().ToString(), "merge_1");
     db_iter->Prev();
     ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
   }
 
   {
@@ -1952,6 +1969,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
     ASSERT_EQ(db_iter->value().ToString(), "merge_1");
     db_iter->Prev();
     ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
   }
 
   {
@@ -1994,6 +2012,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
     ASSERT_EQ(db_iter->value().ToString(), "merge_1");
     db_iter->Prev();
     ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
   }
 
   {
@@ -2041,6 +2060,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
     ASSERT_EQ(db_iter->value().ToString(), "merge_1");
     db_iter->Prev();
     ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
   }
 
   {
@@ -2089,6 +2109,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
     ASSERT_EQ(db_iter->value().ToString(), "merge_1");
     db_iter->Prev();
     ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
   }
 
   {
@@ -2131,6 +2152,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
     ASSERT_EQ(db_iter->value().ToString(), "merge_1");
     db_iter->Prev();
     ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
   }
 
   {
@@ -2179,6 +2201,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
     ASSERT_EQ(db_iter->value().ToString(), "merge_1");
     db_iter->Prev();
     ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
   }
 
   {
@@ -2228,6 +2251,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
     ASSERT_EQ(db_iter->value().ToString(), "merge_1");
     db_iter->Prev();
     ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
   }
 
   {
@@ -2271,6 +2295,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
     ASSERT_EQ(db_iter->value().ToString(), "merge_1");
     db_iter->Prev();
     ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
   }
 }
 
@@ -2440,6 +2465,7 @@ TEST_F(DBIteratorTest, SeekToLastOccurrenceSeq0) {
   ASSERT_EQ(db_iter->value().ToString(), "2");
   db_iter->Next();
   ASSERT_FALSE(db_iter->Valid());
+  ASSERT_OK(db_iter->status());
 }
 
 TEST_F(DBIteratorTest, DBIterator11) {
@@ -2469,6 +2495,7 @@ TEST_F(DBIteratorTest, DBIterator11) {
   ASSERT_EQ(db_iter->key().ToString(), "b");
   db_iter->Next();
   ASSERT_FALSE(db_iter->Valid());
+  ASSERT_OK(db_iter->status());
 }
 
 TEST_F(DBIteratorTest, DBIterator12) {
@@ -2497,6 +2524,7 @@ TEST_F(DBIteratorTest, DBIterator12) {
   ASSERT_EQ(db_iter->value().ToString(), "1");
   db_iter->Prev();
   ASSERT_FALSE(db_iter->Valid());
+  ASSERT_OK(db_iter->status());
 }
 
 TEST_F(DBIteratorTest, DBIterator13) {
@@ -2635,6 +2663,7 @@ TEST_F(DBIterWithMergeIterTest, InnerMergeIterator1) {
   ASSERT_EQ(db_iter_->value().ToString(), "3");
   db_iter_->Next();
   ASSERT_FALSE(db_iter_->Valid());
+  ASSERT_OK(db_iter_->status());
 }
 
 TEST_F(DBIterWithMergeIterTest, InnerMergeIterator2) {

db/db_iterator_test.cc

@@ -183,6 +183,7 @@ TEST_P(DBIteratorTest, NonBlockingIteration) {
       ASSERT_OK(iter->status());
       count++;
     }
+    ASSERT_OK(iter->status());
     ASSERT_EQ(count, 1);
     delete iter;
 
@@ -217,6 +218,7 @@ TEST_P(DBIteratorTest, NonBlockingIteration) {
       ASSERT_OK(iter->status());
       count++;
     }
+    ASSERT_OK(iter->status());
     ASSERT_EQ(count, 1);
     ASSERT_EQ(numopen, TestGetTickerCount(options, NO_FILE_OPENS));
     ASSERT_EQ(cache_added, TestGetTickerCount(options, BLOCK_CACHE_ADD));
@@ -870,6 +872,7 @@ TEST_P(DBIteratorTest, IterWithSnapshot) {
       }
     }
     db_->ReleaseSnapshot(snapshot);
+    ASSERT_OK(iter->status());
     delete iter;
   } while (ChangeOptions());
 }
@@ -1214,6 +1217,7 @@ TEST_P(DBIteratorTest, DBIteratorBoundOptimizationTest) {
 
     iter->Next();
     ASSERT_FALSE(iter->Valid());
+    ASSERT_OK(iter->status());
     ASSERT_EQ(upper_bound_hits, 1);
   }
 }
@@ -1338,6 +1342,7 @@ TEST_P(DBIteratorTest, IndexWithFirstKey) {
 
     iter->Next();
     ASSERT_FALSE(iter->Valid());
+    ASSERT_OK(iter->status());
     EXPECT_EQ(3, stats->getTickerCount(BLOCK_CACHE_DATA_HIT));
     EXPECT_EQ(7, stats->getTickerCount(BLOCK_CACHE_DATA_MISS));
   }
@@ -1579,6 +1584,7 @@ class DBIteratorTestForPinnedData : public DBIteratorTest {
         ASSERT_EQ("1", prop_value);
         all_keys.push_back(iter->key());
       }
+      ASSERT_OK(iter->status());
       ASSERT_EQ(all_keys.size(), true_data.size());
 
       // Verify that all keys slices are valid (backward)
@@ -1682,7 +1688,7 @@ TEST_P(DBIteratorTest, PinnedDataIteratorMultipleFiles) {
     ASSERT_EQ(kv.first, data_iter->first);
     ASSERT_EQ(kv.second, data_iter->second);
   }
-
+  ASSERT_OK(iter->status());
   delete iter;
 }
 
@@ -1728,6 +1734,7 @@ TEST_P(DBIteratorTest, PinnedDataIteratorMergeOperator) {
     ASSERT_EQ("1", prop_value);
     results.emplace_back(iter->key(), iter->value().ToString());
   }
+  ASSERT_OK(iter->status());
 
   ASSERT_EQ(results.size(), 1000);
   for (size_t i = 0; i < results.size(); i++) {
@@ -1785,6 +1792,7 @@ TEST_P(DBIteratorTest, PinnedDataIteratorReadAfterUpdate) {
     ASSERT_EQ("1", prop_value);
     results.emplace_back(iter->key(), iter->value().ToString());
   }
+  ASSERT_OK(iter->status());
 
   auto data_iter = true_data.begin();
   for (size_t i = 0; i < results.size(); i++, data_iter++) {
@@ -2079,6 +2087,7 @@ TEST_P(DBIteratorTest, IterPrevKeyCrossingBlocksRandomized) {
       ASSERT_EQ(iter->value().ToString(), data_iter->second);
       data_iter++;
     }
+    ASSERT_OK(iter->status());
     ASSERT_EQ(data_iter, true_data.rend());
 
     delete iter;
@@ -2136,6 +2145,7 @@ TEST_P(DBIteratorTest, IterPrevKeyCrossingBlocksRandomized) {
       entries_right++;
       data_iter++;
     }
+    ASSERT_OK(iter->status());
     ASSERT_EQ(data_iter, true_data.rend());
 
     delete iter;
@@ -2175,6 +2185,7 @@ TEST_P(DBIteratorTest, IteratorWithLocalStatistics) {
       total_next++;
 
       if (!iter->Valid()) {
+        EXPECT_OK(iter->status());
         break;
       }
       total_next_found++;
@@ -2202,6 +2213,7 @@ TEST_P(DBIteratorTest, IteratorWithLocalStatistics) {
       total_prev++;
 
       if (!iter->Valid()) {
+        EXPECT_OK(iter->status());
         break;
       }
       total_prev_found++;
@@ -2416,6 +2428,7 @@ TEST_P(DBIteratorTest, Refresh) {
   ASSERT_EQ(iter->key().compare(Slice("x")), 0);
   iter->Next();
   ASSERT_FALSE(iter->Valid());
+  ASSERT_OK(iter->status());
 
   iter.reset();
 }
@@ -2469,32 +2482,32 @@ TEST_P(DBIteratorTest, RefreshWithSnapshot) {
     verify_iter(1, 3);
     // Refresh to same snapshot
     ASSERT_OK(iter->Refresh(snapshot));
-    ASSERT_TRUE(iter->status().ok() && !iter->Valid());
+    ASSERT_TRUE(!iter->Valid() && iter->status().ok());
     iter->Seek(Key(3));
     verify_iter(3, 6);
-    ASSERT_TRUE(iter->status().ok() && !iter->Valid());
+    ASSERT_TRUE(!iter->Valid() && iter->status().ok());
 
     // Refresh to a newer snapshot
     ASSERT_OK(iter->Refresh(snapshot2));
-    ASSERT_TRUE(iter->status().ok() && !iter->Valid());
+    ASSERT_TRUE(!iter->Valid() && iter->status().ok());
     iter->SeekToFirst();
     verify_iter(0, 4, /*new_key2=*/true);
-    ASSERT_TRUE(iter->status().ok() && !iter->Valid());
+    ASSERT_TRUE(!iter->Valid() && iter->status().ok());
 
     // Refresh to an older snapshot
     ASSERT_OK(iter->Refresh(snapshot));
-    ASSERT_TRUE(iter->status().ok() && !iter->Valid());
+    ASSERT_TRUE(!iter->Valid() && iter->status().ok());
     iter->Seek(Key(3));
     verify_iter(3, 6);
-    ASSERT_TRUE(iter->status().ok() && !iter->Valid());
+    ASSERT_TRUE(!iter->Valid() && iter->status().ok());
 
     // Refresh to no snapshot
     ASSERT_OK(iter->Refresh());
-    ASSERT_TRUE(iter->status().ok() && !iter->Valid());
+    ASSERT_TRUE(!iter->Valid() && iter->status().ok());
     iter->Seek(Key(2));
     verify_iter(2, 4, /*new_key2=*/true);
     verify_iter(6, 7);
-    ASSERT_TRUE(iter->status().ok() && !iter->Valid());
+    ASSERT_TRUE(!iter->Valid() && iter->status().ok());
 
     // Change LSM shape, new SuperVersion is created.
     ASSERT_OK(Flush());
@@ -2599,6 +2612,7 @@ TEST_P(DBIteratorTest, TableFilter) {
     ASSERT_EQ(IterStatus(iter), "f->6");
     iter->Next();
     ASSERT_FALSE(iter->Valid());
+    ASSERT_OK(iter->status());
     ASSERT_TRUE(unseen.empty());
     delete iter;
   }
@@ -2621,6 +2635,7 @@ TEST_P(DBIteratorTest, TableFilter) {
     ASSERT_EQ(IterStatus(iter), "f->6");
     iter->Next();
     ASSERT_FALSE(iter->Valid());
+    ASSERT_OK(iter->status());
     delete iter;
   }
 }
@@ -2705,6 +2720,7 @@ TEST_P(DBIteratorTest, SkipStatistics) {
     ASSERT_OK(iter->status());
     count++;
   }
+  ASSERT_OK(iter->status());
   ASSERT_EQ(count, 3);
   delete iter;
   skip_count += 8;  // Same as above, but in reverse order
@@ -2740,6 +2756,7 @@ TEST_P(DBIteratorTest, SkipStatistics) {
     ASSERT_OK(iter->status());
     count++;
   }
+  ASSERT_OK(iter->status());
   ASSERT_EQ(count, 2);
   delete iter;
   // 3 deletes + 3 original keys + lower sequence of "a"
@@ -3322,6 +3339,7 @@ TEST_F(DBIteratorTest, BackwardIterationOnInplaceUpdateMemtable) {
     for (iter->SeekToLast(); iter->Valid(); iter->Prev()) {
       ++count;
     }
+    ASSERT_OK(iter->status());
     ASSERT_EQ(kNumKeys, count);
   }
 

db/db_range_del_test.cc

@@ -682,6 +682,7 @@ TEST_F(DBRangeDelTest, TableEvictedDuringScan) {
     // soon as its refcount drops to zero.
     bbto.block_cache->EraseUnRefEntries();
   }
+  ASSERT_OK(iter->status());
   ASSERT_EQ(kNum, expected);
   delete iter;
   db_->ReleaseSnapshot(snapshot);
@@ -840,6 +841,7 @@ TEST_F(DBRangeDelTest, IteratorRemovesCoveredKeys) {
       ++expected;
     }
   }
+  ASSERT_OK(iter->status());
   ASSERT_EQ(kNum, expected);
   delete iter;
 }
@@ -908,6 +910,7 @@ TEST_F(DBRangeDelTest, IteratorIgnoresRangeDeletions) {
     std::string key;
     ASSERT_EQ(expected[i], iter->key());
   }
+  ASSERT_OK(iter->status());
   ASSERT_EQ(3, i);
   delete iter;
   db_->ReleaseSnapshot(snapshot);
@@ -1382,6 +1385,7 @@ TEST_F(DBRangeDelTest, UntruncatedTombstoneDoesNotDeleteNewerKey) {
     for (; iter->Valid(); iter->Next()) {
       ++keys_found;
     }
+    EXPECT_OK(iter->status());
     delete iter;
     return keys_found;
   };
@@ -1485,6 +1489,7 @@ TEST_F(DBRangeDelTest, DeletedMergeOperandReappearsIterPrev) {
   for (; iter->Valid(); iter->Prev()) {
     ++keys_found;
   }
+  ASSERT_OK(iter->status());
   delete iter;
   ASSERT_EQ(kNumKeys, keys_found);
 
@@ -1519,6 +1524,7 @@ TEST_F(DBRangeDelTest, SnapshotPreventsDroppedKeys) {
 
   iter->Next();
   ASSERT_FALSE(iter->Valid());
+  ASSERT_OK(iter->status());
 
   delete iter;
   db_->ReleaseSnapshot(snapshot);
@@ -1564,6 +1570,7 @@ TEST_F(DBRangeDelTest, SnapshotPreventsDroppedKeysInImmMemTables) {
 
   iter->Next();
   ASSERT_FALSE(iter->Valid());
+  ASSERT_OK(iter->status());
 }
 
 TEST_F(DBRangeDelTest, RangeTombstoneWrittenToMinimalSsts) {
@@ -1978,6 +1985,7 @@ TEST_F(DBRangeDelTest, IteratorRefresh) {
     ASSERT_EQ("key1", iter->key());
     iter->Next();
     ASSERT_FALSE(iter->Valid());
+    ASSERT_OK(iter->status());
 
     delete iter;
   }

db/db_rate_limiter_test.cc

@@ -220,6 +220,7 @@ TEST_P(DBRateLimiterOnReadTest, Iterator) {
       ++expected;
     }
   }
+  ASSERT_OK(iter->status());
   // Reverse scan does not read evenly (one block per iteration) due to
   // descending seqno ordering, so wait until after the loop to check total.
   ASSERT_EQ(expected, options_.rate_limiter->GetTotalRequests(Env::IO_USER));

db/db_readonly_with_timestamp_test.cc

@@ -240,6 +240,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, IteratorAndGet) {
       get_value_and_check(db_, read_opts, it->key(), it->value(),
                           write_timestamps[i]);
     }
+    ASSERT_OK(it->status());
     size_t expected_count = kMaxKey - start_keys[i] + 1;
     ASSERT_EQ(expected_count, count);
 
@@ -252,6 +253,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, IteratorAndGet) {
       get_value_and_check(db_, read_opts, it->key(), it->value(),
                           write_timestamps[i]);
     }
+    ASSERT_OK(it->status());
     ASSERT_EQ(static_cast<size_t>(kMaxKey) - start_keys[i] + 1, count);
 
     // SeekToFirst()/SeekToLast() with lower/upper bounds.
@@ -273,6 +275,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, IteratorAndGet) {
         get_value_and_check(db_, read_opts, it->key(), it->value(),
                             write_timestamps[i]);
       }
+      ASSERT_OK(it->status());
       ASSERT_EQ(r - std::max(l, start_keys[i]), count);
 
       for (it->SeekToLast(), key = std::min(r, kMaxKey + 1), count = 0;
@@ -282,6 +285,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, IteratorAndGet) {
         get_value_and_check(db_, read_opts, it->key(), it->value(),
                             write_timestamps[i]);
       }
+      ASSERT_OK(it->status());
       l += (kMaxKey / 100);
       r -= (kMaxKey / 100);
     }
@@ -328,6 +332,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, Iterators) {
     CheckIterUserEntry(iters[0], Key1(key), kTypeValue,
                        "value" + std::to_string(key), write_timestamp);
   }
+  ASSERT_OK(iters[0]->status());
 
   size_t expected_count = kMaxKey - 0 + 1;
   ASSERT_EQ(expected_count, count);

db/db_secondary_test.cc

@@ -200,6 +200,7 @@ TEST_F(DBSecondaryTest, ReopenAsSecondary) {
     }
     ++count;
   }
+  ASSERT_OK(iter->status());
   delete iter;
   ASSERT_EQ(3, count);
 }
@@ -534,6 +535,8 @@ TEST_F(DBSecondaryTest, SecondaryCloseFiles) {
     }
     ASSERT_FALSE(iter1->Valid());
     ASSERT_FALSE(iter2->Valid());
+    ASSERT_OK(iter1->status());
+    ASSERT_OK(iter2->status());
   };
 
   ASSERT_OK(Put("a", "value"));
@@ -806,6 +809,7 @@ TEST_F(DBSecondaryTest, MissingTableFileDuringOpen) {
   for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
     ++count;
   }
+  ASSERT_OK(iter->status());
   ASSERT_EQ(2, count);
   delete iter;
 }
@@ -863,6 +867,7 @@ TEST_F(DBSecondaryTest, MissingTableFile) {
   for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
     ++count;
   }
+  ASSERT_OK(iter->status());
   ASSERT_EQ(2, count);
   delete iter;
 }
@@ -935,6 +940,7 @@ TEST_F(DBSecondaryTest, SwitchManifest) {
       ASSERT_EQ("value_" + std::to_string(kNumFiles - 1),
                 iter->value().ToString());
     }
+    EXPECT_OK(iter->status());
   };
 
   range_scan_db();
@@ -1485,6 +1491,7 @@ TEST_F(DBSecondaryTestWithTimestamp, IteratorAndGet) {
       get_value_and_check(db_, read_opts, it->key(), it->value(),
                           write_timestamps[i]);
     }
+    ASSERT_OK(it->status());
     size_t expected_count = kMaxKey - start_keys[i] + 1;
     ASSERT_EQ(expected_count, count);
 
@@ -1497,6 +1504,7 @@ TEST_F(DBSecondaryTestWithTimestamp, IteratorAndGet) {
       get_value_and_check(db_, read_opts, it->key(), it->value(),
                           write_timestamps[i]);
     }
+    ASSERT_OK(it->status());
     ASSERT_EQ(static_cast<size_t>(kMaxKey) - start_keys[i] + 1, count);
 
     // SeekToFirst()/SeekToLast() with lower/upper bounds.
@@ -1518,6 +1526,7 @@ TEST_F(DBSecondaryTestWithTimestamp, IteratorAndGet) {
         get_value_and_check(db_, read_opts, it->key(), it->value(),
                             write_timestamps[i]);
       }
+      ASSERT_OK(it->status());
       ASSERT_EQ(r - std::max(l, start_keys[i]), count);
 
       for (it->SeekToLast(), key = std::min(r, kMaxKey + 1), count = 0;
@@ -1527,6 +1536,7 @@ TEST_F(DBSecondaryTestWithTimestamp, IteratorAndGet) {
         get_value_and_check(db_, read_opts, it->key(), it->value(),
                             write_timestamps[i]);
       }
+      ASSERT_OK(it->status());
       l += (kMaxKey / 100);
       r -= (kMaxKey / 100);
     }
@@ -1736,6 +1746,7 @@ TEST_F(DBSecondaryTestWithTimestamp, Iterators) {
     CheckIterUserEntry(iters[0], Key1(key), kTypeValue,
                        "value" + std::to_string(key), write_timestamp);
   }
+  ASSERT_OK(iters[0]->status());
 
   size_t expected_count = kMaxKey - 0 + 1;
   ASSERT_EQ(expected_count, count);

db/db_tailing_iter_test.cc

@@ -52,6 +52,7 @@ TEST_P(DBTestTailingIterator, TailingIteratorSingle) {
 
   iter->Next();
   ASSERT_TRUE(!iter->Valid());
+  ASSERT_OK(iter->status());
 }
 
 TEST_P(DBTestTailingIterator, TailingIteratorKeepAdding) {
@@ -361,7 +362,7 @@ TEST_P(DBTestTailingIterator, TailingIteratorDeletes) {
     int count = 0;
     for (; iter->Valid(); iter->Next(), ++count)
       ;
-
+    ASSERT_OK(iter->status());
     ASSERT_EQ(count, num_records);
   }
   Close();
@@ -408,6 +409,7 @@ TEST_P(DBTestTailingIterator, TailingIteratorPrefixSeek) {
 
     iter->Next();
     ASSERT_TRUE(!iter->Valid());
+    ASSERT_OK(iter->status());
   }
   Close();
 }

db/db_test.cc

@@ -127,6 +127,7 @@ TEST_F(DBTest, MockEnvTest) {
     iterator->Next();
   }
   ASSERT_TRUE(!iterator->Valid());
+  ASSERT_OK(iterator->status());
   delete iterator;
 
   DBImpl* dbi = static_cast_with_check<DBImpl>(db);
@@ -171,6 +172,7 @@ TEST_F(DBTest, MemEnvTest) {
     iterator->Next();
   }
   ASSERT_TRUE(!iterator->Valid());
+  ASSERT_OK(iterator->status());
   delete iterator;
 
   DBImpl* dbi = static_cast_with_check<DBImpl>(db);
@@ -2983,6 +2985,7 @@ TEST_F(DBTest, GroupCommitTest) {
       itr->Next();
     }
     ASSERT_TRUE(!itr->Valid());
+    ASSERT_OK(itr->status());
     delete itr;
 
     HistogramData hist_data;
@@ -3511,6 +3514,8 @@ static bool CompareIterators(int step, DB* model, DB* db,
       ok = false;
     }
   }
+  EXPECT_OK(miter->status());
+  EXPECT_OK(dbiter->status());
   (void)count;
   delete miter;
   delete dbiter;
@@ -6008,6 +6013,7 @@ TEST_F(DBTest, MergeTestTime) {
     ASSERT_OK(iter->status());
     ++count;
   }
+  ASSERT_OK(iter->status());
 
   ASSERT_EQ(1, count);
   ASSERT_EQ(4000000, TestGetTickerCount(options, MERGE_OPERATION_TOTAL_TIME));

db/db_test2.cc

@@ -1805,6 +1805,7 @@ TEST_P(CompressionFailuresTest, CompressionFailures) {
       ASSERT_EQ(key_value_written[key], value);
       key_value_written.erase(key);
     }
+    ASSERT_OK(db_iter->status());
     ASSERT_EQ(0, key_value_written.size());
   } else if (compression_failure_type_ == kTestDecompressionFail) {
     ASSERT_EQ(std::string(s.getState()),
@@ -3801,6 +3802,7 @@ TEST_F(DBTest2, MemtableOnlyIterator) {
     count++;
   }
   ASSERT_TRUE(!it->Valid());
+  ASSERT_OK(it->status());
   ASSERT_EQ(2, count);
   delete it;
 
@@ -7588,6 +7590,7 @@ TEST_F(DBTest2, BestEffortsRecoveryWithSstUniqueIdVerification) {
       ASSERT_EQ(std::to_string(cnt), it->key());
       ASSERT_EQ(expected_v, it->value());
     }
+    EXPECT_OK(it->status());
     ASSERT_EQ(expected_count, cnt);
   };
 

db/db_test_util.cc

@@ -943,6 +943,7 @@ std::string DBTestBase::Contents(int cf) {
     EXPECT_EQ(IterStatus(iter), forward[forward.size() - matched - 1]);
     matched++;
   }
+  EXPECT_OK(iter->status());
   EXPECT_EQ(matched, forward.size());
 
   delete iter;
@@ -1365,6 +1366,7 @@ std::string DBTestBase::IterStatus(Iterator* iter) {
   if (iter->Valid()) {
     result = iter->key().ToString() + "->" + iter->value().ToString();
   } else {
+    EXPECT_OK(iter->status());
     result = "(invalid)";
   }
   return result;
@@ -1583,6 +1585,7 @@ void DBTestBase::VerifyDBFromMap(std::map<std::string, std::string> true_data,
       iter_cnt++;
       total_reads++;
     }
+    ASSERT_OK(iter->status());
     ASSERT_EQ(data_iter, true_data.end())
         << iter_cnt << " / " << true_data.size();
     delete iter;
@@ -1606,6 +1609,7 @@ void DBTestBase::VerifyDBFromMap(std::map<std::string, std::string> true_data,
       iter_cnt++;
       total_reads++;
     }
+    ASSERT_OK(iter->status());
     ASSERT_EQ(data_rev, true_data.rend())
         << iter_cnt << " / " << true_data.size();
 

db/db_universal_compaction_test.cc

@@ -1469,6 +1469,7 @@ TEST_P(DBTestUniversalCompaction, IncreaseUniversalCompactionNumLevels) {
       keys_in_db.append(iter->key().ToString());
       keys_in_db.push_back(',');
     }
+    EXPECT_OK(iter->status());
     delete iter;
 
     std::string expected_keys;

db/db_wal_test.cc

@@ -2228,6 +2228,7 @@ TEST_P(DBWALTestWithParamsVaryingRecoveryMode,
       data.push_back(
           std::make_pair(iter->key().ToString(), iter->value().ToString()));
     }
+    EXPECT_OK(iter->status());
     delete iter;
     return data;
   };

db/db_with_timestamp_basic_test.cc

@@ -525,6 +525,7 @@ TEST_F(DBBasicTestWithTimestamp, SimpleIterate) {
       CheckIterUserEntry(it.get(), Key1(key), kTypeValue,
                          "value" + std::to_string(i), write_timestamps[i]);
     }
+    ASSERT_OK(it->status());
     ASSERT_EQ(static_cast<size_t>(kMaxKey) - start_keys[i] + 1, count);
 
     // SeekToFirst()/SeekToLast() with lower/upper bounds.
@@ -544,6 +545,7 @@ TEST_F(DBBasicTestWithTimestamp, SimpleIterate) {
         CheckIterUserEntry(it.get(), Key1(key), kTypeValue,
                            "value" + std::to_string(i), write_timestamps[i]);
       }
+      ASSERT_OK(it->status());
       ASSERT_EQ(r - std::max(l, start_keys[i]), count);
 
       for (it->SeekToLast(), key = std::min(r, kMaxKey + 1), count = 0;
@@ -551,6 +553,7 @@ TEST_F(DBBasicTestWithTimestamp, SimpleIterate) {
         CheckIterUserEntry(it.get(), Key1(key - 1), kTypeValue,
                            "value" + std::to_string(i), write_timestamps[i]);
       }
+      ASSERT_OK(it->status());
       l += (kMaxKey / 100);
       r -= (kMaxKey / 100);
     }
@@ -733,6 +736,7 @@ TEST_P(DBBasicTestWithTimestampTableOptions, GetAndMultiGet) {
       ASSERT_EQ(it->value(), value_from_get);
       ASSERT_EQ(Timestamp(1, 0), timestamp);
     }
+    ASSERT_OK(it->status());
 
     // verify MultiGet()
     constexpr uint64_t step = 2;
@@ -1065,6 +1069,7 @@ TEST_F(DBBasicTestWithTimestamp, SimpleForwardIterateLowerTsBound) {
                        write_timestamps[i - 1]);
       }
     }
+    ASSERT_OK(it->status());
     size_t expected_count = kMaxKey + 1;
     ASSERT_EQ(expected_count, count);
   }
@@ -1143,6 +1148,7 @@ TEST_F(DBBasicTestWithTimestamp, BackwardIterateLowerTsBound) {
                        write_timestamps[1]);
       }
     }
+    ASSERT_OK(it->status());
     size_t expected_count = kMaxKey + 1;
     ASSERT_EQ(expected_count, count);
   }
@@ -1173,6 +1179,7 @@ TEST_F(DBBasicTestWithTimestamp, BackwardIterateLowerTsBound) {
       CheckIterEntry(it.get(), Key1(key), kTypeDeletionWithTimestamp, Slice(),
                      write_timestamp);
     }
+    ASSERT_OK(it->status());
     ASSERT_EQ(kMaxKey + 1, count);
   }
   Close();
@@ -1278,6 +1285,7 @@ TEST_F(DBBasicTestWithTimestamp, BackwardIterateLowerTsBound_Reseek) {
       CheckIterEntry(it.get(), "a", kTypeValue, "v" + std::to_string(4 + i),
                      Timestamp(4 + i, 0));
     }
+    ASSERT_OK(it->status());
   }
 
   Close();
@@ -3145,6 +3153,7 @@ TEST_P(DBBasicTestWithTimestampPrefixSeek, IterateWithPrefix) {
                          "value" + std::to_string(i), write_ts_list[i]);
       iter->Next();
       ASSERT_FALSE(iter->Valid());
+      ASSERT_OK(iter->status());
 
       // Seek to kMinKey
       iter->Seek(Key1(kMinKey));
@@ -3152,6 +3161,7 @@ TEST_P(DBBasicTestWithTimestampPrefixSeek, IterateWithPrefix) {
                          "value" + std::to_string(i), write_ts_list[i]);
       iter->Prev();
       ASSERT_FALSE(iter->Valid());
+      ASSERT_OK(iter->status());
     }
     const std::vector<uint64_t> targets = {kMinKey, kMinKey + 0x10,
                                            kMinKey + 0x100, kMaxKey};
@@ -3190,6 +3200,7 @@ TEST_P(DBBasicTestWithTimestampPrefixSeek, IterateWithPrefix) {
           ++expected_key;
           it->Next();
         }
+        ASSERT_OK(it->status());
         ASSERT_EQ(expected_ub - targets[j] + 1, count);
 
         count = 0;
@@ -3208,6 +3219,7 @@ TEST_P(DBBasicTestWithTimestampPrefixSeek, IterateWithPrefix) {
           --expected_key;
           it->Prev();
         }
+        ASSERT_OK(it->status());
         ASSERT_EQ(targets[j] - std::max(expected_lb, kMinKey) + 1, count);
       }
     }
@@ -3313,6 +3325,7 @@ TEST_P(DBBasicTestWithTsIterTombstones, IterWithDelete) {
       ASSERT_EQ(Key1(key), iter->key());
       ASSERT_EQ("value1" + std::to_string(key), iter->value());
     }
+    ASSERT_OK(iter->status());
     ASSERT_EQ((kMaxKey - kMinKey + 1) / 2, count);
   }
   Close();
@@ -3932,6 +3945,7 @@ TEST_P(DBBasicTestWithTimestampTableOptions, DeleteRangeBaiscReadAndIterate) {
         ++expected;
       }
     }
+    ASSERT_OK(iter->status());
     ASSERT_EQ(kNum, expected);
 
     expected = kNum / 2;
@@ -3939,6 +3953,7 @@ TEST_P(DBBasicTestWithTimestampTableOptions, DeleteRangeBaiscReadAndIterate) {
       ASSERT_EQ(Key1(expected), iter->key());
       ++expected;
     }
+    ASSERT_OK(iter->status());
     ASSERT_EQ(kNum, expected);
 
     expected = kRangeBegin - 1;
@@ -3946,6 +3961,7 @@ TEST_P(DBBasicTestWithTimestampTableOptions, DeleteRangeBaiscReadAndIterate) {
       ASSERT_EQ(Key1(expected), iter->key());
       --expected;
     }
+    ASSERT_OK(iter->status());
     ASSERT_EQ(-1, expected);
 
     read_ts = Timestamp(0, 0);
@@ -4227,6 +4243,7 @@ TEST_F(DBBasicTestWithTimestamp, MergeBasic) {
         ASSERT_EQ(value, it->value());
         ASSERT_EQ(write_ts_strs[i], it->timestamp());
       }
+      EXPECT_OK(it->status());
       ASSERT_EQ(kNumOfUniqKeys, key_int_val);
 
       key_int_val = kNumOfUniqKeys - 1;
@@ -4238,6 +4255,7 @@ TEST_F(DBBasicTestWithTimestamp, MergeBasic) {
         ASSERT_EQ(value, it->value());
         ASSERT_EQ(write_ts_strs[i], it->timestamp());
       }
+      ASSERT_OK(it->status());
       ASSERT_EQ(std::numeric_limits<size_t>::max(), key_int_val);
 
       value_suffix = value_suffix + "." + std::to_string(i + 1);

db/deletefile_test.cc

@@ -575,6 +575,7 @@ TEST_F(DeleteFileTest, DeleteNonDefaultColumnFamily) {
       ASSERT_OK(itr->status());
       ++count;
     }
+    ASSERT_OK(itr->status());
     ASSERT_EQ(count, 1000);
   }
 
@@ -588,6 +589,7 @@ TEST_F(DeleteFileTest, DeleteNonDefaultColumnFamily) {
       ASSERT_OK(itr->status());
       ++count;
     }
+    ASSERT_OK(itr->status());
     ASSERT_EQ(count, 1000);
   }
 }

db/external_sst_file_test.cc

@@ -2623,6 +2623,7 @@ TEST_P(ExternalSSTFileTest,
       "AfterRead");
   ingest_thread.join();
   for (auto* iter : iters) {
+    ASSERT_OK(iter->status());
     delete iter;
   }
   iters.clear();

db/manual_compaction_test.cc

@@ -124,6 +124,7 @@ TEST_F(ManualCompactionTest, CompactTouchesAllKeys) {
     ASSERT_EQ("key3", itr->key().ToString());
     itr->Next();
     ASSERT_TRUE(!itr->Valid());
+    ASSERT_OK(itr->status());
     delete itr;
 
     delete options.compaction_filter;
@@ -179,6 +180,7 @@ TEST_F(ManualCompactionTest, Test) {
   for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
     num_keys++;
   }
+  ASSERT_OK(iter->status());
   delete iter;
   ASSERT_EQ(kNumKeys, num_keys) << "Bad number of keys";
 

db/perf_context_test.cc

@@ -149,6 +149,7 @@ TEST_F(PerfContextTest, SeekIntoDeletion) {
     ASSERT_TRUE(iter->Valid());
     StopWatchNano timer2(SystemClock::Default().get(), true);
     iter->Next();
+    ASSERT_OK(iter->status());
     auto elapsed_nanos2 = timer2.ElapsedNanos();
     if (FLAGS_verbose) {
       std::cout << "next cmp: " << get_perf_context()->user_key_comparison_count
@@ -1092,6 +1093,7 @@ TEST_F(PerfContextTest, MergeOperandCount) {
 
         get_perf_context()->Reset();
       }
+      ASSERT_OK(it->status());
     }
 
     // Backward iteration
@@ -1104,6 +1106,7 @@ TEST_F(PerfContextTest, MergeOperandCount) {
 
         get_perf_context()->Reset();
       }
+      ASSERT_OK(it->status());
     }
   };
 

db/plain_table_db_test.cc

@@ -897,6 +897,7 @@ TEST_P(PlainTableDBTest, IteratorLargeKeys) {
   }
 
   ASSERT_TRUE(!iter->Valid());
+  ASSERT_OK(iter->status());
 
   delete iter;
 }
@@ -945,6 +946,7 @@ TEST_P(PlainTableDBTest, IteratorLargeKeysWithPrefix) {
   }
 
   ASSERT_TRUE(!iter->Valid());
+  ASSERT_OK(iter->status());
 
   delete iter;
 }

db/prefix_test.cc

@@ -782,6 +782,7 @@ TEST_F(PrefixTest, PrefixSeekModePrev) {
         }
       }
     }
+    ASSERT_OK(iter->status());
   }
 }
 

file/prefetch_test.cc

@@ -261,6 +261,7 @@ TEST_P(PrefetchTest, Basic) {
     for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
       num_keys++;
     }
+    ASSERT_OK(iter->status());
     (void)num_keys;
   }
 
@@ -715,6 +716,7 @@ TEST_P(PrefetchTest, ConfigureInternalAutoReadaheadSize) {
         iter->Seek(Key(key_count++));
         iter->Next();
       }
+      ASSERT_OK(iter->status());
 
       buff_prefetch_level_count[level] = buff_prefetch_count;
       if (support_prefetch && !use_direct_io) {
@@ -1071,6 +1073,7 @@ TEST_P(PrefetchTest, PrefetchWhenReseek) {
     do {
       iter->Seek(BuildKey(i));
       if (!iter->Valid()) {
+        ASSERT_OK(iter->status());
         break;
       }
       i = i + 100;
@@ -1090,6 +1093,7 @@ TEST_P(PrefetchTest, PrefetchWhenReseek) {
     auto iter = std::unique_ptr<Iterator>(db_->NewIterator(ReadOptions()));
     for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
     }
+    ASSERT_OK(iter->status());
     if (support_prefetch && !use_direct_io) {
       ASSERT_EQ(fs->GetPrefetchCount(), 13);
       fs->ClearPrefetchCount();
@@ -1577,6 +1581,7 @@ TEST_P(PrefetchTest, DBIterLevelReadAhead) {
       ASSERT_OK(iter->status());
       num_keys++;
     }
+    ASSERT_OK(iter->status());
     ASSERT_EQ(num_keys, total_keys);
 
     // For index and data blocks.
@@ -1691,6 +1696,7 @@ TEST_P(PrefetchTest, DBIterLevelReadAheadWithAsyncIO) {
       ASSERT_OK(iter->status());
       num_keys++;
     }
+    ASSERT_OK(iter->status());
     ASSERT_EQ(num_keys, total_keys);
 
     // For index and data blocks.
@@ -1841,6 +1847,7 @@ TEST_P(PrefetchTest, DBIterAsyncIONoIOUring) {
       ASSERT_OK(iter->status());
       num_keys++;
     }
+    ASSERT_OK(iter->status());
     ASSERT_EQ(num_keys, total_keys);
 
     // Check stats to make sure async prefetch is done.
@@ -1868,6 +1875,7 @@ TEST_P(PrefetchTest, DBIterAsyncIONoIOUring) {
       ASSERT_OK(iter->status());
       num_keys++;
     }
+    ASSERT_OK(iter->status());
     ASSERT_EQ(num_keys, total_keys);
 
     // Check stats to make sure async prefetch is done.
@@ -2464,6 +2472,7 @@ TEST_P(PrefetchTest, IterReadAheadSizeWithUpperBound) {
           iter->Next();
           reseek_keys_with_tuning++;
         }
+        ASSERT_OK(iter->status());
 
         uint64_t readahead_trimmed =
             options.statistics->getAndResetTickerCount(READAHEAD_TRIMMED);
@@ -2508,6 +2517,7 @@ TEST_P(PrefetchTest, IterReadAheadSizeWithUpperBound) {
           iter->Next();
           reseek_keys_without_tuning++;
         }
+        ASSERT_OK(iter->status());
 
         uint64_t readahead_trimmed =
             options.statistics->getAndResetTickerCount(READAHEAD_TRIMMED);
@@ -2753,6 +2763,7 @@ TEST_P(PrefetchTest, ReadAsyncWithPosixFS) {
       ASSERT_OK(iter->status());
       num_keys++;
     }
+    ASSERT_OK(iter->status());
 
     if (read_async_called) {
       ASSERT_EQ(num_keys, total_keys);
@@ -3137,6 +3148,7 @@ TEST_P(PrefetchTest, TraceReadAsyncWithCallbackWrapper) {
       ASSERT_OK(iter->status());
       num_keys++;
     }
+    ASSERT_OK(iter->status());
 
     // End the tracing.
     ASSERT_OK(db_->EndIOTrace());

monitoring/stats_history_test.cc

@@ -206,6 +206,7 @@ TEST_F(StatsHistoryTest, InMemoryStatsHistoryPurging) {
   for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
     ASSERT_TRUE(iterator->key() == iterator->value());
   }
+  ASSERT_OK(iterator->status());
   delete iterator;
   ASSERT_OK(Flush());
   ASSERT_OK(Delete("sol"));
@@ -219,6 +220,7 @@ TEST_F(StatsHistoryTest, InMemoryStatsHistoryPurging) {
   for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
     ASSERT_TRUE(iterator->key() == iterator->value());
   }
+  ASSERT_OK(iterator->status());
   delete iterator;
   ASSERT_OK(Flush());
   ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
@@ -280,6 +282,7 @@ int countkeys(Iterator* iter) {
   for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
     count++;
   }
+  EXPECT_OK(iter->status());
   return count;
 }
 

table/iterator_wrapper.h

@@ -52,6 +52,17 @@ class IteratorWrapperBase {
 
   void DeleteIter(bool is_arena_mode) {
     if (iter_) {
+#ifdef ROCKSDB_ASSERT_STATUS_CHECKED
+      if (!status_checked_after_invalid_) {
+        // If this assertion fails, it is likely that you did not check
+        // iterator status after Valid() returns false.
+        fprintf(stderr,
+                "Failed to check status after Valid() returned false from this "
+                "iterator.\n");
+        port::PrintStack();
+        std::abort();
+      }
+#endif
       if (!is_arena_mode) {
         delete iter_;
       } else {
@@ -61,7 +72,12 @@ class IteratorWrapperBase {
   }
 
   // Iterator interface methods
-  bool Valid() const { return valid_; }
+  bool Valid() const {
+#ifdef ROCKSDB_ASSERT_STATUS_CHECKED
+    status_checked_after_invalid_ = valid_;
+#endif
+    return valid_;
+  }
   Slice key() const {
     assert(Valid());
     return result_.key;
@@ -72,6 +88,9 @@ class IteratorWrapperBase {
   }
   // Methods below require iter() != nullptr
   Status status() const {
+#ifdef ROCKSDB_ASSERT_STATUS_CHECKED
+    status_checked_after_invalid_ = true;
+#endif
     assert(iter_);
     return iter_->status();
   }
@@ -183,6 +202,10 @@ class IteratorWrapperBase {
   InternalIteratorBase<TValue>* iter_;
   IterateResult result_;
   bool valid_;
+
+#ifdef ROCKSDB_ASSERT_STATUS_CHECKED
+  mutable bool status_checked_after_invalid_ = true;
+#endif
 };
 
 using IteratorWrapper = IteratorWrapperBase<Slice>;

table/sst_file_reader_test.cc

@@ -304,6 +304,7 @@ class SstFileReaderTimestampTest : public testing::Test {
     }
 
     ASSERT_FALSE(iter->Valid());
+    ASSERT_OK(iter->status());
   }
 
  protected:

utilities/merge_operators/string_append/stringappend_test.cc

@@ -193,6 +193,7 @@ TEST_P(StringAppendOperatorTest, IteratorTest) {
       ASSERT_EQ(res, "a1,a2,a3");
     }
   }
+  ASSERT_OK(it->status());
 
   // Should release the snapshot and be aware of the new stuff now
   it.reset(db_->NewIterator(ReadOptions()));
@@ -217,6 +218,7 @@ TEST_P(StringAppendOperatorTest, IteratorTest) {
       ASSERT_EQ(res, "a1,a2,a3,a4");
     }
   }
+  ASSERT_OK(it->status());
 
   slists.Append("k3", "g1");
 
@@ -242,6 +244,7 @@ TEST_P(StringAppendOperatorTest, IteratorTest) {
       ASSERT_EQ(res, "g1");
     }
   }
+  ASSERT_OK(it->status());
 }
 
 TEST_P(StringAppendOperatorTest, SimpleTest) {

utilities/option_change_migration/option_change_migration_test.cc

@@ -87,6 +87,7 @@ TEST_P(DBOptionChangeMigrationTests, Migrate1) {
     for (; it->Valid(); it->Next()) {
       keys.insert(it->key().ToString());
     }
+    ASSERT_OK(it->status());
   }
   Close();
 
@@ -124,6 +125,7 @@ TEST_P(DBOptionChangeMigrationTests, Migrate1) {
       it->Next();
     }
     ASSERT_TRUE(!it->Valid());
+    ASSERT_OK(it->status());
   }
 }
 
@@ -165,6 +167,7 @@ TEST_P(DBOptionChangeMigrationTests, Migrate2) {
     for (; it->Valid(); it->Next()) {
       keys.insert(it->key().ToString());
     }
+    ASSERT_OK(it->status());
   }
 
   Close();
@@ -202,6 +205,7 @@ TEST_P(DBOptionChangeMigrationTests, Migrate2) {
       it->Next();
     }
     ASSERT_TRUE(!it->Valid());
+    ASSERT_OK(it->status());
   }
 }
 
@@ -249,6 +253,7 @@ TEST_P(DBOptionChangeMigrationTests, Migrate3) {
     for (; it->Valid(); it->Next()) {
       keys.insert(it->key().ToString());
     }
+    ASSERT_OK(it->status());
   }
   Close();
 
@@ -286,6 +291,7 @@ TEST_P(DBOptionChangeMigrationTests, Migrate3) {
       it->Next();
     }
     ASSERT_TRUE(!it->Valid());
+    ASSERT_OK(it->status());
   }
 }
 
@@ -333,6 +339,7 @@ TEST_P(DBOptionChangeMigrationTests, Migrate4) {
     for (; it->Valid(); it->Next()) {
       keys.insert(it->key().ToString());
     }
+    ASSERT_OK(it->status());
   }
 
   Close();
@@ -370,6 +377,7 @@ TEST_P(DBOptionChangeMigrationTests, Migrate4) {
       it->Next();
     }
     ASSERT_TRUE(!it->Valid());
+    ASSERT_OK(it->status());
   }
 }
 
@@ -509,6 +517,7 @@ TEST_F(DBOptionChangeMigrationTest, CompactedSrcToUniversal) {
     for (; it->Valid(); it->Next()) {
       keys.insert(it->key().ToString());
     }
+    ASSERT_OK(it->status());
   }
 
   Close();