2020-05-29 18:24:19 +00:00
#!/usr/bin/env python3
2019-04-18 17:51:19 +00:00
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
2020-03-25 03:57:53 +00:00
from __future__ import absolute_import , division , print_function , unicode_literals
2022-09-21 00:47:52 +00:00
import argparse
2013-03-13 06:20:14 +00:00
import os
2013-06-08 19:29:43 +00:00
import random
2014-03-20 18:11:08 +00:00
import shutil
2022-09-21 00:47:52 +00:00
import subprocess
import sys
import tempfile
import time
2013-03-13 06:20:14 +00:00
2015-10-19 18:43:14 +00:00
# params overwrite priority:
# for default:
2018-05-09 20:32:03 +00:00
# default_params < {blackbox,whitebox}_default_params < args
2015-10-19 18:43:14 +00:00
# for simple:
2018-05-09 20:32:03 +00:00
# default_params < {blackbox,whitebox}_default_params <
# simple_default_params <
# {blackbox,whitebox}_simple_default_params < args
2019-08-22 23:30:30 +00:00
# for cf_consistency:
2018-10-30 21:01:38 +00:00
# default_params < {blackbox,whitebox}_default_params <
2019-08-22 23:30:30 +00:00
# cf_consistency_params < args
2019-12-11 23:59:45 +00:00
# for txn:
# default_params < {blackbox,whitebox}_default_params < txn_params < args
2022-03-17 02:00:04 +00:00
# for ts:
# default_params < {blackbox,whitebox}_default_params < ts_params < args
# for multiops_txn:
# default_params < {blackbox,whitebox}_default_params < multiops_txn_params < args
2013-03-13 06:20:14 +00:00
2018-04-30 19:23:45 +00:00
2015-10-19 18:43:14 +00:00
default_params = {
2023-12-08 18:22:14 +00:00
" acquire_snapshot_one_in " : lambda : random . choice ( [ 100 , 10000 ] ) ,
2020-09-08 17:46:55 +00:00
" backup_max_size " : 100 * 1024 * 1024 ,
# Consider larger number when backups considered more stable
2023-12-08 18:22:14 +00:00
" backup_one_in " : lambda : random . choice ( [ 1000 , 100000 ] ) ,
Integrity protection for live updates to WriteBatch (#7748)
Summary:
This PR adds the foundation classes for key-value integrity protection and the first use case: protecting live updates from the source buffers added to `WriteBatch` through the destination buffer in `MemTable`. The width of the protection info is not yet configurable -- only eight bytes per key is supported. This PR allows users to enable protection by constructing `WriteBatch` with `protection_bytes_per_key == 8`. It does not yet expose a way for users to get integrity protection via other write APIs (e.g., `Put()`, `Merge()`, `Delete()`, etc.).
The foundation classes (`ProtectionInfo.*`) embed the coverage info in their type, and provide `Protect.*()` and `Strip.*()` functions to navigate between types with different coverage. For making bytes per key configurable (for powers of two up to eight) in the future, these classes are templated on the unsigned integer type used to store the protection info. That integer contains the XOR'd result of hashes with independent seeds for all covered fields. For integer fields, the hash is computed on the raw unadjusted bytes, so the result is endian-dependent. The most significant bytes are truncated when the hash value (8 bytes) is wider than the protection integer.
When `WriteBatch` is constructed with `protection_bytes_per_key == 8`, we hold a `ProtectionInfoKVOTC` (i.e., one that covers key, value, optype aka `ValueType`, timestamp, and CF ID) for each entry added to the batch. The protection info is generated from the original buffers passed by the user, as well as the original metadata generated internally. When writing to memtable, each entry is transformed to a `ProtectionInfoKVOTS` (i.e., dropping coverage of CF ID and adding coverage of sequence number), since at that point we know the sequence number, and have already selected a memtable corresponding to a particular CF. This protection info is verified once the entry is encoded in the `MemTable` buffer.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7748
Test Plan:
- an integration test to verify a wide variety of single-byte changes to the encoded `MemTable` buffer are caught
- add to stress/crash test to verify it works in variety of configs/operations without intentional corruption
- [deferred] unit tests for `ProtectionInfo.*` classes for edge cases like KV swap, `SliceParts` and `Slice` APIs are interchangeable, etc.
Reviewed By: pdillinger
Differential Revision: D25754492
Pulled By: ajkr
fbshipit-source-id: e481bac6c03c2ab268be41359730f1ceb9964866
2021-01-29 20:17:17 +00:00
" batch_protection_bytes_per_key " : lambda : random . choice ( [ 0 , 8 ] ) ,
2022-08-12 20:51:32 +00:00
" memtable_protection_bytes_per_key " : lambda : random . choice ( [ 0 , 1 , 2 , 4 , 8 ] ) ,
2023-04-25 19:08:23 +00:00
" block_protection_bytes_per_key " : lambda : random . choice ( [ 0 , 1 , 2 , 4 , 8 ] ) ,
2015-10-19 18:43:14 +00:00
" block_size " : 16384 ,
2022-09-21 00:47:52 +00:00
" bloom_bits " : lambda : random . choice (
[ random . randint ( 0 , 19 ) , random . lognormvariate ( 2.3 , 1.3 ) ]
) ,
2019-04-30 16:46:40 +00:00
" cache_index_and_filter_blocks " : lambda : random . randint ( 0 , 1 ) ,
2023-10-10 20:12:18 +00:00
" cache_size " : lambda : random . choice ( [ 8388608 , 33554432 ] ) ,
2022-05-17 22:01:51 +00:00
" charge_compression_dictionary_building_buffer " : lambda : random . choice ( [ 0 , 1 ] ) ,
" charge_filter_construction " : lambda : random . choice ( [ 0 , 1 ] ) ,
" charge_table_reader " : lambda : random . choice ( [ 0 , 1 ] ) ,
Account memory of FileMetaData in global memory limit (#9924)
Summary:
**Context/Summary:**
As revealed by heap profiling, allocation of `FileMetaData` for [newly created file added to a Version](https://github.com/facebook/rocksdb/pull/9924/files#diff-a6aa385940793f95a2c5b39cc670bd440c4547fa54fd44622f756382d5e47e43R774) can consume significant heap memory. This PR is to account that toward our global memory limit based on block cache capacity.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9924
Test Plan:
- Previous `make check` verified there are only 2 places where the memory of the allocated `FileMetaData` can be released
- New unit test `TEST_P(ChargeFileMetadataTestWithParam, Basic)`
- db bench (CPU cost of `charge_file_metadata` in write and compact)
- **write micros/op: -0.24%** : `TEST_TMPDIR=/dev/shm/testdb ./db_bench -benchmarks=fillseq -db=$TEST_TMPDIR -charge_file_metadata=1 (remove this option for pre-PR) -disable_auto_compactions=1 -write_buffer_size=100000 -num=4000000 | egrep 'fillseq'`
- **compact micros/op -0.87%** : `TEST_TMPDIR=/dev/shm/testdb ./db_bench -benchmarks=fillseq -db=$TEST_TMPDIR -charge_file_metadata=1 -disable_auto_compactions=1 -write_buffer_size=100000 -num=4000000 -numdistinct=1000 && ./db_bench -benchmarks=compact -db=$TEST_TMPDIR -use_existing_db=1 -charge_file_metadata=1 -disable_auto_compactions=1 | egrep 'compact'`
table 1 - write
#-run | (pre-PR) avg micros/op | std micros/op | (post-PR) micros/op | std micros/op | change (%)
-- | -- | -- | -- | -- | --
10 | 3.9711 | 0.264408 | 3.9914 | 0.254563 | 0.5111933721
20 | 3.83905 | 0.0664488 | 3.8251 | 0.0695456 | -0.3633711465
40 | 3.86625 | 0.136669 | 3.8867 | 0.143765 | 0.5289363078
80 | 3.87828 | 0.119007 | 3.86791 | 0.115674 | **-0.2673865734**
160 | 3.87677 | 0.162231 | 3.86739 | 0.16663 | **-0.2419539978**
table 2 - compact
#-run | (pre-PR) avg micros/op | std micros/op | (post-PR) micros/op | std micros/op | change (%)
-- | -- | -- | -- | -- | --
10 | 2,399,650.00 | 96,375.80 | 2,359,537.00 | 53,243.60 | -1.67
20 | 2,410,480.00 | 89,988.00 | 2,433,580.00 | 91,121.20 | 0.96
40 | 2.41E+06 | 121811 | 2.39E+06 | 131525 | **-0.96**
80 | 2.40E+06 | 134503 | 2.39E+06 | 108799 | **-0.78**
- stress test: `python3 tools/db_crashtest.py blackbox --charge_file_metadata=1 --cache_size=1` killed as normal
Reviewed By: ajkr
Differential Revision: D36055583
Pulled By: hx235
fbshipit-source-id: b60eab94707103cb1322cf815f05810ef0232625
2022-06-14 20:06:40 +00:00
" charge_file_metadata " : lambda : random . choice ( [ 0 , 1 ] ) ,
2023-12-08 18:22:14 +00:00
" checkpoint_one_in " : lambda : random . choice ( [ 10000 , 1000000 ] ) ,
2019-12-16 23:24:26 +00:00
" compression_type " : lambda : random . choice (
2022-09-21 00:47:52 +00:00
[ " none " , " snappy " , " zlib " , " lz4 " , " lz4hc " , " xpress " , " zstd " ]
) ,
" bottommost_compression_type " : lambda : " disable "
if random . randint ( 0 , 1 ) == 0
else random . choice ( [ " none " , " snappy " , " zlib " , " lz4 " , " lz4hc " , " xpress " , " zstd " ] ) ,
" checksum_type " : lambda : random . choice (
[ " kCRC32c " , " kxxHash " , " kxxHash64 " , " kXXH3 " ]
) ,
2018-08-06 22:23:03 +00:00
" compression_max_dict_bytes " : lambda : 16384 * random . randint ( 0 , 1 ) ,
" compression_zstd_max_train_bytes " : lambda : 65536 * random . randint ( 0 , 1 ) ,
2020-05-08 03:50:14 +00:00
# Disabled compression_parallel_threads as the feature is not stable
# lambda: random.choice([1] * 9 + [4])
" compression_parallel_threads " : 1 ,
Limit buffering for collecting samples for compression dictionary (#7970)
Summary:
For dictionary compression, we need to collect some representative samples of the data to be compressed, which we use to either generate or train (when `CompressionOptions::zstd_max_train_bytes > 0`) a dictionary. Previously, the strategy was to buffer all the data blocks during flush, and up to the target file size during compaction. That strategy allowed us to randomly pick samples from as wide a range as possible that'd be guaranteed to land in a single output file.
However, some users try to make huge files in memory-constrained environments, where this strategy can cause OOM. This PR introduces an option, `CompressionOptions::max_dict_buffer_bytes`, that limits how much data blocks are buffered before we switch to unbuffered mode (which means creating the per-SST dictionary, writing out the buffered data, and compressing/writing new blocks as soon as they are built). It is not strict as we currently buffer more than just data blocks -- also keys are buffered. But it does make a step towards giving users predictable memory usage.
Related changes include:
- Changed sampling for dictionary compression to select unique data blocks when there is limited availability of data blocks
- Made use of `BlockBuilder::SwapAndReset()` to save an allocation+memcpy when buffering data blocks for building a dictionary
- Changed `ParseBoolean()` to accept an input containing characters after the boolean. This is necessary since, with this PR, a value for `CompressionOptions::enabled` is no longer necessarily the final component in the `CompressionOptions` string.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7970
Test Plan:
- updated `CompressionOptions` unit tests to verify limit is respected (to the extent expected in the current implementation) in various scenarios of flush/compaction to bottommost/non-bottommost level
- looked at jemalloc heap profiles right before and after switching to unbuffered mode during flush/compaction. Verified memory usage in buffering is proportional to the limit set.
Reviewed By: pdillinger
Differential Revision: D26467994
Pulled By: ajkr
fbshipit-source-id: 3da4ef9fba59974e4ef40e40c01611002c861465
2021-02-19 22:06:59 +00:00
" compression_max_dict_buffer_bytes " : lambda : ( 1 << random . randint ( 0 , 40 ) ) - 1 ,
Support using ZDICT_finalizeDictionary to generate zstd dictionary (#9857)
Summary:
An untrained dictionary is currently simply the concatenation of several samples. The ZSTD API, ZDICT_finalizeDictionary(), can improve such a dictionary's effectiveness at low cost. This PR changes how dictionary is created by calling the ZSTD ZDICT_finalizeDictionary() API instead of creating raw content dictionary (when max_dict_buffer_bytes > 0), and pass in all buffered uncompressed data blocks as samples.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9857
Test Plan:
#### db_bench test for cpu/memory of compression+decompression and space saving on synthetic data:
Set up: change the parameter [here](https://github.com/facebook/rocksdb/blob/fb9a167a55e0970b1ef6f67c1600c8d9c4c6114f/tools/db_bench_tool.cc#L1766) to 16384 to make synthetic data more compressible.
```
# linked local ZSTD with version 1.5.2
# DEBUG_LEVEL=0 ROCKSDB_NO_FBCODE=1 ROCKSDB_DISABLE_ZSTD=1 EXTRA_CXXFLAGS="-DZSTD_STATIC_LINKING_ONLY -DZSTD -I/data/users/changyubi/install/include/" EXTRA_LDFLAGS="-L/data/users/changyubi/install/lib/ -l:libzstd.a" make -j32 db_bench
dict_bytes=16384
train_bytes=1048576
echo "========== No Dictionary =========="
TEST_TMPDIR=/dev/shm ./db_bench -benchmarks=filluniquerandom,compact -num=10000000 -compression_type=zstd -compression_max_dict_bytes=0 -block_size=4096 -max_background_jobs=24 -memtablerep=vector -allow_concurrent_memtable_write=false -disable_wal=true -max_write_buffer_number=8 >/dev/null 2>&1
TEST_TMPDIR=/dev/shm /usr/bin/time ./db_bench -use_existing_db=true -benchmarks=compact -compression_type=zstd -compression_max_dict_bytes=0 -block_size=4096 2>&1 | grep elapsed
du -hc /dev/shm/dbbench/*sst | grep total
echo "========== Raw Content Dictionary =========="
TEST_TMPDIR=/dev/shm ./db_bench_main -benchmarks=filluniquerandom,compact -num=10000000 -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -block_size=4096 -max_background_jobs=24 -memtablerep=vector -allow_concurrent_memtable_write=false -disable_wal=true -max_write_buffer_number=8 >/dev/null 2>&1
TEST_TMPDIR=/dev/shm /usr/bin/time ./db_bench_main -use_existing_db=true -benchmarks=compact -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -block_size=4096 2>&1 | grep elapsed
du -hc /dev/shm/dbbench/*sst | grep total
echo "========== FinalizeDictionary =========="
TEST_TMPDIR=/dev/shm ./db_bench -benchmarks=filluniquerandom,compact -num=10000000 -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes -compression_use_zstd_dict_trainer=false -block_size=4096 -max_background_jobs=24 -memtablerep=vector -allow_concurrent_memtable_write=false -disable_wal=true -max_write_buffer_number=8 >/dev/null 2>&1
TEST_TMPDIR=/dev/shm /usr/bin/time ./db_bench -use_existing_db=true -benchmarks=compact -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes -compression_use_zstd_dict_trainer=false -block_size=4096 2>&1 | grep elapsed
du -hc /dev/shm/dbbench/*sst | grep total
echo "========== TrainDictionary =========="
TEST_TMPDIR=/dev/shm ./db_bench -benchmarks=filluniquerandom,compact -num=10000000 -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes -block_size=4096 -max_background_jobs=24 -memtablerep=vector -allow_concurrent_memtable_write=false -disable_wal=true -max_write_buffer_number=8 >/dev/null 2>&1
TEST_TMPDIR=/dev/shm /usr/bin/time ./db_bench -use_existing_db=true -benchmarks=compact -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes -block_size=4096 2>&1 | grep elapsed
du -hc /dev/shm/dbbench/*sst | grep total
# Result: TrainDictionary is much better on space saving, but FinalizeDictionary seems to use less memory.
# before compression data size: 1.2GB
dict_bytes=16384
max_dict_buffer_bytes = 1048576
space cpu/memory
No Dictionary 468M 14.93user 1.00system 0:15.92elapsed 100%CPU (0avgtext+0avgdata 23904maxresident)k
Raw Dictionary 251M 15.81user 0.80system 0:16.56elapsed 100%CPU (0avgtext+0avgdata 156808maxresident)k
FinalizeDictionary 236M 11.93user 0.64system 0:12.56elapsed 100%CPU (0avgtext+0avgdata 89548maxresident)k
TrainDictionary 84M 7.29user 0.45system 0:07.75elapsed 100%CPU (0avgtext+0avgdata 97288maxresident)k
```
#### Benchmark on 10 sample SST files for spacing saving and CPU time on compression:
FinalizeDictionary is comparable to TrainDictionary in terms of space saving, and takes less time in compression.
```
dict_bytes=16384
train_bytes=1048576
for sst_file in `ls ../temp/myrock-sst/`
do
echo "********** $sst_file **********"
echo "========== No Dictionary =========="
./sst_dump --file="../temp/myrock-sst/$sst_file" --command=recompress --compression_level_from=6 --compression_level_to=6 --compression_types=kZSTD
echo "========== Raw Content Dictionary =========="
./sst_dump --file="../temp/myrock-sst/$sst_file" --command=recompress --compression_level_from=6 --compression_level_to=6 --compression_types=kZSTD --compression_max_dict_bytes=$dict_bytes
echo "========== FinalizeDictionary =========="
./sst_dump --file="../temp/myrock-sst/$sst_file" --command=recompress --compression_level_from=6 --compression_level_to=6 --compression_types=kZSTD --compression_max_dict_bytes=$dict_bytes --compression_zstd_max_train_bytes=$train_bytes --compression_use_zstd_finalize_dict
echo "========== TrainDictionary =========="
./sst_dump --file="../temp/myrock-sst/$sst_file" --command=recompress --compression_level_from=6 --compression_level_to=6 --compression_types=kZSTD --compression_max_dict_bytes=$dict_bytes --compression_zstd_max_train_bytes=$train_bytes
done
010240.sst (Size/Time) 011029.sst 013184.sst 021552.sst 185054.sst 185137.sst 191666.sst 7560381.sst 7604174.sst 7635312.sst
No Dictionary 28165569 / 2614419 32899411 / 2976832 32977848 / 3055542 31966329 / 2004590 33614351 / 1755877 33429029 / 1717042 33611933 / 1776936 33634045 / 2771417 33789721 / 2205414 33592194 / 388254
Raw Content Dictionary 28019950 / 2697961 33748665 / 3572422 33896373 / 3534701 26418431 / 2259658 28560825 / 1839168 28455030 / 1846039 28494319 / 1861349 32391599 / 3095649 33772142 / 2407843 33592230 / 474523
FinalizeDictionary 27896012 / 2650029 33763886 / 3719427 33904283 / 3552793 26008225 / 2198033 28111872 / 1869530 28014374 / 1789771 28047706 / 1848300 32296254 / 3204027 33698698 / 2381468 33592344 / 517433
TrainDictionary 28046089 / 2740037 33706480 / 3679019 33885741 / 3629351 25087123 / 2204558 27194353 / 1970207 27234229 / 1896811 27166710 / 1903119 32011041 / 3322315 32730692 / 2406146 33608631 / 570593
```
#### Decompression/Read test:
With FinalizeDictionary/TrainDictionary, some data structure used for decompression are in stored in dictionary, so they are expected to be faster in terms of decompression/reads.
```
dict_bytes=16384
train_bytes=1048576
echo "No Dictionary"
TEST_TMPDIR=/dev/shm/ ./db_bench -benchmarks=filluniquerandom,compact -compression_type=zstd -compression_max_dict_bytes=0 > /dev/null 2>&1
TEST_TMPDIR=/dev/shm/ ./db_bench -use_existing_db=true -benchmarks=readrandom -cache_size=0 -compression_type=zstd -compression_max_dict_bytes=0 2>&1 | grep MB/s
echo "Raw Dictionary"
TEST_TMPDIR=/dev/shm/ ./db_bench -benchmarks=filluniquerandom,compact -compression_type=zstd -compression_max_dict_bytes=$dict_bytes > /dev/null 2>&1
TEST_TMPDIR=/dev/shm/ ./db_bench -use_existing_db=true -benchmarks=readrandom -cache_size=0 -compression_type=zstd -compression_max_dict_bytes=$dict_bytes 2>&1 | grep MB/s
echo "FinalizeDict"
TEST_TMPDIR=/dev/shm/ ./db_bench -benchmarks=filluniquerandom,compact -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes -compression_use_zstd_dict_trainer=false > /dev/null 2>&1
TEST_TMPDIR=/dev/shm/ ./db_bench -use_existing_db=true -benchmarks=readrandom -cache_size=0 -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes -compression_use_zstd_dict_trainer=false 2>&1 | grep MB/s
echo "Train Dictionary"
TEST_TMPDIR=/dev/shm/ ./db_bench -benchmarks=filluniquerandom,compact -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes > /dev/null 2>&1
TEST_TMPDIR=/dev/shm/ ./db_bench -use_existing_db=true -benchmarks=readrandom -cache_size=0 -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes 2>&1 | grep MB/s
No Dictionary
readrandom : 12.183 micros/op 82082 ops/sec 12.183 seconds 1000000 operations; 9.1 MB/s (1000000 of 1000000 found)
Raw Dictionary
readrandom : 12.314 micros/op 81205 ops/sec 12.314 seconds 1000000 operations; 9.0 MB/s (1000000 of 1000000 found)
FinalizeDict
readrandom : 9.787 micros/op 102180 ops/sec 9.787 seconds 1000000 operations; 11.3 MB/s (1000000 of 1000000 found)
Train Dictionary
readrandom : 9.698 micros/op 103108 ops/sec 9.699 seconds 1000000 operations; 11.4 MB/s (1000000 of 1000000 found)
```
Reviewed By: ajkr
Differential Revision: D35720026
Pulled By: cbi42
fbshipit-source-id: 24d230fdff0fd28a1bb650658798f00dfcfb2a1f
2022-05-20 19:09:09 +00:00
" compression_use_zstd_dict_trainer " : lambda : random . randint ( 0 , 1 ) ,
Add `CompressionOptions::checksum` for enabling ZSTD checksum (#11666)
Summary:
Optionally enable zstd checksum flag (https://github.com/facebook/zstd/blob/d857369028d997c92ff1f1861a4d7f679a125464/lib/zstd.h#L428) to detect corruption during decompression. Main changes are in compression.h:
* User can set CompressionOptions::checksum to true to enable this feature.
* We enable this feature in ZSTD by setting the checksum flag in ZSTD compression context: `ZSTD_CCtx`.
* Uses `ZSTD_compress2()` to do compression since it supports frame parameter like the checksum flag. Compression level is also set in compression context as a flag.
* Error handling during decompression to propagate error message from ZSTD.
* Updated microbench to test read performance impact.
About compatibility, the current compression decoders should continue to work with the data created by the new compression API `ZSTD_compress2()`: https://github.com/facebook/zstd/issues/3711.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/11666
Test Plan:
* Existing unit tests for zstd compression
* Add unit test `DBTest2.ZSTDChecksum` to test the corruption case
* Manually tested that compression levels, parallel compression, dictionary compression, index compression all work with the new ZSTD_compress2() API.
* Manually tested with `sst_dump --command=recompress` that different compression levels and dictionary compression settings all work.
* Manually tested compiling with older versions of ZSTD: v1.3.8, v1.1.0, v0.6.2.
* Perf impact: from public benchmark data: http://fastcompression.blogspot.com/2019/03/presenting-xxh3.html for checksum and https://github.com/facebook/zstd#benchmarks, if decompression is 1700MB/s and checksum computation is 70000MB/s, checksum computation is an additional ~2.4% time for decompression. Compression is slower and checksumming should be less noticeable.
* Microbench:
```
TEST_TMPDIR=/dev/shm ./branch_db_basic_bench --benchmark_filter=DBGet/comp_style:0/max_data:1048576/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:0/mmap:0/compression_type:7/compression_checksum:1/no_blockcache:1/iterations:10000/threads:1 --benchmark_repetitions=100
Min out of 100 runs:
Main:
10390 10436 10456 10484 10499 10535 10544 10545 10565 10568
After this PR, checksum=false
10285 10397 10503 10508 10515 10557 10562 10635 10640 10660
After this PR, checksum=true
10827 10876 10925 10949 10971 11052 11061 11063 11100 11109
```
* db_bench:
```
Write perf
TEST_TMPDIR=/dev/shm/ ./db_bench_ichecksum --benchmarks=fillseq[-X10] --compression_type=zstd --num=10000000 --compression_checksum=..
[FillSeq checksum=0]
fillseq [AVG 10 runs] : 281635 (± 31711) ops/sec; 31.2 (± 3.5) MB/sec
fillseq [MEDIAN 10 runs] : 294027 ops/sec; 32.5 MB/sec
[FillSeq checksum=1]
fillseq [AVG 10 runs] : 286961 (± 34700) ops/sec; 31.7 (± 3.8) MB/sec
fillseq [MEDIAN 10 runs] : 283278 ops/sec; 31.3 MB/sec
Read perf
TEST_TMPDIR=/dev/shm ./db_bench_ichecksum --benchmarks=readrandom[-X20] --num=100000000 --reads=1000000 --use_existing_db=true --readonly=1
[Readrandom checksum=1]
readrandom [AVG 20 runs] : 360928 (± 3579) ops/sec; 4.0 (± 0.0) MB/sec
readrandom [MEDIAN 20 runs] : 362468 ops/sec; 4.0 MB/sec
[Readrandom checksum=0]
readrandom [AVG 20 runs] : 380365 (± 2384) ops/sec; 4.2 (± 0.0) MB/sec
readrandom [MEDIAN 20 runs] : 379800 ops/sec; 4.2 MB/sec
Compression
TEST_TMPDIR=/dev/shm ./db_bench_ichecksum --benchmarks=compress[-X20] --compression_type=zstd --num=100000000 --compression_checksum=1
checksum=1
compress [AVG 20 runs] : 54074 (± 634) ops/sec; 211.2 (± 2.5) MB/sec
compress [MEDIAN 20 runs] : 54396 ops/sec; 212.5 MB/sec
checksum=0
compress [AVG 20 runs] : 54598 (± 393) ops/sec; 213.3 (± 1.5) MB/sec
compress [MEDIAN 20 runs] : 54592 ops/sec; 213.3 MB/sec
Decompression:
TEST_TMPDIR=/dev/shm ./db_bench_ichecksum --benchmarks=uncompress[-X20] --compression_type=zstd --compression_checksum=1
checksum = 0
uncompress [AVG 20 runs] : 167499 (± 962) ops/sec; 654.3 (± 3.8) MB/sec
uncompress [MEDIAN 20 runs] : 167210 ops/sec; 653.2 MB/sec
checksum = 1
uncompress [AVG 20 runs] : 167980 (± 924) ops/sec; 656.2 (± 3.6) MB/sec
uncompress [MEDIAN 20 runs] : 168465 ops/sec; 658.1 MB/sec
```
Reviewed By: ajkr
Differential Revision: D48019378
Pulled By: cbi42
fbshipit-source-id: 674120c6e1853c2ced1436ac8138559d0204feba
2023-08-18 22:01:59 +00:00
" compression_checksum " : lambda : random . randint ( 0 , 1 ) ,
2018-04-30 19:23:45 +00:00
" clear_column_family_one_in " : 0 ,
2023-12-08 18:22:14 +00:00
" compact_files_one_in " : lambda : random . choice ( [ 1000 , 1000000 ] ) ,
" compact_range_one_in " : lambda : random . choice ( [ 1000 , 1000000 ] ) ,
2022-07-01 05:56:58 +00:00
" compaction_pri " : random . randint ( 0 , 4 ) ,
2022-06-21 23:23:58 +00:00
" data_block_index_type " : lambda : random . choice ( [ 0 , 1 ] ) ,
2018-12-18 21:30:56 +00:00
" delpercent " : 4 ,
" delrangepercent " : 1 ,
2015-10-19 18:43:14 +00:00
" destroy_db_initially " : 0 ,
2019-10-30 01:15:08 +00:00
" enable_pipelined_write " : lambda : random . randint ( 0 , 1 ) ,
2020-06-18 16:51:14 +00:00
" enable_compaction_filter " : lambda : random . choice ( [ 0 , 0 , 0 , 1 ] ) ,
2021-09-28 21:12:23 +00:00
" expected_values_dir " : lambda : setup_expected_values_dir ( ) ,
2021-05-05 19:53:42 +00:00
" fail_if_options_file_error " : lambda : random . randint ( 0 , 1 ) ,
2023-12-08 18:22:14 +00:00
" flush_one_in " : lambda : random . choice ( [ 1000 , 1000000 ] ) ,
" manual_wal_flush_one_in " : lambda : random . choice ( [ 0 , 1000 ] ) ,
2020-09-04 06:49:27 +00:00
" file_checksum_impl " : lambda : random . choice ( [ " none " , " crc32c " , " xxh64 " , " big " ] ) ,
2023-12-08 18:22:14 +00:00
" get_live_files_one_in " : lambda : random . choice ( [ 10000 , 1000000 ] ) ,
2020-03-19 00:11:06 +00:00
# Note: the following two are intentionally disabled as the corresponding
# APIs are not guaranteed to succeed.
" get_sorted_wal_files_one_in " : 0 ,
" get_current_wal_file_one_in " : 0 ,
2020-01-23 17:07:50 +00:00
# Temporarily disable hash index
2020-04-20 19:55:13 +00:00
" index_type " : lambda : random . choice ( [ 0 , 0 , 0 , 2 , 2 , 3 ] ) ,
2023-12-08 18:22:14 +00:00
" ingest_external_file_one_in " : lambda : random . choice ( [ 1000 , 1000000 ] ) ,
2020-06-18 16:51:14 +00:00
" iterpercent " : 10 ,
2023-12-08 18:22:14 +00:00
" lock_wal_one_in " : lambda : random . choice ( [ 10000 , 1000000 ] ) ,
2020-08-10 23:16:19 +00:00
" mark_for_compaction_one_file_in " : lambda : 10 * random . randint ( 0 , 1 ) ,
2015-10-19 18:43:14 +00:00
" max_background_compactions " : 20 ,
" max_bytes_for_level_base " : 10485760 ,
2023-12-17 18:46:26 +00:00
# max_key has to be the same across invocations for verification to work, hence no lambda
" max_key " : random . choice ( [ 100000 , 25000000 ] ) ,
2024-03-18 16:05:11 +00:00
" max_sequential_skip_in_iterations " : lambda : random . choice ( [ 1 , 2 , 8 , 16 ] ) ,
2015-10-19 18:43:14 +00:00
" max_write_buffer_number " : 3 ,
" mmap_read " : lambda : random . randint ( 0 , 1 ) ,
2021-10-11 23:22:10 +00:00
# Setting `nooverwritepercent > 0` is only possible because we do not vary
# the random seed, so the same keys are chosen by every run for disallowing
# overwrites.
2018-04-03 22:20:30 +00:00
" nooverwritepercent " : 1 ,
2022-09-21 00:47:52 +00:00
" open_files " : lambda : random . choice ( [ - 1 , - 1 , 100 , 500000 ] ) ,
Minimize memory internal fragmentation for Bloom filters (#6427)
Summary:
New experimental option BBTO::optimize_filters_for_memory builds
filters that maximize their use of "usable size" from malloc_usable_size,
which is also used to compute block cache charges.
Rather than always "rounding up," we track state in the
BloomFilterPolicy object to mix essentially "rounding down" and
"rounding up" so that the average FP rate of all generated filters is
the same as without the option. (YMMV as heavily accessed filters might
be unluckily lower accuracy.)
Thus, the option near-minimizes what the block cache considers as
"memory used" for a given target Bloom filter false positive rate and
Bloom filter implementation. There are no forward or backward
compatibility issues with this change, though it only works on the
format_version=5 Bloom filter.
With Jemalloc, we see about 10% reduction in memory footprint (and block
cache charge) for Bloom filters, but 1-2% increase in storage footprint,
due to encoding efficiency losses (FP rate is non-linear with bits/key).
Why not weighted random round up/down rather than state tracking? By
only requiring malloc_usable_size, we don't actually know what the next
larger and next smaller usable sizes for the allocator are. We pick a
requested size, accept and use whatever usable size it has, and use the
difference to inform our next choice. This allows us to narrow in on the
right balance without tracking/predicting usable sizes.
Why not weight history of generated filter false positive rates by
number of keys? This could lead to excess skew in small filters after
generating a large filter.
Results from filter_bench with jemalloc (irrelevant details omitted):
(normal keys/filter, but high variance)
$ ./filter_bench -quick -impl=2 -average_keys_per_filter=30000 -vary_key_count_ratio=0.9
Build avg ns/key: 29.6278
Number of filters: 5516
Total size (MB): 200.046
Reported total allocated memory (MB): 220.597
Reported internal fragmentation: 10.2732%
Bits/key stored: 10.0097
Average FP rate %: 0.965228
$ ./filter_bench -quick -impl=2 -average_keys_per_filter=30000 -vary_key_count_ratio=0.9 -optimize_filters_for_memory
Build avg ns/key: 30.5104
Number of filters: 5464
Total size (MB): 200.015
Reported total allocated memory (MB): 200.322
Reported internal fragmentation: 0.153709%
Bits/key stored: 10.1011
Average FP rate %: 0.966313
(very few keys / filter, optimization not as effective due to ~59 byte
internal fragmentation in blocked Bloom filter representation)
$ ./filter_bench -quick -impl=2 -average_keys_per_filter=1000 -vary_key_count_ratio=0.9
Build avg ns/key: 29.5649
Number of filters: 162950
Total size (MB): 200.001
Reported total allocated memory (MB): 224.624
Reported internal fragmentation: 12.3117%
Bits/key stored: 10.2951
Average FP rate %: 0.821534
$ ./filter_bench -quick -impl=2 -average_keys_per_filter=1000 -vary_key_count_ratio=0.9 -optimize_filters_for_memory
Build avg ns/key: 31.8057
Number of filters: 159849
Total size (MB): 200
Reported total allocated memory (MB): 208.846
Reported internal fragmentation: 4.42297%
Bits/key stored: 10.4948
Average FP rate %: 0.811006
(high keys/filter)
$ ./filter_bench -quick -impl=2 -average_keys_per_filter=1000000 -vary_key_count_ratio=0.9
Build avg ns/key: 29.7017
Number of filters: 164
Total size (MB): 200.352
Reported total allocated memory (MB): 221.5
Reported internal fragmentation: 10.5552%
Bits/key stored: 10.0003
Average FP rate %: 0.969358
$ ./filter_bench -quick -impl=2 -average_keys_per_filter=1000000 -vary_key_count_ratio=0.9 -optimize_filters_for_memory
Build avg ns/key: 30.7131
Number of filters: 160
Total size (MB): 200.928
Reported total allocated memory (MB): 200.938
Reported internal fragmentation: 0.00448054%
Bits/key stored: 10.1852
Average FP rate %: 0.963387
And from db_bench (block cache) with jemalloc:
$ ./db_bench -db=/dev/shm/dbbench.no_optimize -benchmarks=fillrandom -format_version=5 -value_size=90 -bloom_bits=10 -num=2000000 -threads=8 -compaction_style=2 -fifo_compaction_max_table_files_size_mb=10000 -fifo_compaction_allow_compaction=false
$ ./db_bench -db=/dev/shm/dbbench -benchmarks=fillrandom -format_version=5 -value_size=90 -bloom_bits=10 -num=2000000 -threads=8 -optimize_filters_for_memory -compaction_style=2 -fifo_compaction_max_table_files_size_mb=10000 -fifo_compaction_allow_compaction=false
$ (for FILE in /dev/shm/dbbench.no_optimize/*.sst; do ./sst_dump --file=$FILE --show_properties | grep 'filter block' ; done) | awk '{ t += $4; } END { print t; }'
17063835
$ (for FILE in /dev/shm/dbbench/*.sst; do ./sst_dump --file=$FILE --show_properties | grep 'filter block' ; done) | awk '{ t += $4; } END { print t; }'
17430747
$ #^ 2.1% additional filter storage
$ ./db_bench -db=/dev/shm/dbbench.no_optimize -use_existing_db -benchmarks=readrandom,stats -statistics -bloom_bits=10 -num=2000000 -compaction_style=2 -fifo_compaction_max_table_files_size_mb=10000 -fifo_compaction_allow_compaction=false -duration=10 -cache_index_and_filter_blocks -cache_size=1000000000
rocksdb.block.cache.index.add COUNT : 33
rocksdb.block.cache.index.bytes.insert COUNT : 8440400
rocksdb.block.cache.filter.add COUNT : 33
rocksdb.block.cache.filter.bytes.insert COUNT : 21087528
rocksdb.bloom.filter.useful COUNT : 4963889
rocksdb.bloom.filter.full.positive COUNT : 1214081
rocksdb.bloom.filter.full.true.positive COUNT : 1161999
$ #^ 1.04 % observed FP rate
$ ./db_bench -db=/dev/shm/dbbench -use_existing_db -benchmarks=readrandom,stats -statistics -bloom_bits=10 -num=2000000 -compaction_style=2 -fifo_compaction_max_table_files_size_mb=10000 -fifo_compaction_allow_compaction=false -optimize_filters_for_memory -duration=10 -cache_index_and_filter_blocks -cache_size=1000000000
rocksdb.block.cache.index.add COUNT : 33
rocksdb.block.cache.index.bytes.insert COUNT : 8448592
rocksdb.block.cache.filter.add COUNT : 33
rocksdb.block.cache.filter.bytes.insert COUNT : 18220328
rocksdb.bloom.filter.useful COUNT : 5360933
rocksdb.bloom.filter.full.positive COUNT : 1321315
rocksdb.bloom.filter.full.true.positive COUNT : 1262999
$ #^ 1.08 % observed FP rate, 13.6% less memory usage for filters
(Due to specific key density, this example tends to generate filters that are "worse than average" for internal fragmentation. "Better than average" cases can show little or no improvement.)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6427
Test Plan: unit test added, 'make check' with gcc, clang and valgrind
Reviewed By: siying
Differential Revision: D22124374
Pulled By: pdillinger
fbshipit-source-id: f3e3aa152f9043ddf4fae25799e76341d0d8714e
2020-06-22 20:30:57 +00:00
" optimize_filters_for_memory " : lambda : random . randint ( 0 , 1 ) ,
2019-10-14 17:33:31 +00:00
" partition_filters " : lambda : random . randint ( 0 , 1 ) ,
2020-10-11 21:52:49 +00:00
" partition_pinning " : lambda : random . randint ( 0 , 3 ) ,
2023-12-08 18:22:14 +00:00
" pause_background_one_in " : lambda : random . choice ( [ 10000 , 1000000 ] ) ,
2022-09-21 00:47:52 +00:00
" prefix_size " : lambda : random . choice ( [ - 1 , 1 , 5 , 7 , 8 ] ) ,
2015-10-19 18:43:14 +00:00
" prefixpercent " : 5 ,
" progress_reports " : 0 ,
" readpercent " : 45 ,
2019-03-15 18:58:37 +00:00
" recycle_log_file_num " : lambda : random . randint ( 0 , 1 ) ,
2017-10-20 22:17:03 +00:00
" snapshot_hold_ops " : 100000 ,
2020-06-23 23:24:15 +00:00
" sst_file_manager_bytes_per_sec " : lambda : random . choice ( [ 0 , 104857600 ] ) ,
" sst_file_manager_bytes_per_truncate " : lambda : random . choice ( [ 0 , 1048576 ] ) ,
2019-12-14 23:17:05 +00:00
" long_running_snapshots " : lambda : random . randint ( 0 , 1 ) ,
2018-05-09 20:32:03 +00:00
" subcompactions " : lambda : random . randint ( 1 , 4 ) ,
2023-12-17 18:46:26 +00:00
" target_file_size_base " : lambda : random . choice ( [ 512 * 1024 , 2048 * 1024 ] ) ,
2015-10-19 18:43:14 +00:00
" target_file_size_multiplier " : 2 ,
2022-10-14 01:00:30 +00:00
" test_batches_snapshots " : random . randint ( 0 , 1 ) ,
2020-10-11 21:52:49 +00:00
" top_level_index_pinning " : lambda : random . randint ( 0 , 3 ) ,
" unpartitioned_pinning " : lambda : random . randint ( 0 , 3 ) ,
2018-06-01 23:33:07 +00:00
" use_direct_reads " : lambda : random . randint ( 0 , 1 ) ,
" use_direct_io_for_flush_and_compaction " : lambda : random . randint ( 0 , 1 ) ,
2020-04-25 06:58:13 +00:00
" mock_direct_io " : False ,
Placeholder for AutoHyperClockCache, more (#11692)
Summary:
* The plan is for AutoHyperClockCache to be selected when HyperClockCacheOptions::estimated_entry_charge == 0, and in that case to use a new configuration option min_avg_entry_charge for determining an extreme case maximum size for the hash table. For the placeholder, a hack is in place in HyperClockCacheOptions::MakeSharedCache() to make the unit tests happy despite the new options not really making sense with the current implementation.
* Mostly updating and refactoring tests to test both the current HCC (internal name FixedHyperClockCache) and a placeholder for the new version (internal name AutoHyperClockCache).
* Simplify some existing tests not to depend directly on cache type.
* Type-parameterize the shard-level unit tests, which unfortunately requires more syntax like `this->` in places for disambiguation.
* Added means of choosing auto_hyper_clock_cache to cache_bench, db_bench, and db_stress, including add to crash test.
* Add another templated class BaseHyperClockCache to reduce future copy-paste
* Added ReportProblems support to cache_bench
* Added a DEBUG-level diagnostic to ReportProblems for the variance in load factor throughout the table, which will become more of a concern with linear hashing to be used in the Auto implementation. Example with current Fixed HCC:
```
2023/08/10-13:41:41.602450 6ac36 [DEBUG] [che/clock_cache.cc:1507] Slot occupancy stats: Overall 49% (129008/262144), Min/Max/Window = 39%/60%/500, MaxRun{Pos/Neg} = 18/17
```
In other words, with overall occupancy of 49%, the lowest across any 500 contiguous cells is 39% and highest 60%. Longest run of occupied is 18 and longest run of unoccupied is 17. This seems consistent with random samples from a uniform distribution.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/11692
Test Plan: Shouldn't be any meaningful changes yet to production code or to what is tested, but there is temporary redundancy in testing until the new implementation is plugged in.
Reviewed By: jowlyzhang
Differential Revision: D48247413
Pulled By: pdillinger
fbshipit-source-id: 11541f996d97af403c2e43c92fb67ff22dd0b5da
2023-08-11 23:27:38 +00:00
" cache_type " : lambda : random . choice (
AutoHCC: fix a bug with "blind" Insert (#12046)
Summary:
I have finally tracked down and fixed a bug affecting AutoHCC that was causing CI crash test assertion failures in AutoHCC when using secondary cache, but I was only able to reproduce locally a couple of times, after very long runs/repetitions.
It turns out that the essential feature used by secondary cache to trigger the bug is Insert without keeping a handle, which is otherwise rarely used in RocksDB and not incorporated into cache_bench (also used for targeted correctness stress testing) until this change (new option `-blind_insert_percent`).
The problem was in copying some logic from FixedHCC that makes the entry "sharable" but unreferenced once populated, if no reference is to be saved. The problem in AutoHCC is that we can only add the entry to a chain after it is in the sharable state, and must be removed from the chain while in the "under (de)construction" state and before it is back in the "empty" state. Also, it is possible for Lookup to find entries that are not connected to any chain, by design for efficiency, and for Release to erase_if_last_ref. Therefore, we could have
* Thread 1 starts to Insert a cache entry without keeping ref, and pauses before adding to the chain.
* Thread 2 finds it with Lookup optimizations, and then does Release with `erase_if_last_ref=true` causing it to trigger erasure on the entry. It successfully locks the home chain for the entry and purges any entries pending erasure. It is OK that this entry is not found on the chain, as another thread is allowed to remove it from the chain before we are able to (but after is it marked for (de)construction). And after the purge of the chain, the entry is marked empty.
* Thread 1 resumes in adding the slot (presumed entry) to the home chain for what was being inserted, but that now violates invariants and sets up a race or double-chain-reference as another thread could insert a new entry in the slot and try to insert into a different chain.
This is easily fixed by holding on to a reference until inserted onto the chain.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/12046
Test Plan:
As I don't have a reliable local reproducer, I triggered 20 runs of internal CI on fbcode_blackbox_crash_test that were previously failing in AutoHCC with about 1/3 probability, and they all passed.
Also re-enabling AutoHCC in the crash test with this change. (Revert https://github.com/facebook/rocksdb/issues/12000)
Reviewed By: jowlyzhang
Differential Revision: D51016979
Pulled By: pdillinger
fbshipit-source-id: 3840fb829d65b97c779d8aed62a4a4a433aeff2b
2023-11-07 00:06:01 +00:00
[ " lru_cache " , " fixed_hyper_clock_cache " , " auto_hyper_clock_cache " ,
" auto_hyper_clock_cache " , " tiered_lru_cache " ,
" tiered_fixed_hyper_clock_cache " , " tiered_auto_hyper_clock_cache " ,
" tiered_auto_hyper_clock_cache " ]
Placeholder for AutoHyperClockCache, more (#11692)
Summary:
* The plan is for AutoHyperClockCache to be selected when HyperClockCacheOptions::estimated_entry_charge == 0, and in that case to use a new configuration option min_avg_entry_charge for determining an extreme case maximum size for the hash table. For the placeholder, a hack is in place in HyperClockCacheOptions::MakeSharedCache() to make the unit tests happy despite the new options not really making sense with the current implementation.
* Mostly updating and refactoring tests to test both the current HCC (internal name FixedHyperClockCache) and a placeholder for the new version (internal name AutoHyperClockCache).
* Simplify some existing tests not to depend directly on cache type.
* Type-parameterize the shard-level unit tests, which unfortunately requires more syntax like `this->` in places for disambiguation.
* Added means of choosing auto_hyper_clock_cache to cache_bench, db_bench, and db_stress, including add to crash test.
* Add another templated class BaseHyperClockCache to reduce future copy-paste
* Added ReportProblems support to cache_bench
* Added a DEBUG-level diagnostic to ReportProblems for the variance in load factor throughout the table, which will become more of a concern with linear hashing to be used in the Auto implementation. Example with current Fixed HCC:
```
2023/08/10-13:41:41.602450 6ac36 [DEBUG] [che/clock_cache.cc:1507] Slot occupancy stats: Overall 49% (129008/262144), Min/Max/Window = 39%/60%/500, MaxRun{Pos/Neg} = 18/17
```
In other words, with overall occupancy of 49%, the lowest across any 500 contiguous cells is 39% and highest 60%. Longest run of occupied is 18 and longest run of unoccupied is 17. This seems consistent with random samples from a uniform distribution.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/11692
Test Plan: Shouldn't be any meaningful changes yet to production code or to what is tested, but there is temporary redundancy in testing until the new implementation is plugged in.
Reviewed By: jowlyzhang
Differential Revision: D48247413
Pulled By: pdillinger
fbshipit-source-id: 11541f996d97af403c2e43c92fb67ff22dd0b5da
2023-08-11 23:27:38 +00:00
) ,
2018-05-09 20:32:03 +00:00
" use_full_merge_v1 " : lambda : random . randint ( 0 , 1 ) ,
" use_merge " : lambda : random . randint ( 0 , 1 ) ,
Add the PutEntity API to the stress/crash tests (#10760)
Summary:
The patch adds the `PutEntity` API to the non-batched, batched, and
CF consistency stress tests. Namely, when the new `db_stress` command
line parameter `use_put_entity_one_in` is greater than zero, one in
N writes on average is performed using `PutEntity` rather than `Put`.
The wide-column entity written has the generated value in its default
column; in addition, it contains up to three additional columns where
the original generated value is divided up between the column name and the
column value (with the column name containing the first k characters of
the generated value, and the column value containing the rest). Whether
`PutEntity` is used (and if so, how many columns the entity has) is completely
determined by the "value base" used to generate the value (that is, there is
no randomness involved). Assuming the same `use_put_entity_one_in` setting
is used across `db_stress` invocations, this enables us to reconstruct and
validate the entity during subsequent `db_stress` runs.
Note that `PutEntity` is currently incompatible with `Merge`, transactions, and
user-defined timestamps; these combinations are currently disabled/disallowed.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/10760
Test Plan: Ran some batched, non-batched, and CF consistency stress tests using the script.
Reviewed By: riversand963
Differential Revision: D39939032
Pulled By: ltamasi
fbshipit-source-id: eafdf124e95993fb7d73158e3b006d11819f7fa9
2022-09-30 18:11:07 +00:00
# use_put_entity_one_in has to be the same across invocations for verification to work, hence no lambda
" use_put_entity_one_in " : random . choice ( [ 0 ] * 7 + [ 1 , 5 , 10 ] ) ,
Add Bloom/Ribbon hybrid API support (#8679)
Summary:
This is essentially resurrection and fixing of the part of
https://github.com/facebook/rocksdb/issues/8198 that was reverted in https://github.com/facebook/rocksdb/issues/8212, using data added in https://github.com/facebook/rocksdb/issues/8246. Basically,
when configuring Ribbon filter, you can specify an LSM level before which
Bloom will be used instead of Ribbon. But Bloom is only considered for
Leveled and Universal compaction styles and file going into a known LSM
level. This way, SST file writer, FIFO compaction, etc. use Ribbon filter as
you would expect with NewRibbonFilterPolicy.
So that this can be controlled with a single int value and so that flushes
can be distinguished from intra-L0, we consider flush to go to level -1 for
the purposes of this option. (Explained in API comment.)
I also expect the most common and recommended Ribbon configuration to
use Bloom during flush, to minimize slowing down writes and because according
to my estimates, Ribbon only pays off if the structure lives in memory for
more than an hour. Thus, I have changed the default for NewRibbonFilterPolicy
to be this mild hybrid configuration. I don't really want to add something like
NewHybridFilterPolicy because at least the mild hybrid configuration (Bloom for
flush, Ribbon otherwise) should be considered a natural choice.
C APIs also updated, but because they don't support overloading,
rocksdb_filterpolicy_create_ribbon is kept pure ribbon for clarity and
rocksdb_filterpolicy_create_ribbon_hybrid must be called for a hybrid
configuration. While touching C API, I changed bits per key options from
int to double.
BuiltinFilterPolicy is needed so that LevelThresholdFilterPolicy doesn't inherit
unused fields from BloomFilterPolicy.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8679
Test Plan: new + updated tests, including crash test
Reviewed By: jay-zhuang
Differential Revision: D30445797
Pulled By: pdillinger
fbshipit-source-id: 6f5aeddfd6d79f7e55493b563c2d1d2d568892e1
2021-08-21 00:59:24 +00:00
# 999 -> use Bloom API
2023-09-15 22:46:10 +00:00
" bloom_before_level " : lambda : random . choice ( [ random . randint ( - 1 , 2 ) , random . randint ( - 1 , 10 ) , 0x7fffffff - 1 , 0x7fffffff ] ) ,
2022-01-31 21:19:55 +00:00
" value_size_mult " : 32 ,
2023-08-23 22:24:23 +00:00
" verification_only " : 0 ,
2015-10-19 18:43:14 +00:00
" verify_checksum " : 1 ,
2023-12-17 18:46:26 +00:00
" write_buffer_size " : lambda : random . choice ( [ 1024 * 1024 , 4 * 1024 * 1024 ] ) ,
2015-10-19 18:43:14 +00:00
" writepercent " : 35 ,
format_version=6 and context-aware block checksums (#9058)
Summary:
## Context checksum
All RocksDB checksums currently use 32 bits of checking
power, which should be 1 in 4 billion false negative (FN) probability (failing to
detect corruption). This is true for random corruptions, and in some cases
small corruptions are guaranteed to be detected. But some possible
corruptions, such as in storage metadata rather than storage payload data,
would have a much higher FN rate. For example:
* Data larger than one SST block is replaced by data from elsewhere in
the same or another SST file. Especially with block_align=true, the
probability of exact block size match is probably around 1 in 100, making
the FN probability around that same. Without `block_align=true` the
probability of same block start location is probably around 1 in 10,000,
for FN probability around 1 in a million.
To solve this problem in new format_version=6, we add "context awareness"
to block checksum checks. The stored and expected checksum value is
modified based on the block's position in the file and which file it is in. The
modifications are cleverly chosen so that, for example
* blocks within about 4GB of each other are guaranteed to use different context
* blocks that are offset by exactly some multiple of 4GiB are guaranteed to use
different context
* files generated by the same process are guaranteed to use different context
for the same offsets, until wrap-around after 2^32 - 1 files
Thus, with format_version=6, if a valid SST block and checksum is misplaced,
its checksum FN probability should be essentially ideal, 1 in 4B.
## Footer checksum
This change also adds checksum protection to the SST footer (with
format_version=6), for the first time without relying on whole file checksum.
To prevent a corruption of the format_version in the footer (e.g. 6 -> 5) to
defeat the footer checksum, we change much of the footer data format
including an "extended magic number" in format_version 6 that would be
interpreted as empty index and metaindex block handles in older footer
versions. We also change the encoding of handles to free up space for
other new data in footer.
## More detail: making space in footer
In order to keep footer the same size in format_version=6 (avoid change to IO
patterns), we have to free up some space for new data. We do this two ways:
* Metaindex block handle is encoded down to 4 bytes (from 10) by assuming
it immediately precedes the footer, and by assuming it is < 4GB.
* Index block handle is moved into metaindex. (I don't know why it was
in footer to begin with.)
## Performance
In case of small performance penalty, I've made a "pay as you go" optimization
to compensate: replace `MutableCFOptions` in BlockBasedTableBuilder::Rep
with the only field used in that structure after construction: `prefix_extractor`.
This makes the PR an overall performance improvement (results below).
Nevertheless I'm seeing essentially no difference going from fv=5 to fv=6,
even including that improvement for both. That's based on extreme case table
write performance testing, many files with many blocks. This is relatively
checksum intensive (small blocks) and salt generation intensive (small files).
```
(for I in `seq 1 100`; do TEST_TMPDIR=/dev/shm/dbbench2 ./db_bench -benchmarks=fillseq -memtablerep=vector -disable_wal=1 -allow_concurrent_memtable_write=false -num=3000000 -compaction_style=2 -fifo_compaction_max_table_files_size_mb=10000 -fifo_compaction_allow_compaction=0 -write_buffer_size=100000 -compression_type=none -block_size=1000; done) 2>&1 | grep micros/op | tee out
awk '{ tot += $5; n += 1; } END { print int(1.0 * tot / n) }' < out
```
Each value below is ops/s averaged over 100 runs, run simultaneously with competing
configuration for load fairness
Before -> after (both fv=5): 483530 -> 483673 (negligible)
Re-run 1: 480733 -> 485427 (1.0% faster)
Re-run 2: 483821 -> 484541 (0.1% faster)
Before (fv=5) -> after (fv=6): 482006 -> 485100 (0.6% faster)
Re-run 1: 482212 -> 485075 (0.6% faster)
Re-run 2: 483590 -> 484073 (0.1% faster)
After fv=5 -> after fv=6: 483878 -> 485542 (0.3% faster)
Re-run 1: 485331 -> 483385 (0.4% slower)
Re-run 2: 485283 -> 483435 (0.4% slower)
Re-run 3: 483647 -> 486109 (0.5% faster)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9058
Test Plan:
unit tests included (table_test, db_properties_test, salt in env_test). General DB tests
and crash test updated to test new format_version.
Also temporarily updated the default format version to 6 and saw some test failures. Almost all
were due to an inadvertent additional read in VerifyChecksum to verify the index block checksum,
though it's arguably a bug that VerifyChecksum does not appear to (re-)verify the index block
checksum, just assuming it was verified in opening the index reader (probably *usually* true but
probably not always true). Some other concerns about VerifyChecksum are left in FIXME
comments. The only remaining test failure on change of default (in block_fetcher_test) now
has a comment about how to upgrade the test.
The format compatibility test does not need updating because we have not updated the default
format_version.
Reviewed By: ajkr, mrambacher
Differential Revision: D33100915
Pulled By: pdillinger
fbshipit-source-id: 8679e3e572fa580181a737fd6d113ed53c5422ee
2023-07-30 23:40:01 +00:00
" format_version " : lambda : random . choice ( [ 2 , 3 , 4 , 5 , 6 , 6 ] ) ,
2018-09-19 19:08:42 +00:00
" index_block_restart_interval " : lambda : random . choice ( range ( 1 , 16 ) ) ,
2022-09-21 00:47:52 +00:00
" use_multiget " : lambda : random . randint ( 0 , 1 ) ,
2023-03-17 21:47:29 +00:00
" use_get_entity " : lambda : random . choice ( [ 0 ] * 7 + [ 1 ] ) ,
2023-03-30 03:35:15 +00:00
" use_multi_get_entity " : lambda : random . choice ( [ 0 ] * 7 + [ 1 ] ) ,
2022-09-21 00:47:52 +00:00
" periodic_compaction_seconds " : lambda : random . choice ( [ 0 , 0 , 1 , 2 , 10 , 100 , 1000 ] ) ,
2022-10-06 21:54:21 +00:00
# 0 = never (used by some), 10 = often (for threading bugs), 600 = default
" stats_dump_period_sec " : lambda : random . choice ( [ 0 , 10 , 600 ] ) ,
2022-09-21 00:47:52 +00:00
" compaction_ttl " : lambda : random . choice ( [ 0 , 0 , 1 , 2 , 10 , 100 , 1000 ] ) ,
2023-01-03 19:54:58 +00:00
" fifo_allow_compaction " : lambda : random . randint ( 0 , 1 ) ,
2019-11-14 21:59:43 +00:00
# Test small max_manifest_file_size in a smaller chance, as most of the
# time we wnat manifest history to be preserved to help debug
2022-09-21 00:47:52 +00:00
" max_manifest_file_size " : lambda : random . choice (
[ t * 16384 if t < 3 else 1024 * 1024 * 1024 for t in range ( 1 , 30 ) ]
) ,
2019-12-16 23:24:26 +00:00
# Sync mode might make test runs slower so running it in a smaller chance
2022-09-21 00:47:52 +00:00
" sync " : lambda : random . choice ( [ 1 if t == 0 else 0 for t in range ( 0 , 20 ) ] ) ,
2022-05-05 20:21:03 +00:00
" bytes_per_sync " : lambda : random . choice ( [ 0 , 262144 ] ) ,
" wal_bytes_per_sync " : lambda : random . choice ( [ 0 , 524288 ] ) ,
Fix crash_test_with_best_efforts_recovery (#11938)
Summary:
Thanks ltamasi and ajkr for initial investigations on the test failure. Per the investigations, the following scenario is likely causing the test to fail.
1. Recovery is needed (could be any reason during crash test)
2. Trying to recover from the latest manifest fails (likely due to read error injection)
3. DB opens with recovery from the next manifest which is different from step 2.
4. Expected state is based on the manifest we tried and failed in step 2.
5. Two manifests used in step 2 and 3 are confirmed to have difference in LSM trees (Thanks ltamasi again for the finding).
```
2023/10/05-11:24:18.942189 56341 [db/version_set.cc:6079] Trying to recover from manifest: /dev/shm/rocksdb_test/rocksdb_crashtest_blackbox/MANIFEST-007184
...
2023/10/05-11:24:18.978007 56341 [db/version_set.cc:6079] Trying to recover from manifest: /dev/shm/rocksdb_test/rocksdb_crashtest_blackbox/MANIFEST-007180
```
```
[ltamasi@devbig1024.prn1 /tmp/x]$ ldb manifest_dump --hex --path=MANIFEST-007184_renamed_ > 2
[ltamasi@devbig1024.prn1 /tmp/x]$ ldb manifest_dump --hex --path=MANIFEST-007180_renamed_ > 1
[ltamasi@devbig1024.prn1 /tmp/x]$ diff 1 2
--- 1 2023-10-09 10:29:16.966215207 -0700
+++ 2 2023-10-09 10:29:11.984241645 -0700
@@ -13,7 +13,7 @@
7174:3950254[1875617 .. 2203952]['000000000003415B000000000000012B000000000000007D' seq:1906214, type:1 .. '000000000003CA59000000000000012B000000000000005C' seq:2039838, type:1]
7175:88060[2074748 .. 2203892]['000000000003CA6300000000000000CF78787878787878' seq:2167539, type:2 .. '000000000003D08F000000000000012B0000000000000130' seq:2112478, type:0]
--- level 6 --- version# 1 ---
- 7057:3132633[0 .. 2046144]['0000000000000009000000000000000978' seq:0, type:1 .. '0000000000005F8B000000000000012B00000000000002AC' seq:0, type:1]
+ 7219:2135565[0 .. 2046144]['0000000000000009000000000000000978' seq:0, type:1 .. '0000000000005F8B000000000000012B00000000000002AC' seq:0, type:1]
7061:827724[0 .. 2046131]['0000000000005F95000000000000000778787878787878' seq:0, type:1 .. '000000000000784F000000000000012B0000000000000113' seq:0, type:1]
6763:1352[0 .. 0]['000000000000784F000000000000012B0000000000000129' seq:0, type:1 .. '000000000000784F000000000000012B0000000000000129' seq:0, type:1]
7173:4812291[0 .. 2203957]['000000000000784F000000000000012B0000000000000138' seq:0, type:1 .. '0000000000020FAE787878787878' seq:0, type:1]
@@ -77,4 +77,4 @@
--- level 61 --- version# 1 ---
--- level 62 --- version# 1 ---
--- level 63 --- version# 1 ---
-next_file_number 7182 last_sequence 2203963 prev_log_number 0 max_column_family 0 min_log_number_to_keep 7015
+next_file_number 7221 last_sequence 2203963 prev_log_number 0 max_column_family 0 min_log_number_to_keep 7015
```
We have two options to fix this. Either skip verification against expected state or disable read injection when BE recovery is enabled. I chose to skip verification against expected state per discussion. (See comments in this PR)
Please note that some linter changes were included in this PR.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/11938
Test Plan:
```
TEST_TMPDIR=/dev/shm/rocksdb make crash_test_with_best_efforts_recovery
```
Reviewed By: ltamasi
Differential Revision: D50136341
Pulled By: jaykorean
fbshipit-source-id: ac7434d592aebc148bfc3a4fcaa34936f136b95c
2023-10-11 21:26:10 +00:00
" compaction_readahead_size " : lambda : random . choice (
2023-09-05 17:41:29 +00:00
[ 0 , 0 , 1024 * 1024 ] ) ,
2022-09-21 00:47:52 +00:00
" db_write_buffer_size " : lambda : random . choice (
[ 0 , 0 , 0 , 1024 * 1024 , 8 * 1024 * 1024 , 128 * 1024 * 1024 ]
) ,
Fix crash_test_with_best_efforts_recovery (#11938)
Summary:
Thanks ltamasi and ajkr for initial investigations on the test failure. Per the investigations, the following scenario is likely causing the test to fail.
1. Recovery is needed (could be any reason during crash test)
2. Trying to recover from the latest manifest fails (likely due to read error injection)
3. DB opens with recovery from the next manifest which is different from step 2.
4. Expected state is based on the manifest we tried and failed in step 2.
5. Two manifests used in step 2 and 3 are confirmed to have difference in LSM trees (Thanks ltamasi again for the finding).
```
2023/10/05-11:24:18.942189 56341 [db/version_set.cc:6079] Trying to recover from manifest: /dev/shm/rocksdb_test/rocksdb_crashtest_blackbox/MANIFEST-007184
...
2023/10/05-11:24:18.978007 56341 [db/version_set.cc:6079] Trying to recover from manifest: /dev/shm/rocksdb_test/rocksdb_crashtest_blackbox/MANIFEST-007180
```
```
[ltamasi@devbig1024.prn1 /tmp/x]$ ldb manifest_dump --hex --path=MANIFEST-007184_renamed_ > 2
[ltamasi@devbig1024.prn1 /tmp/x]$ ldb manifest_dump --hex --path=MANIFEST-007180_renamed_ > 1
[ltamasi@devbig1024.prn1 /tmp/x]$ diff 1 2
--- 1 2023-10-09 10:29:16.966215207 -0700
+++ 2 2023-10-09 10:29:11.984241645 -0700
@@ -13,7 +13,7 @@
7174:3950254[1875617 .. 2203952]['000000000003415B000000000000012B000000000000007D' seq:1906214, type:1 .. '000000000003CA59000000000000012B000000000000005C' seq:2039838, type:1]
7175:88060[2074748 .. 2203892]['000000000003CA6300000000000000CF78787878787878' seq:2167539, type:2 .. '000000000003D08F000000000000012B0000000000000130' seq:2112478, type:0]
--- level 6 --- version# 1 ---
- 7057:3132633[0 .. 2046144]['0000000000000009000000000000000978' seq:0, type:1 .. '0000000000005F8B000000000000012B00000000000002AC' seq:0, type:1]
+ 7219:2135565[0 .. 2046144]['0000000000000009000000000000000978' seq:0, type:1 .. '0000000000005F8B000000000000012B00000000000002AC' seq:0, type:1]
7061:827724[0 .. 2046131]['0000000000005F95000000000000000778787878787878' seq:0, type:1 .. '000000000000784F000000000000012B0000000000000113' seq:0, type:1]
6763:1352[0 .. 0]['000000000000784F000000000000012B0000000000000129' seq:0, type:1 .. '000000000000784F000000000000012B0000000000000129' seq:0, type:1]
7173:4812291[0 .. 2203957]['000000000000784F000000000000012B0000000000000138' seq:0, type:1 .. '0000000000020FAE787878787878' seq:0, type:1]
@@ -77,4 +77,4 @@
--- level 61 --- version# 1 ---
--- level 62 --- version# 1 ---
--- level 63 --- version# 1 ---
-next_file_number 7182 last_sequence 2203963 prev_log_number 0 max_column_family 0 min_log_number_to_keep 7015
+next_file_number 7221 last_sequence 2203963 prev_log_number 0 max_column_family 0 min_log_number_to_keep 7015
```
We have two options to fix this. Either skip verification against expected state or disable read injection when BE recovery is enabled. I chose to skip verification against expected state per discussion. (See comments in this PR)
Please note that some linter changes were included in this PR.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/11938
Test Plan:
```
TEST_TMPDIR=/dev/shm/rocksdb make crash_test_with_best_efforts_recovery
```
Reviewed By: ltamasi
Differential Revision: D50136341
Pulled By: jaykorean
fbshipit-source-id: ac7434d592aebc148bfc3a4fcaa34936f136b95c
2023-10-11 21:26:10 +00:00
" use_write_buffer_manager " : lambda : random . randint ( 0 , 1 ) ,
2022-09-21 00:47:52 +00:00
" avoid_unnecessary_blocking_io " : random . randint ( 0 , 1 ) ,
" write_dbid_to_manifest " : random . randint ( 0 , 1 ) ,
" avoid_flush_during_recovery " : lambda : random . choice (
[ 1 if t == 0 else 0 for t in range ( 0 , 8 ) ]
) ,
" max_write_batch_group_size_bytes " : lambda : random . choice (
[ 16 , 64 , 1024 * 1024 , 16 * 1024 * 1024 ]
) ,
2023-03-27 21:55:16 +00:00
" level_compaction_dynamic_level_bytes " : lambda : random . randint ( 0 , 1 ) ,
2023-12-08 18:22:14 +00:00
" verify_checksum_one_in " : lambda : random . choice ( [ 100000 , 1000000 ] ) ,
" verify_file_checksums_one_in " : lambda : random . choice ( [ 100000 , 1000000 ] ) ,
" verify_db_one_in " : lambda : random . choice ( [ 10000 , 100000 ] ) ,
2022-09-21 00:47:52 +00:00
" continuous_verification_interval " : 0 ,
2020-01-10 05:25:40 +00:00
" max_key_len " : 3 ,
2020-04-11 00:18:56 +00:00
" key_len_percent_dist " : " 1,30,69 " ,
2021-09-21 21:47:09 +00:00
" read_fault_one_in " : lambda : random . choice ( [ 0 , 32 , 1000 ] ) ,
2023-10-11 18:28:00 +00:00
" write_fault_one_in " : lambda : random . choice ( [ 0 , 128 , 1000 ] ) ,
2021-07-06 18:04:04 +00:00
" open_metadata_write_fault_one_in " : lambda : random . choice ( [ 0 , 0 , 8 ] ) ,
" open_write_fault_one_in " : lambda : random . choice ( [ 0 , 0 , 16 ] ) ,
" open_read_fault_one_in " : lambda : random . choice ( [ 0 , 0 , 32 ] ) ,
2022-08-31 21:27:23 +00:00
" sync_fault_injection " : lambda : random . randint ( 0 , 1 ) ,
2023-12-08 18:22:14 +00:00
" get_property_one_in " : lambda : random . choice ( [ 100000 , 1000000 ] ) ,
2020-10-09 15:30:44 +00:00
" paranoid_file_checks " : lambda : random . choice ( [ 0 , 1 , 1 , 1 ] ) ,
2020-11-03 21:54:01 +00:00
" max_write_buffer_size_to_maintain " : lambda : random . choice (
2022-09-21 00:47:52 +00:00
[ 0 , 1024 * 1024 , 2 * 1024 * 1024 , 4 * 1024 * 1024 , 8 * 1024 * 1024 ]
) ,
Fix an error while running db_crashtest for non-user-ts tests (#8091)
Summary:
Fix the following error while running `make crash_test`
```
Traceback (most recent call last):
File "tools/db_crashtest.py", line 705, in <module>
main()
File "tools/db_crashtest.py", line 696, in main
blackbox_crash_main(args, unknown_args)
File "tools/db_crashtest.py", line 479, in blackbox_crash_main
+ list({'db': dbname}.items())), unknown_args)
File "tools/db_crashtest.py", line 414, in gen_cmd
finalzied_params = finalize_and_sanitize(params)
File "tools/db_crashtest.py", line 331, in finalize_and_sanitize
dest_params.get("user_timestamp_size") > 0):
TypeError: '>' not supported between instances of 'NoneType' and 'int'
```
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8091
Test Plan: make crash_test
Reviewed By: ltamasi
Differential Revision: D27268276
Pulled By: riversand963
fbshipit-source-id: ed2873b9587ecc51e24abc35ef2bd3d91fb1ed1b
2021-03-23 19:43:19 +00:00
" user_timestamp_size " : 0 ,
2022-09-21 00:47:52 +00:00
" secondary_cache_fault_one_in " : lambda : random . choice ( [ 0 , 0 , 32 ] ) ,
2023-10-10 20:12:18 +00:00
" compressed_secondary_cache_size " : lambda : random . choice ( [ 8388608 , 16777216 ] ) ,
2022-09-21 00:47:52 +00:00
" prepopulate_block_cache " : lambda : random . choice ( [ 0 , 1 ] ) ,
2022-01-27 22:53:39 +00:00
" memtable_prefix_bloom_size_ratio " : lambda : random . choice ( [ 0.001 , 0.01 , 0.1 , 0.5 ] ) ,
" memtable_whole_key_filtering " : lambda : random . randint ( 0 , 1 ) ,
Detect (new) Bloom/Ribbon Filter construction corruption (#9342)
Summary:
Note: rebase on and merge after https://github.com/facebook/rocksdb/pull/9349, https://github.com/facebook/rocksdb/pull/9345, (optional) https://github.com/facebook/rocksdb/pull/9393
**Context:**
(Quoted from pdillinger) Layers of information during new Bloom/Ribbon Filter construction in building block-based tables includes the following:
a) set of keys to add to filter
b) set of hashes to add to filter (64-bit hash applied to each key)
c) set of Bloom indices to set in filter, with duplicates
d) set of Bloom indices to set in filter, deduplicated
e) final filter and its checksum
This PR aims to detect corruption (e.g, unexpected hardware/software corruption on data structures residing in the memory for a long time) from b) to e) and leave a) as future works for application level.
- b)'s corruption is detected by verifying the xor checksum of the hash entries calculated as the entries accumulate before being added to the filter. (i.e, `XXPH3FilterBitsBuilder::MaybeVerifyHashEntriesChecksum()`)
- c) - e)'s corruption is detected by verifying the hash entries indeed exists in the constructed filter by re-querying these hash entries in the filter (i.e, `FilterBitsBuilder::MaybePostVerify()`) after computing the block checksum (except for PartitionFilter, which is done right after each `FilterBitsBuilder::Finish` for impl simplicity - see code comment for more). For this stage of detection, we assume hash entries are not corrupted after checking on b) since the time interval from b) to c) is relatively short IMO.
Option to enable this feature of detection is `BlockBasedTableOptions::detect_filter_construct_corruption` which is false by default.
**Summary:**
- Implemented new functions `XXPH3FilterBitsBuilder::MaybeVerifyHashEntriesChecksum()` and `FilterBitsBuilder::MaybePostVerify()`
- Ensured hash entries, final filter and banding and their [cache reservation ](https://github.com/facebook/rocksdb/issues/9073) are released properly despite corruption
- See [Filter.construction.artifacts.release.point.pdf ](https://github.com/facebook/rocksdb/files/7923487/Design.Filter.construction.artifacts.release.point.pdf) for high-level design
- Bundled and refactored hash entries's related artifact in XXPH3FilterBitsBuilder into `HashEntriesInfo` for better control on lifetime of these artifact during `SwapEntires`, `ResetEntries`
- Ensured RocksDB block-based table builder calls `FilterBitsBuilder::MaybePostVerify()` after constructing the filter by `FilterBitsBuilder::Finish()`
- When encountering such filter construction corruption, stop writing the filter content to files and mark such a block-based table building non-ok by storing the corruption status in the builder.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9342
Test Plan:
- Added new unit test `DBFilterConstructionCorruptionTestWithParam.DetectCorruption`
- Included this new feature in `DBFilterConstructionReserveMemoryTestWithParam.ReserveMemory` as this feature heavily touch ReserveMemory's impl
- For fallback case, I run `./filter_bench -impl=3 -detect_filter_construct_corruption=true -reserve_table_builder_memory=true -strict_capacity_limit=true -quick -runs 10 | grep 'Build avg'` to make sure nothing break.
- Added to `filter_bench`: increased filter construction time by **30%**, mostly by `MaybePostVerify()`
- FastLocalBloom
- Before change: `./filter_bench -impl=2 -quick -runs 10 | grep 'Build avg'`: **28.86643s**
- After change:
- `./filter_bench -impl=2 -detect_filter_construct_corruption=false -quick -runs 10 | grep 'Build avg'` (expect a tiny increase due to MaybePostVerify is always called regardless): **27.6644s (-4% perf improvement might be due to now we don't drop bloom hash entry in `AddAllEntries` along iteration but in bulk later, same with the bypassing-MaybePostVerify case below)**
- `./filter_bench -impl=2 -detect_filter_construct_corruption=true -quick -runs 10 | grep 'Build avg'` (expect acceptable increase): **34.41159s (+20%)**
- `./filter_bench -impl=2 -detect_filter_construct_corruption=true -quick -runs 10 | grep 'Build avg'` (by-passing MaybePostVerify, expect minor increase): **27.13431s (-6%)**
- Standard128Ribbon
- Before change: `./filter_bench -impl=3 -quick -runs 10 | grep 'Build avg'`: **122.5384s**
- After change:
- `./filter_bench -impl=3 -detect_filter_construct_corruption=false -quick -runs 10 | grep 'Build avg'` (expect a tiny increase due to MaybePostVerify is always called regardless - verified by removing MaybePostVerify under this case and found only +-1ns difference): **124.3588s (+2%)**
- `./filter_bench -impl=3 -detect_filter_construct_corruption=true -quick -runs 10 | grep 'Build avg'`(expect acceptable increase): **159.4946s (+30%)**
- `./filter_bench -impl=3 -detect_filter_construct_corruption=true -quick -runs 10 | grep 'Build avg'`(by-passing MaybePostVerify, expect minor increase) : **125.258s (+2%)**
- Added to `db_stress`: `make crash_test`, `./db_stress --detect_filter_construct_corruption=true`
- Manually smoke-tested: manually corrupted the filter construction in some db level tests with basic PUT and background flush. As expected, the error did get returned to users in subsequent PUT and Flush status.
Reviewed By: pdillinger
Differential Revision: D33746928
Pulled By: hx235
fbshipit-source-id: cb056426be5a7debc1cd16f23bc250f36a08ca57
2022-02-02 01:41:20 +00:00
" detect_filter_construct_corruption " : lambda : random . choice ( [ 0 , 1 ] ) ,
2022-03-30 20:52:37 +00:00
" adaptive_readahead " : lambda : random . choice ( [ 0 , 1 ] ) ,
" async_io " : lambda : random . choice ( [ 0 , 1 ] ) ,
2022-06-09 19:08:01 +00:00
" wal_compression " : lambda : random . choice ( [ " none " , " zstd " ] ) ,
2022-05-19 18:04:21 +00:00
" verify_sst_unique_id_in_manifest " : 1 , # always do unique_id verification
2022-09-21 00:47:52 +00:00
" secondary_cache_uri " : lambda : random . choice (
[
" " ,
2023-10-10 20:12:18 +00:00
" " ,
" " ,
2022-09-21 00:47:52 +00:00
" compressed_secondary_cache://capacity=8388608;enable_custom_split_merge=true " ,
]
) ,
2022-06-15 19:38:04 +00:00
" allow_data_in_errors " : True ,
2023-04-21 16:07:18 +00:00
" enable_thread_tracking " : lambda : random . choice ( [ 0 , 1 ] ) ,
2022-09-09 19:52:27 +00:00
" readahead_size " : lambda : random . choice ( [ 0 , 16384 , 524288 ] ) ,
" initial_auto_readahead_size " : lambda : random . choice ( [ 0 , 16384 , 524288 ] ) ,
" max_auto_readahead_size " : lambda : random . choice ( [ 0 , 16384 , 524288 ] ) ,
" num_file_reads_for_auto_readahead " : lambda : random . choice ( [ 0 , 1 , 2 ] ) ,
2022-10-07 01:08:19 +00:00
" min_write_buffer_number_to_merge " : lambda : random . choice ( [ 1 , 2 ] ) ,
2022-10-16 16:28:43 +00:00
" preserve_internal_time_seconds " : lambda : random . choice ( [ 0 , 60 , 3600 , 36000 ] ) ,
2023-08-03 02:58:56 +00:00
" memtable_max_range_deletions " : lambda : random . choice ( [ 0 ] * 6 + [ 100 , 1000 ] ) ,
Delay bottommost level single file compactions (#11701)
Summary:
For leveled compaction, RocksDB has a special kind of compaction with reason "kBottommmostFiles" that compacts bottommost level files to clear data held by snapshots (more detail in https://github.com/facebook/rocksdb/issues/3009). Such compactions can happen soon after a relevant snapshot is released. For some use cases, a bottommost file may contain only a small amount of keys that can be cleared, so compacting such a file has a high write amp. In addition, these bottommost files may be compacted in compactions with reason other than "kBottommmostFiles" if we wait for some time (so that enough data is ingested to trigger such a compaction). This PR introduces an option `bottommost_file_compaction_delay` to specify the delay of these bottommost level single file compactions.
* The main change is in `VersionStorageInfo::ComputeBottommostFilesMarkedForCompaction()` where we only add a file to `bottommost_files_marked_for_compaction_` if it oldest_snapshot is larger than its non-zero largest_seqno **and** the file is old enough. Note that if a file is not old enough but its largest_seqno is less than oldest_snapshot, we exclude it from the calculation of `bottommost_files_mark_threshold_`. This makes the change simpler, but such a file's eligibility for compaction will only be checked the next time `ComputeBottommostFilesMarkedForCompaction()` is called. This happens when a new Version is created (compaction, flush, SetOptions()...), a new enough snapshot is released (`VersionStorageInfo::UpdateOldestSnapshot()`) or when a compaction is picked and compaction score has to be re-calculated.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/11701
Test Plan:
* Add two unit tests to test when bottommost_file_compaction_delay > 0.
* Ran crash test with the new option.
Reviewed By: jaykorean, ajkr
Differential Revision: D48331564
Pulled By: cbi42
fbshipit-source-id: c584f3dc5f6354fce3ed65f4c6366dc450b15ba8
2023-08-17 00:45:44 +00:00
# 0 (disable) is the default and more commonly used value.
Fix crash_test_with_best_efforts_recovery (#11938)
Summary:
Thanks ltamasi and ajkr for initial investigations on the test failure. Per the investigations, the following scenario is likely causing the test to fail.
1. Recovery is needed (could be any reason during crash test)
2. Trying to recover from the latest manifest fails (likely due to read error injection)
3. DB opens with recovery from the next manifest which is different from step 2.
4. Expected state is based on the manifest we tried and failed in step 2.
5. Two manifests used in step 2 and 3 are confirmed to have difference in LSM trees (Thanks ltamasi again for the finding).
```
2023/10/05-11:24:18.942189 56341 [db/version_set.cc:6079] Trying to recover from manifest: /dev/shm/rocksdb_test/rocksdb_crashtest_blackbox/MANIFEST-007184
...
2023/10/05-11:24:18.978007 56341 [db/version_set.cc:6079] Trying to recover from manifest: /dev/shm/rocksdb_test/rocksdb_crashtest_blackbox/MANIFEST-007180
```
```
[ltamasi@devbig1024.prn1 /tmp/x]$ ldb manifest_dump --hex --path=MANIFEST-007184_renamed_ > 2
[ltamasi@devbig1024.prn1 /tmp/x]$ ldb manifest_dump --hex --path=MANIFEST-007180_renamed_ > 1
[ltamasi@devbig1024.prn1 /tmp/x]$ diff 1 2
--- 1 2023-10-09 10:29:16.966215207 -0700
+++ 2 2023-10-09 10:29:11.984241645 -0700
@@ -13,7 +13,7 @@
7174:3950254[1875617 .. 2203952]['000000000003415B000000000000012B000000000000007D' seq:1906214, type:1 .. '000000000003CA59000000000000012B000000000000005C' seq:2039838, type:1]
7175:88060[2074748 .. 2203892]['000000000003CA6300000000000000CF78787878787878' seq:2167539, type:2 .. '000000000003D08F000000000000012B0000000000000130' seq:2112478, type:0]
--- level 6 --- version# 1 ---
- 7057:3132633[0 .. 2046144]['0000000000000009000000000000000978' seq:0, type:1 .. '0000000000005F8B000000000000012B00000000000002AC' seq:0, type:1]
+ 7219:2135565[0 .. 2046144]['0000000000000009000000000000000978' seq:0, type:1 .. '0000000000005F8B000000000000012B00000000000002AC' seq:0, type:1]
7061:827724[0 .. 2046131]['0000000000005F95000000000000000778787878787878' seq:0, type:1 .. '000000000000784F000000000000012B0000000000000113' seq:0, type:1]
6763:1352[0 .. 0]['000000000000784F000000000000012B0000000000000129' seq:0, type:1 .. '000000000000784F000000000000012B0000000000000129' seq:0, type:1]
7173:4812291[0 .. 2203957]['000000000000784F000000000000012B0000000000000138' seq:0, type:1 .. '0000000000020FAE787878787878' seq:0, type:1]
@@ -77,4 +77,4 @@
--- level 61 --- version# 1 ---
--- level 62 --- version# 1 ---
--- level 63 --- version# 1 ---
-next_file_number 7182 last_sequence 2203963 prev_log_number 0 max_column_family 0 min_log_number_to_keep 7015
+next_file_number 7221 last_sequence 2203963 prev_log_number 0 max_column_family 0 min_log_number_to_keep 7015
```
We have two options to fix this. Either skip verification against expected state or disable read injection when BE recovery is enabled. I chose to skip verification against expected state per discussion. (See comments in this PR)
Please note that some linter changes were included in this PR.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/11938
Test Plan:
```
TEST_TMPDIR=/dev/shm/rocksdb make crash_test_with_best_efforts_recovery
```
Reviewed By: ltamasi
Differential Revision: D50136341
Pulled By: jaykorean
fbshipit-source-id: ac7434d592aebc148bfc3a4fcaa34936f136b95c
2023-10-11 21:26:10 +00:00
" bottommost_file_compaction_delay " : lambda : random . choice (
[ 0 , 0 , 0 , 600 , 3600 , 86400 ]
) ,
2023-10-03 21:41:26 +00:00
" auto_readahead_size " : lambda : random . choice ( [ 0 , 1 ] ) ,
2023-11-03 23:27:11 +00:00
" verify_iterator_with_expected_state_one_in " : 5 ,
Add missing db crash options (#12414)
Summary:
**Context/Summary:**
We are doing a sweep in all public options, including but not limited to the `Options`, `Read/WriteOptions`, `IngestExternalFileOptions`, cache options.., to find and add the uncovered ones into db crash. The options included in this PR require minimum changes to db crash other than adding the options themselves.
A bonus change: to surface new issues by improved coverage in stderror, we decided to fail/terminate crash test for manual compactions (CompactFiles, CompactRange()) on meaningful errors. See https://github.com/facebook/rocksdb/pull/12414/files#diff-5c4ced6afb6a90e27fec18ab03b2cd89e8f99db87791b4ecc6fa2694284d50c0R2528-R2532, https://github.com/facebook/rocksdb/pull/12414/files#diff-5c4ced6afb6a90e27fec18ab03b2cd89e8f99db87791b4ecc6fa2694284d50c0R2330-R2336 for more.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/12414
Test Plan:
- Run `python3 ./tools/db_crashtest.py --simple blackbox` for 10 minutes to ensure no trivial failure
- Run `python3 tools/db_crashtest.py --simple blackbox --compact_files_one_in=1 --compact_range_one_in=1 --read_fault_one_in=1 --write_fault_one_in=1 --interval=50` for a while to ensure the bonus change does not result in trivial crash/termination of stress test
Reviewed By: ajkr, jowlyzhang, cbi42
Differential Revision: D54691774
Pulled By: hx235
fbshipit-source-id: 50443dfb6aaabd8e24c79a2e42b68c6de877be88
2024-03-13 00:24:12 +00:00
" allow_fallocate " : lambda : random . choice ( [ 0 , 1 ] ) ,
" table_cache_numshardbits " : lambda : random . choice ( [ 6 ] * 3 + [ - 1 ] * 2 + [ 0 ] ) ,
" enable_write_thread_adaptive_yield " : lambda : random . choice ( [ 0 , 1 ] ) ,
" log_readahead_size " : lambda : random . choice ( [ 0 , 16 * 1024 * 1024 ] ) ,
" bgerror_resume_retry_interval " : lambda : random . choice ( [ 10000 , 1000000 ] ) ,
" delete_obsolete_files_period_micros " : lambda : random . choice ( [ 6 * 60 * 60 * 1000000 , 30 * 1000000 ] ) ,
" max_log_file_size " : lambda : random . choice ( [ 0 , 1024 * 1024 ] ) ,
" log_file_time_to_roll " : lambda : random . choice ( [ 0 , 60 ] ) ,
" use_adaptive_mutex " : lambda : random . choice ( [ 0 , 1 ] ) ,
" advise_random_on_open " : lambda : random . choice ( [ 0 ] + [ 1 ] * 3 ) ,
" WAL_ttl_seconds " : lambda : random . choice ( [ 0 , 60 ] ) ,
" WAL_size_limit_MB " : lambda : random . choice ( [ 0 , 1 ] ) ,
" wal_bytes_per_sync " : lambda : random . choice ( [ 0 , 1024 * 1024 ] ) ,
" strict_bytes_per_sync " : lambda : random . choice ( [ 0 , 1 ] ) ,
" avoid_flush_during_shutdown " : lambda : random . choice ( [ 0 , 1 ] ) ,
" fill_cache " : lambda : random . choice ( [ 0 , 1 ] ) ,
" optimize_multiget_for_io " : lambda : random . choice ( [ 0 , 1 ] ) ,
" memtable_insert_hint_per_batch " : lambda : random . choice ( [ 0 , 1 ] ) ,
" dump_malloc_stats " : lambda : random . choice ( [ 0 , 1 ] ) ,
" stats_history_buffer_size " : lambda : random . choice ( [ 0 , 1024 * 1024 ] ) ,
" skip_stats_update_on_db_open " : lambda : random . choice ( [ 0 , 1 ] ) ,
" optimize_filters_for_hits " : lambda : random . choice ( [ 0 , 1 ] ) ,
" sample_for_compression " : lambda : random . choice ( [ 0 , 5 ] ) ,
" report_bg_io_stats " : lambda : random . choice ( [ 0 , 1 ] ) ,
" cache_index_and_filter_blocks_with_high_priority " : lambda : random . choice ( [ 0 , 1 ] ) ,
" use_delta_encoding " : lambda : random . choice ( [ 0 , 1 ] ) ,
" verify_compression " : lambda : random . choice ( [ 0 , 1 ] ) ,
" read_amp_bytes_per_bit " : lambda : random . choice ( [ 0 , 32 ] ) ,
" enable_index_compression " : lambda : random . choice ( [ 0 , 1 ] ) ,
" index_shortening " : lambda : random . choice ( [ 0 , 1 , 2 ] ) ,
" metadata_charge_policy " : lambda : random . choice ( [ 0 , 1 ] ) ,
" use_adaptive_mutex_lru " : lambda : random . choice ( [ 0 , 1 ] ) ,
" compress_format_version " : lambda : random . choice ( [ 1 , 2 ] ) ,
" manifest_preallocation_size " : lambda : random . choice ( [ 0 , 5 * 1024 ] ) ,
2024-03-14 01:03:55 +00:00
" enable_checksum_handoff " : 0 ,
Add missing db crash options (#12414)
Summary:
**Context/Summary:**
We are doing a sweep in all public options, including but not limited to the `Options`, `Read/WriteOptions`, `IngestExternalFileOptions`, cache options.., to find and add the uncovered ones into db crash. The options included in this PR require minimum changes to db crash other than adding the options themselves.
A bonus change: to surface new issues by improved coverage in stderror, we decided to fail/terminate crash test for manual compactions (CompactFiles, CompactRange()) on meaningful errors. See https://github.com/facebook/rocksdb/pull/12414/files#diff-5c4ced6afb6a90e27fec18ab03b2cd89e8f99db87791b4ecc6fa2694284d50c0R2528-R2532, https://github.com/facebook/rocksdb/pull/12414/files#diff-5c4ced6afb6a90e27fec18ab03b2cd89e8f99db87791b4ecc6fa2694284d50c0R2330-R2336 for more.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/12414
Test Plan:
- Run `python3 ./tools/db_crashtest.py --simple blackbox` for 10 minutes to ensure no trivial failure
- Run `python3 tools/db_crashtest.py --simple blackbox --compact_files_one_in=1 --compact_range_one_in=1 --read_fault_one_in=1 --write_fault_one_in=1 --interval=50` for a while to ensure the bonus change does not result in trivial crash/termination of stress test
Reviewed By: ajkr, jowlyzhang, cbi42
Differential Revision: D54691774
Pulled By: hx235
fbshipit-source-id: 50443dfb6aaabd8e24c79a2e42b68c6de877be88
2024-03-13 00:24:12 +00:00
" max_total_wal_size " : lambda : random . choice ( [ 0 ] * 4 + [ 64 * 1024 * 1024 ] ) ,
" high_pri_pool_ratio " : lambda : random . choice ( [ 0 , 0.5 ] ) ,
" low_pri_pool_ratio " : lambda : random . choice ( [ 0 , 0.5 ] ) ,
" soft_pending_compaction_bytes_limit " : lambda : random . choice ( [ 1024 * 1024 ] + [ 64 * 1073741824 ] * 4 ) ,
" hard_pending_compaction_bytes_limit " : lambda : random . choice ( [ 2 * 1024 * 1024 ] + [ 256 * 1073741824 ] * 4 ) ,
2015-10-19 18:43:14 +00:00
}
2022-09-21 00:47:52 +00:00
_TEST_DIR_ENV_VAR = " TEST_TMPDIR "
2023-05-23 16:42:35 +00:00
# If TEST_TMPDIR_EXPECTED is not specified, default value will be TEST_TMPDIR
_TEST_EXPECTED_DIR_ENV_VAR = " TEST_TMPDIR_EXPECTED "
2022-09-21 00:47:52 +00:00
_DEBUG_LEVEL_ENV_VAR = " DEBUG_LEVEL "
2020-04-25 06:58:13 +00:00
2021-06-28 06:53:47 +00:00
stress_cmd = " ./db_stress "
2022-10-28 02:47:01 +00:00
cleanup_cmd = None
2020-04-25 06:58:13 +00:00
2022-09-21 00:47:52 +00:00
2020-04-25 06:58:13 +00:00
def is_release_mode ( ) :
return os . environ . get ( _DEBUG_LEVEL_ENV_VAR ) == " 0 "
2018-06-04 04:28:41 +00:00
2013-04-10 19:15:30 +00:00
2015-10-19 18:43:14 +00:00
def get_dbname ( test_name ) :
2020-04-25 06:58:13 +00:00
test_dir_name = " rocksdb_crashtest_ " + test_name
2018-06-04 04:28:41 +00:00
test_tmpdir = os . environ . get ( _TEST_DIR_ENV_VAR )
2015-08-04 18:35:44 +00:00
if test_tmpdir is None or test_tmpdir == " " :
2020-04-25 06:58:13 +00:00
dbname = tempfile . mkdtemp ( prefix = test_dir_name )
2015-08-04 18:35:44 +00:00
else :
2020-04-25 06:58:13 +00:00
dbname = test_tmpdir + " / " + test_dir_name
2015-08-04 19:20:38 +00:00
shutil . rmtree ( dbname , True )
2022-10-28 02:47:01 +00:00
if cleanup_cmd is not None :
print ( " Running DB cleanup command - %s \n " % cleanup_cmd )
# Ignore failure
os . system ( cleanup_cmd )
2023-05-23 16:42:35 +00:00
try :
os . mkdir ( dbname )
except OSError :
pass
2015-10-19 18:43:14 +00:00
return dbname
2022-09-21 00:47:52 +00:00
2021-09-28 21:12:23 +00:00
expected_values_dir = None
2022-09-21 00:47:52 +00:00
2021-09-28 21:12:23 +00:00
def setup_expected_values_dir ( ) :
global expected_values_dir
if expected_values_dir is not None :
return expected_values_dir
expected_dir_prefix = " rocksdb_crashtest_expected_ "
2023-05-23 16:42:35 +00:00
test_exp_tmpdir = os . environ . get ( _TEST_EXPECTED_DIR_ENV_VAR )
# set the value to _TEST_DIR_ENV_VAR if _TEST_EXPECTED_DIR_ENV_VAR is not
# specified.
2023-06-17 23:27:37 +00:00
if test_exp_tmpdir is None or test_exp_tmpdir == " " :
2023-05-23 16:42:35 +00:00
test_exp_tmpdir = os . environ . get ( _TEST_DIR_ENV_VAR )
if test_exp_tmpdir is None or test_exp_tmpdir == " " :
2022-09-21 00:47:52 +00:00
expected_values_dir = tempfile . mkdtemp ( prefix = expected_dir_prefix )
2020-10-12 21:08:35 +00:00
else :
2021-09-28 21:12:23 +00:00
# if tmpdir is specified, store the expected_values_dir under that dir
2023-05-23 16:42:35 +00:00
expected_values_dir = test_exp_tmpdir + " /rocksdb_crashtest_expected "
2021-09-28 21:12:23 +00:00
if os . path . exists ( expected_values_dir ) :
shutil . rmtree ( expected_values_dir )
os . mkdir ( expected_values_dir )
return expected_values_dir
2020-10-12 21:08:35 +00:00
2022-09-21 00:47:52 +00:00
2022-03-17 02:00:04 +00:00
multiops_txn_key_spaces_file = None
2022-09-21 00:47:52 +00:00
2022-03-17 02:00:04 +00:00
def setup_multiops_txn_key_spaces_file ( ) :
global multiops_txn_key_spaces_file
if multiops_txn_key_spaces_file is not None :
return multiops_txn_key_spaces_file
key_spaces_file_prefix = " rocksdb_crashtest_multiops_txn_key_spaces "
2023-05-23 16:42:35 +00:00
test_exp_tmpdir = os . environ . get ( _TEST_EXPECTED_DIR_ENV_VAR )
# set the value to _TEST_DIR_ENV_VAR if _TEST_EXPECTED_DIR_ENV_VAR is not
# specified.
2023-06-17 23:27:37 +00:00
if test_exp_tmpdir is None or test_exp_tmpdir == " " :
2023-05-23 16:42:35 +00:00
test_exp_tmpdir = os . environ . get ( _TEST_DIR_ENV_VAR )
if test_exp_tmpdir is None or test_exp_tmpdir == " " :
2022-09-21 00:47:52 +00:00
multiops_txn_key_spaces_file = tempfile . mkstemp ( prefix = key_spaces_file_prefix ) [
1
]
2022-03-17 02:00:04 +00:00
else :
2023-05-23 16:42:35 +00:00
if not os . path . exists ( test_exp_tmpdir ) :
os . mkdir ( test_exp_tmpdir )
2022-03-17 02:00:04 +00:00
multiops_txn_key_spaces_file = tempfile . mkstemp (
2023-05-23 16:42:35 +00:00
prefix = key_spaces_file_prefix , dir = test_exp_tmpdir
2022-09-21 00:47:52 +00:00
) [ 1 ]
2022-03-17 02:00:04 +00:00
return multiops_txn_key_spaces_file
2018-06-01 23:33:07 +00:00
def is_direct_io_supported ( dbname ) :
with tempfile . NamedTemporaryFile ( dir = dbname ) as f :
try :
os . open ( f . name , os . O_DIRECT )
2020-03-25 03:57:53 +00:00
except BaseException :
2018-06-01 23:33:07 +00:00
return False
return True
2015-10-19 18:43:14 +00:00
blackbox_default_params = {
2022-01-06 00:12:12 +00:00
" disable_wal " : lambda : random . choice ( [ 0 , 0 , 0 , 1 ] ) ,
2015-10-19 18:43:14 +00:00
# total time for this script to test db_stress
" duration " : 6000 ,
# time for one db_stress instance to run
" interval " : 120 ,
# since we will be killing anyway, use large value for ops_per_thread
" ops_per_thread " : 100000000 ,
2022-01-06 00:12:12 +00:00
" reopen " : 0 ,
2015-10-19 18:43:14 +00:00
" set_options_one_in " : 10000 ,
}
whitebox_default_params = {
2023-08-16 16:43:20 +00:00
# TODO: enable this at random once we figure out two things. First, we need
# to ensure the kill odds in WAL-disabled runs result in regular crashing
# before the fifteen minute timeout. When WAL is disabled there are very few
# calls to write functions since writes to SST files are buffered and other
# writes (e.g., MANIFEST) are infrequent. Crashing in reasonable time might
# currently assume killpoints in write functions are reached frequently.
#
# Second, we need to make sure disabling WAL works with `-reopen > 0`.
2022-01-06 00:12:12 +00:00
" disable_wal " : 0 ,
2015-10-19 18:43:14 +00:00
" duration " : 10000 ,
" log2_keys_per_lock " : 10 ,
" ops_per_thread " : 200000 ,
2016-07-22 18:46:13 +00:00
" random_kill_odd " : 888887 ,
2022-01-06 00:12:12 +00:00
" reopen " : 20 ,
2015-10-19 18:43:14 +00:00
}
simple_default_params = {
2016-02-04 23:21:32 +00:00
" allow_concurrent_memtable_write " : lambda : random . randint ( 0 , 1 ) ,
2018-05-09 20:32:03 +00:00
" column_families " : 1 ,
2022-06-25 00:11:27 +00:00
# TODO: re-enable once internal task T124324915 is fixed.
# "experimental_mempurge_threshold": lambda: 10.0*random.random(),
2015-10-19 18:43:14 +00:00
" max_background_compactions " : 1 ,
" max_bytes_for_level_base " : 67108864 ,
" memtablerep " : " skip_list " ,
" target_file_size_base " : 16777216 ,
" target_file_size_multiplier " : 1 ,
" test_batches_snapshots " : 0 ,
" write_buffer_size " : 32 * 1024 * 1024 ,
2023-03-27 21:55:16 +00:00
" level_compaction_dynamic_level_bytes " : lambda : random . randint ( 0 , 1 ) ,
2020-10-09 15:30:44 +00:00
" paranoid_file_checks " : lambda : random . choice ( [ 0 , 1 , 1 , 1 ] ) ,
2015-10-19 18:43:14 +00:00
}
blackbox_simple_default_params = {
" open_files " : - 1 ,
" set_options_one_in " : 0 ,
}
2018-05-09 20:32:03 +00:00
whitebox_simple_default_params = { }
2015-10-19 18:43:14 +00:00
2019-08-22 23:30:30 +00:00
cf_consistency_params = {
" disable_wal " : lambda : random . randint ( 0 , 1 ) ,
2018-10-30 21:01:38 +00:00
" reopen " : 0 ,
2019-08-22 23:30:30 +00:00
" test_cf_consistency " : 1 ,
2018-10-30 21:01:38 +00:00
# use small value for write_buffer_size so that RocksDB triggers flush
# more frequently
" write_buffer_size " : 1024 * 1024 ,
2019-10-30 01:15:08 +00:00
" enable_pipelined_write " : lambda : random . randint ( 0 , 1 ) ,
2020-06-19 23:03:56 +00:00
# Snapshots are used heavily in this test mode, while they are incompatible
# with compaction filter.
" enable_compaction_filter " : 0 ,
2022-05-27 00:41:30 +00:00
# `CfConsistencyStressTest::TestIngestExternalFile()` is not implemented.
" ingest_external_file_one_in " : 0 ,
2023-11-03 23:27:11 +00:00
# `CfConsistencyStressTest::TestIterateAgainstExpected()` is not implemented.
" verify_iterator_with_expected_state_one_in " : 0 ,
2018-10-30 21:01:38 +00:00
}
2023-06-17 23:27:37 +00:00
# For pessimistic transaction db
2019-12-11 23:59:45 +00:00
txn_params = {
2022-09-21 00:47:52 +00:00
" use_txn " : 1 ,
2023-06-17 23:27:37 +00:00
" use_optimistic_txn " : 0 ,
2019-12-12 18:34:52 +00:00
# Avoid lambda to set it once for the entire test
" txn_write_policy " : random . randint ( 0 , 2 ) ,
2019-12-13 18:23:01 +00:00
" unordered_write " : random . randint ( 0 , 1 ) ,
2022-01-06 00:12:12 +00:00
# TODO: there is such a thing as transactions with WAL disabled. We should
# cover that case.
2019-12-11 23:59:45 +00:00
" disable_wal " : 0 ,
# OpenReadOnly after checkpoint is not currnetly compatible with WritePrepared txns
" checkpoint_one_in " : 0 ,
# pipeline write is not currnetly compatible with WritePrepared txns
" enable_pipelined_write " : 0 ,
Snapshots with user-specified timestamps (#9879)
Summary:
In RocksDB, keys are associated with (internal) sequence numbers which denote when the keys are written
to the database. Sequence numbers in different RocksDB instances are unrelated, thus not comparable.
It is nice if we can associate sequence numbers with their corresponding actual timestamps. One thing we can
do is to support user-defined timestamp, which allows the applications to specify the format of custom timestamps
and encode a timestamp with each key. More details can be found at https://github.com/facebook/rocksdb/wiki/User-defined-Timestamp-%28Experimental%29.
This PR provides a different but complementary approach. We can associate rocksdb snapshots (defined in
https://github.com/facebook/rocksdb/blob/7.2.fb/include/rocksdb/snapshot.h#L20) with **user-specified** timestamps.
Since a snapshot is essentially an object representing a sequence number, this PR establishes a bi-directional mapping between sequence numbers and timestamps.
In the past, snapshots are usually taken by readers. The current super-version is grabbed, and a `rocksdb::Snapshot`
object is created with the last published sequence number of the super-version. You can see that the reader actually
has no good idea of what timestamp to assign to this snapshot, because by the time the `GetSnapshot()` is called,
an arbitrarily long period of time may have already elapsed since the last write, which is when the last published
sequence number is written.
This observation motivates the creation of "timestamped" snapshots on the write path. Currently, this functionality is
exposed only to the layer of `TransactionDB`. Application can tell RocksDB to create a snapshot when a transaction
commits, effectively associating the last sequence number with a timestamp. It is also assumed that application will
ensure any two snapshots with timestamps should satisfy the following:
```
snapshot1.seq < snapshot2.seq iff. snapshot1.ts < snapshot2.ts
```
If the application can guarantee that when a reader takes a timestamped snapshot, there is no active writes going on
in the database, then we also allow the user to use a new API `TransactionDB::CreateTimestampedSnapshot()` to create
a snapshot with associated timestamp.
Code example
```cpp
// Create a timestamped snapshot when committing transaction.
txn->SetCommitTimestamp(100);
txn->SetSnapshotOnNextOperation();
txn->Commit();
// A wrapper API for convenience
Status Transaction::CommitAndTryCreateSnapshot(
std::shared_ptr<TransactionNotifier> notifier,
TxnTimestamp ts,
std::shared_ptr<const Snapshot>* ret);
// Create a timestamped snapshot if caller guarantees no concurrent writes
std::pair<Status, std::shared_ptr<const Snapshot>> snapshot = txn_db->CreateTimestampedSnapshot(100);
```
The snapshots created in this way will be managed by RocksDB with ref-counting and potentially shared with
other readers. We provide the following APIs for readers to retrieve a snapshot given a timestamp.
```cpp
// Return the timestamped snapshot correponding to given timestamp. If ts is
// kMaxTxnTimestamp, then we return the latest timestamped snapshot if present.
// Othersise, we return the snapshot whose timestamp is equal to `ts`. If no
// such snapshot exists, then we return null.
std::shared_ptr<const Snapshot> TransactionDB::GetTimestampedSnapshot(TxnTimestamp ts) const;
// Return the latest timestamped snapshot if present.
std::shared_ptr<const Snapshot> TransactionDB::GetLatestTimestampedSnapshot() const;
```
We also provide two additional APIs for stats collection and reporting purposes.
```cpp
Status TransactionDB::GetAllTimestampedSnapshots(
std::vector<std::shared_ptr<const Snapshot>>& snapshots) const;
// Return timestamped snapshots whose timestamps fall in [ts_lb, ts_ub) and store them in `snapshots`.
Status TransactionDB::GetTimestampedSnapshots(
TxnTimestamp ts_lb,
TxnTimestamp ts_ub,
std::vector<std::shared_ptr<const Snapshot>>& snapshots) const;
```
To prevent the number of timestamped snapshots from growing infinitely, we provide the following API to release
timestamped snapshots whose timestamps are older than or equal to a given threshold.
```cpp
void TransactionDB::ReleaseTimestampedSnapshotsOlderThan(TxnTimestamp ts);
```
Before shutdown, RocksDB will release all timestamped snapshots.
Comparison with user-defined timestamp and how they can be combined:
User-defined timestamp persists every key with a timestamp, while timestamped snapshots maintain a volatile
mapping between snapshots (sequence numbers) and timestamps.
Different internal keys with the same user key but different timestamps will be treated as different by compaction,
thus a newer version will not hide older versions (with smaller timestamps) unless they are eligible for garbage collection.
In contrast, taking a timestamped snapshot at a certain sequence number and timestamp prevents all the keys visible in
this snapshot from been dropped by compaction. Here, visible means (seq < snapshot and most recent).
The timestamped snapshot supports the semantics of reading at an exact point in time.
Timestamped snapshots can also be used with user-defined timestamp.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9879
Test Plan:
```
make check
TEST_TMPDIR=/dev/shm make crash_test_with_txn
```
Reviewed By: siying
Differential Revision: D35783919
Pulled By: riversand963
fbshipit-source-id: 586ad905e169189e19d3bfc0cb0177a7239d1bd4
2022-06-10 23:07:03 +00:00
" create_timestamped_snapshot_one_in " : random . choice ( [ 0 , 20 ] ) ,
Add the PutEntity API to the stress/crash tests (#10760)
Summary:
The patch adds the `PutEntity` API to the non-batched, batched, and
CF consistency stress tests. Namely, when the new `db_stress` command
line parameter `use_put_entity_one_in` is greater than zero, one in
N writes on average is performed using `PutEntity` rather than `Put`.
The wide-column entity written has the generated value in its default
column; in addition, it contains up to three additional columns where
the original generated value is divided up between the column name and the
column value (with the column name containing the first k characters of
the generated value, and the column value containing the rest). Whether
`PutEntity` is used (and if so, how many columns the entity has) is completely
determined by the "value base" used to generate the value (that is, there is
no randomness involved). Assuming the same `use_put_entity_one_in` setting
is used across `db_stress` invocations, this enables us to reconstruct and
validate the entity during subsequent `db_stress` runs.
Note that `PutEntity` is currently incompatible with `Merge`, transactions, and
user-defined timestamps; these combinations are currently disabled/disallowed.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/10760
Test Plan: Ran some batched, non-batched, and CF consistency stress tests using the script.
Reviewed By: riversand963
Differential Revision: D39939032
Pulled By: ltamasi
fbshipit-source-id: eafdf124e95993fb7d73158e3b006d11819f7fa9
2022-09-30 18:11:07 +00:00
# PutEntity in transactions is not yet implemented
2023-06-17 23:27:37 +00:00
" use_put_entity_one_in " : 0 ,
}
# For optimistic transaction db
optimistic_txn_params = {
" use_txn " : 1 ,
" use_optimistic_txn " : 1 ,
" occ_validation_policy " : random . randint ( 0 , 1 ) ,
" share_occ_lock_buckets " : random . randint ( 0 , 1 ) ,
" occ_lock_bucket_count " : lambda : random . choice ( [ 10 , 100 , 500 ] ) ,
# PutEntity in transactions is not yet implemented
" use_put_entity_one_in " : 0 ,
2019-12-11 23:59:45 +00:00
}
2015-10-19 18:43:14 +00:00
2020-06-13 02:24:11 +00:00
best_efforts_recovery_params = {
2022-05-13 19:29:20 +00:00
" best_efforts_recovery " : 1 ,
" disable_wal " : 1 ,
" column_families " : 1 ,
Fix crash_test_with_best_efforts_recovery (#11938)
Summary:
Thanks ltamasi and ajkr for initial investigations on the test failure. Per the investigations, the following scenario is likely causing the test to fail.
1. Recovery is needed (could be any reason during crash test)
2. Trying to recover from the latest manifest fails (likely due to read error injection)
3. DB opens with recovery from the next manifest which is different from step 2.
4. Expected state is based on the manifest we tried and failed in step 2.
5. Two manifests used in step 2 and 3 are confirmed to have difference in LSM trees (Thanks ltamasi again for the finding).
```
2023/10/05-11:24:18.942189 56341 [db/version_set.cc:6079] Trying to recover from manifest: /dev/shm/rocksdb_test/rocksdb_crashtest_blackbox/MANIFEST-007184
...
2023/10/05-11:24:18.978007 56341 [db/version_set.cc:6079] Trying to recover from manifest: /dev/shm/rocksdb_test/rocksdb_crashtest_blackbox/MANIFEST-007180
```
```
[ltamasi@devbig1024.prn1 /tmp/x]$ ldb manifest_dump --hex --path=MANIFEST-007184_renamed_ > 2
[ltamasi@devbig1024.prn1 /tmp/x]$ ldb manifest_dump --hex --path=MANIFEST-007180_renamed_ > 1
[ltamasi@devbig1024.prn1 /tmp/x]$ diff 1 2
--- 1 2023-10-09 10:29:16.966215207 -0700
+++ 2 2023-10-09 10:29:11.984241645 -0700
@@ -13,7 +13,7 @@
7174:3950254[1875617 .. 2203952]['000000000003415B000000000000012B000000000000007D' seq:1906214, type:1 .. '000000000003CA59000000000000012B000000000000005C' seq:2039838, type:1]
7175:88060[2074748 .. 2203892]['000000000003CA6300000000000000CF78787878787878' seq:2167539, type:2 .. '000000000003D08F000000000000012B0000000000000130' seq:2112478, type:0]
--- level 6 --- version# 1 ---
- 7057:3132633[0 .. 2046144]['0000000000000009000000000000000978' seq:0, type:1 .. '0000000000005F8B000000000000012B00000000000002AC' seq:0, type:1]
+ 7219:2135565[0 .. 2046144]['0000000000000009000000000000000978' seq:0, type:1 .. '0000000000005F8B000000000000012B00000000000002AC' seq:0, type:1]
7061:827724[0 .. 2046131]['0000000000005F95000000000000000778787878787878' seq:0, type:1 .. '000000000000784F000000000000012B0000000000000113' seq:0, type:1]
6763:1352[0 .. 0]['000000000000784F000000000000012B0000000000000129' seq:0, type:1 .. '000000000000784F000000000000012B0000000000000129' seq:0, type:1]
7173:4812291[0 .. 2203957]['000000000000784F000000000000012B0000000000000138' seq:0, type:1 .. '0000000000020FAE787878787878' seq:0, type:1]
@@ -77,4 +77,4 @@
--- level 61 --- version# 1 ---
--- level 62 --- version# 1 ---
--- level 63 --- version# 1 ---
-next_file_number 7182 last_sequence 2203963 prev_log_number 0 max_column_family 0 min_log_number_to_keep 7015
+next_file_number 7221 last_sequence 2203963 prev_log_number 0 max_column_family 0 min_log_number_to_keep 7015
```
We have two options to fix this. Either skip verification against expected state or disable read injection when BE recovery is enabled. I chose to skip verification against expected state per discussion. (See comments in this PR)
Please note that some linter changes were included in this PR.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/11938
Test Plan:
```
TEST_TMPDIR=/dev/shm/rocksdb make crash_test_with_best_efforts_recovery
```
Reviewed By: ltamasi
Differential Revision: D50136341
Pulled By: jaykorean
fbshipit-source-id: ac7434d592aebc148bfc3a4fcaa34936f136b95c
2023-10-11 21:26:10 +00:00
" skip_verifydb " : 1 ,
" verify_db_one_in " : 0
2020-06-13 02:24:11 +00:00
}
2021-02-02 19:39:20 +00:00
blob_params = {
" allow_setting_blob_options_dynamically " : 1 ,
# Enable blob files and GC with a 75% chance initially; note that they might still be
# enabled/disabled during the test via SetOptions
" enable_blob_files " : lambda : random . choice ( [ 0 ] + [ 1 ] * 3 ) ,
2021-03-22 21:36:07 +00:00
" min_blob_size " : lambda : random . choice ( [ 0 , 8 , 16 ] ) ,
2021-02-02 19:39:20 +00:00
" blob_file_size " : lambda : random . choice ( [ 1048576 , 16777216 , 268435456 , 1073741824 ] ) ,
" blob_compression_type " : lambda : random . choice ( [ " none " , " snappy " , " lz4 " , " zstd " ] ) ,
" enable_blob_garbage_collection " : lambda : random . choice ( [ 0 ] + [ 1 ] * 3 ) ,
2022-09-21 00:47:52 +00:00
" blob_garbage_collection_age_cutoff " : lambda : random . choice (
[ 0.0 , 0.25 , 0.5 , 0.75 , 1.0 ]
) ,
Make it possible to force the garbage collection of the oldest blob files (#8994)
Summary:
The current BlobDB garbage collection logic works by relocating the valid
blobs from the oldest blob files as they are encountered during compaction,
and cleaning up blob files once they contain nothing but garbage. However,
with sufficiently skewed workloads, it is theoretically possible to end up in a
situation when few or no compactions get scheduled for the SST files that contain
references to the oldest blob files, which can lead to increased space amp due
to the lack of GC.
In order to efficiently handle such workloads, the patch adds a new BlobDB
configuration option called `blob_garbage_collection_force_threshold`,
which signals to BlobDB to schedule targeted compactions for the SST files
that keep alive the oldest batch of blob files if the overall ratio of garbage in
the given blob files meets the threshold *and* all the given blob files are
eligible for GC based on `blob_garbage_collection_age_cutoff`. (For example,
if the new option is set to 0.9, targeted compactions will get scheduled if the
sum of garbage bytes meets or exceeds 90% of the sum of total bytes in the
oldest blob files, assuming all affected blob files are below the age-based cutoff.)
The net result of these targeted compactions is that the valid blobs in the oldest
blob files are relocated and the oldest blob files themselves cleaned up (since
*all* SST files that rely on them get compacted away).
These targeted compactions are similar to periodic compactions in the sense
that they force certain SST files that otherwise would not get picked up to undergo
compaction and also in the sense that instead of merging files from multiple levels,
they target a single file. (Note: such compactions might still include neighboring files
from the same level due to the need of having a "clean cut" boundary but they never
include any files from any other level.)
This functionality is currently only supported with the leveled compaction style
and is inactive by default (since the default value is set to 1.0, i.e. 100%).
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8994
Test Plan: Ran `make check` and tested using `db_bench` and the stress/crash tests.
Reviewed By: riversand963
Differential Revision: D31489850
Pulled By: ltamasi
fbshipit-source-id: 44057d511726a0e2a03c5d9313d7511b3f0c4eab
2021-10-12 01:00:44 +00:00
" blob_garbage_collection_force_threshold " : lambda : random . choice ( [ 0.5 , 0.75 , 1.0 ] ) ,
2021-11-20 01:52:42 +00:00
" blob_compaction_readahead_size " : lambda : random . choice ( [ 0 , 1048576 , 4194304 ] ) ,
2022-09-21 00:47:52 +00:00
" blob_file_starting_level " : lambda : random . choice (
[ 0 ] * 4 + [ 1 ] * 3 + [ 2 ] * 2 + [ 3 ]
) ,
2022-06-22 23:04:03 +00:00
" use_blob_cache " : lambda : random . randint ( 0 , 1 ) ,
" use_shared_block_and_blob_cache " : lambda : random . randint ( 0 , 1 ) ,
" blob_cache_size " : lambda : random . choice ( [ 1048576 , 2097152 , 4194304 , 8388608 ] ) ,
2022-07-17 14:13:59 +00:00
" prepopulate_blob_cache " : lambda : random . randint ( 0 , 1 ) ,
2021-02-02 19:39:20 +00:00
}
Add user-defined timestamps to db_stress (#8061)
Summary:
Add some basic test for user-defined timestamp to db_stress. Currently,
read with timestamp always tries to read using the current timestamp.
Due to the per-key timestamp-sequence ordering constraint, we only add timestamp-
related tests to the `NonBatchedOpsStressTest` since this test serializes accesses
to the same key and uses a file to cross-check data correctness.
The timestamp feature is not supported in a number of components, e.g. Merge, SingleDelete,
DeleteRange, CompactionFilter, Readonly instance, secondary instance, SST file ingestion, transaction,
etc. Therefore, db_stress should exit if user enables both timestamp and these features at the same
time. The (currently) incompatible features can be found in
`CheckAndSetOptionsForUserTimestamp`.
This PR also fixes a bug triggered when timestamp is enabled together with
`index_type=kBinarySearchWithFirstKey`. This bug fix will also be in another separate PR
with more unit tests coverage. Fixing it here because I do not want to exclude the index type
from crash test.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8061
Test Plan: make crash_test_with_ts
Reviewed By: jay-zhuang
Differential Revision: D27056282
Pulled By: riversand963
fbshipit-source-id: c3e00ad1023fdb9ebbdf9601ec18270c5e2925a9
2021-03-23 12:12:04 +00:00
ts_params = {
" test_cf_consistency " : 0 ,
" test_batches_snapshots " : 0 ,
" user_timestamp_size " : 8 ,
2023-12-12 17:35:29 +00:00
# Below flag is randomly picked once and kept consistent in following runs.
" persist_user_defined_timestamps " : random . choice ( [ 0 , 1 , 1 ] ) ,
Add user-defined timestamps to db_stress (#8061)
Summary:
Add some basic test for user-defined timestamp to db_stress. Currently,
read with timestamp always tries to read using the current timestamp.
Due to the per-key timestamp-sequence ordering constraint, we only add timestamp-
related tests to the `NonBatchedOpsStressTest` since this test serializes accesses
to the same key and uses a file to cross-check data correctness.
The timestamp feature is not supported in a number of components, e.g. Merge, SingleDelete,
DeleteRange, CompactionFilter, Readonly instance, secondary instance, SST file ingestion, transaction,
etc. Therefore, db_stress should exit if user enables both timestamp and these features at the same
time. The (currently) incompatible features can be found in
`CheckAndSetOptionsForUserTimestamp`.
This PR also fixes a bug triggered when timestamp is enabled together with
`index_type=kBinarySearchWithFirstKey`. This bug fix will also be in another separate PR
with more unit tests coverage. Fixing it here because I do not want to exclude the index type
from crash test.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8061
Test Plan: make crash_test_with_ts
Reviewed By: jay-zhuang
Differential Revision: D27056282
Pulled By: riversand963
fbshipit-source-id: c3e00ad1023fdb9ebbdf9601ec18270c5e2925a9
2021-03-23 12:12:04 +00:00
" use_merge " : 0 ,
" use_full_merge_v1 " : 0 ,
" use_txn " : 0 ,
" ingest_external_file_one_in " : 0 ,
Add the PutEntity API to the stress/crash tests (#10760)
Summary:
The patch adds the `PutEntity` API to the non-batched, batched, and
CF consistency stress tests. Namely, when the new `db_stress` command
line parameter `use_put_entity_one_in` is greater than zero, one in
N writes on average is performed using `PutEntity` rather than `Put`.
The wide-column entity written has the generated value in its default
column; in addition, it contains up to three additional columns where
the original generated value is divided up between the column name and the
column value (with the column name containing the first k characters of
the generated value, and the column value containing the rest). Whether
`PutEntity` is used (and if so, how many columns the entity has) is completely
determined by the "value base" used to generate the value (that is, there is
no randomness involved). Assuming the same `use_put_entity_one_in` setting
is used across `db_stress` invocations, this enables us to reconstruct and
validate the entity during subsequent `db_stress` runs.
Note that `PutEntity` is currently incompatible with `Merge`, transactions, and
user-defined timestamps; these combinations are currently disabled/disallowed.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/10760
Test Plan: Ran some batched, non-batched, and CF consistency stress tests using the script.
Reviewed By: riversand963
Differential Revision: D39939032
Pulled By: ltamasi
fbshipit-source-id: eafdf124e95993fb7d73158e3b006d11819f7fa9
2022-09-30 18:11:07 +00:00
# PutEntity with timestamps is not yet implemented
2023-06-17 23:27:37 +00:00
" use_put_entity_one_in " : 0 ,
Add user-defined timestamps to db_stress (#8061)
Summary:
Add some basic test for user-defined timestamp to db_stress. Currently,
read with timestamp always tries to read using the current timestamp.
Due to the per-key timestamp-sequence ordering constraint, we only add timestamp-
related tests to the `NonBatchedOpsStressTest` since this test serializes accesses
to the same key and uses a file to cross-check data correctness.
The timestamp feature is not supported in a number of components, e.g. Merge, SingleDelete,
DeleteRange, CompactionFilter, Readonly instance, secondary instance, SST file ingestion, transaction,
etc. Therefore, db_stress should exit if user enables both timestamp and these features at the same
time. The (currently) incompatible features can be found in
`CheckAndSetOptionsForUserTimestamp`.
This PR also fixes a bug triggered when timestamp is enabled together with
`index_type=kBinarySearchWithFirstKey`. This bug fix will also be in another separate PR
with more unit tests coverage. Fixing it here because I do not want to exclude the index type
from crash test.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8061
Test Plan: make crash_test_with_ts
Reviewed By: jay-zhuang
Differential Revision: D27056282
Pulled By: riversand963
fbshipit-source-id: c3e00ad1023fdb9ebbdf9601ec18270c5e2925a9
2021-03-23 12:12:04 +00:00
}
2022-08-08 20:08:35 +00:00
tiered_params = {
" enable_tiered_storage " : 1 ,
2022-09-15 15:17:16 +00:00
# Set tiered compaction hot data time as: 1 minute, 1 hour, 10 hour
" preclude_last_level_data_seconds " : lambda : random . choice ( [ 60 , 3600 , 36000 ] ) ,
2022-08-08 20:08:35 +00:00
# only test universal compaction for now, level has known issue of
# endless compaction
" compaction_style " : 1 ,
2022-09-15 15:17:16 +00:00
# tiered storage doesn't support blob db yet
2022-09-19 22:39:31 +00:00
" enable_blob_files " : 0 ,
2022-09-15 15:17:16 +00:00
" use_blob_db " : 0 ,
2022-08-08 20:08:35 +00:00
}
2022-03-17 02:00:04 +00:00
multiops_txn_default_params = {
" test_cf_consistency " : 0 ,
" test_batches_snapshots " : 0 ,
" test_multi_ops_txns " : 1 ,
" use_txn " : 1 ,
" two_write_queues " : lambda : random . choice ( [ 0 , 1 ] ) ,
# TODO: enable write-prepared
" disable_wal " : 0 ,
" use_only_the_last_commit_time_batch_for_recovery " : lambda : random . choice ( [ 0 , 1 ] ) ,
" clear_column_family_one_in " : 0 ,
" column_families " : 1 ,
" enable_pipelined_write " : lambda : random . choice ( [ 0 , 1 ] ) ,
# This test already acquires snapshots in reads
" acquire_snapshot_one_in " : 0 ,
" backup_one_in " : 0 ,
" writepercent " : 0 ,
" delpercent " : 0 ,
" delrangepercent " : 0 ,
" customopspercent " : 80 ,
" readpercent " : 5 ,
" iterpercent " : 15 ,
" prefixpercent " : 0 ,
" verify_db_one_in " : 1000 ,
" continuous_verification_interval " : 1000 ,
" delay_snapshot_read_one_in " : 3 ,
# 65536 is the smallest possible value for write_buffer_size. Smaller
# values will be sanitized to 65536 during db open. SetOptions currently
# does not sanitize options, but very small write_buffer_size may cause
# assertion failure in
# https://github.com/facebook/rocksdb/blob/7.0.fb/db/memtable.cc#L117.
" write_buffer_size " : 65536 ,
# flush more frequently to generate more files, thus trigger more
# compactions.
" flush_one_in " : 1000 ,
" key_spaces_path " : setup_multiops_txn_key_spaces_file ( ) ,
2022-09-21 00:47:52 +00:00
" rollback_one_in " : 4 ,
2022-05-02 20:25:45 +00:00
# Re-enable once we have a compaction for MultiOpsTxnStressTest
" enable_compaction_filter " : 0 ,
2022-07-11 19:44:08 +00:00
" create_timestamped_snapshot_one_in " : 50 ,
Support WriteCommit policy with sync_fault_injection=1 (#10624)
Summary:
**Context:**
Prior to this PR, correctness testing with un-sync data loss [disabled](https://github.com/facebook/rocksdb/pull/10605) transaction (`use_txn=1`) thus all of the `txn_write_policy` . This PR improved that by adding support for one policy - WriteCommit (`txn_write_policy=0`).
**Summary:**
They key to this support is (a) handle Mark{Begin, End}Prepare/MarkCommit/MarkRollback in constructing ExpectedState under WriteCommit policy correctly and (b) monitor CI jobs and solve any test incompatibility issue till jobs are stable. (b) will be part of the test plan.
For (a)
- During prepare (i.e, between `MarkBeginPrepare()` and `MarkEndPrepare(xid)`), `ExpectedStateTraceRecordHandler` will buffer all writes by adding all writes to an internal `WriteBatch`.
- On `MarkEndPrepare()`, that `WriteBatch` will be associated with the transaction's `xid`.
- During the commit (i.e, on `MarkCommit(xid)`), `ExpectedStateTraceRecordHandler` will retrieve and iterate the internal `WriteBatch` and finally apply those writes to `ExpectedState`
- During the rollback (i.e, on `MarkRollback(xid)`), `ExpectedStateTraceRecordHandler` will erase the internal `WriteBatch` from the map.
For (b) - one major issue described below:
- TransactionsDB in db stress recovers prepared-but-not-committed txns from the previous crashed run by randomly committing or rolling back it at the start of the current run, see a historical [PR](https://github.com/facebook/rocksdb/commit/6d06be22c083ccf185fd38dba49fde73b644b4c1) predated correctness testing.
- And we will verify those processed keys in a recovered db against their expected state.
- However since now we turn on `sync_fault_injection=1` where the expected state is constructed from the trace instead of using the LATEST.state from previous run. The expected state now used to verify those processed keys won't contain UNKNOWN_SENTINEL as they should - see test 1 for a failed case.
- Therefore, we decided to manually update its expected state to be UNKNOWN_SENTINEL as part of the processing.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/10624
Test Plan:
1. Test exposed the major issue described above. This test will fail without setting UNKNOWN_SENTINEL in expected state during the processing and pass after
```
db=/dev/shm/rocksdb_crashtest_blackbox
exp=/dev/shm/rocksdb_crashtest_expected
dbt=$db.tmp
expt=$exp.tmp
rm -rf $db $exp
mkdir -p $exp
echo "RUN 1"
./db_stress \
--clear_column_family_one_in=0 --column_families=1 --db=$db --delpercent=10 --delrangepercent=0 --destroy_db_initially=0 --expected_values_dir=$exp --iterpercent=0 --key_len_percent_dist=1,30,69 --max_key=1000000 --max_key_len=3 --prefixpercent=0 --readpercent=0 --reopen=0 --ops_per_thread=100000000 --test_batches_snapshots=0 --value_size_mult=32 --writepercent=90 \
--use_txn=1 --txn_write_policy=0 --sync_fault_injection=1 &
pid=$!
sleep 0.2
sleep 20
kill $pid
sleep 0.2
echo "RUN 2"
./db_stress \
--clear_column_family_one_in=0 --column_families=1 --db=$db --delpercent=10 --delrangepercent=0 --destroy_db_initially=0 --expected_values_dir=$exp --iterpercent=0 --key_len_percent_dist=1,30,69 --max_key=1000000 --max_key_len=3 --prefixpercent=0 --readpercent=0 --reopen=0 --ops_per_thread=100000000 --test_batches_snapshots=0 --value_size_mult=32 --writepercent=90 \
--use_txn=1 --txn_write_policy=0 --sync_fault_injection=1 &
pid=$!
sleep 0.2
sleep 20
kill $pid
sleep 0.2
echo "RUN 3"
./db_stress \
--clear_column_family_one_in=0 --column_families=1 --db=$db --delpercent=10 --delrangepercent=0 --destroy_db_initially=0 --expected_values_dir=$exp --iterpercent=0 --key_len_percent_dist=1,30,69 --max_key=1000000 --max_key_len=3 --prefixpercent=0 --readpercent=0 --reopen=0 --ops_per_thread=100000000 --test_batches_snapshots=0 --value_size_mult=32 --writepercent=90 \
--use_txn=1 --txn_write_policy=0 --sync_fault_injection=1
```
2. Manual testing to ensure ExpectedState is constructed correctly during recovery by verifying it against previously crashed TransactionDB's WAL.
- Run the following command to crash a TransactionDB with WriteCommit policy. Then `./ldb dump_wal` on its WAL file
```
db=/dev/shm/rocksdb_crashtest_blackbox
exp=/dev/shm/rocksdb_crashtest_expected
rm -rf $db $exp
mkdir -p $exp
./db_stress \
--clear_column_family_one_in=0 --column_families=1 --db=$db --delpercent=10 --delrangepercent=0 --destroy_db_initially=0 --expected_values_dir=$exp --iterpercent=0 --key_len_percent_dist=1,30,69 --max_key=1000000 --max_key_len=3 --prefixpercent=0 --readpercent=0 --reopen=0 --ops_per_thread=100000000 --test_batches_snapshots=0 --value_size_mult=32 --writepercent=90 \
--use_txn=1 --txn_write_policy=0 --sync_fault_injection=1 &
pid=$!
sleep 30
kill $pid
sleep 1
```
- Run the following command to verify recovery of the crashed db under debugger. Compare the step-wise result with WAL records (e.g, WriteBatch content, xid, prepare/commit/rollback marker)
```
./db_stress \
--clear_column_family_one_in=0 --column_families=1 --db=$db --delpercent=10 --delrangepercent=0 --destroy_db_initially=0 --expected_values_dir=$exp --iterpercent=0 --key_len_percent_dist=1,30,69 --max_key=1000000 --max_key_len=3 --prefixpercent=0 --readpercent=0 --reopen=0 --ops_per_thread=100000000 --test_batches_snapshots=0 --value_size_mult=32 --writepercent=90 \
--use_txn=1 --txn_write_policy=0 --sync_fault_injection=1
```
3. Automatic testing by triggering all RocksDB stress/crash test jobs for 3 rounds with no failure.
Reviewed By: ajkr, riversand963
Differential Revision: D39199373
Pulled By: hx235
fbshipit-source-id: 7a1dec0e3e2ee6ea86ddf5dd19ceb5543a3d6f0c
2022-09-27 01:01:59 +00:00
" sync_fault_injection " : 0 ,
2023-09-18 23:23:26 +00:00
# This test has aggressive flush frequency and small write buffer size.
# Disabling write fault to avoid writes being stopped.
" write_fault_one_in " : 0 ,
Add the PutEntity API to the stress/crash tests (#10760)
Summary:
The patch adds the `PutEntity` API to the non-batched, batched, and
CF consistency stress tests. Namely, when the new `db_stress` command
line parameter `use_put_entity_one_in` is greater than zero, one in
N writes on average is performed using `PutEntity` rather than `Put`.
The wide-column entity written has the generated value in its default
column; in addition, it contains up to three additional columns where
the original generated value is divided up between the column name and the
column value (with the column name containing the first k characters of
the generated value, and the column value containing the rest). Whether
`PutEntity` is used (and if so, how many columns the entity has) is completely
determined by the "value base" used to generate the value (that is, there is
no randomness involved). Assuming the same `use_put_entity_one_in` setting
is used across `db_stress` invocations, this enables us to reconstruct and
validate the entity during subsequent `db_stress` runs.
Note that `PutEntity` is currently incompatible with `Merge`, transactions, and
user-defined timestamps; these combinations are currently disabled/disallowed.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/10760
Test Plan: Ran some batched, non-batched, and CF consistency stress tests using the script.
Reviewed By: riversand963
Differential Revision: D39939032
Pulled By: ltamasi
fbshipit-source-id: eafdf124e95993fb7d73158e3b006d11819f7fa9
2022-09-30 18:11:07 +00:00
# PutEntity in transactions is not yet implemented
2023-06-17 23:27:37 +00:00
" use_put_entity_one_in " : 0 ,
" use_get_entity " : 0 ,
" use_multi_get_entity " : 0 ,
2023-11-03 23:27:11 +00:00
# `MultiOpsTxnsStressTest::TestIterateAgainstExpected()` is not implemented.
" verify_iterator_with_expected_state_one_in " : 0 ,
2022-03-17 02:00:04 +00:00
}
multiops_wc_txn_params = {
" txn_write_policy " : 0 ,
# TODO re-enable pipelined write. Not well tested atm
" enable_pipelined_write " : 0 ,
}
multiops_wp_txn_params = {
" txn_write_policy " : 1 ,
" wp_snapshot_cache_bits " : 1 ,
# try small wp_commit_cache_bits, e.g. 0 once we explore storing full
# commit sequence numbers in commit cache
" wp_commit_cache_bits " : 10 ,
# pipeline write is not currnetly compatible with WritePrepared txns
" enable_pipelined_write " : 0 ,
# OpenReadOnly after checkpoint is not currnetly compatible with WritePrepared txns
" checkpoint_one_in " : 0 ,
2022-04-28 00:50:54 +00:00
# Required to be 1 in order to use commit-time-batch
" use_only_the_last_commit_time_batch_for_recovery " : 1 ,
" clear_wp_commit_cache_one_in " : 10 ,
2022-07-11 19:44:08 +00:00
" create_timestamped_snapshot_one_in " : 0 ,
2024-01-11 23:54:11 +00:00
# sequence number can be advanced in SwitchMemtable::WriteRecoverableState() for WP.
# disable it for now until we find another way to test LockWAL().
" lock_wal_one_in " : 0 ,
2022-03-17 02:00:04 +00:00
}
2016-02-04 23:21:32 +00:00
def finalize_and_sanitize ( src_params ) :
2022-09-21 20:37:51 +00:00
dest_params = { k : v ( ) if callable ( v ) else v for ( k , v ) in src_params . items ( ) }
2022-05-06 18:17:08 +00:00
if is_release_mode ( ) :
2022-09-21 00:47:52 +00:00
dest_params [ " read_fault_one_in " ] = 0
Limit buffering for collecting samples for compression dictionary (#7970)
Summary:
For dictionary compression, we need to collect some representative samples of the data to be compressed, which we use to either generate or train (when `CompressionOptions::zstd_max_train_bytes > 0`) a dictionary. Previously, the strategy was to buffer all the data blocks during flush, and up to the target file size during compaction. That strategy allowed us to randomly pick samples from as wide a range as possible that'd be guaranteed to land in a single output file.
However, some users try to make huge files in memory-constrained environments, where this strategy can cause OOM. This PR introduces an option, `CompressionOptions::max_dict_buffer_bytes`, that limits how much data blocks are buffered before we switch to unbuffered mode (which means creating the per-SST dictionary, writing out the buffered data, and compressing/writing new blocks as soon as they are built). It is not strict as we currently buffer more than just data blocks -- also keys are buffered. But it does make a step towards giving users predictable memory usage.
Related changes include:
- Changed sampling for dictionary compression to select unique data blocks when there is limited availability of data blocks
- Made use of `BlockBuilder::SwapAndReset()` to save an allocation+memcpy when buffering data blocks for building a dictionary
- Changed `ParseBoolean()` to accept an input containing characters after the boolean. This is necessary since, with this PR, a value for `CompressionOptions::enabled` is no longer necessarily the final component in the `CompressionOptions` string.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7970
Test Plan:
- updated `CompressionOptions` unit tests to verify limit is respected (to the extent expected in the current implementation) in various scenarios of flush/compaction to bottommost/non-bottommost level
- looked at jemalloc heap profiles right before and after switching to unbuffered mode during flush/compaction. Verified memory usage in buffering is proportional to the limit set.
Reviewed By: pdillinger
Differential Revision: D26467994
Pulled By: ajkr
fbshipit-source-id: 3da4ef9fba59974e4ef40e40c01611002c861465
2021-02-19 22:06:59 +00:00
if dest_params . get ( " compression_max_dict_bytes " ) == 0 :
dest_params [ " compression_zstd_max_train_bytes " ] = 0
dest_params [ " compression_max_dict_buffer_bytes " ] = 0
if dest_params . get ( " compression_type " ) != " zstd " :
2018-08-06 22:23:03 +00:00
dest_params [ " compression_zstd_max_train_bytes " ] = 0
2016-02-04 23:21:32 +00:00
if dest_params . get ( " allow_concurrent_memtable_write " , 1 ) == 1 :
2016-02-05 22:30:18 +00:00
dest_params [ " memtablerep " ] = " skip_list "
2020-04-25 06:58:13 +00:00
if dest_params [ " mmap_read " ] == 1 :
2018-06-01 23:33:07 +00:00
dest_params [ " use_direct_io_for_flush_and_compaction " ] = 0
dest_params [ " use_direct_reads " ] = 0
2022-09-21 00:47:52 +00:00
if (
dest_params [ " use_direct_io_for_flush_and_compaction " ] == 1
or dest_params [ " use_direct_reads " ] == 1
) and not is_direct_io_supported ( dest_params [ " db " ] ) :
2020-04-25 06:58:13 +00:00
if is_release_mode ( ) :
2022-09-21 00:47:52 +00:00
print (
" {} does not support direct IO. Disabling use_direct_reads and "
" use_direct_io_for_flush_and_compaction. \n " . format ( dest_params [ " db " ] )
)
2020-11-10 18:48:46 +00:00
dest_params [ " use_direct_reads " ] = 0
dest_params [ " use_direct_io_for_flush_and_compaction " ] = 0
2020-04-25 06:58:13 +00:00
else :
dest_params [ " mock_direct_io " ] = True
2022-10-17 21:32:59 +00:00
if dest_params [ " test_batches_snapshots " ] == 1 :
dest_params [ " enable_compaction_filter " ] = 0
if dest_params [ " prefix_size " ] < 0 :
dest_params [ " prefix_size " ] = 1
2022-05-26 17:31:37 +00:00
# Multi-key operations are not currently compatible with transactions or
# timestamp.
2023-06-17 23:27:37 +00:00
if (
dest_params . get ( " test_batches_snapshots " ) == 1
or dest_params . get ( " use_txn " ) == 1
or dest_params . get ( " user_timestamp_size " ) > 0
) :
2022-09-30 23:13:03 +00:00
dest_params [ " ingest_external_file_one_in " ] = 0
2023-06-17 23:27:37 +00:00
if (
dest_params . get ( " test_batches_snapshots " ) == 1
or dest_params . get ( " use_txn " ) == 1
) :
2018-12-18 21:30:56 +00:00
dest_params [ " delpercent " ] + = dest_params [ " delrangepercent " ]
dest_params [ " delrangepercent " ] = 0
2022-09-21 00:47:52 +00:00
if (
dest_params . get ( " disable_wal " ) == 1
or dest_params . get ( " sync_fault_injection " ) == 1
Add manual_wal_flush, FlushWAL() to stress/crash test (#10698)
Summary:
**Context/Summary:**
Introduce `manual_wal_flush_one_in` as titled.
- When `manual_wal_flush_one_in > 0`, we also need tracing to correctly verify recovery because WAL data can be lost in this case when `FlushWAL()` is not explicitly called by users of RocksDB (in our case, db stress) and the recovery from such potential WAL data loss is a prefix recovery that requires tracing to verify. As another consequence, we need to disable features can't run under unsync data loss with `manual_wal_flush_one_in`
Incompatibilities fixed along the way:
```
db_stress: db/db_impl/db_impl_open.cc:2063: static rocksdb::Status rocksdb::DBImpl::Open(const rocksdb::DBOptions&, const string&, const std::vector<rocksdb::ColumnFamilyDescriptor>&, std::vector<rocksdb::ColumnFamilyHandle*>*, rocksdb::DB**, bool, bool): Assertion `impl->TEST_WALBufferIsEmpty()' failed.
```
- It turns out that `Writer::AddCompressionTypeRecord` before this assertion `EmitPhysicalRecord(kSetCompressionType, encode.data(), encode.size());` but do not trigger flush if `manual_wal_flush` is set . This leads to `impl->TEST_WALBufferIsEmpty()' is false.
- As suggested, assertion is removed and violation case is handled by `FlushWAL(sync=true)` along with refactoring `TEST_WALBufferIsEmpty()` to be `WALBufferIsEmpty()` since it is used in prod code now.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/10698
Test Plan:
- Locally running `python3 tools/db_crashtest.py blackbox --manual_wal_flush_one_in=1 --manual_wal_flush=1 --sync_wal_one_in=100 --atomic_flush=1 --flush_one_in=100 --column_families=3`
- Joined https://github.com/facebook/rocksdb/pull/10624 in auto CI testings with all RocksDB stress/crash test jobs
Reviewed By: ajkr
Differential Revision: D39593752
Pulled By: ajkr
fbshipit-source-id: 3a2135bb792c52d2ffa60257d4fbc557fb04d2ce
2022-09-30 22:48:33 +00:00
or dest_params . get ( " manual_wal_flush_one_in " ) > 0
2022-09-21 00:47:52 +00:00
) :
2022-08-25 04:50:34 +00:00
# File ingestion does not guarantee prefix-recoverability when unsynced
# data can be lost. Ingesting a file syncs data immediately that is
# newer than unsynced memtable data that can be lost on restart.
#
# Even if the above issue is fixed or worked around, our
# trace-and-replay does not trace file ingestion, so in its current form
# it would not recover the expected state to the correct point in time.
2022-05-26 17:31:37 +00:00
dest_params [ " ingest_external_file_one_in " ] = 0
2022-08-25 04:50:34 +00:00
# The `DbStressCompactionFilter` can apply memtable updates to SST
# files, which would be problematic when unsynced data can be lost in
# crash recoveries.
dest_params [ " enable_compaction_filter " ] = 0
2019-12-13 18:23:01 +00:00
# Only under WritePrepared txns, unordered_write would provide the same guarnatees as vanilla rocksdb
if dest_params . get ( " unordered_write " , 0 ) == 1 :
dest_params [ " txn_write_policy " ] = 1
dest_params [ " allow_concurrent_memtable_write " ] = 1
2019-08-22 23:30:30 +00:00
if dest_params . get ( " disable_wal " , 0 ) == 1 :
dest_params [ " atomic_flush " ] = 1
2019-12-18 02:23:26 +00:00
dest_params [ " sync " ] = 0
2020-12-17 19:51:04 +00:00
dest_params [ " write_fault_one_in " ] = 0
2019-08-26 22:00:04 +00:00
if dest_params . get ( " open_files " , 1 ) != - 1 :
# Compaction TTL and periodic compactions are only compatible
# with open_files = -1
dest_params [ " compaction_ttl " ] = 0
dest_params [ " periodic_compaction_seconds " ] = 0
2019-08-28 00:54:18 +00:00
if dest_params . get ( " compaction_style " , 0 ) == 2 :
# Disable compaction TTL in FIFO compaction, because right
# now assertion failures are triggered.
dest_params [ " compaction_ttl " ] = 0
2019-11-01 00:26:46 +00:00
dest_params [ " periodic_compaction_seconds " ] = 0
2019-09-11 21:11:38 +00:00
if dest_params [ " partition_filters " ] == 1 :
2019-10-14 17:33:31 +00:00
if dest_params [ " index_type " ] != 2 :
dest_params [ " partition_filters " ] = 0
2019-10-30 18:34:38 +00:00
if dest_params . get ( " atomic_flush " , 0 ) == 1 :
# disable pipelined write when atomic flush is used.
dest_params [ " enable_pipelined_write " ] = 0
2020-06-23 23:24:15 +00:00
if dest_params . get ( " sst_file_manager_bytes_per_sec " , 0 ) == 0 :
dest_params [ " sst_file_manager_bytes_per_truncate " ] = 0
2020-06-18 16:51:14 +00:00
if dest_params . get ( " enable_compaction_filter " , 0 ) == 1 :
# Compaction filter is incompatible with snapshots. Need to avoid taking
# snapshots, as well as avoid operations that use snapshots for
# verification.
dest_params [ " acquire_snapshot_one_in " ] = 0
dest_params [ " compact_range_one_in " ] = 0
# Give the iterator ops away to reads.
dest_params [ " readpercent " ] + = dest_params . get ( " iterpercent " , 10 )
dest_params [ " iterpercent " ] = 0
2022-01-28 07:10:26 +00:00
if dest_params . get ( " prefix_size " ) == - 1 :
dest_params [ " readpercent " ] + = dest_params . get ( " prefixpercent " , 20 )
dest_params [ " prefixpercent " ] = 0
2022-09-21 00:47:52 +00:00
if (
dest_params . get ( " prefix_size " ) == - 1
and dest_params . get ( " memtable_whole_key_filtering " ) == 0
) :
2022-01-27 22:53:39 +00:00
dest_params [ " memtable_prefix_bloom_size_ratio " ] = 0
2022-03-17 02:00:04 +00:00
if dest_params . get ( " two_write_queues " ) == 1 :
dest_params [ " enable_pipelined_write " ] = 0
2022-05-13 19:29:20 +00:00
if dest_params . get ( " best_efforts_recovery " ) == 1 :
2022-06-02 01:00:28 +00:00
dest_params [ " disable_wal " ] = 1
dest_params [ " enable_compaction_filter " ] = 0
dest_params [ " sync " ] = 0
dest_params [ " write_fault_one_in " ] = 0
Fix crash_test_with_best_efforts_recovery (#11938)
Summary:
Thanks ltamasi and ajkr for initial investigations on the test failure. Per the investigations, the following scenario is likely causing the test to fail.
1. Recovery is needed (could be any reason during crash test)
2. Trying to recover from the latest manifest fails (likely due to read error injection)
3. DB opens with recovery from the next manifest which is different from step 2.
4. Expected state is based on the manifest we tried and failed in step 2.
5. Two manifests used in step 2 and 3 are confirmed to have difference in LSM trees (Thanks ltamasi again for the finding).
```
2023/10/05-11:24:18.942189 56341 [db/version_set.cc:6079] Trying to recover from manifest: /dev/shm/rocksdb_test/rocksdb_crashtest_blackbox/MANIFEST-007184
...
2023/10/05-11:24:18.978007 56341 [db/version_set.cc:6079] Trying to recover from manifest: /dev/shm/rocksdb_test/rocksdb_crashtest_blackbox/MANIFEST-007180
```
```
[ltamasi@devbig1024.prn1 /tmp/x]$ ldb manifest_dump --hex --path=MANIFEST-007184_renamed_ > 2
[ltamasi@devbig1024.prn1 /tmp/x]$ ldb manifest_dump --hex --path=MANIFEST-007180_renamed_ > 1
[ltamasi@devbig1024.prn1 /tmp/x]$ diff 1 2
--- 1 2023-10-09 10:29:16.966215207 -0700
+++ 2 2023-10-09 10:29:11.984241645 -0700
@@ -13,7 +13,7 @@
7174:3950254[1875617 .. 2203952]['000000000003415B000000000000012B000000000000007D' seq:1906214, type:1 .. '000000000003CA59000000000000012B000000000000005C' seq:2039838, type:1]
7175:88060[2074748 .. 2203892]['000000000003CA6300000000000000CF78787878787878' seq:2167539, type:2 .. '000000000003D08F000000000000012B0000000000000130' seq:2112478, type:0]
--- level 6 --- version# 1 ---
- 7057:3132633[0 .. 2046144]['0000000000000009000000000000000978' seq:0, type:1 .. '0000000000005F8B000000000000012B00000000000002AC' seq:0, type:1]
+ 7219:2135565[0 .. 2046144]['0000000000000009000000000000000978' seq:0, type:1 .. '0000000000005F8B000000000000012B00000000000002AC' seq:0, type:1]
7061:827724[0 .. 2046131]['0000000000005F95000000000000000778787878787878' seq:0, type:1 .. '000000000000784F000000000000012B0000000000000113' seq:0, type:1]
6763:1352[0 .. 0]['000000000000784F000000000000012B0000000000000129' seq:0, type:1 .. '000000000000784F000000000000012B0000000000000129' seq:0, type:1]
7173:4812291[0 .. 2203957]['000000000000784F000000000000012B0000000000000138' seq:0, type:1 .. '0000000000020FAE787878787878' seq:0, type:1]
@@ -77,4 +77,4 @@
--- level 61 --- version# 1 ---
--- level 62 --- version# 1 ---
--- level 63 --- version# 1 ---
-next_file_number 7182 last_sequence 2203963 prev_log_number 0 max_column_family 0 min_log_number_to_keep 7015
+next_file_number 7221 last_sequence 2203963 prev_log_number 0 max_column_family 0 min_log_number_to_keep 7015
```
We have two options to fix this. Either skip verification against expected state or disable read injection when BE recovery is enabled. I chose to skip verification against expected state per discussion. (See comments in this PR)
Please note that some linter changes were included in this PR.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/11938
Test Plan:
```
TEST_TMPDIR=/dev/shm/rocksdb make crash_test_with_best_efforts_recovery
```
Reviewed By: ltamasi
Differential Revision: D50136341
Pulled By: jaykorean
fbshipit-source-id: ac7434d592aebc148bfc3a4fcaa34936f136b95c
2023-10-11 21:26:10 +00:00
dest_params [ " skip_verifydb " ] = 1
dest_params [ " verify_db_one_in " ] = 0
Snapshots with user-specified timestamps (#9879)
Summary:
In RocksDB, keys are associated with (internal) sequence numbers which denote when the keys are written
to the database. Sequence numbers in different RocksDB instances are unrelated, thus not comparable.
It is nice if we can associate sequence numbers with their corresponding actual timestamps. One thing we can
do is to support user-defined timestamp, which allows the applications to specify the format of custom timestamps
and encode a timestamp with each key. More details can be found at https://github.com/facebook/rocksdb/wiki/User-defined-Timestamp-%28Experimental%29.
This PR provides a different but complementary approach. We can associate rocksdb snapshots (defined in
https://github.com/facebook/rocksdb/blob/7.2.fb/include/rocksdb/snapshot.h#L20) with **user-specified** timestamps.
Since a snapshot is essentially an object representing a sequence number, this PR establishes a bi-directional mapping between sequence numbers and timestamps.
In the past, snapshots are usually taken by readers. The current super-version is grabbed, and a `rocksdb::Snapshot`
object is created with the last published sequence number of the super-version. You can see that the reader actually
has no good idea of what timestamp to assign to this snapshot, because by the time the `GetSnapshot()` is called,
an arbitrarily long period of time may have already elapsed since the last write, which is when the last published
sequence number is written.
This observation motivates the creation of "timestamped" snapshots on the write path. Currently, this functionality is
exposed only to the layer of `TransactionDB`. Application can tell RocksDB to create a snapshot when a transaction
commits, effectively associating the last sequence number with a timestamp. It is also assumed that application will
ensure any two snapshots with timestamps should satisfy the following:
```
snapshot1.seq < snapshot2.seq iff. snapshot1.ts < snapshot2.ts
```
If the application can guarantee that when a reader takes a timestamped snapshot, there is no active writes going on
in the database, then we also allow the user to use a new API `TransactionDB::CreateTimestampedSnapshot()` to create
a snapshot with associated timestamp.
Code example
```cpp
// Create a timestamped snapshot when committing transaction.
txn->SetCommitTimestamp(100);
txn->SetSnapshotOnNextOperation();
txn->Commit();
// A wrapper API for convenience
Status Transaction::CommitAndTryCreateSnapshot(
std::shared_ptr<TransactionNotifier> notifier,
TxnTimestamp ts,
std::shared_ptr<const Snapshot>* ret);
// Create a timestamped snapshot if caller guarantees no concurrent writes
std::pair<Status, std::shared_ptr<const Snapshot>> snapshot = txn_db->CreateTimestampedSnapshot(100);
```
The snapshots created in this way will be managed by RocksDB with ref-counting and potentially shared with
other readers. We provide the following APIs for readers to retrieve a snapshot given a timestamp.
```cpp
// Return the timestamped snapshot correponding to given timestamp. If ts is
// kMaxTxnTimestamp, then we return the latest timestamped snapshot if present.
// Othersise, we return the snapshot whose timestamp is equal to `ts`. If no
// such snapshot exists, then we return null.
std::shared_ptr<const Snapshot> TransactionDB::GetTimestampedSnapshot(TxnTimestamp ts) const;
// Return the latest timestamped snapshot if present.
std::shared_ptr<const Snapshot> TransactionDB::GetLatestTimestampedSnapshot() const;
```
We also provide two additional APIs for stats collection and reporting purposes.
```cpp
Status TransactionDB::GetAllTimestampedSnapshots(
std::vector<std::shared_ptr<const Snapshot>>& snapshots) const;
// Return timestamped snapshots whose timestamps fall in [ts_lb, ts_ub) and store them in `snapshots`.
Status TransactionDB::GetTimestampedSnapshots(
TxnTimestamp ts_lb,
TxnTimestamp ts_ub,
std::vector<std::shared_ptr<const Snapshot>>& snapshots) const;
```
To prevent the number of timestamped snapshots from growing infinitely, we provide the following API to release
timestamped snapshots whose timestamps are older than or equal to a given threshold.
```cpp
void TransactionDB::ReleaseTimestampedSnapshotsOlderThan(TxnTimestamp ts);
```
Before shutdown, RocksDB will release all timestamped snapshots.
Comparison with user-defined timestamp and how they can be combined:
User-defined timestamp persists every key with a timestamp, while timestamped snapshots maintain a volatile
mapping between snapshots (sequence numbers) and timestamps.
Different internal keys with the same user key but different timestamps will be treated as different by compaction,
thus a newer version will not hide older versions (with smaller timestamps) unless they are eligible for garbage collection.
In contrast, taking a timestamped snapshot at a certain sequence number and timestamp prevents all the keys visible in
this snapshot from been dropped by compaction. Here, visible means (seq < snapshot and most recent).
The timestamped snapshot supports the semantics of reading at an exact point in time.
Timestamped snapshots can also be used with user-defined timestamp.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9879
Test Plan:
```
make check
TEST_TMPDIR=/dev/shm make crash_test_with_txn
```
Reviewed By: siying
Differential Revision: D35783919
Pulled By: riversand963
fbshipit-source-id: 586ad905e169189e19d3bfc0cb0177a7239d1bd4
2022-06-10 23:07:03 +00:00
# Remove the following once write-prepared/write-unprepared with/without
# unordered write supports timestamped snapshots
if dest_params . get ( " create_timestamped_snapshot_one_in " , 0 ) > 0 :
dest_params [ " txn_write_policy " ] = 0
dest_params [ " unordered_write " ] = 0
Support WriteCommit policy with sync_fault_injection=1 (#10624)
Summary:
**Context:**
Prior to this PR, correctness testing with un-sync data loss [disabled](https://github.com/facebook/rocksdb/pull/10605) transaction (`use_txn=1`) thus all of the `txn_write_policy` . This PR improved that by adding support for one policy - WriteCommit (`txn_write_policy=0`).
**Summary:**
They key to this support is (a) handle Mark{Begin, End}Prepare/MarkCommit/MarkRollback in constructing ExpectedState under WriteCommit policy correctly and (b) monitor CI jobs and solve any test incompatibility issue till jobs are stable. (b) will be part of the test plan.
For (a)
- During prepare (i.e, between `MarkBeginPrepare()` and `MarkEndPrepare(xid)`), `ExpectedStateTraceRecordHandler` will buffer all writes by adding all writes to an internal `WriteBatch`.
- On `MarkEndPrepare()`, that `WriteBatch` will be associated with the transaction's `xid`.
- During the commit (i.e, on `MarkCommit(xid)`), `ExpectedStateTraceRecordHandler` will retrieve and iterate the internal `WriteBatch` and finally apply those writes to `ExpectedState`
- During the rollback (i.e, on `MarkRollback(xid)`), `ExpectedStateTraceRecordHandler` will erase the internal `WriteBatch` from the map.
For (b) - one major issue described below:
- TransactionsDB in db stress recovers prepared-but-not-committed txns from the previous crashed run by randomly committing or rolling back it at the start of the current run, see a historical [PR](https://github.com/facebook/rocksdb/commit/6d06be22c083ccf185fd38dba49fde73b644b4c1) predated correctness testing.
- And we will verify those processed keys in a recovered db against their expected state.
- However since now we turn on `sync_fault_injection=1` where the expected state is constructed from the trace instead of using the LATEST.state from previous run. The expected state now used to verify those processed keys won't contain UNKNOWN_SENTINEL as they should - see test 1 for a failed case.
- Therefore, we decided to manually update its expected state to be UNKNOWN_SENTINEL as part of the processing.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/10624
Test Plan:
1. Test exposed the major issue described above. This test will fail without setting UNKNOWN_SENTINEL in expected state during the processing and pass after
```
db=/dev/shm/rocksdb_crashtest_blackbox
exp=/dev/shm/rocksdb_crashtest_expected
dbt=$db.tmp
expt=$exp.tmp
rm -rf $db $exp
mkdir -p $exp
echo "RUN 1"
./db_stress \
--clear_column_family_one_in=0 --column_families=1 --db=$db --delpercent=10 --delrangepercent=0 --destroy_db_initially=0 --expected_values_dir=$exp --iterpercent=0 --key_len_percent_dist=1,30,69 --max_key=1000000 --max_key_len=3 --prefixpercent=0 --readpercent=0 --reopen=0 --ops_per_thread=100000000 --test_batches_snapshots=0 --value_size_mult=32 --writepercent=90 \
--use_txn=1 --txn_write_policy=0 --sync_fault_injection=1 &
pid=$!
sleep 0.2
sleep 20
kill $pid
sleep 0.2
echo "RUN 2"
./db_stress \
--clear_column_family_one_in=0 --column_families=1 --db=$db --delpercent=10 --delrangepercent=0 --destroy_db_initially=0 --expected_values_dir=$exp --iterpercent=0 --key_len_percent_dist=1,30,69 --max_key=1000000 --max_key_len=3 --prefixpercent=0 --readpercent=0 --reopen=0 --ops_per_thread=100000000 --test_batches_snapshots=0 --value_size_mult=32 --writepercent=90 \
--use_txn=1 --txn_write_policy=0 --sync_fault_injection=1 &
pid=$!
sleep 0.2
sleep 20
kill $pid
sleep 0.2
echo "RUN 3"
./db_stress \
--clear_column_family_one_in=0 --column_families=1 --db=$db --delpercent=10 --delrangepercent=0 --destroy_db_initially=0 --expected_values_dir=$exp --iterpercent=0 --key_len_percent_dist=1,30,69 --max_key=1000000 --max_key_len=3 --prefixpercent=0 --readpercent=0 --reopen=0 --ops_per_thread=100000000 --test_batches_snapshots=0 --value_size_mult=32 --writepercent=90 \
--use_txn=1 --txn_write_policy=0 --sync_fault_injection=1
```
2. Manual testing to ensure ExpectedState is constructed correctly during recovery by verifying it against previously crashed TransactionDB's WAL.
- Run the following command to crash a TransactionDB with WriteCommit policy. Then `./ldb dump_wal` on its WAL file
```
db=/dev/shm/rocksdb_crashtest_blackbox
exp=/dev/shm/rocksdb_crashtest_expected
rm -rf $db $exp
mkdir -p $exp
./db_stress \
--clear_column_family_one_in=0 --column_families=1 --db=$db --delpercent=10 --delrangepercent=0 --destroy_db_initially=0 --expected_values_dir=$exp --iterpercent=0 --key_len_percent_dist=1,30,69 --max_key=1000000 --max_key_len=3 --prefixpercent=0 --readpercent=0 --reopen=0 --ops_per_thread=100000000 --test_batches_snapshots=0 --value_size_mult=32 --writepercent=90 \
--use_txn=1 --txn_write_policy=0 --sync_fault_injection=1 &
pid=$!
sleep 30
kill $pid
sleep 1
```
- Run the following command to verify recovery of the crashed db under debugger. Compare the step-wise result with WAL records (e.g, WriteBatch content, xid, prepare/commit/rollback marker)
```
./db_stress \
--clear_column_family_one_in=0 --column_families=1 --db=$db --delpercent=10 --delrangepercent=0 --destroy_db_initially=0 --expected_values_dir=$exp --iterpercent=0 --key_len_percent_dist=1,30,69 --max_key=1000000 --max_key_len=3 --prefixpercent=0 --readpercent=0 --reopen=0 --ops_per_thread=100000000 --test_batches_snapshots=0 --value_size_mult=32 --writepercent=90 \
--use_txn=1 --txn_write_policy=0 --sync_fault_injection=1
```
3. Automatic testing by triggering all RocksDB stress/crash test jobs for 3 rounds with no failure.
Reviewed By: ajkr, riversand963
Differential Revision: D39199373
Pulled By: hx235
fbshipit-source-id: 7a1dec0e3e2ee6ea86ddf5dd19ceb5543a3d6f0c
2022-09-27 01:01:59 +00:00
# For TransactionDB, correctness testing with unsync data loss is currently
# compatible with only write committed policy
2023-06-17 23:27:37 +00:00
if dest_params . get ( " use_txn " ) == 1 and dest_params . get ( " txn_write_policy " ) != 0 :
Support WriteCommit policy with sync_fault_injection=1 (#10624)
Summary:
**Context:**
Prior to this PR, correctness testing with un-sync data loss [disabled](https://github.com/facebook/rocksdb/pull/10605) transaction (`use_txn=1`) thus all of the `txn_write_policy` . This PR improved that by adding support for one policy - WriteCommit (`txn_write_policy=0`).
**Summary:**
They key to this support is (a) handle Mark{Begin, End}Prepare/MarkCommit/MarkRollback in constructing ExpectedState under WriteCommit policy correctly and (b) monitor CI jobs and solve any test incompatibility issue till jobs are stable. (b) will be part of the test plan.
For (a)
- During prepare (i.e, between `MarkBeginPrepare()` and `MarkEndPrepare(xid)`), `ExpectedStateTraceRecordHandler` will buffer all writes by adding all writes to an internal `WriteBatch`.
- On `MarkEndPrepare()`, that `WriteBatch` will be associated with the transaction's `xid`.
- During the commit (i.e, on `MarkCommit(xid)`), `ExpectedStateTraceRecordHandler` will retrieve and iterate the internal `WriteBatch` and finally apply those writes to `ExpectedState`
- During the rollback (i.e, on `MarkRollback(xid)`), `ExpectedStateTraceRecordHandler` will erase the internal `WriteBatch` from the map.
For (b) - one major issue described below:
- TransactionsDB in db stress recovers prepared-but-not-committed txns from the previous crashed run by randomly committing or rolling back it at the start of the current run, see a historical [PR](https://github.com/facebook/rocksdb/commit/6d06be22c083ccf185fd38dba49fde73b644b4c1) predated correctness testing.
- And we will verify those processed keys in a recovered db against their expected state.
- However since now we turn on `sync_fault_injection=1` where the expected state is constructed from the trace instead of using the LATEST.state from previous run. The expected state now used to verify those processed keys won't contain UNKNOWN_SENTINEL as they should - see test 1 for a failed case.
- Therefore, we decided to manually update its expected state to be UNKNOWN_SENTINEL as part of the processing.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/10624
Test Plan:
1. Test exposed the major issue described above. This test will fail without setting UNKNOWN_SENTINEL in expected state during the processing and pass after
```
db=/dev/shm/rocksdb_crashtest_blackbox
exp=/dev/shm/rocksdb_crashtest_expected
dbt=$db.tmp
expt=$exp.tmp
rm -rf $db $exp
mkdir -p $exp
echo "RUN 1"
./db_stress \
--clear_column_family_one_in=0 --column_families=1 --db=$db --delpercent=10 --delrangepercent=0 --destroy_db_initially=0 --expected_values_dir=$exp --iterpercent=0 --key_len_percent_dist=1,30,69 --max_key=1000000 --max_key_len=3 --prefixpercent=0 --readpercent=0 --reopen=0 --ops_per_thread=100000000 --test_batches_snapshots=0 --value_size_mult=32 --writepercent=90 \
--use_txn=1 --txn_write_policy=0 --sync_fault_injection=1 &
pid=$!
sleep 0.2
sleep 20
kill $pid
sleep 0.2
echo "RUN 2"
./db_stress \
--clear_column_family_one_in=0 --column_families=1 --db=$db --delpercent=10 --delrangepercent=0 --destroy_db_initially=0 --expected_values_dir=$exp --iterpercent=0 --key_len_percent_dist=1,30,69 --max_key=1000000 --max_key_len=3 --prefixpercent=0 --readpercent=0 --reopen=0 --ops_per_thread=100000000 --test_batches_snapshots=0 --value_size_mult=32 --writepercent=90 \
--use_txn=1 --txn_write_policy=0 --sync_fault_injection=1 &
pid=$!
sleep 0.2
sleep 20
kill $pid
sleep 0.2
echo "RUN 3"
./db_stress \
--clear_column_family_one_in=0 --column_families=1 --db=$db --delpercent=10 --delrangepercent=0 --destroy_db_initially=0 --expected_values_dir=$exp --iterpercent=0 --key_len_percent_dist=1,30,69 --max_key=1000000 --max_key_len=3 --prefixpercent=0 --readpercent=0 --reopen=0 --ops_per_thread=100000000 --test_batches_snapshots=0 --value_size_mult=32 --writepercent=90 \
--use_txn=1 --txn_write_policy=0 --sync_fault_injection=1
```
2. Manual testing to ensure ExpectedState is constructed correctly during recovery by verifying it against previously crashed TransactionDB's WAL.
- Run the following command to crash a TransactionDB with WriteCommit policy. Then `./ldb dump_wal` on its WAL file
```
db=/dev/shm/rocksdb_crashtest_blackbox
exp=/dev/shm/rocksdb_crashtest_expected
rm -rf $db $exp
mkdir -p $exp
./db_stress \
--clear_column_family_one_in=0 --column_families=1 --db=$db --delpercent=10 --delrangepercent=0 --destroy_db_initially=0 --expected_values_dir=$exp --iterpercent=0 --key_len_percent_dist=1,30,69 --max_key=1000000 --max_key_len=3 --prefixpercent=0 --readpercent=0 --reopen=0 --ops_per_thread=100000000 --test_batches_snapshots=0 --value_size_mult=32 --writepercent=90 \
--use_txn=1 --txn_write_policy=0 --sync_fault_injection=1 &
pid=$!
sleep 30
kill $pid
sleep 1
```
- Run the following command to verify recovery of the crashed db under debugger. Compare the step-wise result with WAL records (e.g, WriteBatch content, xid, prepare/commit/rollback marker)
```
./db_stress \
--clear_column_family_one_in=0 --column_families=1 --db=$db --delpercent=10 --delrangepercent=0 --destroy_db_initially=0 --expected_values_dir=$exp --iterpercent=0 --key_len_percent_dist=1,30,69 --max_key=1000000 --max_key_len=3 --prefixpercent=0 --readpercent=0 --reopen=0 --ops_per_thread=100000000 --test_batches_snapshots=0 --value_size_mult=32 --writepercent=90 \
--use_txn=1 --txn_write_policy=0 --sync_fault_injection=1
```
3. Automatic testing by triggering all RocksDB stress/crash test jobs for 3 rounds with no failure.
Reviewed By: ajkr, riversand963
Differential Revision: D39199373
Pulled By: hx235
fbshipit-source-id: 7a1dec0e3e2ee6ea86ddf5dd19ceb5543a3d6f0c
2022-09-27 01:01:59 +00:00
dest_params [ " sync_fault_injection " ] = 0
Add manual_wal_flush, FlushWAL() to stress/crash test (#10698)
Summary:
**Context/Summary:**
Introduce `manual_wal_flush_one_in` as titled.
- When `manual_wal_flush_one_in > 0`, we also need tracing to correctly verify recovery because WAL data can be lost in this case when `FlushWAL()` is not explicitly called by users of RocksDB (in our case, db stress) and the recovery from such potential WAL data loss is a prefix recovery that requires tracing to verify. As another consequence, we need to disable features can't run under unsync data loss with `manual_wal_flush_one_in`
Incompatibilities fixed along the way:
```
db_stress: db/db_impl/db_impl_open.cc:2063: static rocksdb::Status rocksdb::DBImpl::Open(const rocksdb::DBOptions&, const string&, const std::vector<rocksdb::ColumnFamilyDescriptor>&, std::vector<rocksdb::ColumnFamilyHandle*>*, rocksdb::DB**, bool, bool): Assertion `impl->TEST_WALBufferIsEmpty()' failed.
```
- It turns out that `Writer::AddCompressionTypeRecord` before this assertion `EmitPhysicalRecord(kSetCompressionType, encode.data(), encode.size());` but do not trigger flush if `manual_wal_flush` is set . This leads to `impl->TEST_WALBufferIsEmpty()' is false.
- As suggested, assertion is removed and violation case is handled by `FlushWAL(sync=true)` along with refactoring `TEST_WALBufferIsEmpty()` to be `WALBufferIsEmpty()` since it is used in prod code now.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/10698
Test Plan:
- Locally running `python3 tools/db_crashtest.py blackbox --manual_wal_flush_one_in=1 --manual_wal_flush=1 --sync_wal_one_in=100 --atomic_flush=1 --flush_one_in=100 --column_families=3`
- Joined https://github.com/facebook/rocksdb/pull/10624 in auto CI testings with all RocksDB stress/crash test jobs
Reviewed By: ajkr
Differential Revision: D39593752
Pulled By: ajkr
fbshipit-source-id: 3a2135bb792c52d2ffa60257d4fbc557fb04d2ce
2022-09-30 22:48:33 +00:00
dest_params [ " manual_wal_flush_one_in " ] = 0
2023-09-29 15:54:50 +00:00
# Wide column stress tests require FullMergeV3
Add the PutEntity API to the stress/crash tests (#10760)
Summary:
The patch adds the `PutEntity` API to the non-batched, batched, and
CF consistency stress tests. Namely, when the new `db_stress` command
line parameter `use_put_entity_one_in` is greater than zero, one in
N writes on average is performed using `PutEntity` rather than `Put`.
The wide-column entity written has the generated value in its default
column; in addition, it contains up to three additional columns where
the original generated value is divided up between the column name and the
column value (with the column name containing the first k characters of
the generated value, and the column value containing the rest). Whether
`PutEntity` is used (and if so, how many columns the entity has) is completely
determined by the "value base" used to generate the value (that is, there is
no randomness involved). Assuming the same `use_put_entity_one_in` setting
is used across `db_stress` invocations, this enables us to reconstruct and
validate the entity during subsequent `db_stress` runs.
Note that `PutEntity` is currently incompatible with `Merge`, transactions, and
user-defined timestamps; these combinations are currently disabled/disallowed.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/10760
Test Plan: Ran some batched, non-batched, and CF consistency stress tests using the script.
Reviewed By: riversand963
Differential Revision: D39939032
Pulled By: ltamasi
fbshipit-source-id: eafdf124e95993fb7d73158e3b006d11819f7fa9
2022-09-30 18:11:07 +00:00
if dest_params [ " use_put_entity_one_in " ] != 0 :
dest_params [ " use_full_merge_v1 " ] = 0
Group rocksdb.sst.read.micros stat by different user read IOActivity + misc (#11444)
Summary:
**Context/Summary:**
- Similar to https://github.com/facebook/rocksdb/pull/11288 but for user read such as `Get(), MultiGet(), DBIterator::XXX(), Verify(File)Checksum()`.
- For this, I refactored some user-facing `MultiGet` calls in `TransactionBase` and various types of `DB` so that it does not call a user-facing `Get()` but `GetImpl()` for passing the `ReadOptions::io_activity` check (see PR conversation)
- New user read stats breakdown are guarded by `kExceptDetailedTimers` since measurement shows they have 4-5% regression to the upstream/main.
- Misc
- More refactoring: with https://github.com/facebook/rocksdb/pull/11288, we complete passing `ReadOptions/IOOptions` to FS level. So we can now replace the previously [added](https://github.com/facebook/rocksdb/pull/9424) `rate_limiter_priority` parameter in `RandomAccessFileReader`'s `Read/MultiRead/Prefetch()` with `IOOptions::rate_limiter_priority`
- Also, `ReadAsync()` call time is measured in `SST_READ_MICRO` now
Pull Request resolved: https://github.com/facebook/rocksdb/pull/11444
Test Plan:
- CI fake db crash/stress test
- Microbenchmarking
**Build** `make clean && ROCKSDB_NO_FBCODE=1 DEBUG_LEVEL=0 make -jN db_basic_bench`
- google benchmark version: https://github.com/google/benchmark/commit/604f6fd3f4b34a84ec4eb4db81d842fa4db829cd
- db_basic_bench_base: upstream
- db_basic_bench_pr: db_basic_bench_base + this PR
- asyncread_db_basic_bench_base: upstream + [db basic bench patch for IteratorNext](https://github.com/facebook/rocksdb/compare/main...hx235:rocksdb:micro_bench_async_read)
- asyncread_db_basic_bench_pr: asyncread_db_basic_bench_base + this PR
**Test**
Get
```
TEST_TMPDIR=/dev/shm ./db_basic_bench_{null_stat|base|pr} --benchmark_filter=DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/mmap:1/threads:1 --benchmark_repetitions=1000
```
Result
```
Coming soon
```
AsyncRead
```
TEST_TMPDIR=/dev/shm ./asyncread_db_basic_bench_{base|pr} --benchmark_filter=IteratorNext/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/async_io:1/include_detailed_timers:0 --benchmark_repetitions=1000 > syncread_db_basic_bench_{base|pr}.out
```
Result
```
Base:
1956,1956,1968,1977,1979,1986,1988,1988,1988,1990,1991,1991,1993,1993,1993,1993,1994,1996,1997,1997,1997,1998,1999,2001,2001,2002,2004,2007,2007,2008,
PR (2.3% regression, due to measuring `SST_READ_MICRO` that wasn't measured before):
1993,2014,2016,2022,2024,2027,2027,2028,2028,2030,2031,2031,2032,2032,2038,2039,2042,2044,2044,2047,2047,2047,2048,2049,2050,2052,2052,2052,2053,2053,
```
Reviewed By: ajkr
Differential Revision: D45918925
Pulled By: hx235
fbshipit-source-id: 58a54560d9ebeb3a59b6d807639692614dad058a
2023-08-09 00:26:50 +00:00
if dest_params [ " file_checksum_impl " ] == " none " :
dest_params [ " verify_file_checksums_one_in " ] = 0
2023-09-18 23:23:26 +00:00
if dest_params [ " write_fault_one_in " ] > 0 :
# background work may be disabled while DB is resuming after some error
2023-09-19 15:33:05 +00:00
dest_params [ " max_write_buffer_number " ] = max ( dest_params [ " max_write_buffer_number " ] , 10 )
2023-10-10 20:12:18 +00:00
if dest_params [ " secondary_cache_uri " ] . find ( " compressed_secondary_cache " ) > = 0 :
dest_params [ " compressed_secondary_cache_size " ] = 0
dest_params [ " compressed_secondary_cache_ratio " ] = 0.0
if dest_params [ " cache_type " ] . find ( " tiered_ " ) > = 0 :
if dest_params [ " compressed_secondary_cache_size " ] > 0 :
dest_params [ " compressed_secondary_cache_ratio " ] = \
float ( dest_params [ " compressed_secondary_cache_size " ] / \
( dest_params [ " cache_size " ] + dest_params [ " compressed_secondary_cache_size " ] ) )
dest_params [ " compressed_secondary_cache_size " ] = 0
else :
dest_params [ " compressed_secondary_cache_ratio " ] = 0.0
dest_params [ " cache_type " ] = dest_params [ " cache_type " ] . replace ( " tiered_ " , " " )
2023-10-17 00:28:36 +00:00
else :
if dest_params [ " secondary_cache_uri " ] :
dest_params [ " compressed_secondary_cache_size " ] = 0
dest_params [ " compressed_secondary_cache_ratio " ] = 0.0
2023-10-10 20:12:18 +00:00
if dest_params [ " use_write_buffer_manager " ] :
if ( dest_params [ " cache_size " ] < = 0
or dest_params [ " db_write_buffer_size " ] < = 0 ) :
dest_params [ " use_write_buffer_manager " ] = 0
2023-12-12 17:35:29 +00:00
if dest_params [ " user_timestamp_size " ] > 0 and dest_params [ " persist_user_defined_timestamps " ] == 0 :
# Features that are not compatible with UDT in memtable only feature.
dest_params [ " enable_blob_files " ] = 0
2024-01-10 01:37:01 +00:00
dest_params [ " allow_setting_blob_options_dynamically " ] = 0
2023-12-12 17:35:29 +00:00
dest_params [ " atomic_flush " ] = 0
dest_params [ " allow_concurrent_memtable_write " ] = 0
dest_params [ " block_protection_bytes_per_key " ] = 0
# TODO(yuzhangyu): make stress test logic handle this and enable testing
# these APIs.
# These operations need to compare side to side one operation with another.
# It's hard to guarantee their consistency because when timestamps can be
# collapsed, only operations using the same SuperVersion can be consistent
# with each other. There is no external APIs to ensure that.
dest_params [ " use_multiget " ] = 0
dest_params [ " use_multi_get_entity " ] = 0
dest_params [ " readpercent " ] + = dest_params . get ( " iterpercent " , 10 ) ;
dest_params [ " iterpercent " ] = 0
# Only best efforts recovery test support disabling wal and
# disable atomic flush.
if dest_params [ " test_best_efforts_recovery " ] == 0 :
dest_params [ " disable_wal " ] = 0
2016-02-04 23:21:32 +00:00
return dest_params
2022-09-21 00:47:52 +00:00
2015-10-19 18:43:14 +00:00
def gen_cmd_params ( args ) :
params = { }
2018-05-09 20:32:03 +00:00
params . update ( default_params )
2022-09-21 00:47:52 +00:00
if args . test_type == " blackbox " :
2018-05-09 20:32:03 +00:00
params . update ( blackbox_default_params )
2022-09-21 00:47:52 +00:00
if args . test_type == " whitebox " :
2018-05-09 20:32:03 +00:00
params . update ( whitebox_default_params )
2015-10-19 18:43:14 +00:00
if args . simple :
params . update ( simple_default_params )
2022-09-21 00:47:52 +00:00
if args . test_type == " blackbox " :
2015-10-19 18:43:14 +00:00
params . update ( blackbox_simple_default_params )
2022-09-21 00:47:52 +00:00
if args . test_type == " whitebox " :
2015-10-19 18:43:14 +00:00
params . update ( whitebox_simple_default_params )
2019-08-22 23:30:30 +00:00
if args . cf_consistency :
params . update ( cf_consistency_params )
2019-12-11 23:59:45 +00:00
if args . txn :
params . update ( txn_params )
2023-06-17 23:27:37 +00:00
if args . optimistic_txn :
params . update ( optimistic_txn_params )
2020-06-13 02:24:11 +00:00
if args . test_best_efforts_recovery :
params . update ( best_efforts_recovery_params )
Add user-defined timestamps to db_stress (#8061)
Summary:
Add some basic test for user-defined timestamp to db_stress. Currently,
read with timestamp always tries to read using the current timestamp.
Due to the per-key timestamp-sequence ordering constraint, we only add timestamp-
related tests to the `NonBatchedOpsStressTest` since this test serializes accesses
to the same key and uses a file to cross-check data correctness.
The timestamp feature is not supported in a number of components, e.g. Merge, SingleDelete,
DeleteRange, CompactionFilter, Readonly instance, secondary instance, SST file ingestion, transaction,
etc. Therefore, db_stress should exit if user enables both timestamp and these features at the same
time. The (currently) incompatible features can be found in
`CheckAndSetOptionsForUserTimestamp`.
This PR also fixes a bug triggered when timestamp is enabled together with
`index_type=kBinarySearchWithFirstKey`. This bug fix will also be in another separate PR
with more unit tests coverage. Fixing it here because I do not want to exclude the index type
from crash test.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8061
Test Plan: make crash_test_with_ts
Reviewed By: jay-zhuang
Differential Revision: D27056282
Pulled By: riversand963
fbshipit-source-id: c3e00ad1023fdb9ebbdf9601ec18270c5e2925a9
2021-03-23 12:12:04 +00:00
if args . enable_ts :
params . update ( ts_params )
2022-03-17 02:00:04 +00:00
if args . test_multiops_txn :
params . update ( multiops_txn_default_params )
2022-09-21 00:47:52 +00:00
if args . write_policy == " write_committed " :
2022-03-17 02:00:04 +00:00
params . update ( multiops_wc_txn_params )
2022-09-21 00:47:52 +00:00
elif args . write_policy == " write_prepared " :
2022-03-17 02:00:04 +00:00
params . update ( multiops_wp_txn_params )
2022-09-09 00:39:22 +00:00
if args . test_tiered_storage :
2022-08-08 20:08:35 +00:00
params . update ( tiered_params )
2015-10-19 18:43:14 +00:00
2023-02-03 00:22:32 +00:00
# Best-effort recovery, tiered storage are currently incompatible with BlobDB.
# Test BE recovery if specified on the command line; otherwise, apply BlobDB
# related overrides with a 10% chance.
2022-09-21 00:47:52 +00:00
if (
not args . test_best_efforts_recovery
and not args . test_tiered_storage
and random . choice ( [ 0 ] * 9 + [ 1 ] ) == 1
) :
2021-02-02 19:39:20 +00:00
params . update ( blob_params )
2015-10-19 18:43:14 +00:00
for k , v in vars ( args ) . items ( ) :
if v is not None :
params [ k ] = v
return params
2018-05-09 20:32:03 +00:00
def gen_cmd ( params , unknown_params ) :
2019-12-21 00:13:19 +00:00
finalzied_params = finalize_and_sanitize ( params )
2022-09-21 00:47:52 +00:00
cmd = (
[ stress_cmd ]
+ [
" -- {0} = {1} " . format ( k , v )
for k , v in [ ( k , finalzied_params [ k ] ) for k in sorted ( finalzied_params ) ]
if k
not in {
2022-09-21 20:37:51 +00:00
" test_type " ,
" simple " ,
" duration " ,
" interval " ,
" random_kill_odd " ,
" cf_consistency " ,
" txn " ,
2023-06-17 23:27:37 +00:00
" optimistic_txn " ,
2022-09-21 20:37:51 +00:00
" test_best_efforts_recovery " ,
" enable_ts " ,
" test_multiops_txn " ,
" write_policy " ,
" stress_cmd " ,
" test_tiered_storage " ,
2022-10-28 02:47:01 +00:00
" cleanup_cmd " ,
Fix crash_test_with_best_efforts_recovery (#11938)
Summary:
Thanks ltamasi and ajkr for initial investigations on the test failure. Per the investigations, the following scenario is likely causing the test to fail.
1. Recovery is needed (could be any reason during crash test)
2. Trying to recover from the latest manifest fails (likely due to read error injection)
3. DB opens with recovery from the next manifest which is different from step 2.
4. Expected state is based on the manifest we tried and failed in step 2.
5. Two manifests used in step 2 and 3 are confirmed to have difference in LSM trees (Thanks ltamasi again for the finding).
```
2023/10/05-11:24:18.942189 56341 [db/version_set.cc:6079] Trying to recover from manifest: /dev/shm/rocksdb_test/rocksdb_crashtest_blackbox/MANIFEST-007184
...
2023/10/05-11:24:18.978007 56341 [db/version_set.cc:6079] Trying to recover from manifest: /dev/shm/rocksdb_test/rocksdb_crashtest_blackbox/MANIFEST-007180
```
```
[ltamasi@devbig1024.prn1 /tmp/x]$ ldb manifest_dump --hex --path=MANIFEST-007184_renamed_ > 2
[ltamasi@devbig1024.prn1 /tmp/x]$ ldb manifest_dump --hex --path=MANIFEST-007180_renamed_ > 1
[ltamasi@devbig1024.prn1 /tmp/x]$ diff 1 2
--- 1 2023-10-09 10:29:16.966215207 -0700
+++ 2 2023-10-09 10:29:11.984241645 -0700
@@ -13,7 +13,7 @@
7174:3950254[1875617 .. 2203952]['000000000003415B000000000000012B000000000000007D' seq:1906214, type:1 .. '000000000003CA59000000000000012B000000000000005C' seq:2039838, type:1]
7175:88060[2074748 .. 2203892]['000000000003CA6300000000000000CF78787878787878' seq:2167539, type:2 .. '000000000003D08F000000000000012B0000000000000130' seq:2112478, type:0]
--- level 6 --- version# 1 ---
- 7057:3132633[0 .. 2046144]['0000000000000009000000000000000978' seq:0, type:1 .. '0000000000005F8B000000000000012B00000000000002AC' seq:0, type:1]
+ 7219:2135565[0 .. 2046144]['0000000000000009000000000000000978' seq:0, type:1 .. '0000000000005F8B000000000000012B00000000000002AC' seq:0, type:1]
7061:827724[0 .. 2046131]['0000000000005F95000000000000000778787878787878' seq:0, type:1 .. '000000000000784F000000000000012B0000000000000113' seq:0, type:1]
6763:1352[0 .. 0]['000000000000784F000000000000012B0000000000000129' seq:0, type:1 .. '000000000000784F000000000000012B0000000000000129' seq:0, type:1]
7173:4812291[0 .. 2203957]['000000000000784F000000000000012B0000000000000138' seq:0, type:1 .. '0000000000020FAE787878787878' seq:0, type:1]
@@ -77,4 +77,4 @@
--- level 61 --- version# 1 ---
--- level 62 --- version# 1 ---
--- level 63 --- version# 1 ---
-next_file_number 7182 last_sequence 2203963 prev_log_number 0 max_column_family 0 min_log_number_to_keep 7015
+next_file_number 7221 last_sequence 2203963 prev_log_number 0 max_column_family 0 min_log_number_to_keep 7015
```
We have two options to fix this. Either skip verification against expected state or disable read injection when BE recovery is enabled. I chose to skip verification against expected state per discussion. (See comments in this PR)
Please note that some linter changes were included in this PR.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/11938
Test Plan:
```
TEST_TMPDIR=/dev/shm/rocksdb make crash_test_with_best_efforts_recovery
```
Reviewed By: ltamasi
Differential Revision: D50136341
Pulled By: jaykorean
fbshipit-source-id: ac7434d592aebc148bfc3a4fcaa34936f136b95c
2023-10-11 21:26:10 +00:00
" skip_tmpdir_check " ,
print stderr separately per option (#12301)
Summary:
While working on Meta's internal test triaging process, I found that `db_crashtest.py` was printing out `stdout` and `stderr` altogether. Adding an option to print `stderr` separately so that it's easy to extract only `stderr` from the test run.
`print_stderr_separately` is introduced as an optional parameter with default value `False` to keep the existing behavior as is (except a few minor changes).
Minor changes to the existing behavior
- We no longer print `stderr has error message:` and `***` prefix to each line. We simply print `stderr:` before printing `stderr` if stderr is printed in stdout and print `stderr` as is.
- We no longer print `times error occurred in output is ...` which doesn't appear to have any values
Pull Request resolved: https://github.com/facebook/rocksdb/pull/12301
Test Plan:
**Default Behavior (blackbox)**
Run printed everything as is
```
$> python3 tools/db_crashtest.py blackbox --simple --max_key=25000000 --write_buffer_size=4194304 2> /tmp/error.log
Running blackbox-crash-test with
interval_between_crash=120
total-duration=6000
...
Integrated BlobDB: blob files enabled 0, min blob size 0, blob file size 268435456, blob compression type NoCompression, blob GC enabled 0, cutoff 0.250000, force threshold 1.000000, blob compaction readahead size 0, blob file starting level 0
Integrated BlobDB: blob cache disabled
DB path: [/tmp/jewoongh/rocksdb_crashtest_blackboxwh7yxpec]
(Re-)verified 0 unique IDs
2024/01/29-09:16:30 Initializing worker threads
Crash-recovery verification passed :)
2024/01/29-09:16:35 Starting database operations
2024/01/29-09:16:35 Starting verification
Stress Test : 543.600 micros/op 8802 ops/sec
: Wrote 0.00 MB (0.27 MB/sec) (50% of 10 ops)
: Wrote 5 times
: Deleted 1 times
: Single deleted 0 times
: 4 read and 0 found the key
: Prefix scanned 0 times
: Iterator size sum is 0
: Iterated 0 times
: Deleted 0 key-ranges
: Range deletions covered 0 keys
: Got errors 0 times
: 0 CompactFiles() succeed
: 0 CompactFiles() did not succeed
stderr:
WARNING: prefix_size is non-zero but memtablerep != prefix_hash
Error : jewoongh injected test error This is not a real failure.
Verification failed :(
```
Nothing in stderr
```
$> cat /tmp/error.log
```
**Default Behavior (whitebox)**
Run printed everything as is
```
$> python3 tools/db_crashtest.py whitebox --simple --max_key=25000000 --write_buffer_size=4194304 2> /tmp/error.log
Running whitebox-crash-test with
total-duration=10000
...
(Re-)verified 571 unique IDs
2024/01/29-09:33:53 Initializing worker threads
Crash-recovery verification passed :)
2024/01/29-09:35:16 Starting database operations
2024/01/29-09:35:16 Starting verification
Stress Test : 97248.125 micros/op 10 ops/sec
: Wrote 0.00 MB (0.00 MB/sec) (12% of 8 ops)
: Wrote 1 times
: Deleted 0 times
: Single deleted 0 times
: 4 read and 1 found the key
: Prefix scanned 1 times
: Iterator size sum is 120868
: Iterated 4 times
: Deleted 0 key-ranges
: Range deletions covered 0 keys
: Got errors 0 times
: 0 CompactFiles() succeed
: 0 CompactFiles() did not succeed
stderr:
WARNING: prefix_size is non-zero but memtablerep != prefix_hash
Error : jewoongh injected test error This is not a real failure.
New cache capacity = 4865393
Verification failed :(
TEST FAILED. See kill option and exit code above!!!
```
Nothing in stderr
```
$> cat /tmp/error.log
```
**New option added (blackbox)**
```
$> python3 tools/db_crashtest.py blackbox --simple --max_key=25000000 --write_buffer_size=4194304 --print_stderr_separately 2> /tmp/error.log
Running blackbox-crash-test with
interval_between_crash=120
total-duration=6000
...
Integrated BlobDB: blob files enabled 0, min blob size 0, blob file size 268435456, blob compression type NoCompression, blob GC enabled 0, cutoff 0.250000, force threshold 1.000000, blob compaction readahead size 0, blob file starting level 0
Integrated BlobDB: blob cache disabled
DB path: [/tmp/jewoongh/rocksdb_crashtest_blackbox7ybna32z]
(Re-)verified 0 unique IDs
Compaction filter factory: DbStressCompactionFilterFactory
2024/01/29-09:05:39 Initializing worker threads
Crash-recovery verification passed :)
2024/01/29-09:05:46 Starting database operations
2024/01/29-09:05:46 Starting verification
Stress Test : 235.917 micros/op 16000 ops/sec
: Wrote 0.00 MB (0.16 MB/sec) (16% of 12 ops)
: Wrote 2 times
: Deleted 1 times
: Single deleted 0 times
: 9 read and 0 found the key
: Prefix scanned 0 times
: Iterator size sum is 0
: Iterated 0 times
: Deleted 0 key-ranges
: Range deletions covered 0 keys
: Got errors 0 times
: 0 CompactFiles() succeed
: 0 CompactFiles() did not succeed
```
stderr printed separately
```
$> cat /tmp/error.log
WARNING: prefix_size is non-zero but memtablerep != prefix_hash
Error : jewoongh injected test error This is not a real failure.
New cache capacity = 19461571
Verification failed :(
```
**New option added (whitebox)**
```
$> python3 tools/db_crashtest.py whitebox --simple --max_key=25000000 --write_buffer_size=4194304 --print_stderr_separately 2> /tmp/error.log
Running whitebox-crash-test with
total-duration=10000
...
Integrated BlobDB: blob files enabled 0, min blob size 0, blob file size 268435456, blob compression type NoCompression, blob GC enabled 0, cutoff 0.250000, force threshold 1.000000, blob compaction readahead size 0, blob file starting level 0
Integrated BlobDB: blob cache disabled
DB path: [/tmp/jewoongh/rocksdb_crashtest_whiteboxtwj0ihn6]
(Re-)verified 157 unique IDs
2024/01/29-09:39:59 Initializing worker threads
Crash-recovery verification passed :)
2024/01/29-09:40:16 Starting database operations
2024/01/29-09:40:16 Starting verification
Stress Test : 742.474 micros/op 11801 ops/sec
: Wrote 0.00 MB (0.27 MB/sec) (36% of 19 ops)
: Wrote 7 times
: Deleted 1 times
: Single deleted 0 times
: 8 read and 0 found the key
: Prefix scanned 0 times
: Iterator size sum is 0
: Iterated 4 times
: Deleted 0 key-ranges
: Range deletions covered 0 keys
: Got errors 0 times
: 0 CompactFiles() succeed
: 0 CompactFiles() did not succeed
TEST FAILED. See kill option and exit code above!!!
```
stderr printed separately
```
$> cat /tmp/error.log
WARNING: prefix_size is non-zero but memtablerep != prefix_hash
Error : jewoongh injected test error This is not a real failure.
Error : jewoongh injected test error This is not a real failure.
Error : jewoongh injected test error This is not a real failure.
New cache capacity = 4865393
Verification failed :(
```
Reviewed By: akankshamahajan15
Differential Revision: D53187491
Pulled By: jaykorean
fbshipit-source-id: 76f9100d08b96d014e41b7b88b206d69f0ae932b
2024-01-29 19:09:47 +00:00
" print_stderr_separately "
2022-09-21 20:37:51 +00:00
}
2022-09-21 00:47:52 +00:00
and v is not None
]
+ unknown_params
)
2015-10-19 18:43:14 +00:00
return cmd
2023-08-23 22:24:23 +00:00
def execute_cmd ( cmd , timeout = None ) :
2022-09-21 00:47:52 +00:00
child = subprocess . Popen ( cmd , stderr = subprocess . PIPE , stdout = subprocess . PIPE )
print ( " Running db_stress with pid= %d : %s \n \n " % ( child . pid , " " . join ( cmd ) ) )
2021-06-01 16:33:50 +00:00
try :
outs , errs = child . communicate ( timeout = timeout )
hit_timeout = False
2022-09-21 00:47:52 +00:00
print ( " WARNING: db_stress ended before kill: exitcode= %d \n " % child . returncode )
2021-06-01 16:33:50 +00:00
except subprocess . TimeoutExpired :
hit_timeout = True
child . kill ( )
print ( " KILLED %d \n " % child . pid )
outs , errs = child . communicate ( )
2022-09-21 00:47:52 +00:00
return hit_timeout , child . returncode , outs . decode ( " utf-8 " ) , errs . decode ( " utf-8 " )
2021-06-01 16:33:50 +00:00
2020-06-13 02:24:11 +00:00
2024-02-13 22:15:52 +00:00
def print_output_and_exit_on_error ( stdout , stderr , print_stderr_separately = False ) :
print ( " stdout: \n " , stdout )
print stderr separately per option (#12301)
Summary:
While working on Meta's internal test triaging process, I found that `db_crashtest.py` was printing out `stdout` and `stderr` altogether. Adding an option to print `stderr` separately so that it's easy to extract only `stderr` from the test run.
`print_stderr_separately` is introduced as an optional parameter with default value `False` to keep the existing behavior as is (except a few minor changes).
Minor changes to the existing behavior
- We no longer print `stderr has error message:` and `***` prefix to each line. We simply print `stderr:` before printing `stderr` if stderr is printed in stdout and print `stderr` as is.
- We no longer print `times error occurred in output is ...` which doesn't appear to have any values
Pull Request resolved: https://github.com/facebook/rocksdb/pull/12301
Test Plan:
**Default Behavior (blackbox)**
Run printed everything as is
```
$> python3 tools/db_crashtest.py blackbox --simple --max_key=25000000 --write_buffer_size=4194304 2> /tmp/error.log
Running blackbox-crash-test with
interval_between_crash=120
total-duration=6000
...
Integrated BlobDB: blob files enabled 0, min blob size 0, blob file size 268435456, blob compression type NoCompression, blob GC enabled 0, cutoff 0.250000, force threshold 1.000000, blob compaction readahead size 0, blob file starting level 0
Integrated BlobDB: blob cache disabled
DB path: [/tmp/jewoongh/rocksdb_crashtest_blackboxwh7yxpec]
(Re-)verified 0 unique IDs
2024/01/29-09:16:30 Initializing worker threads
Crash-recovery verification passed :)
2024/01/29-09:16:35 Starting database operations
2024/01/29-09:16:35 Starting verification
Stress Test : 543.600 micros/op 8802 ops/sec
: Wrote 0.00 MB (0.27 MB/sec) (50% of 10 ops)
: Wrote 5 times
: Deleted 1 times
: Single deleted 0 times
: 4 read and 0 found the key
: Prefix scanned 0 times
: Iterator size sum is 0
: Iterated 0 times
: Deleted 0 key-ranges
: Range deletions covered 0 keys
: Got errors 0 times
: 0 CompactFiles() succeed
: 0 CompactFiles() did not succeed
stderr:
WARNING: prefix_size is non-zero but memtablerep != prefix_hash
Error : jewoongh injected test error This is not a real failure.
Verification failed :(
```
Nothing in stderr
```
$> cat /tmp/error.log
```
**Default Behavior (whitebox)**
Run printed everything as is
```
$> python3 tools/db_crashtest.py whitebox --simple --max_key=25000000 --write_buffer_size=4194304 2> /tmp/error.log
Running whitebox-crash-test with
total-duration=10000
...
(Re-)verified 571 unique IDs
2024/01/29-09:33:53 Initializing worker threads
Crash-recovery verification passed :)
2024/01/29-09:35:16 Starting database operations
2024/01/29-09:35:16 Starting verification
Stress Test : 97248.125 micros/op 10 ops/sec
: Wrote 0.00 MB (0.00 MB/sec) (12% of 8 ops)
: Wrote 1 times
: Deleted 0 times
: Single deleted 0 times
: 4 read and 1 found the key
: Prefix scanned 1 times
: Iterator size sum is 120868
: Iterated 4 times
: Deleted 0 key-ranges
: Range deletions covered 0 keys
: Got errors 0 times
: 0 CompactFiles() succeed
: 0 CompactFiles() did not succeed
stderr:
WARNING: prefix_size is non-zero but memtablerep != prefix_hash
Error : jewoongh injected test error This is not a real failure.
New cache capacity = 4865393
Verification failed :(
TEST FAILED. See kill option and exit code above!!!
```
Nothing in stderr
```
$> cat /tmp/error.log
```
**New option added (blackbox)**
```
$> python3 tools/db_crashtest.py blackbox --simple --max_key=25000000 --write_buffer_size=4194304 --print_stderr_separately 2> /tmp/error.log
Running blackbox-crash-test with
interval_between_crash=120
total-duration=6000
...
Integrated BlobDB: blob files enabled 0, min blob size 0, blob file size 268435456, blob compression type NoCompression, blob GC enabled 0, cutoff 0.250000, force threshold 1.000000, blob compaction readahead size 0, blob file starting level 0
Integrated BlobDB: blob cache disabled
DB path: [/tmp/jewoongh/rocksdb_crashtest_blackbox7ybna32z]
(Re-)verified 0 unique IDs
Compaction filter factory: DbStressCompactionFilterFactory
2024/01/29-09:05:39 Initializing worker threads
Crash-recovery verification passed :)
2024/01/29-09:05:46 Starting database operations
2024/01/29-09:05:46 Starting verification
Stress Test : 235.917 micros/op 16000 ops/sec
: Wrote 0.00 MB (0.16 MB/sec) (16% of 12 ops)
: Wrote 2 times
: Deleted 1 times
: Single deleted 0 times
: 9 read and 0 found the key
: Prefix scanned 0 times
: Iterator size sum is 0
: Iterated 0 times
: Deleted 0 key-ranges
: Range deletions covered 0 keys
: Got errors 0 times
: 0 CompactFiles() succeed
: 0 CompactFiles() did not succeed
```
stderr printed separately
```
$> cat /tmp/error.log
WARNING: prefix_size is non-zero but memtablerep != prefix_hash
Error : jewoongh injected test error This is not a real failure.
New cache capacity = 19461571
Verification failed :(
```
**New option added (whitebox)**
```
$> python3 tools/db_crashtest.py whitebox --simple --max_key=25000000 --write_buffer_size=4194304 --print_stderr_separately 2> /tmp/error.log
Running whitebox-crash-test with
total-duration=10000
...
Integrated BlobDB: blob files enabled 0, min blob size 0, blob file size 268435456, blob compression type NoCompression, blob GC enabled 0, cutoff 0.250000, force threshold 1.000000, blob compaction readahead size 0, blob file starting level 0
Integrated BlobDB: blob cache disabled
DB path: [/tmp/jewoongh/rocksdb_crashtest_whiteboxtwj0ihn6]
(Re-)verified 157 unique IDs
2024/01/29-09:39:59 Initializing worker threads
Crash-recovery verification passed :)
2024/01/29-09:40:16 Starting database operations
2024/01/29-09:40:16 Starting verification
Stress Test : 742.474 micros/op 11801 ops/sec
: Wrote 0.00 MB (0.27 MB/sec) (36% of 19 ops)
: Wrote 7 times
: Deleted 1 times
: Single deleted 0 times
: 8 read and 0 found the key
: Prefix scanned 0 times
: Iterator size sum is 0
: Iterated 4 times
: Deleted 0 key-ranges
: Range deletions covered 0 keys
: Got errors 0 times
: 0 CompactFiles() succeed
: 0 CompactFiles() did not succeed
TEST FAILED. See kill option and exit code above!!!
```
stderr printed separately
```
$> cat /tmp/error.log
WARNING: prefix_size is non-zero but memtablerep != prefix_hash
Error : jewoongh injected test error This is not a real failure.
Error : jewoongh injected test error This is not a real failure.
Error : jewoongh injected test error This is not a real failure.
New cache capacity = 4865393
Verification failed :(
```
Reviewed By: akankshamahajan15
Differential Revision: D53187491
Pulled By: jaykorean
fbshipit-source-id: 76f9100d08b96d014e41b7b88b206d69f0ae932b
2024-01-29 19:09:47 +00:00
if len ( stderr ) == 0 :
return
2023-12-01 19:01:29 +00:00
2024-02-07 20:34:40 +00:00
if print_stderr_separately :
2024-02-13 22:15:52 +00:00
print ( " stderr: \n " , stderr , file = sys . stderr )
print stderr separately per option (#12301)
Summary:
While working on Meta's internal test triaging process, I found that `db_crashtest.py` was printing out `stdout` and `stderr` altogether. Adding an option to print `stderr` separately so that it's easy to extract only `stderr` from the test run.
`print_stderr_separately` is introduced as an optional parameter with default value `False` to keep the existing behavior as is (except a few minor changes).
Minor changes to the existing behavior
- We no longer print `stderr has error message:` and `***` prefix to each line. We simply print `stderr:` before printing `stderr` if stderr is printed in stdout and print `stderr` as is.
- We no longer print `times error occurred in output is ...` which doesn't appear to have any values
Pull Request resolved: https://github.com/facebook/rocksdb/pull/12301
Test Plan:
**Default Behavior (blackbox)**
Run printed everything as is
```
$> python3 tools/db_crashtest.py blackbox --simple --max_key=25000000 --write_buffer_size=4194304 2> /tmp/error.log
Running blackbox-crash-test with
interval_between_crash=120
total-duration=6000
...
Integrated BlobDB: blob files enabled 0, min blob size 0, blob file size 268435456, blob compression type NoCompression, blob GC enabled 0, cutoff 0.250000, force threshold 1.000000, blob compaction readahead size 0, blob file starting level 0
Integrated BlobDB: blob cache disabled
DB path: [/tmp/jewoongh/rocksdb_crashtest_blackboxwh7yxpec]
(Re-)verified 0 unique IDs
2024/01/29-09:16:30 Initializing worker threads
Crash-recovery verification passed :)
2024/01/29-09:16:35 Starting database operations
2024/01/29-09:16:35 Starting verification
Stress Test : 543.600 micros/op 8802 ops/sec
: Wrote 0.00 MB (0.27 MB/sec) (50% of 10 ops)
: Wrote 5 times
: Deleted 1 times
: Single deleted 0 times
: 4 read and 0 found the key
: Prefix scanned 0 times
: Iterator size sum is 0
: Iterated 0 times
: Deleted 0 key-ranges
: Range deletions covered 0 keys
: Got errors 0 times
: 0 CompactFiles() succeed
: 0 CompactFiles() did not succeed
stderr:
WARNING: prefix_size is non-zero but memtablerep != prefix_hash
Error : jewoongh injected test error This is not a real failure.
Verification failed :(
```
Nothing in stderr
```
$> cat /tmp/error.log
```
**Default Behavior (whitebox)**
Run printed everything as is
```
$> python3 tools/db_crashtest.py whitebox --simple --max_key=25000000 --write_buffer_size=4194304 2> /tmp/error.log
Running whitebox-crash-test with
total-duration=10000
...
(Re-)verified 571 unique IDs
2024/01/29-09:33:53 Initializing worker threads
Crash-recovery verification passed :)
2024/01/29-09:35:16 Starting database operations
2024/01/29-09:35:16 Starting verification
Stress Test : 97248.125 micros/op 10 ops/sec
: Wrote 0.00 MB (0.00 MB/sec) (12% of 8 ops)
: Wrote 1 times
: Deleted 0 times
: Single deleted 0 times
: 4 read and 1 found the key
: Prefix scanned 1 times
: Iterator size sum is 120868
: Iterated 4 times
: Deleted 0 key-ranges
: Range deletions covered 0 keys
: Got errors 0 times
: 0 CompactFiles() succeed
: 0 CompactFiles() did not succeed
stderr:
WARNING: prefix_size is non-zero but memtablerep != prefix_hash
Error : jewoongh injected test error This is not a real failure.
New cache capacity = 4865393
Verification failed :(
TEST FAILED. See kill option and exit code above!!!
```
Nothing in stderr
```
$> cat /tmp/error.log
```
**New option added (blackbox)**
```
$> python3 tools/db_crashtest.py blackbox --simple --max_key=25000000 --write_buffer_size=4194304 --print_stderr_separately 2> /tmp/error.log
Running blackbox-crash-test with
interval_between_crash=120
total-duration=6000
...
Integrated BlobDB: blob files enabled 0, min blob size 0, blob file size 268435456, blob compression type NoCompression, blob GC enabled 0, cutoff 0.250000, force threshold 1.000000, blob compaction readahead size 0, blob file starting level 0
Integrated BlobDB: blob cache disabled
DB path: [/tmp/jewoongh/rocksdb_crashtest_blackbox7ybna32z]
(Re-)verified 0 unique IDs
Compaction filter factory: DbStressCompactionFilterFactory
2024/01/29-09:05:39 Initializing worker threads
Crash-recovery verification passed :)
2024/01/29-09:05:46 Starting database operations
2024/01/29-09:05:46 Starting verification
Stress Test : 235.917 micros/op 16000 ops/sec
: Wrote 0.00 MB (0.16 MB/sec) (16% of 12 ops)
: Wrote 2 times
: Deleted 1 times
: Single deleted 0 times
: 9 read and 0 found the key
: Prefix scanned 0 times
: Iterator size sum is 0
: Iterated 0 times
: Deleted 0 key-ranges
: Range deletions covered 0 keys
: Got errors 0 times
: 0 CompactFiles() succeed
: 0 CompactFiles() did not succeed
```
stderr printed separately
```
$> cat /tmp/error.log
WARNING: prefix_size is non-zero but memtablerep != prefix_hash
Error : jewoongh injected test error This is not a real failure.
New cache capacity = 19461571
Verification failed :(
```
**New option added (whitebox)**
```
$> python3 tools/db_crashtest.py whitebox --simple --max_key=25000000 --write_buffer_size=4194304 --print_stderr_separately 2> /tmp/error.log
Running whitebox-crash-test with
total-duration=10000
...
Integrated BlobDB: blob files enabled 0, min blob size 0, blob file size 268435456, blob compression type NoCompression, blob GC enabled 0, cutoff 0.250000, force threshold 1.000000, blob compaction readahead size 0, blob file starting level 0
Integrated BlobDB: blob cache disabled
DB path: [/tmp/jewoongh/rocksdb_crashtest_whiteboxtwj0ihn6]
(Re-)verified 157 unique IDs
2024/01/29-09:39:59 Initializing worker threads
Crash-recovery verification passed :)
2024/01/29-09:40:16 Starting database operations
2024/01/29-09:40:16 Starting verification
Stress Test : 742.474 micros/op 11801 ops/sec
: Wrote 0.00 MB (0.27 MB/sec) (36% of 19 ops)
: Wrote 7 times
: Deleted 1 times
: Single deleted 0 times
: 8 read and 0 found the key
: Prefix scanned 0 times
: Iterator size sum is 0
: Iterated 4 times
: Deleted 0 key-ranges
: Range deletions covered 0 keys
: Got errors 0 times
: 0 CompactFiles() succeed
: 0 CompactFiles() did not succeed
TEST FAILED. See kill option and exit code above!!!
```
stderr printed separately
```
$> cat /tmp/error.log
WARNING: prefix_size is non-zero but memtablerep != prefix_hash
Error : jewoongh injected test error This is not a real failure.
Error : jewoongh injected test error This is not a real failure.
Error : jewoongh injected test error This is not a real failure.
New cache capacity = 4865393
Verification failed :(
```
Reviewed By: akankshamahajan15
Differential Revision: D53187491
Pulled By: jaykorean
fbshipit-source-id: 76f9100d08b96d014e41b7b88b206d69f0ae932b
2024-01-29 19:09:47 +00:00
else :
2024-02-13 22:15:52 +00:00
print ( " stderr: \n " , stderr )
2023-12-01 19:01:29 +00:00
2024-02-07 20:34:40 +00:00
sys . exit ( 2 )
2023-12-01 19:01:29 +00:00
2024-01-20 00:25:39 +00:00
def cleanup_after_success ( dbname ) :
shutil . rmtree ( dbname , True )
if cleanup_cmd is not None :
print ( " Running DB cleanup command - %s \n " % cleanup_cmd )
ret = os . system ( cleanup_cmd )
if ret != 0 :
print ( " TEST FAILED. DB cleanup returned error %d \n " % ret )
sys . exit ( 1 )
2015-10-19 18:43:14 +00:00
# This script runs and kills db_stress multiple times. It checks consistency
# in case of unsafe crashes in RocksDB.
2018-05-09 20:32:03 +00:00
def blackbox_crash_main ( args , unknown_args ) :
2015-10-19 18:43:14 +00:00
cmd_params = gen_cmd_params ( args )
2022-09-21 00:47:52 +00:00
dbname = get_dbname ( " blackbox " )
exit_time = time . time ( ) + cmd_params [ " duration " ]
print (
" Running blackbox-crash-test with \n "
+ " interval_between_crash= "
+ str ( cmd_params [ " interval " ] )
+ " \n "
+ " total-duration= "
+ str ( cmd_params [ " duration " ] )
+ " \n "
)
2014-03-20 18:11:08 +00:00
2013-03-13 06:20:14 +00:00
while time . time ( ) < exit_time :
2022-09-21 00:47:52 +00:00
cmd = gen_cmd (
dict ( list ( cmd_params . items ( ) ) + list ( { " db " : dbname } . items ( ) ) ) , unknown_args
)
2013-08-21 00:37:49 +00:00
2022-09-21 00:47:52 +00:00
hit_timeout , retcode , outs , errs = execute_cmd ( cmd , cmd_params [ " interval " ] )
2021-06-01 16:33:50 +00:00
if not hit_timeout :
2022-09-21 00:47:52 +00:00
print ( " Exit Before Killing " )
2024-02-13 22:15:52 +00:00
print_output_and_exit_on_error ( outs , errs , args . print_stderr_separately )
2021-06-01 16:33:50 +00:00
sys . exit ( 2 )
2024-02-13 22:15:52 +00:00
print_output_and_exit_on_error ( outs , errs , args . print_stderr_separately )
2023-10-24 18:46:18 +00:00
2013-08-21 00:37:49 +00:00
time . sleep ( 1 ) # time to stabilize before the next run
2013-03-13 06:20:14 +00:00
2020-06-13 02:24:11 +00:00
time . sleep ( 1 ) # time to stabilize before the next run
2023-08-23 22:24:23 +00:00
# We should run the test one more time with VerifyOnly setup and no-timeout
# Only do this if the tests are not failed for total-duration
print ( " Running final time for verification " )
cmd_params . update ( { " verification_only " : 1 } )
cmd_params . update ( { " skip_verifydb " : 0 } )
cmd = gen_cmd (
dict ( list ( cmd_params . items ( ) ) + list ( { " db " : dbname } . items ( ) ) ) , unknown_args
)
hit_timeout , retcode , outs , errs = execute_cmd ( cmd )
2024-02-13 22:15:52 +00:00
# For the final run
print_output_and_exit_on_error ( outs , errs , args . print_stderr_separately )
2023-10-24 18:46:18 +00:00
2014-03-20 18:11:08 +00:00
# we need to clean up after ourselves -- only do this on test success
2024-01-20 00:25:39 +00:00
cleanup_after_success ( dbname )
2014-03-20 18:11:08 +00:00
2015-10-19 18:43:14 +00:00
# This python script runs db_stress multiple times. Some runs with
# kill_random_test that causes rocksdb to crash at various points in code.
2018-05-09 20:32:03 +00:00
def whitebox_crash_main ( args , unknown_args ) :
2015-10-19 18:43:14 +00:00
cmd_params = gen_cmd_params ( args )
2022-09-21 00:47:52 +00:00
dbname = get_dbname ( " whitebox " )
2015-10-19 18:43:14 +00:00
cur_time = time . time ( )
2022-09-21 00:47:52 +00:00
exit_time = cur_time + cmd_params [ " duration " ]
half_time = cur_time + cmd_params [ " duration " ] / / 2
2015-10-19 18:43:14 +00:00
2022-09-21 00:47:52 +00:00
print (
" Running whitebox-crash-test with \n "
+ " total-duration= "
+ str ( cmd_params [ " duration " ] )
+ " \n "
)
2015-10-19 18:43:14 +00:00
total_check_mode = 4
check_mode = 0
2022-09-21 00:47:52 +00:00
kill_random_test = cmd_params [ " random_kill_odd " ]
2015-10-19 18:43:14 +00:00
kill_mode = 0
2022-09-27 19:18:28 +00:00
prev_compaction_style = - 1
2024-01-20 00:25:39 +00:00
succeeded = True
hit_timeout = False
2015-10-19 18:43:14 +00:00
while time . time ( ) < exit_time :
if check_mode == 0 :
additional_opts = {
# use large ops per thread since we will kill it anyway
2022-09-21 00:47:52 +00:00
" ops_per_thread " : 100
* cmd_params [ " ops_per_thread " ] ,
2015-10-19 18:43:14 +00:00
}
2015-10-26 23:02:32 +00:00
# run with kill_random_test, with three modes.
# Mode 0 covers all kill points. Mode 1 covers less kill points but
# increases change of triggering them. Mode 2 covers even less
# frequent kill points and further increases triggering change.
2015-10-19 18:43:14 +00:00
if kill_mode == 0 :
2022-09-21 00:47:52 +00:00
additional_opts . update (
{
" kill_random_test " : kill_random_test ,
}
)
2015-10-19 18:43:14 +00:00
elif kill_mode == 1 :
2022-09-21 00:47:52 +00:00
if cmd_params . get ( " disable_wal " , 0 ) == 1 :
2020-03-25 03:57:53 +00:00
my_kill_odd = kill_random_test / / 50 + 1
2019-08-19 17:50:25 +00:00
else :
2020-03-25 03:57:53 +00:00
my_kill_odd = kill_random_test / / 10 + 1
2022-09-21 00:47:52 +00:00
additional_opts . update (
{
" kill_random_test " : my_kill_odd ,
" kill_exclude_prefixes " : " WritableFileWriter::Append, "
+ " WritableFileWriter::WriteBuffered " ,
}
)
2015-10-26 23:02:32 +00:00
elif kill_mode == 2 :
2016-07-22 18:46:13 +00:00
# TODO: May need to adjust random odds if kill_random_test
# is too small.
2022-09-21 00:47:52 +00:00
additional_opts . update (
{
" kill_random_test " : ( kill_random_test / / 5000 + 1 ) ,
" kill_exclude_prefixes " : " WritableFileWriter::Append, "
" WritableFileWriter::WriteBuffered, "
" PosixMmapFile::Allocate,WritableFileWriter::Flush " ,
}
)
2015-10-26 23:02:32 +00:00
# Run kill mode 0, 1 and 2 by turn.
kill_mode = ( kill_mode + 1 ) % 3
2015-10-19 18:43:14 +00:00
elif check_mode == 1 :
# normal run with universal compaction mode
additional_opts = {
" kill_random_test " : None ,
2022-09-21 00:47:52 +00:00
" ops_per_thread " : cmd_params [ " ops_per_thread " ] ,
2015-10-19 18:43:14 +00:00
" compaction_style " : 1 ,
}
2020-05-07 01:06:04 +00:00
# Single level universal has a lot of special logic. Ensure we cover
# it sometimes.
if random . randint ( 0 , 1 ) == 1 :
2022-09-21 00:47:52 +00:00
additional_opts . update (
{
" num_levels " : 1 ,
}
)
2015-10-19 18:43:14 +00:00
elif check_mode == 2 :
# normal run with FIFO compaction mode
# ops_per_thread is divided by 5 because FIFO compaction
# style is quite a bit slower on reads with lot of files
additional_opts = {
" kill_random_test " : None ,
2022-09-21 00:47:52 +00:00
" ops_per_thread " : cmd_params [ " ops_per_thread " ] / / 5 ,
2015-10-19 18:43:14 +00:00
" compaction_style " : 2 ,
}
else :
# normal run
2018-06-26 19:35:26 +00:00
additional_opts = {
2015-10-19 18:43:14 +00:00
" kill_random_test " : None ,
2022-09-21 00:47:52 +00:00
" ops_per_thread " : cmd_params [ " ops_per_thread " ] ,
2015-10-19 18:43:14 +00:00
}
2023-06-17 23:27:37 +00:00
cur_compaction_style = additional_opts . get (
" compaction_style " , cmd_params . get ( " compaction_style " , 0 )
)
if (
prev_compaction_style != - 1
and prev_compaction_style != cur_compaction_style
) :
print (
" `compaction_style` is changed in current run so `destroy_db_initially` is set to 1 as a short-term solution to avoid cycling through previous db of different compaction style. "
+ " \n "
)
2022-09-27 19:18:28 +00:00
additional_opts [ " destroy_db_initially " ] = 1
prev_compaction_style = cur_compaction_style
2022-09-21 00:47:52 +00:00
cmd = gen_cmd (
dict (
list ( cmd_params . items ( ) )
+ list ( additional_opts . items ( ) )
+ list ( { " db " : dbname } . items ( ) )
) ,
unknown_args ,
)
2015-10-19 18:43:14 +00:00
2022-09-21 00:47:52 +00:00
print (
" Running: " + " " . join ( cmd ) + " \n "
) # noqa: E999 T25377293 Grandfathered in
2015-10-19 18:43:14 +00:00
2021-06-01 16:33:50 +00:00
# If the running time is 15 minutes over the run time, explicit kill and
# exit even if white box kill didn't hit. This is to guarantee run time
# limit, as if it runs as a job, running too long will create problems
# for job scheduling or execution.
# TODO detect a hanging condition. The job might run too long as RocksDB
# hits a hanging bug.
hit_timeout , retncode , stdoutdata , stderrdata = execute_cmd (
2022-09-21 00:47:52 +00:00
cmd , exit_time - time . time ( ) + 900
)
msg = " check_mode= {0} , kill option= {1} , exitcode= {2} \n " . format (
check_mode , additional_opts [ " kill_random_test " ] , retncode
)
2021-05-06 00:53:16 +00:00
2020-03-25 03:57:53 +00:00
print ( msg )
2024-02-13 22:15:52 +00:00
print_output_and_exit_on_error ( stdoutdata , stderrdata , args . print_stderr_separately )
2015-10-19 18:43:14 +00:00
2021-06-01 16:33:50 +00:00
if hit_timeout :
print ( " Killing the run for running too long " )
break
2024-01-20 00:25:39 +00:00
succeeded = False
2022-09-21 00:47:52 +00:00
if additional_opts [ " kill_random_test " ] is None and ( retncode == 0 ) :
2015-10-19 18:43:14 +00:00
# we expect zero retncode if no kill option
2024-01-20 00:25:39 +00:00
succeeded = True
2022-09-21 00:47:52 +00:00
elif additional_opts [ " kill_random_test " ] is not None and retncode < = 0 :
2019-05-04 00:26:20 +00:00
# When kill option is given, the test MIGHT kill itself.
# If it does, negative retncode is expected. Otherwise 0.
2024-01-20 00:25:39 +00:00
succeeded = True
2015-10-19 18:43:14 +00:00
2024-01-20 00:25:39 +00:00
if not succeeded :
2020-03-25 03:57:53 +00:00
print ( " TEST FAILED. See kill option and exit code above!!! \n " )
2015-10-19 18:43:14 +00:00
sys . exit ( 1 )
# First half of the duration, keep doing kill test. For the next half,
# try different modes.
if time . time ( ) > half_time :
2024-01-20 00:25:39 +00:00
cleanup_after_success ( dbname )
2023-05-23 16:42:35 +00:00
try :
os . mkdir ( dbname )
except OSError :
pass
2023-06-17 23:27:37 +00:00
if expected_values_dir is not None :
2022-09-29 23:29:51 +00:00
shutil . rmtree ( expected_values_dir , True )
os . mkdir ( expected_values_dir )
2015-10-19 18:43:14 +00:00
check_mode = ( check_mode + 1 ) % total_check_mode
time . sleep ( 1 ) # time to stabilize after a kill
2024-01-20 00:25:39 +00:00
# If successfully finished or timed out (we currently treat timed out test as passing)
# Clean up after ourselves
if succeeded or hit_timeout :
cleanup_after_success ( dbname )
2015-10-19 18:43:14 +00:00
def main ( ) :
2021-06-28 06:53:47 +00:00
global stress_cmd
2022-10-28 02:47:01 +00:00
global cleanup_cmd
2021-06-28 06:53:47 +00:00
2022-09-21 00:47:52 +00:00
parser = argparse . ArgumentParser (
description = " This script runs and kills \
db_stress multiple times "
)
2015-10-19 18:43:14 +00:00
parser . add_argument ( " test_type " , choices = [ " blackbox " , " whitebox " ] )
parser . add_argument ( " --simple " , action = " store_true " )
2022-09-21 00:47:52 +00:00
parser . add_argument ( " --cf_consistency " , action = " store_true " )
parser . add_argument ( " --txn " , action = " store_true " )
2023-06-17 23:27:37 +00:00
parser . add_argument ( " --optimistic_txn " , action = " store_true " )
2022-09-21 00:47:52 +00:00
parser . add_argument ( " --test_best_efforts_recovery " , action = " store_true " )
parser . add_argument ( " --enable_ts " , action = " store_true " )
parser . add_argument ( " --test_multiops_txn " , action = " store_true " )
2022-03-17 02:00:04 +00:00
parser . add_argument ( " --write_policy " , choices = [ " write_committed " , " write_prepared " ] )
2021-06-28 06:53:47 +00:00
parser . add_argument ( " --stress_cmd " )
2022-09-21 00:47:52 +00:00
parser . add_argument ( " --test_tiered_storage " , action = " store_true " )
2022-10-28 02:47:01 +00:00
parser . add_argument ( " --cleanup_cmd " )
2023-06-27 19:30:19 +00:00
parser . add_argument ( " --skip_tmpdir_check " , action = " store_true " )
print stderr separately per option (#12301)
Summary:
While working on Meta's internal test triaging process, I found that `db_crashtest.py` was printing out `stdout` and `stderr` altogether. Adding an option to print `stderr` separately so that it's easy to extract only `stderr` from the test run.
`print_stderr_separately` is introduced as an optional parameter with default value `False` to keep the existing behavior as is (except a few minor changes).
Minor changes to the existing behavior
- We no longer print `stderr has error message:` and `***` prefix to each line. We simply print `stderr:` before printing `stderr` if stderr is printed in stdout and print `stderr` as is.
- We no longer print `times error occurred in output is ...` which doesn't appear to have any values
Pull Request resolved: https://github.com/facebook/rocksdb/pull/12301
Test Plan:
**Default Behavior (blackbox)**
Run printed everything as is
```
$> python3 tools/db_crashtest.py blackbox --simple --max_key=25000000 --write_buffer_size=4194304 2> /tmp/error.log
Running blackbox-crash-test with
interval_between_crash=120
total-duration=6000
...
Integrated BlobDB: blob files enabled 0, min blob size 0, blob file size 268435456, blob compression type NoCompression, blob GC enabled 0, cutoff 0.250000, force threshold 1.000000, blob compaction readahead size 0, blob file starting level 0
Integrated BlobDB: blob cache disabled
DB path: [/tmp/jewoongh/rocksdb_crashtest_blackboxwh7yxpec]
(Re-)verified 0 unique IDs
2024/01/29-09:16:30 Initializing worker threads
Crash-recovery verification passed :)
2024/01/29-09:16:35 Starting database operations
2024/01/29-09:16:35 Starting verification
Stress Test : 543.600 micros/op 8802 ops/sec
: Wrote 0.00 MB (0.27 MB/sec) (50% of 10 ops)
: Wrote 5 times
: Deleted 1 times
: Single deleted 0 times
: 4 read and 0 found the key
: Prefix scanned 0 times
: Iterator size sum is 0
: Iterated 0 times
: Deleted 0 key-ranges
: Range deletions covered 0 keys
: Got errors 0 times
: 0 CompactFiles() succeed
: 0 CompactFiles() did not succeed
stderr:
WARNING: prefix_size is non-zero but memtablerep != prefix_hash
Error : jewoongh injected test error This is not a real failure.
Verification failed :(
```
Nothing in stderr
```
$> cat /tmp/error.log
```
**Default Behavior (whitebox)**
Run printed everything as is
```
$> python3 tools/db_crashtest.py whitebox --simple --max_key=25000000 --write_buffer_size=4194304 2> /tmp/error.log
Running whitebox-crash-test with
total-duration=10000
...
(Re-)verified 571 unique IDs
2024/01/29-09:33:53 Initializing worker threads
Crash-recovery verification passed :)
2024/01/29-09:35:16 Starting database operations
2024/01/29-09:35:16 Starting verification
Stress Test : 97248.125 micros/op 10 ops/sec
: Wrote 0.00 MB (0.00 MB/sec) (12% of 8 ops)
: Wrote 1 times
: Deleted 0 times
: Single deleted 0 times
: 4 read and 1 found the key
: Prefix scanned 1 times
: Iterator size sum is 120868
: Iterated 4 times
: Deleted 0 key-ranges
: Range deletions covered 0 keys
: Got errors 0 times
: 0 CompactFiles() succeed
: 0 CompactFiles() did not succeed
stderr:
WARNING: prefix_size is non-zero but memtablerep != prefix_hash
Error : jewoongh injected test error This is not a real failure.
New cache capacity = 4865393
Verification failed :(
TEST FAILED. See kill option and exit code above!!!
```
Nothing in stderr
```
$> cat /tmp/error.log
```
**New option added (blackbox)**
```
$> python3 tools/db_crashtest.py blackbox --simple --max_key=25000000 --write_buffer_size=4194304 --print_stderr_separately 2> /tmp/error.log
Running blackbox-crash-test with
interval_between_crash=120
total-duration=6000
...
Integrated BlobDB: blob files enabled 0, min blob size 0, blob file size 268435456, blob compression type NoCompression, blob GC enabled 0, cutoff 0.250000, force threshold 1.000000, blob compaction readahead size 0, blob file starting level 0
Integrated BlobDB: blob cache disabled
DB path: [/tmp/jewoongh/rocksdb_crashtest_blackbox7ybna32z]
(Re-)verified 0 unique IDs
Compaction filter factory: DbStressCompactionFilterFactory
2024/01/29-09:05:39 Initializing worker threads
Crash-recovery verification passed :)
2024/01/29-09:05:46 Starting database operations
2024/01/29-09:05:46 Starting verification
Stress Test : 235.917 micros/op 16000 ops/sec
: Wrote 0.00 MB (0.16 MB/sec) (16% of 12 ops)
: Wrote 2 times
: Deleted 1 times
: Single deleted 0 times
: 9 read and 0 found the key
: Prefix scanned 0 times
: Iterator size sum is 0
: Iterated 0 times
: Deleted 0 key-ranges
: Range deletions covered 0 keys
: Got errors 0 times
: 0 CompactFiles() succeed
: 0 CompactFiles() did not succeed
```
stderr printed separately
```
$> cat /tmp/error.log
WARNING: prefix_size is non-zero but memtablerep != prefix_hash
Error : jewoongh injected test error This is not a real failure.
New cache capacity = 19461571
Verification failed :(
```
**New option added (whitebox)**
```
$> python3 tools/db_crashtest.py whitebox --simple --max_key=25000000 --write_buffer_size=4194304 --print_stderr_separately 2> /tmp/error.log
Running whitebox-crash-test with
total-duration=10000
...
Integrated BlobDB: blob files enabled 0, min blob size 0, blob file size 268435456, blob compression type NoCompression, blob GC enabled 0, cutoff 0.250000, force threshold 1.000000, blob compaction readahead size 0, blob file starting level 0
Integrated BlobDB: blob cache disabled
DB path: [/tmp/jewoongh/rocksdb_crashtest_whiteboxtwj0ihn6]
(Re-)verified 157 unique IDs
2024/01/29-09:39:59 Initializing worker threads
Crash-recovery verification passed :)
2024/01/29-09:40:16 Starting database operations
2024/01/29-09:40:16 Starting verification
Stress Test : 742.474 micros/op 11801 ops/sec
: Wrote 0.00 MB (0.27 MB/sec) (36% of 19 ops)
: Wrote 7 times
: Deleted 1 times
: Single deleted 0 times
: 8 read and 0 found the key
: Prefix scanned 0 times
: Iterator size sum is 0
: Iterated 4 times
: Deleted 0 key-ranges
: Range deletions covered 0 keys
: Got errors 0 times
: 0 CompactFiles() succeed
: 0 CompactFiles() did not succeed
TEST FAILED. See kill option and exit code above!!!
```
stderr printed separately
```
$> cat /tmp/error.log
WARNING: prefix_size is non-zero but memtablerep != prefix_hash
Error : jewoongh injected test error This is not a real failure.
Error : jewoongh injected test error This is not a real failure.
Error : jewoongh injected test error This is not a real failure.
New cache capacity = 4865393
Verification failed :(
```
Reviewed By: akankshamahajan15
Differential Revision: D53187491
Pulled By: jaykorean
fbshipit-source-id: 76f9100d08b96d014e41b7b88b206d69f0ae932b
2024-01-29 19:09:47 +00:00
parser . add_argument ( " --print_stderr_separately " , action = " store_true " , default = False )
2022-09-21 00:47:52 +00:00
all_params = dict (
list ( default_params . items ( ) )
+ list ( blackbox_default_params . items ( ) )
+ list ( whitebox_default_params . items ( ) )
+ list ( simple_default_params . items ( ) )
+ list ( blackbox_simple_default_params . items ( ) )
+ list ( whitebox_simple_default_params . items ( ) )
+ list ( blob_params . items ( ) )
+ list ( ts_params . items ( ) )
+ list ( multiops_txn_default_params . items ( ) )
+ list ( multiops_wc_txn_params . items ( ) )
+ list ( multiops_wp_txn_params . items ( ) )
+ list ( best_efforts_recovery_params . items ( ) )
+ list ( cf_consistency_params . items ( ) )
+ list ( tiered_params . items ( ) )
+ list ( txn_params . items ( ) )
2023-06-17 23:27:37 +00:00
+ list ( optimistic_txn_params . items ( ) )
2022-09-21 00:47:52 +00:00
)
2015-10-19 18:43:14 +00:00
for k , v in all_params . items ( ) :
parser . add_argument ( " -- " + k , type = type ( v ( ) if callable ( v ) else v ) )
2018-05-09 20:32:03 +00:00
# unknown_args are passed directly to db_stress
args , unknown_args = parser . parse_known_args ( )
2015-10-19 18:43:14 +00:00
2018-06-04 04:28:41 +00:00
test_tmpdir = os . environ . get ( _TEST_DIR_ENV_VAR )
2023-06-27 19:30:19 +00:00
if test_tmpdir is not None and not args . skip_tmpdir_check :
2023-05-23 16:42:35 +00:00
isdir = False
try :
isdir = os . path . isdir ( test_tmpdir )
if not isdir :
print (
2023-06-27 19:30:19 +00:00
" ERROR: %s env var is set to a non-existent directory: %s . Update it to correct directory path. "
2023-05-23 16:42:35 +00:00
% ( _TEST_DIR_ENV_VAR , test_tmpdir )
)
sys . exit ( 1 )
except OSError :
pass
2018-06-04 04:28:41 +00:00
2021-06-28 06:53:47 +00:00
if args . stress_cmd :
stress_cmd = args . stress_cmd
2022-10-28 02:47:01 +00:00
if args . cleanup_cmd :
cleanup_cmd = args . cleanup_cmd
2022-09-21 00:47:52 +00:00
if args . test_type == " blackbox " :
2018-05-09 20:32:03 +00:00
blackbox_crash_main ( args , unknown_args )
2022-09-21 00:47:52 +00:00
if args . test_type == " whitebox " :
2018-05-09 20:32:03 +00:00
whitebox_crash_main ( args , unknown_args )
2021-09-28 21:12:23 +00:00
# Only delete the `expected_values_dir` if test passes
if expected_values_dir is not None :
shutil . rmtree ( expected_values_dir )
2022-03-17 02:00:04 +00:00
if multiops_txn_key_spaces_file is not None :
os . remove ( multiops_txn_key_spaces_file )
2020-10-12 21:08:35 +00:00
2015-10-19 18:43:14 +00:00
2022-09-21 00:47:52 +00:00
if __name__ == " __main__ " :
2015-10-19 18:43:14 +00:00
main ( )
