mirror of
https://github.com/facebook/rocksdb.git
synced 2024-11-26 16:30:56 +00:00
d150e01474
Summary:
This is a new API added to db.h to allow for fetching all merge operands associated with a Key. The main motivation for this API is to support use cases where doing a full online merge is not necessary as it is performance sensitive. Example use-cases:
1. Update subset of columns and read subset of columns -
Imagine a SQL Table, a row is encoded as a K/V pair (as it is done in MyRocks). If there are many columns and users only updated one of them, we can use merge operator to reduce write amplification. While users only read one or two columns in the read query, this feature can avoid a full merging of the whole row, and save some CPU.
2. Updating very few attributes in a value which is a JSON-like document -
Updating one attribute can be done efficiently using merge operator, while reading back one attribute can be done more efficiently if we don't need to do a full merge.
----------------------------------------------------------------------------------------------------
API :
Status GetMergeOperands(
const ReadOptions& options, ColumnFamilyHandle* column_family,
const Slice& key, PinnableSlice* merge_operands,
GetMergeOperandsOptions* get_merge_operands_options,
int* number_of_operands)
Example usage :
int size = 100;
int number_of_operands = 0;
std::vector<PinnableSlice> values(size);
GetMergeOperandsOptions merge_operands_info;
db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), "k1", values.data(), merge_operands_info, &number_of_operands);
Description :
Returns all the merge operands corresponding to the key. If the number of merge operands in DB is greater than merge_operands_options.expected_max_number_of_operands no merge operands are returned and status is Incomplete. Merge operands returned are in the order of insertion.
merge_operands-> Points to an array of at-least merge_operands_options.expected_max_number_of_operands and the caller is responsible for allocating it. If the status returned is Incomplete then number_of_operands will contain the total number of merge operands found in DB for key.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5604
Test Plan:
Added unit test and perf test in db_bench that can be run using the command:
./db_bench -benchmarks=getmergeoperands --merge_operator=sortlist
Differential Revision: D16657366
Pulled By: vjnadimpalli
fbshipit-source-id: 0faadd752351745224ee12d4ae9ef3cb529951bf
68 lines
2.7 KiB
YAML
68 lines
2.7 KiB
YAML
version: 1.0.{build}
|
|
|
|
image: Visual Studio 2017
|
|
|
|
environment:
|
|
JAVA_HOME: C:\Program Files\Java\jdk1.8.0
|
|
THIRDPARTY_HOME: $(APPVEYOR_BUILD_FOLDER)\thirdparty
|
|
SNAPPY_HOME: $(THIRDPARTY_HOME)\snappy-1.1.7
|
|
SNAPPY_INCLUDE: $(SNAPPY_HOME);$(SNAPPY_HOME)\build
|
|
SNAPPY_LIB_DEBUG: $(SNAPPY_HOME)\build\Debug\snappy.lib
|
|
SNAPPY_LIB_RELEASE: $(SNAPPY_HOME)\build\Release\snappy.lib
|
|
LZ4_HOME: $(THIRDPARTY_HOME)\lz4-1.8.3
|
|
LZ4_INCLUDE: $(LZ4_HOME)\lib
|
|
LZ4_LIB_DEBUG: $(LZ4_HOME)\visual\VS2010\bin\x64_Debug\liblz4_static.lib
|
|
LZ4_LIB_RELEASE: $(LZ4_HOME)\visual\VS2010\bin\x64_Release\liblz4_static.lib
|
|
ZSTD_HOME: $(THIRDPARTY_HOME)\zstd-1.4.0
|
|
ZSTD_INCLUDE: $(ZSTD_HOME)\lib;$(ZSTD_HOME)\lib\dictBuilder
|
|
ZSTD_LIB_DEBUG: $(ZSTD_HOME)\build\VS2010\bin\x64_Debug\libzstd_static.lib
|
|
ZSTD_LIB_RELEASE: $(ZSTD_HOME)\build\VS2010\bin\x64_Release\libzstd_static.lib
|
|
|
|
install:
|
|
- md %THIRDPARTY_HOME%
|
|
- echo "Building Snappy dependency..."
|
|
- cd %THIRDPARTY_HOME%
|
|
- curl -fsSL -o snappy-1.1.7.zip https://github.com/google/snappy/archive/1.1.7.zip
|
|
- unzip snappy-1.1.7.zip
|
|
- cd snappy-1.1.7
|
|
- mkdir build
|
|
- cd build
|
|
- cmake -DCMAKE_GENERATOR_PLATFORM=x64 ..
|
|
- msbuild Snappy.sln /p:Configuration=Debug /p:Platform=x64
|
|
- msbuild Snappy.sln /p:Configuration=Release /p:Platform=x64
|
|
- echo "Building LZ4 dependency..."
|
|
- cd %THIRDPARTY_HOME%
|
|
- curl -fsSL -o lz4-1.8.3.zip https://github.com/lz4/lz4/archive/v1.8.3.zip
|
|
- unzip lz4-1.8.3.zip
|
|
- cd lz4-1.8.3\visual\VS2010
|
|
- ps: $CMD="C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\IDE\devenv.com"; & $CMD lz4.sln /upgrade
|
|
- msbuild lz4.sln /p:Configuration=Debug /p:Platform=x64
|
|
- msbuild lz4.sln /p:Configuration=Release /p:Platform=x64
|
|
- echo "Building ZStd dependency..."
|
|
- cd %THIRDPARTY_HOME%
|
|
- curl -fsSL -o zstd-1.4.0.zip https://github.com/facebook/zstd/archive/v1.4.0.zip
|
|
- unzip zstd-1.4.0.zip
|
|
- cd zstd-1.4.0\build\VS2010
|
|
- ps: $CMD="C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\IDE\devenv.com"; & $CMD zstd.sln /upgrade
|
|
- msbuild zstd.sln /p:Configuration=Debug /p:Platform=x64
|
|
- msbuild zstd.sln /p:Configuration=Release /p:Platform=x64
|
|
|
|
before_build:
|
|
- md %APPVEYOR_BUILD_FOLDER%\build
|
|
- cd %APPVEYOR_BUILD_FOLDER%\build
|
|
- cmake -G "Visual Studio 15 Win64" -DOPTDBG=1 -DPORTABLE=1 -DSNAPPY=1 -DLZ4=1 -DZSTD=1 -DXPRESS=1 -DJNI=1 ..
|
|
- cd ..
|
|
build:
|
|
project: build\rocksdb.sln
|
|
parallel: true
|
|
verbosity: normal
|
|
|
|
test:
|
|
|
|
test_script:
|
|
- ps: build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,db_test2,db_test,env_basic_test,env_test,db_merge_operand_test -Concurrency 8
|
|
|
|
on_failure:
|
|
- cmd: 7z a build-failed.zip %APPVEYOR_BUILD_FOLDER%\build\ && appveyor PushArtifact build-failed.zip
|
|
|