mirror of https://github.com/facebook/rocksdb.git
Added support for SstFileManager to RocksJava
Summary: Closes https://github.com/facebook/rocksdb/pull/3666 Differential Revision: D7457634 Pulled By: sagar0 fbshipit-source-id: 47741e2ee66e9255c580f4e38cfb86b284c27c2f
This commit is contained in:
parent
74767deec3
commit
ca87aef82d
|
@ -13,6 +13,7 @@
|
|||
|
||||
### Java API Changes
|
||||
* Add `BlockBasedTableConfig.setBlockCache` to allow sharing a block cache across DB instances.
|
||||
* Added SstFileManager to the Java API to allow managing SST files across DB instances.
|
||||
|
||||
## 5.13.0 (3/20/2018)
|
||||
### Public API Change
|
||||
|
|
|
@ -37,6 +37,7 @@ set(JNI_NATIVE_SOURCES
|
|||
rocksjni/rocksjni.cc
|
||||
rocksjni/slice.cc
|
||||
rocksjni/snapshot.cc
|
||||
rocksjni/sst_file_manager.cc
|
||||
rocksjni/sst_file_writerjni.cc
|
||||
rocksjni/statistics.cc
|
||||
rocksjni/statisticsjni.cc
|
||||
|
@ -111,6 +112,7 @@ set(NATIVE_JAVA_CLASSES
|
|||
org.rocksdb.Slice
|
||||
org.rocksdb.Snapshot
|
||||
org.rocksdb.SnapshotTest
|
||||
org.rocksdb.SstFileManager
|
||||
org.rocksdb.SstFileWriter
|
||||
org.rocksdb.Statistics
|
||||
org.rocksdb.StringAppendOperator
|
||||
|
@ -232,6 +234,7 @@ add_jar(
|
|||
src/main/java/org/rocksdb/SkipListMemTableConfig.java
|
||||
src/main/java/org/rocksdb/Slice.java
|
||||
src/main/java/org/rocksdb/Snapshot.java
|
||||
src/main/java/org/rocksdb/SstFileManager.java
|
||||
src/main/java/org/rocksdb/SstFileWriter.java
|
||||
src/main/java/org/rocksdb/Statistics.java
|
||||
src/main/java/org/rocksdb/StatsLevel.java
|
||||
|
|
|
@ -47,6 +47,7 @@ NATIVE_JAVA_CLASSES = org.rocksdb.AbstractCompactionFilter\
|
|||
org.rocksdb.RocksMemEnv\
|
||||
org.rocksdb.SkipListMemTableConfig\
|
||||
org.rocksdb.Slice\
|
||||
org.rocksdb.SstFileManager\
|
||||
org.rocksdb.SstFileWriter\
|
||||
org.rocksdb.Statistics\
|
||||
org.rocksdb.Transaction\
|
||||
|
@ -132,6 +133,7 @@ JAVA_TESTS = org.rocksdb.BackupableDBOptionsTest\
|
|||
org.rocksdb.util.SizeUnitTest\
|
||||
org.rocksdb.SliceTest\
|
||||
org.rocksdb.SnapshotTest\
|
||||
org.rocksdb.SstFileManagerTest\
|
||||
org.rocksdb.SstFileWriterTest\
|
||||
org.rocksdb.TransactionTest\
|
||||
org.rocksdb.TransactionDBTest\
|
||||
|
|
|
@ -895,6 +895,20 @@ void Java_org_rocksdb_Options_setRateLimiter(
|
|||
rate_limiter = *pRateLimiter;
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_Options
|
||||
* Method: setSstFileManager
|
||||
* Signature: (JJ)V
|
||||
*/
|
||||
void Java_org_rocksdb_Options_setSstFileManager(
|
||||
JNIEnv* env, jobject job, jlong jhandle,
|
||||
jlong jsst_file_manager_handle) {
|
||||
auto* sptr_sst_file_manager =
|
||||
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jsst_file_manager_handle);
|
||||
reinterpret_cast<rocksdb::Options*>(jhandle)->
|
||||
sst_file_manager = *sptr_sst_file_manager;
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_Options
|
||||
* Method: setLogger
|
||||
|
@ -4399,6 +4413,20 @@ void Java_org_rocksdb_DBOptions_setRateLimiter(
|
|||
reinterpret_cast<rocksdb::DBOptions*>(jhandle)->rate_limiter = *pRateLimiter;
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_DBOptions
|
||||
* Method: setSstFileManager
|
||||
* Signature: (JJ)V
|
||||
*/
|
||||
void Java_org_rocksdb_DBOptions_setSstFileManager(
|
||||
JNIEnv* env, jobject jobj, jlong jhandle,
|
||||
jlong jsst_file_manager_handle) {
|
||||
auto* sptr_sst_file_manager =
|
||||
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jsst_file_manager_handle);
|
||||
reinterpret_cast<rocksdb::DBOptions*>(jhandle)->
|
||||
sst_file_manager = *sptr_sst_file_manager;
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_DBOptions
|
||||
* Method: setLogger
|
||||
|
|
|
@ -0,0 +1,217 @@
|
|||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
//
|
||||
// This file implements the "bridge" between Java and C++ and enables
|
||||
// calling C++ rocksdb::SstFileManager methods
|
||||
// from Java side.
|
||||
|
||||
#include <jni.h>
|
||||
#include <memory>
|
||||
|
||||
#include "include/org_rocksdb_SstFileManager.h"
|
||||
#include "rocksdb/sst_file_manager.h"
|
||||
#include "rocksjni/portal.h"
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_SstFileManager
|
||||
* Method: newSstFileManager
|
||||
* Signature: (JJJDJ)J
|
||||
*/
|
||||
jlong Java_org_rocksdb_SstFileManager_newSstFileManager(
|
||||
JNIEnv* jnienv, jclass jcls, jlong jenv_handle, jlong jlogger_handle,
|
||||
jlong jrate_bytes, jdouble jmax_trash_db_ratio,
|
||||
jlong jmax_delete_chunk_bytes) {
|
||||
|
||||
auto* env = reinterpret_cast<rocksdb::Env*>(jenv_handle);
|
||||
rocksdb::Status s;
|
||||
rocksdb::SstFileManager* sst_file_manager = nullptr;
|
||||
|
||||
if (jlogger_handle != 0) {
|
||||
auto* sptr_logger =
|
||||
reinterpret_cast<std::shared_ptr<rocksdb::Logger> *>(jlogger_handle);
|
||||
sst_file_manager = rocksdb::NewSstFileManager(env, *sptr_logger, "",
|
||||
jrate_bytes, true, &s, jmax_trash_db_ratio,
|
||||
jmax_delete_chunk_bytes);
|
||||
} else {
|
||||
sst_file_manager = rocksdb::NewSstFileManager(env, nullptr, "",
|
||||
jrate_bytes, true, &s, jmax_trash_db_ratio,
|
||||
jmax_delete_chunk_bytes);
|
||||
}
|
||||
|
||||
if (!s.ok()) {
|
||||
if (sst_file_manager != nullptr) {
|
||||
delete sst_file_manager;
|
||||
}
|
||||
rocksdb::RocksDBExceptionJni::ThrowNew(jnienv, s);
|
||||
}
|
||||
auto* sptr_sst_file_manager
|
||||
= new std::shared_ptr<rocksdb::SstFileManager>(sst_file_manager);
|
||||
|
||||
return reinterpret_cast<jlong>(sptr_sst_file_manager);
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_SstFileManager
|
||||
* Method: setMaxAllowedSpaceUsage
|
||||
* Signature: (JJ)V
|
||||
*/
|
||||
void Java_org_rocksdb_SstFileManager_setMaxAllowedSpaceUsage(
|
||||
JNIEnv* env, jobject jobj, jlong jhandle, jlong jmax_allowed_space) {
|
||||
auto* sptr_sst_file_manager =
|
||||
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jhandle);
|
||||
sptr_sst_file_manager->get()->SetMaxAllowedSpaceUsage(jmax_allowed_space);
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_SstFileManager
|
||||
* Method: setCompactionBufferSize
|
||||
* Signature: (JJ)V
|
||||
*/
|
||||
void Java_org_rocksdb_SstFileManager_setCompactionBufferSize(
|
||||
JNIEnv* env, jobject jobj, jlong jhandle, jlong jcompaction_buffer_size) {
|
||||
auto* sptr_sst_file_manager =
|
||||
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jhandle);
|
||||
sptr_sst_file_manager->get()->SetCompactionBufferSize(jcompaction_buffer_size);
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_SstFileManager
|
||||
* Method: isMaxAllowedSpaceReached
|
||||
* Signature: (J)Z
|
||||
*/
|
||||
jboolean Java_org_rocksdb_SstFileManager_isMaxAllowedSpaceReached(
|
||||
JNIEnv* env, jobject jobj, jlong jhandle) {
|
||||
auto* sptr_sst_file_manager =
|
||||
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jhandle);
|
||||
return sptr_sst_file_manager->get()->IsMaxAllowedSpaceReached();
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_SstFileManager
|
||||
* Method: isMaxAllowedSpaceReachedIncludingCompactions
|
||||
* Signature: (J)Z
|
||||
*/
|
||||
jboolean Java_org_rocksdb_SstFileManager_isMaxAllowedSpaceReachedIncludingCompactions(
|
||||
JNIEnv* env, jobject jobj, jlong jhandle) {
|
||||
auto* sptr_sst_file_manager =
|
||||
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jhandle);
|
||||
return sptr_sst_file_manager->get()->IsMaxAllowedSpaceReachedIncludingCompactions();
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_SstFileManager
|
||||
* Method: getTotalSize
|
||||
* Signature: (J)J
|
||||
*/
|
||||
jlong Java_org_rocksdb_SstFileManager_getTotalSize(
|
||||
JNIEnv* env, jobject jobj, jlong jhandle) {
|
||||
auto* sptr_sst_file_manager =
|
||||
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jhandle);
|
||||
return sptr_sst_file_manager->get()->GetTotalSize();
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_SstFileManager
|
||||
* Method: getTrackedFiles
|
||||
* Signature: (J)Ljava/util/Map;
|
||||
*/
|
||||
jobject Java_org_rocksdb_SstFileManager_getTrackedFiles(
|
||||
JNIEnv* env, jobject jobj, jlong jhandle) {
|
||||
auto* sptr_sst_file_manager =
|
||||
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jhandle);
|
||||
auto tracked_files = sptr_sst_file_manager->get()->GetTrackedFiles();
|
||||
|
||||
const jobject jtracked_files = rocksdb::HashMapJni::construct(env,
|
||||
static_cast<uint32_t>(tracked_files.size()));
|
||||
if (jtracked_files == nullptr) {
|
||||
// exception occurred
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
const rocksdb::HashMapJni::FnMapKV<const std::string, const uint64_t> fn_map_kv =
|
||||
[env, &tracked_files](const std::pair<const std::string, const uint64_t>& pair) {
|
||||
const jstring jtracked_file_path = env->NewStringUTF(pair.first.c_str());
|
||||
if (jtracked_file_path == nullptr) {
|
||||
// an error occurred
|
||||
return std::unique_ptr<std::pair<jobject, jobject>>(nullptr);
|
||||
}
|
||||
const jobject jtracked_file_size =
|
||||
rocksdb::LongJni::valueOf(env, pair.second);
|
||||
if (jtracked_file_size == nullptr) {
|
||||
// an error occurred
|
||||
return std::unique_ptr<std::pair<jobject, jobject>>(nullptr);
|
||||
}
|
||||
return std::unique_ptr<std::pair<jobject, jobject>>(new std::pair<jobject, jobject>(jtracked_file_path,
|
||||
jtracked_file_size));
|
||||
};
|
||||
|
||||
if(!rocksdb::HashMapJni::putAll(env, jtracked_files,
|
||||
tracked_files.begin(), tracked_files.end(), fn_map_kv)) {
|
||||
// exception occcurred
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
return jtracked_files;
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_SstFileManager
|
||||
* Method: getDeleteRateBytesPerSecond
|
||||
* Signature: (J)J
|
||||
*/
|
||||
jlong Java_org_rocksdb_SstFileManager_getDeleteRateBytesPerSecond(
|
||||
JNIEnv* env, jobject jobj, jlong jhandle) {
|
||||
auto* sptr_sst_file_manager =
|
||||
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jhandle);
|
||||
return sptr_sst_file_manager->get()->GetDeleteRateBytesPerSecond();
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_SstFileManager
|
||||
* Method: setDeleteRateBytesPerSecond
|
||||
* Signature: (JJ)V
|
||||
*/
|
||||
void Java_org_rocksdb_SstFileManager_setDeleteRateBytesPerSecond(
|
||||
JNIEnv* env, jobject jobj, jlong jhandle, jlong jdelete_rate) {
|
||||
auto* sptr_sst_file_manager =
|
||||
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jhandle);
|
||||
sptr_sst_file_manager->get()->SetDeleteRateBytesPerSecond(jdelete_rate);
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_SstFileManager
|
||||
* Method: getMaxTrashDBRatio
|
||||
* Signature: (J)D
|
||||
*/
|
||||
jdouble Java_org_rocksdb_SstFileManager_getMaxTrashDBRatio(
|
||||
JNIEnv* env, jobject jobj, jlong jhandle) {
|
||||
auto* sptr_sst_file_manager =
|
||||
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jhandle);
|
||||
return sptr_sst_file_manager->get()->GetMaxTrashDBRatio();
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_SstFileManager
|
||||
* Method: setMaxTrashDBRatio
|
||||
* Signature: (JD)V
|
||||
*/
|
||||
void Java_org_rocksdb_SstFileManager_setMaxTrashDBRatio(
|
||||
JNIEnv* env, jobject jobj, jlong jhandle, jdouble jratio) {
|
||||
auto* sptr_sst_file_manager =
|
||||
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jhandle);
|
||||
sptr_sst_file_manager->get()->SetMaxTrashDBRatio(jratio);
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_SstFileManager
|
||||
* Method: disposeInternal
|
||||
* Signature: (J)V
|
||||
*/
|
||||
void Java_org_rocksdb_SstFileManager_disposeInternal(
|
||||
JNIEnv* env, jobject jobj, jlong jhandle) {
|
||||
auto* sptr_sst_file_manager =
|
||||
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jhandle);
|
||||
delete sptr_sst_file_manager;
|
||||
}
|
|
@ -12,6 +12,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
|
|||
* {@link AbstractNativeReference} which have an immutable reference to the
|
||||
* underlying native C++ object
|
||||
*/
|
||||
//@ThreadSafe
|
||||
public abstract class AbstractImmutableNativeReference
|
||||
extends AbstractNativeReference {
|
||||
|
||||
|
|
|
@ -178,6 +178,13 @@ public class DBOptions
|
|||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public DBOptions setSstFileManager(final SstFileManager sstFileManager) {
|
||||
assert(isOwningHandle());
|
||||
setSstFileManager(nativeHandle_, sstFileManager.nativeHandle_);
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public DBOptions setLogger(final Logger logger) {
|
||||
assert(isOwningHandle());
|
||||
|
@ -988,6 +995,8 @@ public class DBOptions
|
|||
private native boolean paranoidChecks(long handle);
|
||||
private native void setRateLimiter(long handle,
|
||||
long rateLimiterHandle);
|
||||
private native void setSstFileManager(final long handle,
|
||||
final long sstFileManagerHandle);
|
||||
private native void setLogger(long handle,
|
||||
long loggerHandle);
|
||||
private native void setInfoLogLevel(long handle, byte logLevel);
|
||||
|
|
|
@ -158,6 +158,25 @@ public interface DBOptionsInterface<T extends DBOptionsInterface> {
|
|||
*/
|
||||
T setRateLimiter(RateLimiter rateLimiter);
|
||||
|
||||
/**
|
||||
* Use to track SST files and control their file deletion rate.
|
||||
*
|
||||
* Features:
|
||||
* - Throttle the deletion rate of the SST files.
|
||||
* - Keep track the total size of all SST files.
|
||||
* - Set a maximum allowed space limit for SST files that when reached
|
||||
* the DB wont do any further flushes or compactions and will set the
|
||||
* background error.
|
||||
* - Can be shared between multiple dbs.
|
||||
*
|
||||
* Limitations:
|
||||
* - Only track and throttle deletes of SST files in
|
||||
* first db_path (db_name if db_paths is empty).
|
||||
*
|
||||
* @param sstFileManager The SST File Manager for the db.
|
||||
*/
|
||||
T setSstFileManager(SstFileManager sstFileManager);
|
||||
|
||||
/**
|
||||
* <p>Any internal progress/error information generated by
|
||||
* the db will be written to the Logger if it is non-nullptr,
|
||||
|
|
|
@ -1032,6 +1032,13 @@ public class Options extends RocksObject
|
|||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Options setSstFileManager(final SstFileManager sstFileManager) {
|
||||
assert(isOwningHandle());
|
||||
setSstFileManager(nativeHandle_, sstFileManager.nativeHandle_);
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Options setLogger(final Logger logger) {
|
||||
assert(isOwningHandle());
|
||||
|
@ -1588,6 +1595,8 @@ public class Options extends RocksObject
|
|||
private native boolean paranoidChecks(long handle);
|
||||
private native void setRateLimiter(long handle,
|
||||
long rateLimiterHandle);
|
||||
private native void setSstFileManager(final long handle,
|
||||
final long sstFileManagerHandle);
|
||||
private native void setLogger(long handle,
|
||||
long loggerHandle);
|
||||
private native void setInfoLogLevel(long handle, byte logLevel);
|
||||
|
|
|
@ -0,0 +1,241 @@
|
|||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
|
||||
package org.rocksdb;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* SstFileManager is used to track SST files in the DB and control their
|
||||
* deletion rate.
|
||||
*
|
||||
* All SstFileManager public functions are thread-safe.
|
||||
*
|
||||
* SstFileManager is not extensible.
|
||||
*/
|
||||
//@ThreadSafe
|
||||
public final class SstFileManager extends RocksObject {
|
||||
|
||||
public static final long RATE_BYTES_PER_SEC_DEFAULT = 0;
|
||||
public static final boolean DELETE_EXISTING_TRASH_DEFAULT = true;
|
||||
public static final double MAX_TRASH_DB_RATION_DEFAULT = 0.25;
|
||||
public static final long BYTES_MAX_DELETE_CHUNK_DEFAULT = 64 * 1024 * 1024;
|
||||
|
||||
/**
|
||||
* Create a new SstFileManager that can be shared among multiple RocksDB
|
||||
* instances to track SST file and control there deletion rate.
|
||||
*
|
||||
* @param env the environment.
|
||||
*/
|
||||
public SstFileManager(final Env env) throws RocksDBException {
|
||||
this(env, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new SstFileManager that can be shared among multiple RocksDB
|
||||
* instances to track SST file and control there deletion rate.
|
||||
*
|
||||
* @param env the environment.
|
||||
* @param logger if not null, the logger will be used to log errors.
|
||||
*/
|
||||
public SstFileManager(final Env env, /*@Nullable*/ final Logger logger)
|
||||
throws RocksDBException {
|
||||
this(env, logger, RATE_BYTES_PER_SEC_DEFAULT);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new SstFileManager that can be shared among multiple RocksDB
|
||||
* instances to track SST file and control there deletion rate.
|
||||
*
|
||||
* @param env the environment.
|
||||
* @param logger if not null, the logger will be used to log errors.
|
||||
*
|
||||
* == Deletion rate limiting specific arguments ==
|
||||
* @param rateBytesPerSec how many bytes should be deleted per second, If
|
||||
* this value is set to 1024 (1 Kb / sec) and we deleted a file of size
|
||||
* 4 Kb in 1 second, we will wait for another 3 seconds before we delete
|
||||
* other files, Set to 0 to disable deletion rate limiting.
|
||||
*/
|
||||
public SstFileManager(final Env env, /*@Nullable*/ final Logger logger,
|
||||
final long rateBytesPerSec) throws RocksDBException {
|
||||
this(env, logger, rateBytesPerSec, MAX_TRASH_DB_RATION_DEFAULT);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new SstFileManager that can be shared among multiple RocksDB
|
||||
* instances to track SST file and control there deletion rate.
|
||||
*
|
||||
* @param env the environment.
|
||||
* @param logger if not null, the logger will be used to log errors.
|
||||
*
|
||||
* == Deletion rate limiting specific arguments ==
|
||||
* @param rateBytesPerSec how many bytes should be deleted per second, If
|
||||
* this value is set to 1024 (1 Kb / sec) and we deleted a file of size
|
||||
* 4 Kb in 1 second, we will wait for another 3 seconds before we delete
|
||||
* other files, Set to 0 to disable deletion rate limiting.
|
||||
* @param maxTrashDbRatio if the trash size constitutes for more than this
|
||||
* fraction of the total DB size we will start deleting new files passed
|
||||
* to DeleteScheduler immediately.
|
||||
*/
|
||||
public SstFileManager(final Env env, /*@Nullable*/ final Logger logger,
|
||||
final long rateBytesPerSec, final double maxTrashDbRatio)
|
||||
throws RocksDBException {
|
||||
this(env, logger, rateBytesPerSec, maxTrashDbRatio,
|
||||
BYTES_MAX_DELETE_CHUNK_DEFAULT);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new SstFileManager that can be shared among multiple RocksDB
|
||||
* instances to track SST file and control there deletion rate.
|
||||
*
|
||||
* @param env the environment.
|
||||
* @param logger if not null, the logger will be used to log errors.
|
||||
*
|
||||
* == Deletion rate limiting specific arguments ==
|
||||
* @param rateBytesPerSec how many bytes should be deleted per second, If
|
||||
* this value is set to 1024 (1 Kb / sec) and we deleted a file of size
|
||||
* 4 Kb in 1 second, we will wait for another 3 seconds before we delete
|
||||
* other files, Set to 0 to disable deletion rate limiting.
|
||||
* @param maxTrashDbRatio if the trash size constitutes for more than this
|
||||
* fraction of the total DB size we will start deleting new files passed
|
||||
* to DeleteScheduler immediately.
|
||||
* @param bytesMaxDeleteChunk if a single file is larger than delete chunk,
|
||||
* ftruncate the file by this size each time, rather than dropping the whole
|
||||
* file. 0 means to always delete the whole file.
|
||||
*/
|
||||
public SstFileManager(final Env env, /*@Nullable*/final Logger logger,
|
||||
final long rateBytesPerSec, final double maxTrashDbRatio,
|
||||
final long bytesMaxDeleteChunk) throws RocksDBException {
|
||||
super(newSstFileManager(env.nativeHandle_,
|
||||
logger != null ? logger.nativeHandle_ : 0,
|
||||
rateBytesPerSec, maxTrashDbRatio, bytesMaxDeleteChunk));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Update the maximum allowed space that should be used by RocksDB, if
|
||||
* the total size of the SST files exceeds {@code maxAllowedSpace}, writes to
|
||||
* RocksDB will fail.
|
||||
*
|
||||
* Setting {@code maxAllowedSpace} to 0 will disable this feature;
|
||||
* maximum allowed space will be infinite (Default value).
|
||||
*
|
||||
* @param maxAllowedSpace the maximum allowed space that should be used by
|
||||
* RocksDB.
|
||||
*/
|
||||
public void setMaxAllowedSpaceUsage(final long maxAllowedSpace) {
|
||||
setMaxAllowedSpaceUsage(nativeHandle_, maxAllowedSpace);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the amount of buffer room each compaction should be able to leave.
|
||||
* In other words, at its maximum disk space consumption, the compaction
|
||||
* should still leave {@code compactionBufferSize} available on the disk so
|
||||
* that other background functions may continue, such as logging and flushing.
|
||||
*
|
||||
* @param compactionBufferSize the amount of buffer room each compaction
|
||||
* should be able to leave.
|
||||
*/
|
||||
public void setCompactionBufferSize(final long compactionBufferSize) {
|
||||
setCompactionBufferSize(nativeHandle_, compactionBufferSize);
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines if the total size of SST files exceeded the maximum allowed
|
||||
* space usage.
|
||||
*
|
||||
* @return true when the maximum allows space usage has been exceeded.
|
||||
*/
|
||||
public boolean isMaxAllowedSpaceReached() {
|
||||
return isMaxAllowedSpaceReached(nativeHandle_);
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines if the total size of SST files as well as estimated size
|
||||
* of ongoing compactions exceeds the maximums allowed space usage.
|
||||
*
|
||||
* @return true when the total size of SST files as well as estimated size
|
||||
* of ongoing compactions exceeds the maximums allowed space usage.
|
||||
*/
|
||||
public boolean isMaxAllowedSpaceReachedIncludingCompactions() {
|
||||
return isMaxAllowedSpaceReachedIncludingCompactions(nativeHandle_);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the total size of all tracked files.
|
||||
*
|
||||
* @return the total size of all tracked files.
|
||||
*/
|
||||
public long getTotalSize() {
|
||||
return getTotalSize(nativeHandle_);
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets all tracked files and their corresponding sizes.
|
||||
*
|
||||
* @return a map containing all tracked files and there corresponding sizes.
|
||||
*/
|
||||
public Map<String, Long> getTrackedFiles() {
|
||||
return getTrackedFiles(nativeHandle_);
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the delete rate limit.
|
||||
*
|
||||
* @return the delete rate limit (in bytes per second).
|
||||
*/
|
||||
public long getDeleteRateBytesPerSecond() {
|
||||
return getDeleteRateBytesPerSecond(nativeHandle_);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the delete rate limit.
|
||||
*
|
||||
* Zero means disable delete rate limiting and delete files immediately.
|
||||
*
|
||||
* @param deleteRate the delete rate limit (in bytes per second).
|
||||
*/
|
||||
public void setDeleteRateBytesPerSecond(final long deleteRate) {
|
||||
setDeleteRateBytesPerSecond(nativeHandle_, deleteRate);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the trash/DB size ratio where new files will be deleted immediately.
|
||||
*
|
||||
* @return the trash/DB size ratio.
|
||||
*/
|
||||
public double getMaxTrashDBRatio() {
|
||||
return getMaxTrashDBRatio(nativeHandle_);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the trash/DB size ratio where new files will be deleted immediately.
|
||||
*
|
||||
* @param ratio the trash/DB size ratio.
|
||||
*/
|
||||
public void setMaxTrashDBRatio(final double ratio) {
|
||||
setMaxTrashDBRatio(nativeHandle_, ratio);
|
||||
}
|
||||
|
||||
private native static long newSstFileManager(final long handle,
|
||||
final long logger_handle, final long rateBytesPerSec,
|
||||
final double maxTrashDbRatio, final long bytesMaxDeleteChunk)
|
||||
throws RocksDBException;
|
||||
private native void setMaxAllowedSpaceUsage(final long handle,
|
||||
final long maxAllowedSpace);
|
||||
private native void setCompactionBufferSize(final long handle,
|
||||
final long compactionBufferSize);
|
||||
private native boolean isMaxAllowedSpaceReached(final long handle);
|
||||
private native boolean isMaxAllowedSpaceReachedIncludingCompactions(
|
||||
final long handle);
|
||||
private native long getTotalSize(final long handle);
|
||||
private native Map<String, Long> getTrackedFiles(final long handle);
|
||||
private native long getDeleteRateBytesPerSecond(final long handle);
|
||||
private native void setDeleteRateBytesPerSecond(final long handle,
|
||||
final long deleteRate);
|
||||
private native double getMaxTrashDBRatio(final long handle);
|
||||
private native void setMaxTrashDBRatio(final long handle, final double ratio);
|
||||
@Override protected final native void disposeInternal(final long handle);
|
||||
}
|
|
@ -643,6 +643,15 @@ public class DBOptionsTest {
|
|||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void sstFileManager() throws RocksDBException {
|
||||
try (final DBOptions options = new DBOptions();
|
||||
final SstFileManager sstFileManager =
|
||||
new SstFileManager(Env.getDefault())) {
|
||||
options.setSstFileManager(sstFileManager);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void statistics() {
|
||||
try(final DBOptions options = new DBOptions()) {
|
||||
|
|
|
@ -999,6 +999,15 @@ public class OptionsTest {
|
|||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void sstFileManager() throws RocksDBException {
|
||||
try (final Options options = new Options();
|
||||
final SstFileManager sstFileManager =
|
||||
new SstFileManager(Env.getDefault())) {
|
||||
options.setSstFileManager(sstFileManager);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldSetTestPrefixExtractor() {
|
||||
try (final Options options = new Options()) {
|
||||
|
|
|
@ -0,0 +1,66 @@
|
|||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
|
||||
package org.rocksdb;
|
||||
|
||||
import org.junit.Test;
|
||||
|
||||
import java.util.Collections;
|
||||
|
||||
import static org.assertj.core.api.Assertions.*;
|
||||
|
||||
public class SstFileManagerTest {
|
||||
|
||||
@Test
|
||||
public void maxAllowedSpaceUsage() throws RocksDBException {
|
||||
try (final SstFileManager sstFileManager = new SstFileManager(Env.getDefault())) {
|
||||
sstFileManager.setMaxAllowedSpaceUsage(1024 * 1024 * 64);
|
||||
assertThat(sstFileManager.isMaxAllowedSpaceReached()).isFalse();
|
||||
assertThat(sstFileManager.isMaxAllowedSpaceReachedIncludingCompactions()).isFalse();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void compactionBufferSize() throws RocksDBException {
|
||||
try (final SstFileManager sstFileManager = new SstFileManager(Env.getDefault())) {
|
||||
sstFileManager.setCompactionBufferSize(1024 * 1024 * 10);
|
||||
assertThat(sstFileManager.isMaxAllowedSpaceReachedIncludingCompactions()).isFalse();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void totalSize() throws RocksDBException {
|
||||
try (final SstFileManager sstFileManager = new SstFileManager(Env.getDefault())) {
|
||||
assertThat(sstFileManager.getTotalSize()).isEqualTo(0);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void trackedFiles() throws RocksDBException {
|
||||
try (final SstFileManager sstFileManager = new SstFileManager(Env.getDefault())) {
|
||||
assertThat(sstFileManager.getTrackedFiles()).isEqualTo(Collections.emptyMap());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void deleteRateBytesPerSecond() throws RocksDBException {
|
||||
try (final SstFileManager sstFileManager = new SstFileManager(Env.getDefault())) {
|
||||
assertThat(sstFileManager.getDeleteRateBytesPerSecond()).isEqualTo(SstFileManager.RATE_BYTES_PER_SEC_DEFAULT);
|
||||
final long ratePerSecond = 1024 * 1024 * 52;
|
||||
sstFileManager.setDeleteRateBytesPerSecond(ratePerSecond);
|
||||
assertThat(sstFileManager.getDeleteRateBytesPerSecond()).isEqualTo(ratePerSecond);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void maxTrashDBRatio() throws RocksDBException {
|
||||
try (final SstFileManager sstFileManager = new SstFileManager(Env.getDefault())) {
|
||||
assertThat(sstFileManager.getMaxTrashDBRatio()).isEqualTo(SstFileManager.MAX_TRASH_DB_RATION_DEFAULT);
|
||||
final double trashRatio = 0.2;
|
||||
sstFileManager.setMaxTrashDBRatio(trashRatio);
|
||||
assertThat(sstFileManager.getMaxTrashDBRatio()).isEqualTo(trashRatio);
|
||||
}
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue